Index: /branches/pap/DataStore/lib/DataStore/FileSet/Parser.pm =================================================================== --- /branches/pap/DataStore/lib/DataStore/FileSet/Parser.pm (revision 28483) +++ /branches/pap/DataStore/lib/DataStore/FileSet/Parser.pm (revision 28484) @@ -100,5 +100,5 @@ $p{base_uri} =~ qr|(.*?/)index.txt|; - $self->{base_uri} = $1; + $self->{base_uri} = $1; return $self; @@ -129,5 +129,4 @@ { type => SCALAR, - regex => qr/\S+/, # string with at least 1 non WS char } ); @@ -139,5 +138,5 @@ next LINE if $line =~ /^\s*$/; - # comment lines + # comment lines next LINE if $line =~ /^\s*\#/; @@ -183,5 +182,5 @@ carp "line $lineno: type $type unknown: $line"; next LINE; - } + } my @extra = @fields[3 .. $#fields] if $#fields >= 3; @@ -192,5 +191,5 @@ datetime => $datetime, type => $type, - extra => \@extra, + extra => \@extra, uri => $self->base_uri . $fileset . '/index.txt', }); Index: /branches/pap/DataStore/lib/DataStore/Product/Parser.pm =================================================================== --- /branches/pap/DataStore/lib/DataStore/Product/Parser.pm (revision 28483) +++ /branches/pap/DataStore/lib/DataStore/Product/Parser.pm (revision 28484) @@ -132,5 +132,4 @@ { type => SCALAR, - regex => qr/\S+/, # string with at least 1 non WS char } ); @@ -142,5 +141,5 @@ next LINE if $line =~ /^\s*$/; - # comment lines + # comment lines next LINE if $line =~ /^\s*\#/; @@ -159,5 +158,5 @@ next LINE; } - + # strip leading and trailing whitespace $field =~ s/^\s+//; @@ -184,5 +183,5 @@ carp "line $lineno: type $type unknown: $line"; next LINE; - } + } # fifo Index: /branches/pap/DataStore/lib/DataStore/Utils.pm =================================================================== --- /branches/pap/DataStore/lib/DataStore/Utils.pm (revision 28483) +++ /branches/pap/DataStore/lib/DataStore/Utils.pm (revision 28484) @@ -44,5 +44,5 @@ %KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits ); %KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS IPP-DIST); -%KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc); +%KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc dqresults); =pod Index: /branches/pap/Nebulous-Server/bin/nebdiskd =================================================================== --- /branches/pap/Nebulous-Server/bin/nebdiskd (revision 28483) +++ /branches/pap/Nebulous-Server/bin/nebdiskd (revision 28484) @@ -224,4 +224,13 @@ } }; + + # fetch stats on the mounted device. this has to be done AFTER + # we determine if it's a valid mountpoint incase + # is_mountpoint() invokes the automounter + my $dev_info = df($mountpoint, 1024); + unless (defined $dev_info) { + $valid_mountpoint = 0; + } + if (!$valid_mountpoint) { # try is_mountpoint() again if $retry > 1 @@ -254,5 +263,5 @@ # we determine if it's a valid mountpoint incase # is_mountpoint() invokes the automounter - my $dev_info = df($mountpoint, 1024); + $dev_info = df($mountpoint, 1024); unless (defined $dev_info) { $log->error("can't find device info for $mountpoint"); Index: /branches/pap/Nebulous-Server/lib/Nebulous/Server.pm =================================================================== --- /branches/pap/Nebulous-Server/lib/Nebulous/Server.pm (revision 28483) +++ /branches/pap/Nebulous-Server/lib/Nebulous/Server.pm (revision 28484) @@ -1598,4 +1598,58 @@ } +sub find_ext_id_by_volume +{ + my $self = shift; + my $log = $self->log; + $log->debug("entered - @_"); + my ($vol_name,$limit) = validate_pos(@_, + { + type => SCALAR, +# # callbacks => { +# # 'is valid volume name' => sub { +# # return 1 if not defined $_[0]; +# # $self->_is_valid_volume_name($_[0]) +# # }, +# }, + }, + { + type => SCALAR|UNDEF, + optional => 1, + }, + ); + unless (defined($limit)) { + $limit = 50000; + } + + my $sql = $self->sql; + my @ext_ids; + my $db = $self->_db_for_index(0); + eval { + my $query; + $query = $db->prepare_cached( $sql->get_ext_id_by_vol_name ); + my $rows = $query->execute($vol_name, 1,$limit); + unless ($rows > 0) { + $query->finish; + $log->logdie("no instances on storage volume or volume is not avaiable for volume: $vol_name"); + } + while (my $row = $query->fetchrow_hashref) { + my $ext_id = $row->{ 'ext_id' }; + push @ext_ids, $ext_id if $ext_id; + } + }; + if ($@) { + $db->rollback; + $log->logdie("database error: $@"); + } + + # XXX remove this? + $log->logdie("no ext_ids found") unless (scalar @ext_ids); + + $log->debug("found: \@ext_ids"); + + $log->debug("leaving"); + + return \@ext_ids; +} sub find_instances Index: /branches/pap/Nebulous-Server/lib/Nebulous/Server/SQL.pm =================================================================== --- /branches/pap/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 28483) +++ /branches/pap/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 28484) @@ -266,4 +266,18 @@ AND mountedvol.name = ? AND mountedvol.available = ? + }, + get_ext_id_by_vol_name => qq{ + SELECT + ext_id + FROM instance + JOIN storage_object + USING (so_id) + JOIN mountedvol + USING(vol_id) + JOIN volume + USING(vol_id) + WHERE volume.name = ? + AND mountedvol.available = ? + LIMIT ? }, # volume handler Index: /branches/pap/Nebulous/Build.PL =================================================================== --- /branches/pap/Nebulous/Build.PL (revision 28483) +++ /branches/pap/Nebulous/Build.PL (revision 28484) @@ -113,7 +113,9 @@ bin/neb-locate bin/neb-ls + bin/neb-migrate bin/neb-mv bin/neb-replicate bin/neb-rm + bin/neb-shift bin/neb-stat bin/neb-swap Index: /branches/pap/Nebulous/MANIFEST =================================================================== --- /branches/pap/Nebulous/MANIFEST (revision 28483) +++ /branches/pap/Nebulous/MANIFEST (revision 28484) @@ -14,7 +14,9 @@ bin/neb-insert bin/neb-ls +bin/neb-migrate bin/neb-mv bin/neb-replicate bin/neb-rm +bin/neb-shift bin/neb-stat bin/neb-swap Index: /branches/pap/Nebulous/bin/neb-migrate =================================================================== --- /branches/pap/Nebulous/bin/neb-migrate (revision 28484) +++ /branches/pap/Nebulous/bin/neb-migrate (revision 28484) @@ -0,0 +1,180 @@ +#!/usr/bin/env perl + +# Copyright (C) 2007-2010 Joshua Hoblitt/Chris Waters + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.03'; + +use Nebulous::Client; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +my ( + $server, + $limit, + $dest_volume, + $no_pretend, +# $recursive, +); + +$server = $ENV{'NEB_SERVER'} unless $server; + +GetOptions( + 'server|s=s' => \$server, + 'limit|l=s' => \$limit, + 'destination|d=s' => \$dest_volume, + 'no_pretend' => \$no_pretend, +) || pod2usage( 2 ); + +if ($limit && $limit <= 0) { + die "Limit must be positive."; +} + + +my $host = shift; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --server", -exitval => 2 ) + unless $server; + +my $neb = Nebulous::Client->new( + proxy => "$server", +); + +die "can't connected to Nebulous Server: $server" + unless defined $neb; + +# Find the keys on this volume. +my $keys; +if ($limit) { + $keys = $neb->find_ext_id_by_volume($host,$limit); +} +else { + $keys = $neb->find_ext_id_by_volume($host); +} + +# Print the keys on this volume, and if we're not pretending, do the neb-shift code to move them off it. +if ($keys) { + foreach my $src (@{ $keys }) { + print "$src"; + if ($no_pretend) { + print "\t"; + my $uris = $neb->find_instances($src, "~$host"); + unless (defined $uris) { + die "No copy of $src exists on $host"; + } + + my $status; + if ($dest_volume) { + $status = $neb->replicate( $src, $dest_volume); + } + else { + $status = $neb->replicate($src); + } + die "Replicate phase failed for $src ($host) ($dest_volume)" unless $status; + + $status = $neb->cull($src,$host); + + warn "Cull phase failed for $src from $host" unless $status; + + print "Success!"; + } + print "\n"; + } +} + +__END__ + +=pod + +=head1 NAME + +neb-migrate - list Nebulous keys on a specific volume and move them off it + +=head1 SYNOPSIS + + neb-migrate [--server ] [--limit ] [--no_pretend] + +=head1 DESCRIPTION + +This program list Nebulous keys found on volume <>, and will move them +to another volume if requested to do so. + +=head1 OPTIONS + +=over 4 + +Optional + +=item * --limit + +Limit the number of results. + +=item * --no_pretend + +Actually move the files off of the specified host. + +=item * --destination|-d + +Specify the destination volume for the keys moved. + +=item * --server|-s + +URL of the Nebulous server to connect to. + +Optional if the appropriate environment variable is set. + +=back + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --server|-s + +=back + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Chris Waters + +=head1 COPYRIGHT + +Copyright (C) 2007-2010 Josh Hoblitt / Chris Waters. All rights reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L, L, +L, C + +=cut Index: /branches/pap/Nebulous/bin/neb-shift =================================================================== --- /branches/pap/Nebulous/bin/neb-shift (revision 28484) +++ /branches/pap/Nebulous/bin/neb-shift (revision 28484) @@ -0,0 +1,146 @@ +#!/usr/bin/env perl + +# Copyright (C) 2007-2008 Joshua Hoblitt +# +# $Id: neb-shift,v 1.5 2008-01-22 21:40:39 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Nebulous::Client; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +my ($volume, $server); + +$server = $ENV{'NEB_SERVER'} unless $server; + +GetOptions( + 'volume=s' => \$volume, + 'server|s=s' => \$server, +) || pod2usage( 2 ); + +my $src = shift; +my $abandon_vol = shift; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --server ", + -exitval => 2 ) + unless defined $server and defined $src and defined $abandon_vol; + +my $neb = Nebulous::Client->new( + proxy => "$server", +); + +die "can't connected to Nebulous Server: $server" + unless defined $neb; + +my $uris = $neb->find_instances($src, "~$abandon_vol"); +unless (defined $uris) { + die "No copy of $src exists on $abandon_vol"; +} + +my $status; +if ($volume) { + $status = $neb->replicate( $src, $volume); +} +else { + $status = $neb->replicate($src); +} +die "Replicate phase failed for $src ($abandon_vol) ($volume)" unless $status; + +$status = $neb->cull($src,$abandon_vol); + +warn "Cull phase failed for $src from $abandon_vol" unless $status; + +__END__ + +=pod + +=head1 NAME + +neb-shift - list Nebulous keys + +=head1 SYNOPSIS + + neb-shift [--server ] [--volume ] + +=head1 DESCRIPTION + +Shift a Nebulous storage key from a specific source volume, optionally assigning it +to a specific destination volume. + +=head1 OPTIONS + +=over 4 + +=item * --server|-s + +URL of the Nebulous server to connect to. + +Optional if the appropriate environment variable is set. + +=item * --volume + +The volume to place the key's new storage instance on. + +Optional. + +=back + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --server|-s + +=back + +=head1 CREDITS + +Just me, myself, and I. + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Joshua Hoblitt + +=head1 COPYRIGHT + +Copyright (C) 2007-2008 Joshua Hoblitt. All rights reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L, L, +L, C + +=cut Index: /branches/pap/Nebulous/lib/Nebulous/Client.pm =================================================================== --- /branches/pap/Nebulous/lib/Nebulous/Client.pm (revision 28483) +++ /branches/pap/Nebulous/lib/Nebulous/Client.pm (revision 28484) @@ -790,4 +790,38 @@ } +sub find_ext_id_by_volume +{ + my $self = shift; + my ($vol_name, $limit) = validate_pos( @_, + { + type => SCALAR, + }, + { + type => SCALAR|UNDEF, + optional => 1, + }, + ); + + $log->debug( "entered - @_" ); + + my $response = $self->{ 'server' }->find_ext_id_by_volume( $vol_name, $limit); + if ( $response->fault ) { + $self->set_err($response->faultstring); + if ($response->faultstring =~ /no instances on storage volume/) { + $log->debug( "leaving" ); + return; + } + + $log->logdie("unhandled fault - ", $self->err); + } + + my $ext_ids = $response->result; + + $log->debug( "server found @$ext_ids" ); + $log->debug( "leaving" ); + + return($ext_ids); +} + sub find_instances_for_cull { Index: /branches/pap/Ohana/Makefile.in =================================================================== --- /branches/pap/Ohana/Makefile.in (revision 28483) +++ /branches/pap/Ohana/Makefile.in (revision 28484) @@ -31,4 +31,5 @@ delstar \ dvosplit \ +dvomerge \ elixir \ gastro \ Index: /branches/pap/Ohana/configure.tcsh =================================================================== --- /branches/pap/Ohana/configure.tcsh (revision 28483) +++ /branches/pap/Ohana/configure.tcsh (revision 28484) @@ -186,5 +186,9 @@ # use_gnu99 -if ($use_gnu99) set CFLAGS = "$CFLAGS -std=gnu99" +if ($use_gnu99) then + set CFLAGS = "$CFLAGS -std=gnu99" +else + set CFLAGS = "$CFLAGS -std=gnu89" +endif # no_largefiles @@ -670,5 +674,5 @@ --use-tcmalloc use the alternate tcmalloc from Google --use-gnu99 use gnu99 flags to ensure C99 compatibility - --use-largefiles ensure large file compatibility + --no-largefiles skip large file compatibility Installation directories: Index: /branches/pap/Ohana/src/addstar/src/ConfigInit.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/ConfigInit.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/ConfigInit.c (revision 28484) @@ -7,5 +7,5 @@ double ZERO_POINT; char *config, *file; - char RadiusWord[80]; + char RadiusWord[80], tmpword[80]; char CatdirPhotcodeFile[256]; char MasterPhotcodeFile[256]; @@ -68,6 +68,29 @@ ScanConfig (config, "CCDNUM-KEYWORD", "%s", 0, CCDNumKeyword); ScanConfig (config, "ST-KEYWORD", "%s", 0, STKeyword); - ScanConfig (config, "OBSERVATORY-LATITUDE", "%lf", 0, &Latitude); - ScanConfig (config, "OBSERVATORY-LONGITUDE", "%lf", 0, &Longitude); + + ScanConfig (config, "OBSERVATORY-LATITUDE", "%s", 0, tmpword); + if (!strcasecmp(tmpword, "NONE")) { + fprintf (stderr, "observatory latitude is not set\n"); + Latitude = NAN; + } else { + ScanConfig (config, "OBSERVATORY-LATITUDE", "%lf", 0, &Latitude); + } + ScanConfig (config, "OBSERVATORY-LONGITUDE", "%s", 0, tmpword); + if (!strcasecmp(tmpword, "NONE")) { + fprintf (stderr, "observatory longitude is not set\n"); + Longitude = NAN; + } else { + ScanConfig (config, "OBSERVATORY-LONGITUDE", "%lf", 0, &Longitude); + } + fprintf (stderr, "observatory @ (%f,%f)\n", Longitude, Latitude); + + if (!strcasecmp(STKeyword, "NONE")) { + if (isnan(Longitude)) { + fprintf (stderr, "WARNING: ST cannot be determined for this image (no ST Keyword, no longitude)\n"); + } else { + fprintf (stderr, "ST Keyword is not defined, ST will be derived from time & longitude\n"); + } + } + ScanConfig (config, "SUBPIX_DATAFILE", "%s", 0, SubpixDatafile); Index: /branches/pap/Ohana/src/addstar/src/FilterStars.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/FilterStars.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/FilterStars.c (revision 28484) @@ -54,5 +54,5 @@ /* calculate accurate per-star airmass and azimuth */ stars[N].measure.airmass = airmass (image[0].secz, stars[N].average.R, stars[N].average.D, image[0].sidtime, image[0].latitude); - stars[N].measure.az = azimuth (image[0].sidtime - stars[N].average.R, stars[N].average.D, image[0].latitude); + stars[N].measure.az = azimuth (15.0*image[0].sidtime - stars[N].average.R, stars[N].average.D, image[0].latitude); stars[N].measure.Mcal = image[0].Mcal; stars[N].measure.t = image[0].tzero + 1e-4*stars[N].measure.Yccd*image[0].trate; /* trate is in 0.1 msec / row */ Index: /branches/pap/Ohana/src/addstar/src/LoadStars.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/LoadStars.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/LoadStars.c (revision 28484) @@ -81,5 +81,5 @@ continue; } - if (VERBOSE) fprintf (stderr, "file %s has %d headers, including %lld images\n", file[i], Nheaders, (long long) NheaderSets); + if (VERBOSE) fprintf (stderr, "file %s has %d headers, including "OFF_T_FMT" images\n", file[i], Nheaders, NheaderSets); /* supplied photcode is incompatible with multi-chip images */ Index: /branches/pap/Ohana/src/addstar/src/ReadImageHeader.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/ReadImageHeader.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/ReadImageHeader.c (revision 28484) @@ -148,5 +148,5 @@ image[0].fwhm_y = tmp * 25.0 * image[0].coords.cdelt1 * 3600.0; - if (STKeyword[0]) { + if (STKeyword[0] && strcasecmp(STKeyword, "NONE")) { /* get ST (used for airmass calculation) */ gfits_scan (header, STKeyword, "%s", 1, line); Index: /branches/pap/Ohana/src/addstar/src/SkyTableFromTychoIndex.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/SkyTableFromTychoIndex.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/SkyTableFromTychoIndex.c (revision 28484) @@ -147,5 +147,5 @@ skytable[0].filename[i] = NULL; } - if (VERBOSE) fprintf (stderr, "loaded %lld tables from tycho index\n", (long long) skytable[0].Nregions); + if (VERBOSE) fprintf (stderr, "loaded "OFF_T_FMT" tables from tycho index\n", skytable[0].Nregions); return (skytable); Index: /branches/pap/Ohana/src/addstar/src/UpdateImageIDs.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/UpdateImageIDs.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/UpdateImageIDs.c (revision 28484) @@ -55,7 +55,7 @@ if (isEmpty) { - dvo_image_addrows (&db, NULL, 0); + if (!dvo_image_addrows (&db, NULL, 0)) Shutdown ("failed to create image table"); SetProtect (TRUE); - dvo_image_update (&db, VERBOSE); + if (!dvo_image_update (&db, VERBOSE)) Shutdown ("failed to update image table"); SetProtect (FALSE); } else { Index: /branches/pap/Ohana/src/addstar/src/addstar.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/addstar.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/addstar.c (revision 28484) @@ -48,5 +48,5 @@ newlist = SkyListByImage (sky, -1, &images[i]); SkyListMerge (&skylist, newlist); - if (VERBOSE) fprintf (stderr, "added %lld regions to yield %lld total\n", (long long) newlist[0].Nregions, (long long) skylist[0].Nregions); + if (VERBOSE) fprintf (stderr, "added "OFF_T_FMT" regions to yield "OFF_T_FMT" total\n", newlist[0].Nregions, skylist[0].Nregions); SkyListFree (newlist); } @@ -80,5 +80,5 @@ skylist = tmp; } - if (VERBOSE) fprintf (stderr, "writing to %lld regions\n", (long long) skylist[0].Nregions); + if (VERBOSE) fprintf (stderr, "writing to "OFF_T_FMT" regions\n", skylist[0].Nregions); /* don't load the object tables for only_images, unless we are getting the calibration. */ @@ -213,5 +213,5 @@ gettimeofday (&stop, NULL); dtime = DTIME (stop, start); - fprintf (stderr, "SUCCESS: elapsed time %9.4f sec for %5d stars (%5d matches), %6lld average, %7lld measure\n", dtime, Nstars, Nmatch, (long long) Naverage, (long long) Nmeasure); + fprintf (stderr, "SUCCESS: elapsed time %9.4f sec for %5d stars (%5d matches), "OFF_T_FMT" average, "OFF_T_FMT" measure\n", dtime, Nstars, Nmatch, Naverage, Nmeasure); exit (0); Index: /branches/pap/Ohana/src/addstar/src/find_matches.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/find_matches.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/find_matches.c (revision 28484) @@ -3,5 +3,5 @@ int find_matches (SkyRegion *region, Stars *stars, unsigned int NstarsIn, Catalog *catalog, AddstarClientOptions options) { - off_t i, j, n, N, J, status, Nstars; + off_t i, j, n, N, J, Nstars; double RADIUS, RADIUS2; double *X1, *Y1, *X2, *Y2; @@ -9,5 +9,5 @@ off_t *N1, *N2, *next_meas; off_t Nave, NAVE, Nmeas, NMEAS, Nmatch; - int Nsecfilt, Nsec; + int Nsecfilt, Nsec, status; unsigned int objID, catID; Coords tcoords; @@ -356,5 +356,5 @@ catalog[0].Nmeasure = Nmeas; catalog[0].Nsecf_mem = Nave*Nsecfilt; - if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: %lld %lld %lld, (%lld matches)\n", (long long) Nstars, (long long) Nave, (long long) Nmeas, (long long) Nmatch); + if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n", Nstars, Nave, Nmeas, Nmatch); free (catalog[0].found); Index: /branches/pap/Ohana/src/addstar/src/find_matches_closest.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/find_matches_closest.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/find_matches_closest.c (revision 28484) @@ -350,5 +350,5 @@ catalog[0].Nmeasure = Nmeas; catalog[0].Nsecf_mem = Nave*Nsecfilt; - if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: %lld %lld %lld, (%lld matches)\n", (long long) Nstars, (long long) Nave, (long long) Nmeas, (long long) Nmatch); + if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n", Nstars, Nave, Nmeas, Nmatch); free (catalog[0].found); Index: /branches/pap/Ohana/src/addstar/src/find_matches_closest_refstars.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/find_matches_closest_refstars.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/find_matches_closest_refstars.c (revision 28484) @@ -361,5 +361,5 @@ catalog[0].Nmeasure = Nmeas; catalog[0].Nsecf_mem = Nave*Nsecfilt; - if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: %lld %lld %lld, (%lld matches)\n", (long long) Nstars, (long long) Nave, (long long) Nmeas, (long long) Nmatch); + if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n", Nstars, Nave, Nmeas, Nmatch); free (catalog[0].found); Index: /branches/pap/Ohana/src/addstar/src/find_matches_refstars.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/find_matches_refstars.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/find_matches_refstars.c (revision 28484) @@ -329,5 +329,5 @@ catalog[0].Nmeasure = Nmeas; catalog[0].Nsecf_mem = Nave*Nsecfilt; - if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: %lld %lld %lld, (%lld matches)\n", (long long) Nstars, (long long) Nave, (long long) Nmeas, (long long) Nmatch); + if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: %d "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n", Nstars, Nave, Nmeas, Nmatch); free (catalog[0].found); Index: /branches/pap/Ohana/src/addstar/src/sky_tessalation.c =================================================================== --- /branches/pap/Ohana/src/addstar/src/sky_tessalation.c (revision 28483) +++ /branches/pap/Ohana/src/addstar/src/sky_tessalation.c (revision 28484) @@ -940,5 +940,5 @@ int dvo_image_clear_vtable (FITS_DB *db) { - int i; + int i, nbytes; // free memory used by the current vtable rows @@ -951,5 +951,6 @@ // reset db[0].theader(NAXIS1) to match Image - gfits_modify (&db[0].theader, "NAXIS1", "%lld", 1, (long long) sizeof(Image)); + nbytes = sizeof(Image); + gfits_modify (&db[0].theader, "NAXIS1", "%d", 1, nbytes); db[0].theader.Naxis[0] = sizeof(Image); Index: /branches/pap/Ohana/src/addstar/test/dvomerge.dvo =================================================================== --- /branches/pap/Ohana/src/addstar/test/dvomerge.dvo (revision 28483) +++ /branches/pap/Ohana/src/addstar/test/dvomerge.dvo (revision 28484) @@ -30,5 +30,8 @@ exec rsync -auc catdir.test2/ catdir.test3/ + date -var t1 -seconds -reftime 1276000000 exec dvomerge catdir.test1 into catdir.test3 + date -var t2 -seconds -reftime 1276000000 + echo "merge time: {$t2 - $t1}" catdir catdir.test3 @@ -105,8 +108,84 @@ end -# create 2 populated catdirs, each with a couple of cmf files -macro test.dvomerge.create - - tapPLAN 21 +# create 1 populated catdir, merge into currently non-existent catdir +macro test.dvomerge.update.new + + tapPLAN 51 + + exec rm -rf catdir.test1 + exec rm -rf catdir.test2 + + $RA = 10.0 + $DEC = 20.0 + + mkinput + exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC + exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf + + exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 02:00:00 -radec $RA $DEC + exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf + + date -var t1 -seconds -reftime 1276000000 + exec dvomerge catdir.test1 into catdir.test2 + date -var t2 -seconds -reftime 1276000000 + echo "merge time: {$t2 - $t1}" + + catdir catdir.test2 + skyregion {$RA-1} {$RA+1} {$DEC-1} {$DEC+1} + mextract ra dec mag + create n 0 ra[] + subset r0 = ra if (n % 2 == 0) + subset r1 = ra if (n % 2 == 1) + + catdir catdir.test1/ + mextract RA DEC MAG + create N 0 RA[] + subset R0 = RA if (N % 2 == 0) + subset R1 = RA if (N % 2 == 1) + + set dr0 = r0 - R0 + vstat -q dr0 + tapOK {abs($MEAN) < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)" + tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)" + + set dr1 = r1 - R1 + vstat -q dr1 + tapOK {abs($MEAN) < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)" + tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)" + + # check on updates to imageID + catdir catdir.test3 + imextract imageID + sort imageID + tapOK {imageID[] == 4} "image IDs exist" + tapOK {imageID[0] == 1} "updated image IDs" + tapOK {imageID[1] == 2} "updated image IDs" + + catdir catdir.test2 + mextract imageID, time + set id = imageID + set t = time + imextract imageID, time + + for i 0 time[] + subset T = t if (id == imageID[$i]) + set dT = T - time[$i] + vstat -q dT + tapOK {abs($MEAN) < 0.00001} "time for measure ID $i (MEAN)" + tapOK {abs($SIGMA) < 0.00001} "time for measure ID $i (SIGMA)" + end + + # exec rm test.in.txt test.cmf + # exec rm -rf catdir.test1 + # exec rm -rf catdir.test2 + # exec rm -rf catdir.test3 + + tapDONE +end + +# create 2 populated catdirs, each with a couple of cmf files -- force some unmatched objects +macro test.dvomerge.update.extras + + tapPLAN 51 exec rm -rf catdir.test1 @@ -124,4 +203,6 @@ exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf + # generate a few extra unmatched sources + mkinput.extras exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 03:00:00 -radec $RA $DEC exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf @@ -130,5 +211,7 @@ exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf - exec dvomerge catdir.test1 and catdir.test2 to catdir.test3 + exec rsync -auc catdir.test2/ catdir.test3/ + + exec dvomerge catdir.test1 into catdir.test3 catdir catdir.test3 @@ -205,4 +288,104 @@ end +# create 2 populated catdirs, each with a couple of cmf files +macro test.dvomerge.create + + tapPLAN 21 + + exec rm -rf catdir.test1 + exec rm -rf catdir.test2 + exec rm -rf catdir.test3 + + $RA = 10.0 + $DEC = 20.0 + + mkinput + exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC + exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf + + exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 02:00:00 -radec $RA $DEC + exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf + + exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 03:00:00 -radec $RA $DEC + exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf + + exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 04:00:00 -radec $RA $DEC + exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf + + exec dvomerge catdir.test1 and catdir.test2 to catdir.test3 + + catdir catdir.test3 + skyregion {$RA-1} {$RA+1} {$DEC-1} {$DEC+1} + mextract ra dec mag + create n 0 ra[] + subset r0 = ra if (n % 4 == 0) + subset r1 = ra if (n % 4 == 1) + subset r2 = ra if (n % 4 == 2) + subset r3 = ra if (n % 4 == 3) + + catdir catdir.test1/ + mextract RA DEC MAG + create N 0 RA[] + subset R0 = RA if (N % 2 == 0) + subset R1 = RA if (N % 2 == 1) + + set dr0 = r0 - R0 + vstat -q dr0 + tapOK {abs($MEAN) < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)" + tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)" + + set dr1 = r1 - R1 + vstat -q dr1 + tapOK {abs($MEAN) < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)" + tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)" + + catdir catdir.test2/ + mextract RA DEC MAG + create N 0 RA[] + subset R2 = RA if (N % 2 == 0) + subset R3 = RA if (N % 2 == 1) + + set dr2 = r2 - R2 + vstat -q dr2 + tapOK {abs($MEAN) < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)" + tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)" + + set dr3 = r3 - R3 + vstat -q dr3 + tapOK {abs($MEAN) < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)" + tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)" + + # check on updates to imageID + catdir catdir.test3 + imextract imageID + sort imageID + tapOK {imageID[] == 4} "image IDs exist" + tapOK {imageID[0] == 1} "updated image IDs" + tapOK {imageID[1] == 2} "updated image IDs" + tapOK {imageID[2] == 3} "updated image IDs" + tapOK {imageID[3] == 4} "updated image IDs" + + catdir catdir.test3 + mextract imageID, time + set id = imageID + set t = time + imextract imageID, time + + for i 0 time[] + subset T = t if (id == imageID[$i]) + set dT = T - time[$i] + vstat -q dT + tapOK {abs($MEAN) < 0.00001} "time for measure ID $i (MEAN)" + tapOK {abs($SIGMA) < 0.00001} "time for measure ID $i (SIGMA)" + end + + # exec rm test.in.txt test.cmf + # exec rm -rf catdir.test1 + # exec rm -rf catdir.test2 + # exec rm -rf catdir.test3 + + tapDONE +end + # make a simple input file for mkcmf macro mkinput.alt @@ -225,4 +408,22 @@ for i 10 1024 100 for j 10 1024 100 + fprintf " %4d %4d %6.2f" $i $j {-15.0 + 2.5*($i + $j)/1000.0} + end + end + output stdout +end + +# make a simple input file for mkcmf +macro mkinput.extras + exec rm -f test.in.txt + + output test.in.txt + for i 10 1024 100 + for j 10 1024 100 + fprintf " %4d %4d %6.2f" $i $j {-15.0 + 2.5*($i + $j)/1000.0} + end + end + for i 20 1024 500 + for j 20 1024 500 fprintf " %4d %4d %6.2f" $i $j {-15.0 + 2.5*($i + $j)/1000.0} end Index: /branches/pap/Ohana/src/delstar/src/delete_imagename.c =================================================================== --- /branches/pap/Ohana/src/delstar/src/delete_imagename.c (revision 28483) +++ /branches/pap/Ohana/src/delstar/src/delete_imagename.c (revision 28484) @@ -77,9 +77,9 @@ Noutimage = Nimage - Nimlist; - if (VERBOSE) fprintf (stderr, "removing %lld images (leaving %lld of %lld)\n", (long long) Nimlist, (long long) Noutimage, (long long) Nimage); + if (VERBOSE) fprintf (stderr, "removing "OFF_T_FMT" images (leaving "OFF_T_FMT" of "OFF_T_FMT")\n", Nimlist, Noutimage, Nimage); // gfits_table_set_Image (&db[0].ftable, outimage, Noutimage); - gfits_modify (&db[0].theader, "NAXIS2", "%lld", 1, (long long) Noutimage); - gfits_modify (&db[0].header, "NIMAGES", "%lld", 1, (long long) Noutimage); + gfits_modify (&db[0].theader, "NAXIS2", OFF_T_FMT, 1, Noutimage); + gfits_modify (&db[0].header, "NIMAGES", OFF_T_FMT, 1, Noutimage); db[0].theader.Naxis[1] = Noutimage; db[0].ftable.buffer = (char *) outimage; Index: /branches/pap/Ohana/src/delstar/src/delete_missed.c =================================================================== --- /branches/pap/Ohana/src/delstar/src/delete_missed.c (revision 28483) +++ /branches/pap/Ohana/src/delstar/src/delete_missed.c (revision 28484) @@ -11,5 +11,5 @@ Nmiss = catalog[0].Nmissing; - if (VERBOSE) fprintf (stderr, "starting with Nave, Nmeas, Nmiss: %lld %lld %lld\n", (long long) Nave, (long long) Nmeas, (long long) Nmiss); + if (VERBOSE) fprintf (stderr, "starting with Nave, Nmeas, Nmiss: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT"\n", Nave, Nmeas, Nmiss); /* set up references for missing to average */ @@ -19,5 +19,5 @@ REALLOCATE (catalog[0].missing, Missing, 1); catalog[0].Nmissing = 0; - if (VERBOSE) fprintf (stderr, " ending with Nave, Nmeas, Nmiss: %lld %lld %lld\n", (long long) Nave, (long long) Nmeas, (long long) Nmiss); + if (VERBOSE) fprintf (stderr, " ending with Nave, Nmeas, Nmiss: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT"\n", Nave, Nmeas, Nmiss); } Index: /branches/pap/Ohana/src/delstar/src/find_matches.c =================================================================== --- /branches/pap/Ohana/src/delstar/src/find_matches.c (revision 28483) +++ /branches/pap/Ohana/src/delstar/src/find_matches.c (revision 28484) @@ -22,5 +22,5 @@ ALLOCATE (ave_miss, off_t, MAX(Nmiss,1)); - if (VERBOSE) fprintf (stderr, "starting with Nave, Nmeas, Nmiss: %lld %lld %lld\n", (long long) Nave, (long long) Nmeas, (long long) Nmiss); + if (VERBOSE) fprintf (stderr, "starting with Nave, Nmeas, Nmiss: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT"\n", Nave, Nmeas, Nmiss); /* set up pointers for linked list of measure */ @@ -75,5 +75,5 @@ if (next_meas[j] != i) { fprintf (stderr, "error? (1) this link seems to have been lost\n"); - fprintf (stderr, "j: %lld, next_meas[j]: %lld, i: %lld\n", (long long) j, (long long) next_meas[j], (long long) i); + fprintf (stderr, "j: "OFF_T_FMT", next_meas[j]: "OFF_T_FMT", i: "OFF_T_FMT"\n", j, next_meas[j], i); exit (1); } @@ -108,5 +108,5 @@ if (next_miss[j] != m) { fprintf (stderr, "error? (2) this link seems to have been lost\n"); - fprintf (stderr, "j: %lld, next_miss[j]: %lld, i: %lld\n", (long long) j, (long long) next_miss[j], (long long) i); + fprintf (stderr, "j: "OFF_T_FMT", next_miss[j]: "OFF_T_FMT", i: "OFF_T_FMT"\n", j, next_miss[j], i); exit (1); } @@ -131,5 +131,5 @@ } } - fprintf (stderr, "found %lld meas to remove\n", (long long) Nmeasfound); + fprintf (stderr, "found "OFF_T_FMT" meas to remove\n", Nmeasfound); if (VERBOSE) fprintf (stderr, "fixing missing..."); @@ -146,5 +146,5 @@ if (next_miss[j] != i) { fprintf (stderr, "error? (3) this link seems to have been lost\n"); - fprintf (stderr, "j: %lld, next_miss[j]: %lld, i: %lld\n", (long long) j, (long long) next_miss[j], (long long) i); + fprintf (stderr, "j: "OFF_T_FMT", next_miss[j]: "OFF_T_FMT", i: "OFF_T_FMT"\n", j, next_miss[j], i); exit (1); } @@ -249,5 +249,5 @@ catalog[0].Nsecf_mem = Nave*Nsecfilt; - if (VERBOSE) fprintf (stderr, " ending with Nave, Nmeas, Nmiss: %lld %lld %lld\n", (long long) Nave, (long long) Nmeas, (long long) Nmiss); + if (VERBOSE) fprintf (stderr, " ending with Nave, Nmeas, Nmiss: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT"\n", Nave, Nmeas, Nmiss); free (next_meas); Index: /branches/pap/Ohana/src/dvomerge/src/LoadCatalog.c =================================================================== --- /branches/pap/Ohana/src/dvomerge/src/LoadCatalog.c (revision 28483) +++ /branches/pap/Ohana/src/dvomerge/src/LoadCatalog.c (revision 28484) @@ -7,9 +7,6 @@ catalog[0].Nsecfilt = GetPhotcodeNsecfilt (); - if (!strcmp (mode, "w")) { - catalog[0].catflags = LOAD_NONE; - } else { - catalog[0].catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; - } + // always load all of the data (if any exists) + catalog[0].catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; catalog[0].catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data Index: /branches/pap/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c =================================================================== --- /branches/pap/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c (revision 28483) +++ /branches/pap/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c (revision 28484) @@ -1,3 +1,5 @@ # include "dvomerge.h" + +void sort_IDmap (IDmapType *IDmap); // merge db2 into db1 @@ -17,6 +19,6 @@ /* adjust header */ Nout = 0; - gfits_scan (&out[0].header, "NIMAGES", "%lld", 1, (long long *) &Nout); - status = gfits_scan (&out[0].header, "IMAGEID", "%lld", 1, (long long *) &IDstart); + gfits_scan (&out[0].header, "NIMAGES", OFF_T_FMT, 1, &Nout); + status = gfits_scan (&out[0].header, "IMAGEID", OFF_T_FMT, 1, &IDstart); if (!status) { IDstart = 1; @@ -29,4 +31,7 @@ } + // sort IDmap->old,new on the basis of IDmap->old: + sort_IDmap (IDmap); + if (!out[0].swapped) { gfits_convert_Image ((Image *) out[0].ftable.buffer, sizeof(Image), Nout); @@ -36,6 +41,6 @@ Nout += Nimages; IDstart += Nimages; - gfits_modify (&out[0].header, "NIMAGES", "%lld", 1, (long long) Nout); - gfits_modify (&out[0].header, "IMAGEID", "%lld", 1, (long long) IDstart); + gfits_modify (&out[0].header, "NIMAGES", OFF_T_FMT, 1, Nout); + gfits_modify (&out[0].header, "IMAGEID", OFF_T_FMT, 1, IDstart); gfits_add_rows (&out[0].ftable, (char *) images, Nimages, sizeof(Image)); @@ -43,12 +48,32 @@ } -// optimize with sort and bisection +// XXX isn't the map just ID_new = ID_old + offset ?? off_t dvo_map_image_ID (IDmapType *IDmap, off_t oldID) { - off_t i; + // off_t i; + // + // for (i = 0; i < IDmap->Nmap; i++) { + // if (IDmap->old[i] != oldID) continue; + // return (IDmap->new[i]); + // } - for (i = 0; i < IDmap->Nmap; i++) { - if (IDmap->old[i] != oldID) continue; - return (IDmap->new[i]); + off_t Nlo, Nhi, N; + + // find the a close entry below desired ID + Nlo = 0; Nhi = IDmap->Nmap; + while (Nhi - Nlo > 10) { + N = 0.5*(Nlo + Nhi); + if (IDmap->old[N] < oldID) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N + 1, IDmap->Nmap); + } + } + + // search for the desired ID starting from Nlo, give up at Nhi + for (N = Nlo; N < Nhi; N++) { + if (IDmap->old[N] < oldID) continue; + if (IDmap->old[N] > oldID) return 0; + return (IDmap->new[N]); } return 0; @@ -65,5 +90,5 @@ newID = dvo_map_image_ID (IDmap, oldID); if (newID == 0) { - fprintf (stderr, "cannot find image ID %lld\n", (long long) oldID); + fprintf (stderr, "cannot find image ID "OFF_T_FMT"\n", oldID); exit (2); } @@ -72,2 +97,19 @@ return TRUE; } + +// sort two times vectors and an index by first time vector +void sort_IDmap (IDmapType *IDmap) { + +# define SWAPFUNC(A,B){ off_t tmp_old, tmp_new; \ + tmp_old = IDmap->old[A]; IDmap->old[A] = IDmap->old[B]; IDmap->old[B] = tmp_old; \ + tmp_new = IDmap->new[A]; IDmap->new[A] = IDmap->new[B]; IDmap->new[B] = tmp_new; \ +} +# define COMPARE(A,B)(IDmap->old[A] < IDmap->old[B]) + + OHANA_SORT (IDmap->Nmap, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + Index: /branches/pap/Ohana/src/dvomerge/src/dvoconvert.c =================================================================== --- /branches/pap/Ohana/src/dvomerge/src/dvoconvert.c (revision 28483) +++ /branches/pap/Ohana/src/dvomerge/src/dvoconvert.c (revision 28484) @@ -150,7 +150,7 @@ // update additional metadata - gfits_scan (&inDB.header, "IMAGEID", "%lld", 1, (long long *) &ID); - gfits_modify (&outDB.header, "NIMAGES", "%lld", 1, (long long) Nimages); - gfits_modify (&outDB.header, "IMAGEID", "%lld", 1, (long long) ID); + gfits_scan (&inDB.header, "IMAGEID", OFF_T_FMT, 1, &ID); + gfits_modify (&outDB.header, "NIMAGES", OFF_T_FMT, 1, Nimages); + gfits_modify (&outDB.header, "IMAGEID", OFF_T_FMT, 1, ID); // copy input rows to output table Index: /branches/pap/Ohana/src/dvomerge/src/dvomerge.c =================================================================== --- /branches/pap/Ohana/src/dvomerge/src/dvomerge.c (revision 28483) +++ /branches/pap/Ohana/src/dvomerge/src/dvomerge.c (revision 28484) @@ -7,4 +7,6 @@ ConfigInit (&argc, argv); dvomerge_args (&argc, argv); + + // XXX require both inputs to be sorted? if (argc == 6) dvomergeCreate (argc, argv); Index: /branches/pap/Ohana/src/dvomerge/src/dvomergeUpdate.c =================================================================== --- /branches/pap/Ohana/src/dvomerge/src/dvomergeUpdate.c (revision 28483) +++ /branches/pap/Ohana/src/dvomerge/src/dvomergeUpdate.c (revision 28484) @@ -16,9 +16,23 @@ output = argv[3]; - // the first input defines the photcode table & db layout - sprintf (filename, "%s/Photcodes.dat", input); + // since we are merging the input db into the output db, the output defines the photcode + // table & db layout but, this requires the output to exist. if it does not, instead use the + // input. + sprintf (filename, "%s/Photcodes.dat", output); if (!LoadPhotcodes (filename, NULL, FALSE)) { - fprintf (stderr, "error loading photcode table %s\n", filename); - exit (1); + sprintf (filename, "%s/Photcodes.dat", input); + if (!LoadPhotcodes (filename, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table: tried %s/Photcodes.dat and %s/Photcodes.dat\n", output, input); + exit (1); + } + if (!check_dir_access (output, VERBOSE)) { + fprintf (stderr, "error creating output database directory %s\n", output); + exit (1); + } + sprintf (filename, "%s/Photcodes.dat", output); + if (!SavePhotcodesFITS (filename)) { + fprintf (stderr, "error saving photcode table in %s/Photcodes.dat\n", output); + exit (1); + } } @@ -27,8 +41,14 @@ // load the sky table for the existing database insky = SkyTableLoadOptimal (input, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE); + if (!insky) { + Shutdown ("can't read SkyTable for %s", input); + } SkyTableSetFilenames (insky, input, "cpt"); // generate an output table populated at the desired depth - outsky = SkyTableLoadOptimal (output, NULL, NULL, TRUE, SKY_DEPTH, VERBOSE); + outsky = SkyTableLoadOptimal (output, NULL, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); + if (!outsky) { + Shutdown ("can't read or create SkyTable for %s", output); + } SkyTableSetFilenames (outsky, output, "cpt"); @@ -43,11 +63,14 @@ if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name); - // load / create output catalog - LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "a"); + // load / create output catalog (if catalog does not exist, it will be created) + LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w"); // combine only tables at equal or larger depth // load in all of the tables from input for this region - inlist = SkyListByBounds (insky, depth, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax); + // SkyListByBounds will return neighbor catalogs if the boundaries exactly match (due to rounding). Since the regions are not infinitely small, + // compare to a slightly reduced footprint + float dPos = 2.0/3600.0; + inlist = SkyListByBounds (insky, depth, outsky[0].regions[i].Rmin + dPos, outsky[0].regions[i].Rmax - dPos, outsky[0].regions[i].Dmin + dPos, outsky[0].regions[i].Dmax - dPos); for (j = 0; j < inlist[0].Nregions; j++) { if (VERBOSE) fprintf (stderr, "input : %s\n", inlist[0].regions[j][0].name); @@ -66,4 +89,6 @@ dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); + + fprintf (stderr, "merged %s into %s\n", outsky[0].regions[i].name, inlist[0].regions[j][0].name); } SkyListFree (inlist); @@ -102,8 +127,8 @@ // load the image table if (inDB.dbstate == LCK_EMPTY) { - Shutdown ("can't find input (1) image catalog %s", inDB.filename); + Shutdown ("can't find input image catalog %s", inDB.filename); } if (!dvo_image_load (&inDB, VERBOSE, TRUE)) { - Shutdown ("can't read input (1) image catalog %s", inDB.filename); + Shutdown ("can't read input image catalog %s", inDB.filename); } @@ -117,8 +142,9 @@ /* load the image table */ if (outDB.dbstate == LCK_EMPTY) { - Shutdown ("can't find input (2) image catalog %s", outDB.filename); - } - if (!dvo_image_load (&outDB, VERBOSE, TRUE)) { - Shutdown ("can't read input (2) image catalog %s", outDB.filename); + dvo_image_create (&outDB, GetZeroPoint()); + } else { + if (!dvo_image_load (&outDB, VERBOSE, TRUE)) { + Shutdown ("can't read output image catalog %s", outDB.filename); + } } Index: /branches/pap/Ohana/src/dvomerge/src/dvoverify.c =================================================================== --- /branches/pap/Ohana/src/dvomerge/src/dvoverify.c (revision 28484) +++ /branches/pap/Ohana/src/dvomerge/src/dvoverify.c (revision 28484) @@ -0,0 +1,173 @@ +# include "dvomerge.h" + +/* things we can verify easily: + table sizes: (NAXIS1 vs EXTTYPE; NAXIS2 vs data size) + sum of catalog.average.Nmeasure == catalog.Nmeasure + averef, obj_id consistent between average and measure + do we need a checksum? +*/ + +int main (int argc, char **argv) { + + char filename[256], *input, *output; + + int depth; + off_t i, j, Ns, Ne; + SkyTable *outsky, *insky; + SkyList *inlist; + Catalog incatalog, outcatalog; + + SetSignals (); + dvoconvert_help (argc, argv); + ConfigInit (&argc, argv); + dvoconvert_args (&argc, argv); + + if (strcasecmp (argv[2], "to")) dvoconvert_usage(); + input = argv[1]; + output = argv[3]; + + // load input images, save to output images + dvoConvert_copy_images (input, output); + + // copy photcode table + { + // the first input defines the photcode table & db layout + sprintf (filename, "%s/Photcodes.dat", input); + if (!LoadPhotcodes (filename, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table %s\n", filename); + exit (1); + } + // save the photcodes in the output catdir + sprintf (filename, "%s/Photcodes.dat", output); + if (!check_file_access (filename, TRUE, TRUE, VERBOSE)) { + fprintf (stderr, "error creating output catdir %s\n", output); + exit (1); + } + if (!SavePhotcodesFITS (filename)) { + fprintf (stderr, "error saving photcode table %s\n", filename); + exit (1); + } + } + + // copy skytable + { + // load the sky table for the existing database + insky = SkyTableLoadOptimal (input, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE); + SkyTableSetFilenames (insky, input, "cpt"); + + // generate an output table populated at the desired depth + // XXX force this to match the input? + outsky = SkyTableLoadOptimal (output, NULL, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); + SkyTableSetFilenames (outsky, output, "cpt"); + + SkyTablePopulatedRange (&Ns, &Ne, insky, 0); + depth = insky[0].regions[Ns].depth; + // XXX this seems to imply that insky is a uniform depth... + } + + // loop over all input catalogs, save to output catalogs + // loop over the populatable output regions + for (i = 0; i < outsky[0].Nregions; i++) { + if (!outsky[0].regions[i].table) continue; + if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name); + + // load / create output catalog + LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w"); + + // combine only tables at equal or larger depth + + // load in all of the tables from input for this region + inlist = SkyListByBounds (insky, depth, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax); + for (j = 0; j < inlist[0].Nregions; j++) { + if (VERBOSE) fprintf (stderr, "input : %s\n", inlist[0].regions[j][0].name); + + // load input catalog + LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r"); + + // skip empty input catalogs + if (!incatalog.Naves_disk) { + dvo_catalog_unlock (&incatalog); + dvo_catalog_free (&incatalog); + continue; + } + merge_catalogs_new (&outsky[0].regions[i], &outcatalog, &incatalog); + dvo_catalog_unlock (&incatalog); + dvo_catalog_free (&incatalog); + } + SkyListFree (inlist); + + outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + dvo_catalog_save (&outcatalog, VERBOSE); + dvo_catalog_unlock (&outcatalog); + dvo_catalog_free (&outcatalog); + } + + // save the output sky table copy + sprintf (filename, "%s/SkyTable.fits", output); + check_file_access (filename, TRUE, TRUE, VERBOSE); + if (!SkyTableSave (outsky, filename)) { + fprintf (stderr, "ERROR: failed to save sky table for %s\n", output); + exit (1); + } + + exit (0); +} + +int dvoConvert_copy_images (char *input, char *output) { + + FITS_DB inDB; + FITS_DB outDB; + int status; + + Image *images; + off_t Nimages; + off_t ID; + + /*** load output/Images.dat ***/ + sprintf (ImageCat, "%s/Images.dat", output); + outDB.mode = dvo_catalog_catmode (CATMODE); + outDB.format = dvo_catalog_catformat (CATFORMAT); + status = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD); // shorter timeout? + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename); + + // output image table should not already exist + if (outDB.dbstate != LCK_EMPTY) { + Shutdown ("ERROR: image table %s already exists", outDB.filename); + } + dvo_image_create (&outDB, GetZeroPoint()); + + /*** load input/Images.dat ***/ + sprintf (ImageCat, "%s/Images.dat", input); + // inDB.mode = dvo_catalog_catmode (CATMODE); + // inDB.format = dvo_catalog_catformat (CATFORMAT); + status = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD); // shorter timeout? + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename); + + // load the image table + if (inDB.dbstate == LCK_EMPTY) { + Shutdown ("can't find input image catalog %s", inDB.filename); + } + if (!dvo_image_load (&inDB, VERBOSE, TRUE)) { + Shutdown ("can't read input image catalog %s", inDB.filename); + } + + // convert the raw image table to Image type (byteswap if needed) + images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.swapped); + + // update additional metadata + gfits_scan (&inDB.header, "IMAGEID", OFF_T_FMT, 1, &ID); + gfits_modify (&outDB.header, "NIMAGES", OFF_T_FMT, 1, Nimages); + gfits_modify (&outDB.header, "IMAGEID", OFF_T_FMT, 1, ID); + + // copy input rows to output table + gfits_add_rows (&outDB.ftable, (char *) images, Nimages, sizeof(Image)); + + // write out the image table to disk + SetProtect (TRUE); + dvo_image_save (&outDB, VERBOSE); + SetProtect (FALSE); + dvo_image_unlock (&outDB); // unlock output + dvo_image_unlock (&inDB); // unlock input1 + + return TRUE; +} Index: /branches/pap/Ohana/src/dvomerge/src/merge_catalogs_new.c =================================================================== --- /branches/pap/Ohana/src/dvomerge/src/merge_catalogs_new.c (revision 28483) +++ /branches/pap/Ohana/src/dvomerge/src/merge_catalogs_new.c (revision 28484) @@ -72,8 +72,8 @@ if (VERBOSE) { - fprintf (stderr, "%lld: using %lld stars (%lld measures) for catalog\n", - (long long) i, - (long long) output[0].Naverage, - (long long) output[0].Nmeasure); + fprintf (stderr, OFF_T_FMT": using "OFF_T_FMT" stars ("OFF_T_FMT" measures) for catalog\n", + i, + output[0].Naverage, + output[0].Nmeasure); } return (TRUE); Index: /branches/pap/Ohana/src/dvomerge/src/merge_catalogs_old.c =================================================================== --- /branches/pap/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 28483) +++ /branches/pap/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 28484) @@ -2,7 +2,15 @@ # define PSPS_ID TRUE -# define IN_REGION(R,D) ( \ -((D) >= region[0].Dmin) && ((D) < region[0].Dmax) && \ -((R) >= region[0].Rmin) && ((R) < region[0].Rmax)) +# define MARKTIME(MSG,...) { \ + float dtime; \ + gettimeofday (&stop, (void *) NULL); \ + dtime = DTIME (stop, start); \ + fprintf (stderr, MSG, __VA_ARGS__); \ + gettimeofday (&start, (void *) NULL); \ + } + +# define IN_REGION(R,D) ( \ + ((D) >= region[0].Dmin) && ((D) < region[0].Dmax) && \ + ((R) >= region[0].Rmin) && ((R) < region[0].Rmax)) // merge the input data into the output catalog @@ -10,5 +18,5 @@ int merge_catalogs_old (SkyRegion *region, Catalog *output, Catalog *input, double RADIUS) { - off_t i, j, Nin, offset, J, Jmin, status, Nstars; + off_t i, j, k, Nin, offset, J, Jmin, status, Nstars; double RADIUS2, Rmin, Rin, Din; double *X1, *Y1, *X2, *Y2; @@ -19,4 +27,7 @@ unsigned int objID, catID; Coords tcoords; + + // struct timeval start, stop; + // gettimeofday (&start, (void *) NULL); Nsecfilt = output[0].Nsecfilt; @@ -64,4 +75,6 @@ strcpy (tcoords.ctype, "RA---ARC"); + if (VERBOSE) fprintf (stderr, "merging %s into %s\n", input[0].filename, output[0].filename); + /* build spatial index (RA sort) referencing input array sequence */ Nstars = 0; @@ -96,5 +109,5 @@ /* set up pointers for linked list of measure */ - if (output[0].sorted && (output[0].Nmeasure == output[0].Nmeas_disk)) { + if (output[0].sorted && (output[0].Nmeasure >= output[0].Nmeas_disk)) { // this version is only valid if we have done a full catalog load, and if the catalog // is sorted while processed @@ -106,4 +119,6 @@ /* choose a radius for matches */ RADIUS2 = RADIUS*RADIUS; + + // MARKTIME("set up structures: %f sec\n", dtime); /** find matched stars **/ @@ -162,4 +177,6 @@ } + // 4) average properties from the input and the output db need to be properly merged. + /** add ALL measurements for this input average object **/ for (Nin = 0; Nin < input[0].average[N].Nmeasure; Nin ++) { @@ -183,5 +200,7 @@ output[0].measure[Nmeas].catID = output[0].catID; - // fprintf (stderr, "Nave : %lld, Nmeas : %lld, dR: %f, dD: %f, catID: %d\n", (long long) n, (long long) Nmeas, output[0].measure[Nmeas].dR, output[0].measure[Nmeas].dD, output[0].measure[i].catID); + assert (output[0].measure[Nmeas].averef < Nave); + + // fprintf (stderr, "Nave : "OFF_T_FMT", Nmeas : "OFF_T_FMT", dR: %f, dD: %f, catID: %d\n", n, Nmeas, output[0].measure[Nmeas].dR, output[0].measure[Nmeas].dD, output[0].measure[i].catID); // rationalize dR @@ -207,4 +226,12 @@ } + // update the average properties to reflect the incoming entries: + // if the original value is NAN but the input value is not, accept the input: + for (k = 0; k < Nsecfilt; k++) { + if ( isfinite(output[0].secfilt[n*Nsecfilt+k].M)) continue; + if (!isfinite( input[0].secfilt[N*Nsecfilt+k].M)) continue; + output[0].secfilt[n*Nsecfilt+k] = input[0].secfilt[N*Nsecfilt+k]; + } + /* Nm is updated, but not written out in -update mode (for existing entries) Nm is recalculated in build_meas_links if loaded table is not sorted */ @@ -212,11 +239,5 @@ i++; } - -# if (0) - fprintf (stderr, "--- 1 ---\n"); - for (i = 0; i < Nmeas; i++) { - fprintf (stderr, "Nave : %d, Nmeas : %lld, dR: %f, dD: %f, catID: %d\n", output[0].measure[i].averef, (long long) i, output[0].measure[i].dR, output[0].measure[i].dD, output[0].measure[i].catID); - } -# endif + // MARKTIME("find matched stars: %f sec for "OFF_T_FMT","OFF_T_FMT" stars\n", dtime, Nstars, Nave); /** incorporate unmatched image stars, if this star is in field of this catalog **/ @@ -240,4 +261,6 @@ if (!IN_REGION (input[0].average[N].R, input[0].average[N].D)) continue; + // XXX should we accept the input measurements for these fields? + output[0].average[Nave].R = input[0].average[N].R; output[0].average[Nave].D = input[0].average[N].D; @@ -245,5 +268,5 @@ output[0].average[Nave].dD = 0; - output[0].average[Nave].Nmeasure = 1; + output[0].average[Nave].Nmeasure = 0; // this value is update as the measurements are associated with this entry below output[0].average[Nave].Nmissing = 0; output[0].average[Nave].Nextend = 0; @@ -280,11 +303,21 @@ for (j = 0; j < Nsecfilt; j++) { - output[0].secfilt[Nave*Nsecfilt+j].M = NAN; - output[0].secfilt[Nave*Nsecfilt+j].dM = NAN; - output[0].secfilt[Nave*Nsecfilt+j].Xm = NAN_S_SHORT; - output[0].secfilt[Nave*Nsecfilt+j].M_20 = NAN_S_SHORT; - output[0].secfilt[Nave*Nsecfilt+j].M_80 = NAN_S_SHORT; - output[0].secfilt[Nave*Nsecfilt+j].Ncode = 0; - output[0].secfilt[Nave*Nsecfilt+j].Nused = 0; + if (isfinite(input[0].secfilt[N*Nsecfilt+j].M)) { + output[0].secfilt[Nave*Nsecfilt+j].M = input[0].secfilt[N*Nsecfilt+j].M; + output[0].secfilt[Nave*Nsecfilt+j].dM = input[0].secfilt[N*Nsecfilt+j].dM; + output[0].secfilt[Nave*Nsecfilt+j].Xm = input[0].secfilt[N*Nsecfilt+j].Xm; + output[0].secfilt[Nave*Nsecfilt+j].M_20 = input[0].secfilt[N*Nsecfilt+j].M_20; + output[0].secfilt[Nave*Nsecfilt+j].M_80 = input[0].secfilt[N*Nsecfilt+j].M_80; + output[0].secfilt[Nave*Nsecfilt+j].Ncode = input[0].secfilt[N*Nsecfilt+j].Ncode; + output[0].secfilt[Nave*Nsecfilt+j].Nused = input[0].secfilt[N*Nsecfilt+j].Nused; + } else { + output[0].secfilt[Nave*Nsecfilt+j].M = NAN; + output[0].secfilt[Nave*Nsecfilt+j].dM = NAN; + output[0].secfilt[Nave*Nsecfilt+j].Xm = NAN_S_SHORT; + output[0].secfilt[Nave*Nsecfilt+j].M_20 = NAN_S_SHORT; + output[0].secfilt[Nave*Nsecfilt+j].M_80 = NAN_S_SHORT; + output[0].secfilt[Nave*Nsecfilt+j].Ncode = 0; + output[0].secfilt[Nave*Nsecfilt+j].Nused = 0; + } } @@ -302,6 +335,8 @@ output[0].measure[Nmeas].catID = output[0].catID; - /* next[Nmeas] should always be -1 in this context (it is always the only - measurement for the star) */ + // as we add measurements, update Nmeasure to match + output[0].average[Nave].Nmeasure ++; + + /* we set next[Nmeas] to -1 here, and update correctly below */ input[0].found[N] = Nmeas; next_meas[Nmeas] = -1; @@ -310,20 +345,15 @@ int Ngroup = input[0].average[N].Nmeasure; for (j = 0; j < Ngroup - 1; j++) { - next_meas[Nmeas - Ngroup + j] = Nmeas - Ngroup + j + 1; + next_meas[Nmeas - Ngroup + j] = Nmeas - Ngroup + j + 1; } Nave ++; } + // MARKTIME("save unmatched stars: %f sec\n", dtime); + REALLOCATE (output[0].average, Average, Nave); REALLOCATE (output[0].measure, Measure, Nmeas); -# if (0) - fprintf (stderr, "--- 2 ---\n"); - for (i = 0; i < Nmeas; i++) { - fprintf (stderr, "Nave : %d, Nmeas : %lld, dR: %f, dD: %f, catID: %d\n", output[0].measure[i].averef, (long long) i, output[0].measure[i].dR, output[0].measure[i].dD, output[0].measure[i].catID); - } -# endif - -# define NOSORT FALSE +# define NOSORT 0 if (NOSORT) { output[0].sorted = FALSE; @@ -332,11 +362,4 @@ output[0].measure = sort_measure (output[0].average, Nave, output[0].measure, Nmeas, next_meas); } - -# if (0) - fprintf (stderr, "--- 3 ---\n"); - for (i = 0; i < Nmeas; i++) { - fprintf (stderr, "Nave : %d, Nmeas : %lld, dR: %f, dD: %f, catID: %d\n", output[0].measure[i].averef, (long long) i, output[0].measure[i].dR, output[0].measure[i].dD, output[0].measure[i].catID); - } -# endif /* note stars which have been found in this catalog */ @@ -354,5 +377,5 @@ output[0].Nmeasure = Nmeas; output[0].Nsecf_mem = Nave*Nsecfilt; - if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: %lld %lld %lld, (%lld matches)\n", (long long) Nstars, (long long) Nave, (long long) Nmeas, (long long) Nmatch); + if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n", Nstars, Nave, Nmeas, Nmatch); free (next_meas); @@ -366,4 +389,6 @@ free (Y1); free (N1); + + // MARKTIME("cleanup: %f sec\n", dtime); return (Nmatch); } Index: /branches/pap/Ohana/src/fixcat/src/wcatalog.c =================================================================== --- /branches/pap/Ohana/src/fixcat/src/wcatalog.c (revision 28483) +++ /branches/pap/Ohana/src/fixcat/src/wcatalog.c (revision 28484) @@ -34,7 +34,7 @@ chmod (filename, mode); - gfits_modify (&catalog[0].header, "NSTARS", "%lld", 1, catalog[0].Naverage); - gfits_modify (&catalog[0].header, "NMEAS", "%lld", 1, catalog[0].Nmeasure); - gfits_modify (&catalog[0].header, "NMISS", "%lld", 1, catalog[0].Nmissing); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, catalog[0].Naverage); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, catalog[0].Nmeasure); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, catalog[0].Nmissing); gfits_modify_alt (&catalog[0].header, "MARKSTAR", "%t", 1, TRUE); Index: /branches/pap/Ohana/src/getstar/src/MatchImages.c =================================================================== --- /branches/pap/Ohana/src/getstar/src/MatchImages.c (revision 28483) +++ /branches/pap/Ohana/src/getstar/src/MatchImages.c (revision 28484) @@ -132,5 +132,5 @@ } - if (VERBOSE) fprintf (stderr, "found %lld overlapping images\n", (long long) nmatch); + if (VERBOSE) fprintf (stderr, "found "OFF_T_FMT" overlapping images\n", nmatch); *Nmatch = nmatch; Index: /branches/pap/Ohana/src/getstar/src/SelectImages.c =================================================================== --- /branches/pap/Ohana/src/getstar/src/SelectImages.c (revision 28483) +++ /branches/pap/Ohana/src/getstar/src/SelectImages.c (revision 28484) @@ -31,5 +31,5 @@ } - if (VERBOSE) fprintf (stderr, "found %lld matching images\n", (long long) nmatch); + if (VERBOSE) fprintf (stderr, "found "OFF_T_FMT" matching images\n", nmatch); *Nmatch = nmatch; Index: /branches/pap/Ohana/src/getstar/src/select_by_region.c =================================================================== --- /branches/pap/Ohana/src/getstar/src/select_by_region.c (revision 28483) +++ /branches/pap/Ohana/src/getstar/src/select_by_region.c (revision 28484) @@ -161,6 +161,6 @@ output[0].Nsecf_mem = Nave*Nsecfilt; - fprintf (stderr, "output catalog has %lld stars (%lld measures, %d secfilt)\n", - (long long) output[0].Naverage, (long long) output[0].Nmeasure, output[0].Nsecfilt); + fprintf (stderr, "output catalog has "OFF_T_FMT" stars ("OFF_T_FMT" measures, %d secfilt)\n", + output[0].Naverage, output[0].Nmeasure, output[0].Nsecfilt); return (TRUE); } Index: /branches/pap/Ohana/src/imregister/detrend/delete.c =================================================================== --- /branches/pap/Ohana/src/imregister/detrend/delete.c (revision 28483) +++ /branches/pap/Ohana/src/imregister/detrend/delete.c (revision 28484) @@ -10,5 +10,5 @@ ALLOCATE (keep, off_t, MAX (Nimage, 1)); for (i = 0; i < Nimage; i++) keep[i] = TRUE; - fprintf (stderr, "total of %lld detrend images\n", (long long) Nimage); + fprintf (stderr, "total of "OFF_T_FMT" detrend images\n", Nimage); Ndel = 0; @@ -20,5 +20,5 @@ delete_image (&image[i]); } - fprintf (stderr, "delete %lld images\n", (long long) Ndel); + fprintf (stderr, "delete "OFF_T_FMT" images\n", Ndel); if (Ndel == 0) { fprintf (stderr, "SUCCESS\n"); @@ -30,5 +30,5 @@ Nsubset = Nimage - Ndel; ALLOCATE (subset, DetReg, MAX (1, Nsubset)); - fprintf (stderr, "keeping %lld images\n", (long long) Nsubset); + fprintf (stderr, "keeping "OFF_T_FMT" images\n", Nsubset); for (j = i = 0; i < Nimage; i++) { if (!keep[i]) continue; Index: /branches/pap/Ohana/src/imregister/imphot/rtext.c =================================================================== --- /branches/pap/Ohana/src/imregister/imphot/rtext.c (revision 28483) +++ /branches/pap/Ohana/src/imregister/imphot/rtext.c (revision 28484) @@ -10,5 +10,5 @@ /* check that file size makes sense */ Nimage = 0; - gfits_scan (&db[0].header, "NIMAGES", "%lld", 1, (long long *) &Nimage); + gfits_scan (&db[0].header, "NIMAGES", OFF_T_FMT, 1, &Nimage); if (stat (db[0].filename, &filestatus) == -1) { if (VERBOSE) fprintf (stderr, "ERROR: failed to get status of image catalog\n"); @@ -21,5 +21,5 @@ Ndata = (filestatus.st_size - db[0].header.datasize) / sizeof (Image); if (VERBOSE) fprintf (stderr, "ERROR: image catalog has inconsistent size\n"); - if (VERBOSE) fprintf (stderr, "header: %lld, data: %d\n", (long long) Nimage, Ndata); + if (VERBOSE) fprintf (stderr, "header: "OFF_T_FMT", data: %d\n", Nimage, Ndata); if (!FORCE_READ) exit (1); Nimage = Ndata; @@ -41,5 +41,5 @@ } db[0].ftable.buffer = (char *) image; - gfits_modify (&db[0].theader, "NAXIS2", "%lld", 1, (long long) Nimage); + gfits_modify (&db[0].theader, "NAXIS2", OFF_T_FMT, 1, Nimage); db[0].theader.Naxis[1] = Nimage; db[0].ftable.datasize = gfits_data_size (&db[0].theader); Index: /branches/pap/Ohana/src/imregister/imreg/delete.c =================================================================== --- /branches/pap/Ohana/src/imregister/imreg/delete.c (revision 28483) +++ /branches/pap/Ohana/src/imregister/imreg/delete.c (revision 28484) @@ -10,5 +10,5 @@ ALLOCATE (keep, off_t, MAX (Nimage, 1)); for (i = 0; i < Nimage; i++) keep[i] = TRUE; - fprintf (stderr, "total of %lld images\n", (long long) Nimage); + fprintf (stderr, "total of "OFF_T_FMT" images\n", Nimage); Ndel = 0; @@ -19,5 +19,5 @@ Ndel ++; } - fprintf (stderr, "delete %lld images\n", (long long) Ndel); + fprintf (stderr, "delete "OFF_T_FMT" images\n", Ndel); if (Ndel == 0) { fprintf (stderr, "SUCCESS\n"); @@ -29,5 +29,5 @@ Nsubset = Nimage - Ndel; ALLOCATE (subset, RegImage, MAX (1, Nsubset)); - fprintf (stderr, "keeping %lld images\n", (long long) Nsubset); + fprintf (stderr, "keeping "OFF_T_FMT" images\n", Nsubset); for (j = i = 0; i < Nimage; i++) { if (!keep[i]) continue; Index: /branches/pap/Ohana/src/imregister/photreg/delete.c =================================================================== --- /branches/pap/Ohana/src/imregister/photreg/delete.c (revision 28483) +++ /branches/pap/Ohana/src/imregister/photreg/delete.c (revision 28484) @@ -10,5 +10,5 @@ ALLOCATE (keep, off_t, MAX (Nphotdata, 1)); for (i = 0; i < Nphotdata; i++) keep[i] = TRUE; - fprintf (stderr, "total of %lld photdata\n", (long long) Nphotdata); + fprintf (stderr, "total of "OFF_T_FMT" photdata\n", Nphotdata); Ndel = 0; @@ -19,5 +19,5 @@ Ndel ++; } - fprintf (stderr, "delete %lld photdata\n", (long long) Ndel); + fprintf (stderr, "delete "OFF_T_FMT" photdata\n", Ndel); if (Ndel == 0) { @@ -31,5 +31,5 @@ Nsubset = Nphotdata - Ndel; ALLOCATE (subset, PhotPars, MAX (1, Nsubset)); - fprintf (stderr, "keeping %lld photdata\n", (long long) Nsubset); + fprintf (stderr, "keeping "OFF_T_FMT" photdata\n", Nsubset); for (j = i = 0; i < Nphotdata; i++) { if (!keep[i]) continue; Index: /branches/pap/Ohana/src/imregister/spreg/delete.c =================================================================== --- /branches/pap/Ohana/src/imregister/spreg/delete.c (revision 28483) +++ /branches/pap/Ohana/src/imregister/spreg/delete.c (revision 28484) @@ -10,5 +10,5 @@ ALLOCATE (keep, off_t, MAX (Nspectrum, 1)); for (i = 0; i < Nspectrum; i++) keep[i] = TRUE; - fprintf (stderr, "total of %lld spectra\n", (long long) Nspectrum); + fprintf (stderr, "total of "OFF_T_FMT" spectra\n", Nspectrum); Nbad = 0; @@ -19,5 +19,5 @@ Nbad ++; } - fprintf (stderr, "delete %lld spectra\n", (long long) Nbad); + fprintf (stderr, "delete "OFF_T_FMT" spectra\n", Nbad); if (Nbad == 0) { fprintf (stderr, "SUCCESS\n"); @@ -28,5 +28,5 @@ Nsubset = Nspectrum - Nbad; ALLOCATE (subset, Spectrum, MAX (1, Nsubset)); - fprintf (stderr, "keeping %lld spectra\n", (long long) Nsubset); + fprintf (stderr, "keeping "OFF_T_FMT" spectra\n", Nsubset); for (j = i = 0; i < Nspectrum; i++) { if (!keep[i]) continue; Index: /branches/pap/Ohana/src/imregister/src/convertimreg.c =================================================================== --- /branches/pap/Ohana/src/imregister/src/convertimreg.c (revision 28483) +++ /branches/pap/Ohana/src/imregister/src/convertimreg.c (revision 28484) @@ -41,9 +41,9 @@ /* load existing data from database */ - gfits_scan (&header, "NIMAGES", "%lld", 1, (long long *) &Nimage); + gfits_scan (&header, "NIMAGES", OFF_T_FMT, 1, &Nimage); ALLOCATE (pimage, RegImage, Nimage); status = fread (pimage, sizeof(RegImage), Nimage, f); if (status != Nimage) { - fprintf (stderr, "ERROR: header and data in dB don't match (%lld vs %lld)\n", (long long) Nimage, (long long) status); + fprintf (stderr, "ERROR: header and data in dB don't match ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nimage, status); fclearlockfile (argv[1], f, lockstate, &dbstate); exit (1); Index: /branches/pap/Ohana/src/imregister/src/imphotmerge.c =================================================================== --- /branches/pap/Ohana/src/imregister/src/imphotmerge.c (revision 28483) +++ /branches/pap/Ohana/src/imregister/src/imphotmerge.c (revision 28484) @@ -113,5 +113,5 @@ } Nimage += Nin; - gfits_modify (&header, "NIMAGES", "%lld", 1, (long long) Nimage); + gfits_modify (&header, "NIMAGES", OFF_T_FMT, 1, Nimage); /* position to begining of file to write header */ @@ -152,5 +152,5 @@ /* check that file size makes sense */ Nimage = 0; - gfits_scan (header, "NIMAGES", "%lld", 1, (long long *) &Nimage); + gfits_scan (header, "NIMAGES", OFF_T_FMT, 1, &Nimage); if (fstat (fileno(f), &filestatus) == -1) { fprintf (stderr, "ERROR: failed to get status of image catalog\n"); @@ -161,5 +161,5 @@ Ndata = (filestatus.st_size - header[0].datasize) / sizeof (Image); fprintf (stderr, "ERROR: image catalog has inconsistent size\n"); - fprintf (stderr, "header: %lld, data: %lld\n", (long long) Nimage, (long long) Ndata); + fprintf (stderr, "header: "OFF_T_FMT", data: "OFF_T_FMT"\n", Nimage, Ndata); Nimage = Ndata; } Index: /branches/pap/Ohana/src/kapa2/src/LoadPicture.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/LoadPicture.c (revision 28483) +++ /branches/pap/Ohana/src/kapa2/src/LoadPicture.c (revision 28484) @@ -32,5 +32,5 @@ gfits_init_header (&header); header.Naxes = 2; - KiiScanMessage (sock, "%lld %lld", (long long *) &header.Naxis[0], (long long *) &header.Naxis[1]); + KiiScanMessage (sock, OFF_T_FMT" "OFF_T_FMT, &header.Naxis[0], &header.Naxis[1]); // internal image are 32 bit floats; sender must send in this format @@ -41,5 +41,5 @@ KiiScanMessage (sock, "%lf %lf %s %s", &image[0].image[0].zero, &image[0].image[0].range, image[0].image[0].name, image[0].image[0].file); - KiiScanMessage (sock, "%lf %lf %lld", &image[0].image[0].min, &image[0].image[0].max, (long long *) &header.datasize); + KiiScanMessage (sock, "%lf %lf "OFF_T_FMT, &image[0].image[0].min, &image[0].image[0].max, &header.datasize); KiiScanMessage (sock, "%lf %f %f %f %f", &image[0].image[0].coords.crval1, &image[0].image[0].coords.crpix1, &image[0].image[0].coords.cdelt1, &image[0].image[0].coords.pc1_1, &image[0].image[0].coords.pc1_2); KiiScanMessage (sock, "%lf %f %f %f %f", &image[0].image[0].coords.crval2, &image[0].image[0].coords.crpix2, &image[0].image[0].coords.cdelt2, &image[0].image[0].coords.pc2_1, &image[0].image[0].coords.pc2_2); @@ -85,8 +85,8 @@ fcntl (sock, F_SETFL, !O_NONBLOCK); - if (DEBUG) fprintf (stderr, "read %lld bytes\n", (long long) image[0].image[0].matrix.datasize); + if (DEBUG) fprintf (stderr, "read "OFF_T_FMT" bytes\n", image[0].image[0].matrix.datasize); /* it it not obvious this condition should kill kapa, but ... */ if (image[0].image[0].matrix.datasize != header.datasize) { - fprintf (stderr, "error: expected %lld bytes, but got only %lld\n", (long long) header.datasize, (long long) image[0].image[0].matrix.datasize); + fprintf (stderr, "error: expected "OFF_T_FMT" bytes, but got only "OFF_T_FMT"\n", header.datasize, image[0].image[0].matrix.datasize); return (FALSE); } Index: /branches/pap/Ohana/src/kapa2/src/PSFrame.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/PSFrame.c (revision 28483) +++ /branches/pap/Ohana/src/kapa2/src/PSFrame.c (revision 28484) @@ -4,5 +4,5 @@ int PSFrame (KapaGraphWidget *graph, FILE *f) { - int i, Nticks, P; + int i, j, Nticks, P; double fx, fy, dfx, dfy, lweight; Graphic *graphic; @@ -34,6 +34,6 @@ if (graph[0].axis[i].areticks) { ticks = CreateAxisTicks (&graph[0].axis[i], &Nticks); - for (i = 0; i < Nticks; i++) { - PSTick (graphic, &graph[0].axis[i], P, &ticks[i], i, f); + for (j = 0; j < Nticks; j++) { + PSTick (graphic, &graph[0].axis[i], P, &ticks[j], i, f); } FREE (ticks); @@ -72,5 +72,5 @@ y = fy + (value-min)*dfy/(max - min); - dir = ((naxis == 0) || (naxis == 1)) ? -1 : +1; + dir = ((naxis == 0) || (naxis == 1)) ? +1 : -1; dx = dir*size*dfy*n; dy = dir*size*dfx*n; @@ -86,6 +86,6 @@ /* temporarily assume rectilinear axes */ - if (naxis == 0) { dx = 0; dy = +pad; pos = 1; } - if (naxis == 2) { dx = 0; dy = -pad; pos = 7; } + if (naxis == 0) { dx = 0; dy = -pad; pos = 1; } + if (naxis == 2) { dx = 0; dy = +pad; pos = 7; } if (naxis == 1) { dy = 0; dx = -pad; pos = 3; } Index: /branches/pap/Ohana/src/kapa2/src/bDrawFrame.c =================================================================== --- /branches/pap/Ohana/src/kapa2/src/bDrawFrame.c (revision 28483) +++ /branches/pap/Ohana/src/kapa2/src/bDrawFrame.c (revision 28484) @@ -4,5 +4,5 @@ int bDrawFrame (KapaGraphWidget *graph) { - int i, Nticks, P; + int i, j, Nticks, P; double fx, fy, dfx, dfy, lweight; // Graphic graphic; is not needed @@ -33,6 +33,6 @@ if (graph[0].axis[i].areticks) { ticks = CreateAxisTicks (&graph[0].axis[i], &Nticks); - for (i = 0; i < Nticks; i++) { - bDrawTick (&graph[0].axis[i], P, &ticks[i], i); + for (j = 0; j < Nticks; j++) { + bDrawTick (&graph[0].axis[i], P, &ticks[j], i); } FREE (ticks); Index: /branches/pap/Ohana/src/libautocode/def/autocode.c =================================================================== --- /branches/pap/Ohana/src/libautocode/def/autocode.c (revision 28483) +++ /branches/pap/Ohana/src/libautocode/def/autocode.c (revision 28484) @@ -7,5 +7,5 @@ if (size != $SIZE) { - fprintf (stderr, "WARNING: mismatch in data types $STRUCT: %lld vs %lld\n", (long long) size, (long long) $SIZE); + fprintf (stderr, "WARNING: mismatch in data types $STRUCT: "OFF_T_FMT" vs %d\n", size, $SIZE); return (FALSE); } @@ -32,5 +32,5 @@ if ($SIZE > size) { - fprintf (stderr, "ERROR: uncorrectable mismatch in data types $STRUCT: %lld vs %lld\n", (long long) size, (long long) $SIZE); + fprintf (stderr, "ERROR: uncorrectable mismatch in data types $STRUCT: "OFF_T_FMT" vs %d\n", size, $SIZE); exit (1); } @@ -60,5 +60,12 @@ $STRUCT *gfits_table_get_$STRUCT (FTable *ftable, off_t *Ndata, char *swapped) { + int Ncols; $STRUCT *data, *output; + + Ncols = ftable[0].header[0].Naxis[0]; + if (Ncols != $SIZE) { + fprintf (stderr, "ERROR: mis-match in table size: width is %d but should be %d bytes\n", Ncols, $SIZE); + exit (1); + } *Ndata = ftable[0].header[0].Naxis[1]; @@ -84,5 +91,5 @@ /* create table header */ - gfits_create_table_header (header, "$TYPE", "$EXTNAME"); + if (!gfits_create_table_header (header, "$TYPE", "$EXTNAME")) return (FALSE); /* define table layout */ @@ -90,10 +97,10 @@ /* create table */ - gfits_create_table (header, ftable); + if (!gfits_create_table (header, ftable)) return (FALSE); /* add data values */ - gfits_table_scale_data (ftable); - gfits_convert_$STRUCT (data, sizeof ($STRUCT), Ndata); - gfits_add_rows (ftable, (char *) data, Ndata, sizeof ($STRUCT)); + if (!gfits_table_scale_data (ftable)) return (FALSE); + if (!gfits_convert_$STRUCT (data, sizeof ($STRUCT), Ndata)) return (FALSE); + if (!gfits_add_rows (ftable, (char *) data, Ndata, sizeof ($STRUCT))) return (FALSE); return (TRUE); @@ -103,5 +110,5 @@ /* create table header */ - gfits_create_table_header (header, "$TYPE", "$EXTNAME"); + if (!gfits_create_table_header (header, "$TYPE", "$EXTNAME")) return (FALSE); /* define table layout */ @@ -125,5 +132,5 @@ } - gfits_convert_$STRUCT (tmpdata, sizeof ($STRUCT), Ndata); + if (!gfits_convert_$STRUCT (tmpdata, sizeof ($STRUCT), Ndata)) return (FALSE); SendCommand (device, 16, "NVALUE: %6d", Ndata); @@ -153,5 +160,5 @@ tmpdata = ($STRUCT *) message.buffer; - gfits_convert_$STRUCT (tmpdata, sizeof ($STRUCT), ndata); + if (!gfits_convert_$STRUCT (tmpdata, sizeof ($STRUCT), ndata)) return (FALSE); /* double-check data length? */ Index: /branches/pap/Ohana/src/libautocode/def/common.h =================================================================== --- /branches/pap/Ohana/src/libautocode/def/common.h (revision 28483) +++ /branches/pap/Ohana/src/libautocode/def/common.h (revision 28484) @@ -22,18 +22,8 @@ tmp = byte[X+3]; byte[X+3] = byte[X+4]; byte[X+4] = tmp; -# ifdef linux -# define BYTE_SWAP +# ifndef BYTE_SWAP +# ifndef NOT_BYTE_SWAP +# error "neither BYTE_SWAP not NOT_BYTE_SWAP is set" # endif - -# ifdef darwin_x86 -# define BYTE_SWAP -# endif - -# ifdef sid -# define BYTE_SWAP -# endif - -# ifdef dec -# define BYTE_SWAP # endif Index: /branches/pap/Ohana/src/libdvo/src/LoadPhotcodes.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/LoadPhotcodes.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/LoadPhotcodes.c (revision 28484) @@ -15,5 +15,5 @@ if (LoadPhotcodesText (master_file)) { if (!check_file_access (catdir_file, TRUE, TRUE, TRUE)) return TRUE; - SavePhotcodesFITS (catdir_file); + if (!SavePhotcodesFITS (catdir_file)) return FALSE; return TRUE; } Index: /branches/pap/Ohana/src/libdvo/src/SavePhotcodesFITS.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 28484) @@ -32,8 +32,8 @@ /* convert FITS format data to internal format (byteswaps & EXTNAME) */ - gfits_db_create (&db); - gfits_table_set_PhotCode_PS1_V2 (&db.ftable, photcode_output, table[0].Ncode); - gfits_db_save (&db); - gfits_db_close (&db); + if (!gfits_db_create (&db)) return (FALSE); + if (!gfits_table_set_PhotCode_PS1_V2 (&db.ftable, photcode_output, table[0].Ncode)) return (FALSE); + if (!gfits_db_save (&db)) return (FALSE); + if (!gfits_db_close (&db)) return (FALSE); free (photcode_output); Index: /branches/pap/Ohana/src/libdvo/src/SavePhotcodesText.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/SavePhotcodesText.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/SavePhotcodesText.c (revision 28484) @@ -69,5 +69,5 @@ type, table[0].code[i].C*SCALE, - table[0].code[i].K*SCALE, + table[0].code[i].K, table[0].code[i].dC*SCALE); Index: /branches/pap/Ohana/src/libdvo/src/cmf-ps1-v1-alt.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/cmf-ps1-v1-alt.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/cmf-ps1-v1-alt.c (revision 28484) @@ -13,5 +13,5 @@ // this function is a special case : it must have Nx = 136 if (ftable[0].header[0].Naxis[0] != 136) { - fprintf (stderr, "ERROR: wrong format for CMF_PS1_V1_Alt: %lld vs %d\n", (long long) ftable[0].header[0].Naxis[0], 136); + fprintf (stderr, "ERROR: wrong format for CMF_PS1_V1_Alt: "OFF_T_FMT" vs %d\n", ftable[0].header[0].Naxis[0], 136); exit (2); } Index: /branches/pap/Ohana/src/libdvo/src/dvo_catalog_mef.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 28484) @@ -24,7 +24,7 @@ } /* get the components from the header */ - if (!gfits_scan (&catalog[0].header, "NSTARS", "%lld", 1, (long long *) &Naverage)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMEAS", "%lld", 1, (long long *) &Nmeasure)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMISS", "%lld", 1, (long long *) &Nmissing)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, &Naverage)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, &Nmeasure)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMISS", OFF_T_FMT, 1, &Nmissing)) return (FALSE); if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; @@ -73,5 +73,5 @@ catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); if (Naverage != catalog[0].Naves_disk) { - fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%lld vs %lld)\n", (long long) Naverage, (long long) catalog[0].Naves_disk); + fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naves_disk); } catalog[0].Naverage = catalog[0].Naves_disk; @@ -100,5 +100,5 @@ catalog[0].measure = FtableToMeasure (&ftable, &catalog[0].Nmeasure, &catalog[0].catformat); if (Nmeasure != catalog[0].Nmeas_disk) { - fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%lld vs %lld)\n", (long long) Nmeasure, (long long) catalog[0].Nmeas_disk); + fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmeasure, catalog[0].Nmeas_disk); } catalog[0].Nmeasure = catalog[0].Nmeas_disk; @@ -126,5 +126,5 @@ catalog[0].missing = gfits_table_get_Missing (&ftable, &catalog[0].Nmissing, NULL); if (Nmissing != catalog[0].Nmiss_disk) { - fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%lld vs %lld)\n", (long long) Nmissing, (long long) catalog[0].Nmiss_disk); + fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmissing, catalog[0].Nmiss_disk); } catalog[0].Nmissing = catalog[0].Nmiss_disk; @@ -154,5 +154,5 @@ catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); if (Nexpect != Nitems) { - fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers (%lld vs %lld)\n", (long long) Nexpect, (long long) Nitems); + fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nexpect, Nitems); } @@ -238,7 +238,7 @@ /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", "%lld", 1, (long long) catalog[0].Naverage); - gfits_modify (&catalog[0].header, "NMEAS", "%lld", 1, (long long) catalog[0].Nmeasure); - gfits_modify (&catalog[0].header, "NMISS", "%lld", 1, (long long) catalog[0].Nmissing); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, catalog[0].Naverage); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, catalog[0].Nmeasure); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, catalog[0].Nmissing); gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); Index: /branches/pap/Ohana/src/libdvo/src/dvo_catalog_raw.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 28484) @@ -21,7 +21,7 @@ /* get the components from the header */ catalog[0].Naverage = catalog[0].Nmeasure = catalog[0].Nmissing = catalog[0].Nsecfilt = 0; - if (!gfits_scan (&catalog[0].header, "NSTARS", "%lld", 1, (long long *) &catalog[0].Naverage)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMEAS", "%lld", 1, (long long *) &catalog[0].Nmeasure)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMISS", "%lld", 1, (long long *) &catalog[0].Nmissing)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, &catalog[0].Naverage)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, &catalog[0].Nmeasure)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMISS", OFF_T_FMT, 1, &catalog[0].Nmissing)) return (FALSE); if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &catalog[0].Nsecfilt)) catalog[0].Nsecfilt = 0; @@ -113,9 +113,9 @@ if (VERBOSE) { fprintf (stderr, "star catalog has inconsistent size\n"); - fprintf (stderr, "average: %lld = %lld bytes\n", (long long) catalog[0].Naverage, (long long) catalog[0].Naverage*AverageSize); - fprintf (stderr, "measure: %lld = %lld bytes\n", (long long) catalog[0].Nmeasure, (long long) catalog[0].Nmeasure*MeasureSize); - fprintf (stderr, "missing: %lld = %lld bytes\n", (long long) catalog[0].Nmissing, (long long) catalog[0].Nmissing*MissingSize); - fprintf (stderr, "secfilt: %lld = %lld bytes\n", (long long) catalog[0].Nsecfilt, (long long) catalog[0].Nsecfilt*SecFiltSize*catalog[0].Naverage); - fprintf (stderr, "expect: %lld, found: %lld\n", (long long) size, (long long) filestatus.st_size); + fprintf (stderr, "average: "OFF_T_FMT" = "OFF_T_FMT" bytes\n", catalog[0].Naverage, catalog[0].Naverage*AverageSize); + fprintf (stderr, "measure: "OFF_T_FMT" = "OFF_T_FMT" bytes\n", catalog[0].Nmeasure, catalog[0].Nmeasure*MeasureSize); + fprintf (stderr, "missing: "OFF_T_FMT" = "OFF_T_FMT" bytes\n", catalog[0].Nmissing, catalog[0].Nmissing*MissingSize); + fprintf (stderr, "secfilt: %d = "OFF_T_FMT" bytes\n", catalog[0].Nsecfilt, catalog[0].Nsecfilt*SecFiltSize*catalog[0].Naverage); + fprintf (stderr, "expect: "OFF_T_FMT", found: "OFF_T_FMT"\n", size, filestatus.st_size); } return (FALSE); @@ -155,5 +155,5 @@ nitems = fread (catalog[0].missing, MissingSize, Nitems, f); if (nitems != Nitems) { - if (VERBOSE) fprintf (stderr, "failed to read missing from catalog file %s (%lld vs %lld)\n", catalog[0].filename, (long long) nitems, (long long) Nitems); + if (VERBOSE) fprintf (stderr, "failed to read missing from catalog file %s ("OFF_T_FMT" vs "OFF_T_FMT")\n", catalog[0].filename, nitems, Nitems); return (FALSE); } @@ -198,10 +198,10 @@ } - if (VERBOSE) fprintf (stderr, "read %lld stars from catalog file %s (%lld measurements, %lld missing, %lld secondary filters)\n", - (long long) catalog[0].Naverage, + if (VERBOSE) fprintf (stderr, "read "OFF_T_FMT" stars from catalog file %s ("OFF_T_FMT" measurements, "OFF_T_FMT" missing, %d secondary filters)\n", + catalog[0].Naverage, catalog[0].filename, - (long long) catalog[0].Nmeasure, - (long long) catalog[0].Nmissing, - (long long) catalog[0].Nsecfilt); + catalog[0].Nmeasure, + catalog[0].Nmissing, + catalog[0].Nsecfilt); /* check data integrity */ @@ -214,6 +214,6 @@ if (VERBOSE) { fprintf (stderr, "****** data in catalog %s is corrupt, sums don't check\n", catalog[0].filename); - fprintf (stderr, "****** Nmeas: %lld, %lld\n", (long long) Nmeas, (long long) catalog[0].Nmeasure); - fprintf (stderr, "****** Nmiss: %lld, %lld\n", (long long) Nmiss, (long long) catalog[0].Nmissing); + fprintf (stderr, "****** Nmeas: "OFF_T_FMT", "OFF_T_FMT"\n", Nmeas, catalog[0].Nmeasure); + fprintf (stderr, "****** Nmiss: "OFF_T_FMT", "OFF_T_FMT"\n", Nmiss, catalog[0].Nmissing); } return (FALSE); @@ -264,8 +264,8 @@ /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", "%lld", 1, (long long) catalog[0].Naverage); - gfits_modify (&catalog[0].header, "NMEAS", "%lld", 1, (long long) catalog[0].Nmeasure); - gfits_modify (&catalog[0].header, "NMISS", "%lld", 1, (long long) catalog[0].Nmissing); - gfits_modify (&catalog[0].header, "NSECFILT", "%lld", 1, (long long) catalog[0].Nsecfilt); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, catalog[0].Naverage); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, catalog[0].Nmeasure); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, catalog[0].Nmissing); + gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, catalog[0].Nsecfilt); gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); @@ -353,5 +353,5 @@ nitems = fread (tmpAverage, sizeof(Average_##TYPE), Naverage, f); \ if (nitems != Naverage) { \ - fprintf (stderr, "failed to read averages (%lld vs %lld)\n", (long long) nitems, (long long) Naverage); \ + fprintf (stderr, "failed to read averages ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Naverage); \ return (NULL); \ } \ @@ -367,5 +367,5 @@ nitems = fread (average, sizeof(Average), Naverage, f); if (nitems != Naverage) { - fprintf (stderr, "failed to read averages (%lld vs %lld)\n", (long long) nitems, (long long) Naverage); + fprintf (stderr, "failed to read averages ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Naverage); return (NULL); } @@ -406,5 +406,5 @@ free (tmpAverage); \ if (nitems != Naverage) { \ - fprintf (stderr, "failed to write averages (%lld vs %lld)\n", (long long) nitems, (long long) Naverage); \ + fprintf (stderr, "failed to write averages ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Naverage); \ return (FALSE); \ } \ @@ -417,5 +417,5 @@ nitems = fwrite (average, sizeof(Average), Naverage, f); if (nitems != Naverage) { - fprintf (stderr, "failed to write averages (%lld vs %lld)\n", (long long) nitems, (long long) Naverage); + fprintf (stderr, "failed to write averages ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Naverage); return (FALSE); } @@ -455,5 +455,5 @@ nitems = fread (tmpMeasure, sizeof(Measure_##TYPE), Nmeasure, f); \ if (nitems != Nmeasure) { \ - fprintf (stderr, "failed to read measures (%lld vs %lld)\n", (long long) nitems, (long long) Nmeasure); \ + fprintf (stderr, "failed to read measures ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Nmeasure); \ return (NULL); \ } \ @@ -469,5 +469,5 @@ nitems = fread (measure, sizeof(Measure), Nmeasure, f); if (nitems != Nmeasure) { - fprintf (stderr, "failed to read measures (%lld vs %lld)\n", (long long) nitems, (long long) Nmeasure); + fprintf (stderr, "failed to read measures ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Nmeasure); return (NULL); } @@ -508,5 +508,5 @@ free (tmpMeasure); \ if (nitems != Nmeasure) { \ - fprintf (stderr, "failed to write measures (%lld vs %lld)\n", (long long) nitems, (long long) Nmeasure); \ + fprintf (stderr, "failed to write measures ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Nmeasure); \ return (FALSE); \ } \ @@ -519,5 +519,5 @@ nitems = fwrite (measure, sizeof(Measure), Nmeasure, f); if (nitems != Nmeasure) { - fprintf (stderr, "failed to write measures (%lld vs %lld)\n", (long long) nitems, (long long) Nmeasure); + fprintf (stderr, "failed to write measures ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Nmeasure); return (FALSE); } @@ -557,5 +557,5 @@ nitems = fread (tmpSecFilt, sizeof(SecFilt_##TYPE), Nsecfilt, f); \ if (nitems != Nsecfilt) { \ - fprintf (stderr, "failed to read secfilts (%lld vs %lld)\n", (long long) nitems, (long long) Nsecfilt); \ + fprintf (stderr, "failed to read secfilts ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Nsecfilt); \ return (NULL); \ } \ @@ -571,5 +571,5 @@ nitems = fread (secfilt, sizeof(SecFilt), Nsecfilt, f); if (nitems != Nsecfilt) { - fprintf (stderr, "failed to read secfilts (%lld vs %lld)\n", (long long) nitems, (long long) Nsecfilt); + fprintf (stderr, "failed to read secfilts ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Nsecfilt); return (NULL); } @@ -610,5 +610,5 @@ free (tmpSecFilt); \ if (nitems != Nsecfilt) { \ - fprintf (stderr, "failed to write secfilts (%lld vs %lld)\n", (long long) nitems, (long long) Nsecfilt); \ + fprintf (stderr, "failed to write secfilts ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Nsecfilt); \ return (FALSE); \ } \ @@ -621,5 +621,5 @@ nitems = fwrite (secfilt, sizeof(SecFilt), Nsecfilt, f); if (nitems != Nsecfilt) { - fprintf (stderr, "failed to write secfilts (%lld vs %lld)\n", (long long) nitems, (long long) Nsecfilt); + fprintf (stderr, "failed to write secfilts ("OFF_T_FMT" vs "OFF_T_FMT")\n", nitems, Nsecfilt); return (FALSE); } Index: /branches/pap/Ohana/src/libdvo/src/dvo_catalog_split.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 28484) @@ -174,7 +174,7 @@ /* get the components from the header - these duplicate information in the split files (NAXIS2) */ - if (!gfits_scan (&catalog[0].header, "NSTARS", "%lld", 1, (long long *) &Naverage)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMEAS", "%lld", 1, (long long *) &Nmeasure)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMISS", "%lld", 1, (long long *) &Nmissing)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, &Naverage)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, &Nmeasure)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMISS", OFF_T_FMT, 1, &Nmissing)) return (FALSE); if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; @@ -227,5 +227,5 @@ catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); if (Naverage != catalog[0].Naves_disk) { - fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%lld vs %lld)\n", (long long) Naverage, (long long) catalog[0].Naves_disk); + fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naves_disk); } gfits_free_header (&header); @@ -256,5 +256,5 @@ catalog[0].measure = FtableToMeasure (&ftable, &Nmeasure, &catalog[0].catformat); if (Nmeasure != catalog[0].Nmeas_disk) { - fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%lld vs %lld)\n", (long long) Nmeasure, (long long) catalog[0].Nmeas_disk); + fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmeasure, catalog[0].Nmeas_disk); } catalog[0].Nmeasure = catalog[0].Nmeas_disk; @@ -286,5 +286,5 @@ catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL); if (Nmissing != catalog[0].Nmiss_disk) { - fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%lld vs %lld)\n", (long long) Nmissing, (long long) catalog[0].Nmiss_disk); + fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmissing, catalog[0].Nmiss_disk); } catalog[0].Nmissing = catalog[0].Nmiss_disk; @@ -315,5 +315,5 @@ catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); if (Nitems != catalog[0].Nsecf_disk) { - fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers (%lld vs %lld)\n", (long long) Nitems, (long long) catalog[0].Nsecf_disk); + fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nitems, catalog[0].Nsecf_disk); } catalog[0].Nsecf_mem = catalog[0].Nsecf_disk; @@ -408,5 +408,5 @@ catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); if (Nitems != Nexpect) { - fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers (%lld vs %lld)\n", (long long) Nitems, (long long) Nexpect); + fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nitems, Nexpect); } catalog[0].Nsecf_mem = catalog[0].Naverage * catalog[0].Nsecfilt; @@ -444,5 +444,5 @@ catalog[0].measure = FtableToMeasure (&ftable, &Nmeasure, &catalog[0].catformat); if (Nmeasure != Nrows) { - fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%lld vs %lld)\n", (long long) Nmeasure, (long long) Nrows); + fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmeasure, Nrows); } gfits_free_header (&header); @@ -474,5 +474,5 @@ catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL); if (Nmissing != Nrows) { - fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%lld vs %lld)\n", (long long) Nmissing, (long long) Nrows); + fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmissing, Nrows); } gfits_free_header (&header); @@ -524,7 +524,7 @@ /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", "%lld", 1, (long long) Naves_disk_new); - gfits_modify (&catalog[0].header, "NMEAS", "%lld", 1, (long long) Nmeas_disk_new); - gfits_modify (&catalog[0].header, "NMISS", "%lld", 1, (long long) Nmiss_disk_new); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naves_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeas_disk_new); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmiss_disk_new); gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); @@ -723,7 +723,7 @@ /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", "%lld", 1, (long long) Naves_disk_new); - gfits_modify (&catalog[0].header, "NMEAS", "%lld", 1, (long long) Nmeas_disk_new); - gfits_modify (&catalog[0].header, "NMISS", "%lld", 1, (long long) Nmiss_disk_new); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naves_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeas_disk_new); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmiss_disk_new); gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); @@ -902,7 +902,7 @@ /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", "%lld", 1, (long long) Naves_disk_new); - gfits_modify (&catalog[0].header, "NMEAS", "%lld", 1, (long long) Nmeas_disk_new); - gfits_modify (&catalog[0].header, "NMISS", "%lld", 1, (long long) Nmiss_disk_new); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naves_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeas_disk_new); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmiss_disk_new); gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); Index: /branches/pap/Ohana/src/libdvo/src/dvo_convert.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/dvo_convert.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/dvo_convert.c (revision 28484) @@ -267,5 +267,5 @@ gfits_free_header (theader); gfits_table_mkheader_Image (theader); - gfits_modify (theader, "NAXIS2", "%lld", 1, (long long) Nimage); + gfits_modify (theader, "NAXIS2", OFF_T_FMT, 1, Nimage); theader[0].Naxis[1] = Nimage; ftable[0].datasize = gfits_data_size (theader); @@ -288,5 +288,5 @@ gfits_free_header (theader); \ gfits_table_mkheader_Image (theader); \ - gfits_modify (theader, "NAXIS2", "%lld", 1, (long long) Nimage); \ + gfits_modify (theader, "NAXIS2", OFF_T_FMT, 1, Nimage); \ theader[0].Naxis[1] = Nimage; \ ftable[0].datasize = gfits_data_size (theader); \ @@ -373,8 +373,8 @@ } \ /* convert header from old format to new format */ \ - gfits_scan (theader, "NAXIS2", "%lld", 1, (long long *) &Nimage); \ + gfits_scan (theader, "NAXIS2", OFF_T_FMT, 1, &Nimage); \ gfits_free_header (theader); \ gfits_table_mkheader_Image_##TYPE (theader); \ - gfits_modify (theader, "NAXIS2", "%lld", 1, (long long) Nimage); \ + gfits_modify (theader, "NAXIS2", OFF_T_FMT, 1, Nimage); \ theader[0].Naxis[1] = Nimage; \ vtable[0].datasize = gfits_data_size (theader); \ Index: /branches/pap/Ohana/src/libdvo/src/dvo_image.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/dvo_image.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/dvo_image.c (revision 28484) @@ -120,14 +120,17 @@ /* adjust header */ Nimages = 0; - gfits_scan (&db[0].header, "NIMAGES", "%lld", 1, (long long *) &Nimages); + if (!gfits_scan (&db[0].header, "NIMAGES", OFF_T_FMT, 1, &Nimages)) return (FALSE); Nimages += Nnew; - gfits_modify (&db[0].header, "NIMAGES", "%lld", 1, (long long) Nimages); - - gfits_table_to_vtable (&db[0].ftable, &db[0].vtable, 0, 0); - gfits_vadd_rows (&db[0].vtable, (char *) new, Nnew, sizeof(Image)); + + if (!gfits_modify (&db[0].header, "NIMAGES", OFF_T_FMT, 1, Nimages)) return (FALSE); + + if (!gfits_table_to_vtable (&db[0].ftable, &db[0].vtable, 0, 0)) return (FALSE); + + if (!gfits_vadd_rows (&db[0].vtable, (char *) new, Nnew, sizeof(Image))) return (FALSE); /* check that primary header and table header agree */ if (Nimages != db[0].theader.Naxis[1]) { fprintf (stderr, "header / table length mismatch!\n"); + return (FALSE); } return (TRUE); @@ -157,5 +160,5 @@ exit (2); } - return (TRUE); + return (status); } @@ -195,5 +198,5 @@ gfits_table_set_Image (&db[0].ftable, NULL, 0); - gfits_modify (&db[0].header, "NIMAGES", "%lld", 1, 0LL); + gfits_modify (&db[0].header, "NIMAGES", "%d", 1, 0); gfits_modify (&db[0].header, "ZERO_PT", "%lf", 1, ZeroPoint); Index: /branches/pap/Ohana/src/libdvo/src/dvo_image_raw.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/dvo_image_raw.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/dvo_image_raw.c (revision 28484) @@ -39,5 +39,5 @@ /* find number of images */ Nimage = 0; - gfits_scan (&db[0].header, "NIMAGES", "%lld", 1, (long long *) &Nimage); + gfits_scan (&db[0].header, "NIMAGES", OFF_T_FMT, 1, &Nimage); if (stat (db[0].filename, &filestatus) == -1) { if (VERBOSE) fprintf (stderr, "ERROR: failed to get status of image catalog\n"); @@ -64,5 +64,5 @@ Ndata = (filestatus.st_size - db[0].header.datasize) / ImageSize; if (VERBOSE) fprintf (stderr, "ERROR: image catalog has inconsistent size\n"); - if (VERBOSE) fprintf (stderr, "header: %lld, data: %lld\n", (long long) Nimage, (long long) Ndata); + if (VERBOSE) fprintf (stderr, "header: "OFF_T_FMT", data: "OFF_T_FMT"\n", Nimage, Ndata); if (!FORCE_READ) exit (1); Nimage = Ndata; @@ -96,5 +96,5 @@ } - gfits_modify (&db[0].theader, "NAXIS2", "%lld", 1, (long long) Nimage); + gfits_modify (&db[0].theader, "NAXIS2", OFF_T_FMT, 1, Nimage); db[0].theader.Naxis[1] = Nimage; db[0].ftable.datasize = gfits_data_size (&db[0].theader); @@ -110,5 +110,5 @@ off_t *row; - if (VERBOSE) fprintf (stderr, "writing out %lld images\n", (long long) db[0].vtable.Nrow); + if (VERBOSE) fprintf (stderr, "writing out "OFF_T_FMT" images\n", db[0].vtable.Nrow); /* position to start of file */ @@ -125,6 +125,6 @@ Nrow = db[0].vtable.Nrow; row = db[0].vtable.row; - gfits_scan (db[0].vtable.header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (db[0].vtable.header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (db[0].vtable.header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (db[0].vtable.header, "NAXIS2", OFF_T_FMT, 1, &Ny); /* file pointer is at beginning of desired table data */ @@ -146,5 +146,5 @@ int status; - if (VERBOSE) fprintf (stderr, "writing out %lld images\n", (long long) db[0].theader.Naxis[1]); + if (VERBOSE) fprintf (stderr, "writing out "OFF_T_FMT" images\n", db[0].theader.Naxis[1]); /* position to start of file */ @@ -158,6 +158,6 @@ } - gfits_scan (db[0].ftable.header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (db[0].ftable.header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (db[0].ftable.header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (db[0].ftable.header, "NAXIS2", OFF_T_FMT, 1, &Ny); size = Nx * Ny; Nbytes = fwrite (db[0].ftable.buffer, sizeof(char), size, db[0].f); Index: /branches/pap/Ohana/src/libdvo/src/fits_db.c =================================================================== --- /branches/pap/Ohana/src/libdvo/src/fits_db.c (revision 28483) +++ /branches/pap/Ohana/src/libdvo/src/fits_db.c (revision 28484) @@ -20,9 +20,9 @@ /* create an empty db */ int gfits_db_create (FITS_DB *db) { - gfits_init_header (&db[0].header); + if (!gfits_init_header (&db[0].header)) return (FALSE); db[0].header.extend = TRUE; - gfits_create_header (&db[0].header); - gfits_create_matrix (&db[0].header, &db[0].matrix); - gfits_print (&db[0].header, "NEXTEND", "%d", 1, 1); + if (!gfits_create_header (&db[0].header)) return (FALSE); + if (!gfits_create_matrix (&db[0].header, &db[0].matrix)) return (FALSE); + if (!gfits_print (&db[0].header, "NEXTEND", "%d", 1, 1)) return (FALSE); db[0].ftable.header = &db[0].theader; return (TRUE); Index: /branches/pap/Ohana/src/libfits/header/F_create_H.c =================================================================== --- /branches/pap/Ohana/src/libfits/header/F_create_H.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/header/F_create_H.c (revision 28484) @@ -23,5 +23,5 @@ for (i = 0; i < header[0].Naxes; i++) { snprintf (axis, 10, "NAXIS%d", i + 1); - gfits_modify (header, axis, "%lld", 1, (long long) header[0].Naxis[i]); + gfits_modify (header, axis, OFF_T_FMT, 1, header[0].Naxis[i]); } Index: /branches/pap/Ohana/src/libfits/header/F_modify.c =================================================================== --- /branches/pap/Ohana/src/libfits/header/F_modify.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/header/F_modify.c (revision 28484) @@ -51,19 +51,20 @@ /* write the numeric modes */ - if (!strcmp (mode, "%d")) snprintf (string, 81, "%-8s= %20d / %-s ", field, va_arg (argp, int), comment); - if (!strcmp (mode, "%ld")) snprintf (string, 81, "%-8s= %20ld / %-s ", field, va_arg (argp, long), comment); - if (!strcmp (mode, "%lld")) snprintf (string, 81, "%-8s= %20lld / %-s ", field, va_arg (argp, long long), comment); - if (!strcmp (mode, "%Ld")) snprintf (string, 81, "%-8s= %20lld / %-s ", field, va_arg (argp, long long), comment); - if (!strcmp (mode, "%u")) snprintf (string, 81, "%-8s= %20u / %-s ", field, va_arg (argp, unsigned), comment); - if (!strcmp (mode, "%lu")) snprintf (string, 81, "%-8s= %20lu / %-s ", field, va_arg (argp, unsigned long), comment); - if (!strcmp (mode, "%llu")) snprintf (string, 81, "%-8s= %20llu / %-s ", field, va_arg (argp, unsigned long long), comment); - if (!strcmp (mode, "%Lu")) snprintf (string, 81, "%-8s= %20llu / %-s ", field, va_arg (argp, unsigned long long), comment); - if (!strcmp (mode, "%hd")) snprintf (string, 81, "%-8s= %20d / %-s ", field, va_arg (argp, int), comment); - if (!strcmp (mode, "%f")) snprintf (string, 81, "%-8s= %20.10f / %-s ", field, va_arg (argp, double), comment); - if (!strcmp (mode, "%lf")) snprintf (string, 81, "%-8s= %20.10f / %-s ", field, va_arg (argp, double), comment); - if (!strcmp (mode, "%e")) snprintf (string, 81, "%-8s= %20.10E / %-s ", field, va_arg (argp, double), comment); - if (!strcmp (mode, "%le")) snprintf (string, 81, "%-8s= %20.10E / %-s ", field, va_arg (argp, double), comment); - if (!strcmp (mode, "%g")) snprintf (string, 81, "%-8s= %20.10G / %-s ", field, va_arg (argp, double), comment); - if (!strcmp (mode, "%lg")) snprintf (string, 81, "%-8s= %20.10G / %-s ", field, va_arg (argp, double), comment); + if (!strcmp (mode, "%d")) { snprintf (string, 81, "%-8s= %20d / %-s ", field, va_arg (argp, int), comment); goto found_it; } + if (!strcmp (mode, "%ld")) { snprintf (string, 81, "%-8s= %20ld / %-s ", field, va_arg (argp, long), comment); goto found_it; } + if (!strcmp (mode, "%lld")) { snprintf (string, 81, "%-8s= %20lld / %-s ", field, va_arg (argp, long long), comment); goto found_it; } + if (!strcmp (mode, "%Ld")) { snprintf (string, 81, "%-8s= %20lld / %-s ", field, va_arg (argp, long long), comment); goto found_it; } + if (!strcmp (mode, "%u")) { snprintf (string, 81, "%-8s= %20u / %-s ", field, va_arg (argp, unsigned), comment); goto found_it; } + if (!strcmp (mode, "%lu")) { snprintf (string, 81, "%-8s= %20lu / %-s ", field, va_arg (argp, unsigned long), comment); goto found_it; } + if (!strcmp (mode, "%llu")) { snprintf (string, 81, "%-8s= %20llu / %-s ", field, va_arg (argp, unsigned long long), comment); goto found_it; } + if (!strcmp (mode, "%Lu")) { snprintf (string, 81, "%-8s= %20llu / %-s ", field, va_arg (argp, unsigned long long), comment); goto found_it; } + if (!strcmp (mode, "%hd")) { snprintf (string, 81, "%-8s= %20d / %-s ", field, va_arg (argp, int), comment); goto found_it; } + if (!strcmp (mode, "%f")) { snprintf (string, 81, "%-8s= %20.10f / %-s ", field, va_arg (argp, double), comment); goto found_it; } + if (!strcmp (mode, "%lf")) { snprintf (string, 81, "%-8s= %20.10f / %-s ", field, va_arg (argp, double), comment); goto found_it; } + if (!strcmp (mode, "%e")) { snprintf (string, 81, "%-8s= %20.10E / %-s ", field, va_arg (argp, double), comment); goto found_it; } + if (!strcmp (mode, "%le")) { snprintf (string, 81, "%-8s= %20.10E / %-s ", field, va_arg (argp, double), comment); goto found_it; } + if (!strcmp (mode, "%g")) { snprintf (string, 81, "%-8s= %20.10G / %-s ", field, va_arg (argp, double), comment); goto found_it; } + if (!strcmp (mode, "%lg")) { snprintf (string, 81, "%-8s= %20.10G / %-s ", field, va_arg (argp, double), comment); goto found_it; } + if (!strcmp (mode, "%jd")) { snprintf (string, 81, "%-8s= %20jd / %-s ", field, va_arg (argp, intmax_t), comment); goto found_it; } /* string value. Quotes must be at least 18 chars apart */ @@ -71,6 +72,11 @@ strncpy (data, va_arg (argp, char *), 68); snprintf (string, 81, "%-8s= '%-18s' / %-s ", field, data, comment); - } - + goto found_it; + } + + /* failed to find mode */ + return (FALSE); + +found_it: strncpy (p, string, 80); va_end (argp); Index: /branches/pap/Ohana/src/libfits/header/F_print.c =================================================================== --- /branches/pap/Ohana/src/libfits/header/F_print.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/header/F_print.c (revision 28484) @@ -46,19 +46,20 @@ /* write the numeric modes */ - if (!strcmp (mode, "%d")) { snprintf (string, 81, "%-8s= %20d / %46s ", field, va_arg (argp, int), blank); } - if (!strcmp (mode, "%ld")) { snprintf (string, 81, "%-8s= %20ld / %46s ", field, va_arg (argp, long), blank); } - if (!strcmp (mode, "%lld")){ snprintf (string, 81, "%-8s= %20lld / %46s ", field, va_arg (argp, long long), blank); } - if (!strcmp (mode, "%Ld")) { snprintf (string, 81, "%-8s= %20lld / %46s ", field, va_arg (argp, long long), blank); } - if (!strcmp (mode, "%u")) { snprintf (string, 81, "%-8s= %20u / %46s ", field, va_arg (argp, unsigned), blank); } - if (!strcmp (mode, "%lu")) { snprintf (string, 81, "%-8s= %20lu / %46s ", field, va_arg (argp, unsigned long), blank); } - if (!strcmp (mode, "%llu")){ snprintf (string, 81, "%-8s= %20llu / %46s ", field, va_arg (argp, unsigned long long), blank); } - if (!strcmp (mode, "%Lu")) { snprintf (string, 81, "%-8s= %20llu / %46s ", field, va_arg (argp, unsigned long long), blank); } - if (!strcmp (mode, "%hd")) { snprintf (string, 81, "%-8s= %20d / %46s ", field, va_arg (argp, int), blank); } - if (!strcmp (mode, "%f")) { snprintf (string, 81, "%-8s= %20.10f / %46s ", field, va_arg (argp, double), blank); } - if (!strcmp (mode, "%lf")) { snprintf (string, 81, "%-8s= %20.10f / %46s ", field, va_arg (argp, double), blank); } - if (!strcmp (mode, "%e")) { snprintf (string, 81, "%-8s= %20.10E / %46s ", field, va_arg (argp, double), blank); } - if (!strcmp (mode, "%le")) { snprintf (string, 81, "%-8s= %20.10E / %46s ", field, va_arg (argp, double), blank); } - if (!strcmp (mode, "%g")) { snprintf (string, 81, "%-8s= %20.10G / %46s ", field, va_arg (argp, double), blank); } - if (!strcmp (mode, "%lg")) { snprintf (string, 81, "%-8s= %20.10G / %46s ", field, va_arg (argp, double), blank); } + if (!strcmp (mode, "%d")) { snprintf (string, 81, "%-8s= %20d / %46s ", field, va_arg (argp, int), blank); goto found_it; } + if (!strcmp (mode, "%ld")) { snprintf (string, 81, "%-8s= %20ld / %46s ", field, va_arg (argp, long), blank); goto found_it; } + if (!strcmp (mode, "%lld")){ snprintf (string, 81, "%-8s= %20lld / %46s ", field, va_arg (argp, long long), blank); goto found_it; } + if (!strcmp (mode, "%Ld")) { snprintf (string, 81, "%-8s= %20lld / %46s ", field, va_arg (argp, long long), blank); goto found_it; } + if (!strcmp (mode, "%u")) { snprintf (string, 81, "%-8s= %20u / %46s ", field, va_arg (argp, unsigned), blank); goto found_it; } + if (!strcmp (mode, "%lu")) { snprintf (string, 81, "%-8s= %20lu / %46s ", field, va_arg (argp, unsigned long), blank); goto found_it; } + if (!strcmp (mode, "%llu")){ snprintf (string, 81, "%-8s= %20llu / %46s ", field, va_arg (argp, unsigned long long), blank); goto found_it; } + if (!strcmp (mode, "%Lu")) { snprintf (string, 81, "%-8s= %20llu / %46s ", field, va_arg (argp, unsigned long long), blank); goto found_it; } + if (!strcmp (mode, "%hd")) { snprintf (string, 81, "%-8s= %20d / %46s ", field, va_arg (argp, int), blank); goto found_it; } + if (!strcmp (mode, "%f")) { snprintf (string, 81, "%-8s= %20.10f / %46s ", field, va_arg (argp, double), blank); goto found_it; } + if (!strcmp (mode, "%lf")) { snprintf (string, 81, "%-8s= %20.10f / %46s ", field, va_arg (argp, double), blank); goto found_it; } + if (!strcmp (mode, "%e")) { snprintf (string, 81, "%-8s= %20.10E / %46s ", field, va_arg (argp, double), blank); goto found_it; } + if (!strcmp (mode, "%le")) { snprintf (string, 81, "%-8s= %20.10E / %46s ", field, va_arg (argp, double), blank); goto found_it; } + if (!strcmp (mode, "%g")) { snprintf (string, 81, "%-8s= %20.10G / %46s ", field, va_arg (argp, double), blank); goto found_it; } + if (!strcmp (mode, "%lg")) { snprintf (string, 81, "%-8s= %20.10G / %46s ", field, va_arg (argp, double), blank); goto found_it; } + if (!strcmp (mode, "%jd")) { snprintf (string, 81, "%-8s= %20jd / %46s ", field, va_arg (argp, intmax_t), blank); goto found_it; } /* string value. Quotes must be at least 18 chars apart. Longer lines will this should be fixed to allow arbitrary string lengths, up to 69 chars */ @@ -67,8 +68,10 @@ line[68] = 0; snprintf (string, 81, "%-8s= '%-18s' / %46s ", field, line, blank); + goto found_it; } + return (FALSE); +found_it: strncpy (p, string, 80); - va_end (argp); return (TRUE); Index: /branches/pap/Ohana/src/libfits/header/F_read_H.c =================================================================== --- /branches/pap/Ohana/src/libfits/header/F_read_H.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/header/F_read_H.c (revision 28484) @@ -54,7 +54,12 @@ } + /* if these are not found in the header, they should be set to FALSE */ + header[0].simple = FALSE; + header[0].extend = FALSE; + header[0].unsign = gfits_get_unsign_mode(); header[0].bscale = 1; header[0].bzero = 0; + for (i = 0; i < FT_MAX_NAXES; i++) header[0].Naxis[i] = 0; @@ -69,14 +74,14 @@ gfits_scan (header, "BZERO", "%lf", 1, &header[0].bzero); - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &header[0].Naxis[0]); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &header[0].Naxis[1]); - gfits_scan (header, "NAXIS3", "%lld", 1, (long long *) &header[0].Naxis[2]); - gfits_scan (header, "NAXIS4", "%lld", 1, (long long *) &header[0].Naxis[3]); - gfits_scan (header, "NAXIS5", "%lld", 1, (long long *) &header[0].Naxis[4]); - gfits_scan (header, "NAXIS6", "%lld", 1, (long long *) &header[0].Naxis[5]); - gfits_scan (header, "NAXIS7", "%lld", 1, (long long *) &header[0].Naxis[6]); - gfits_scan (header, "NAXIS8", "%lld", 1, (long long *) &header[0].Naxis[7]); - gfits_scan (header, "NAXIS9", "%lld", 1, (long long *) &header[0].Naxis[8]); - gfits_scan (header, "NAXIS10", "%lld", 1, (long long *) &header[0].Naxis[9]); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &header[0].Naxis[0]); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &header[0].Naxis[1]); + gfits_scan (header, "NAXIS3", OFF_T_FMT, 1, &header[0].Naxis[2]); + gfits_scan (header, "NAXIS4", OFF_T_FMT, 1, &header[0].Naxis[3]); + gfits_scan (header, "NAXIS5", OFF_T_FMT, 1, &header[0].Naxis[4]); + gfits_scan (header, "NAXIS6", OFF_T_FMT, 1, &header[0].Naxis[5]); + gfits_scan (header, "NAXIS7", OFF_T_FMT, 1, &header[0].Naxis[6]); + gfits_scan (header, "NAXIS8", OFF_T_FMT, 1, &header[0].Naxis[7]); + gfits_scan (header, "NAXIS9", OFF_T_FMT, 1, &header[0].Naxis[8]); + gfits_scan (header, "NAXIS10", OFF_T_FMT, 1, &header[0].Naxis[9]); if (!gfits_scan (header, "PCOUNT", "%d", 1, &header[0].pcount)) { Index: /branches/pap/Ohana/src/libfits/header/F_scan.c =================================================================== --- /branches/pap/Ohana/src/libfits/header/F_scan.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/header/F_scan.c (revision 28484) @@ -84,4 +84,7 @@ if (!strcmp (mode, "%hd")) { *va_arg (argp, short *) = value; return (TRUE); } + // XXX is this safe for 64bit off_t and 32bit off_t? + if (!strcmp (mode, "%jd")) { *va_arg (argp, intmax_t *) = value; return (TRUE); } + /* no valid mode found */ return (FALSE); @@ -219,5 +222,5 @@ if (!strcmp (mode, "%d")) { *va_arg (argp, int *) = value; return (TRUE); } if (!strcmp (mode, "%ld")) { *va_arg (argp, long *) = value; return (TRUE); } - if (!strcmp (mode, "%lld")) { *va_arg (argp, long long *) = value; return (TRUE); } + if (!strcmp (mode, OFF_T_FMT)) { *va_arg (argp, long long *) = value; return (TRUE); } if (!strcmp (mode, "%Ld")) { *va_arg (argp, long long *) = value; return (TRUE); } if (!strcmp (mode, "%u")) { *va_arg (argp, unsigned *) = value; return (TRUE); } Index: /branches/pap/Ohana/src/libfits/include/gfitsio.h =================================================================== --- /branches/pap/Ohana/src/libfits/include/gfitsio.h (revision 28483) +++ /branches/pap/Ohana/src/libfits/include/gfitsio.h (revision 28484) @@ -7,23 +7,9 @@ # define GFITSIO -/* also defined in libautocode/def/common.h */ -/* what about lin64 ?? - 'linux' might be defined automatically by linux */ # ifndef BYTE_SWAP -# ifdef linux -# define BYTE_SWAP +# ifndef NOT_BYTE_SWAP +# error "neither BYTE_SWAP not NOT_BYTE_SWAP is set" # endif - -# ifdef sid -# define BYTE_SWAP # endif - -# ifdef darwin_x86 -# define BYTE_SWAP -# endif - -# ifdef dec -# define BYTE_SWAP -# endif -# endif /* BYTE_SWAP */ # ifndef NEWLINE Index: /branches/pap/Ohana/src/libfits/matrix/F_add_M.c =================================================================== --- /branches/pap/Ohana/src/libfits/matrix/F_add_M.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/matrix/F_add_M.c (revision 28484) @@ -13,11 +13,11 @@ if ((x + array[0].Naxis[0] > matrix[0].Naxis[0]) || (y + array[0].Naxis[1] > matrix[0].Naxis[1])) { - fprintf (stderr, "can't add array here: (%lld,%lld) - (%lld,%lld) vs (%lld,%lld)\n", - (long long) x, - (long long) y, - (long long) x + array[0].Naxis[0], - (long long) y + array[0].Naxis[1], - (long long) matrix[0].Naxis[0], - (long long) matrix[0].Naxis[1]); + fprintf (stderr, "can't add array here: ("OFF_T_FMT","OFF_T_FMT") - ("OFF_T_FMT","OFF_T_FMT") vs ("OFF_T_FMT","OFF_T_FMT")\n", + x, + y, + x + array[0].Naxis[0], + y + array[0].Naxis[1], + matrix[0].Naxis[0], + matrix[0].Naxis[1]); return; } Index: /branches/pap/Ohana/src/libfits/matrix/F_compress_M.c =================================================================== --- /branches/pap/Ohana/src/libfits/matrix/F_compress_M.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/matrix/F_compress_M.c (revision 28484) @@ -77,5 +77,5 @@ snprintf (zaxis, 10, "ZNAXIS%d", i + 1); snprintf (naxis, 10, "NAXIS%d", i + 1); - MOD_KEYWORD_REQUIRED (zaxis, naxis, "%lld", (long long *) &header->Naxis[i], (long long) header->Naxis[i]); + MOD_KEYWORD_REQUIRED (zaxis, naxis, OFF_T_FMT, &header->Naxis[i], header->Naxis[i]); } @@ -203,7 +203,7 @@ if (!gfits_varlength_column_define (ftable, &zdef, zcol)) ESCAPE; gfits_delete (header, "TFIELDS", 1); - snprintf (key, 10, "TTYPE%lld", (long long) zcol); + snprintf (key, 10, "TTYPE"OFF_T_FMT, zcol); gfits_delete (header, key, 1); - snprintf (key, 10, "TFORM%lld", (long long) zcol); + snprintf (key, 10, "TFORM"OFF_T_FMT, zcol); gfits_delete (header, key, 1); Index: /branches/pap/Ohana/src/libfits/matrix/F_insert_M.c =================================================================== --- /branches/pap/Ohana/src/libfits/matrix/F_insert_M.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/matrix/F_insert_M.c (revision 28484) @@ -12,11 +12,11 @@ if ((x + array[0].Naxis[0] > matrix[0].Naxis[0]) || (y + array[0].Naxis[1] > matrix[0].Naxis[1])) { - fprintf (stderr, "can't insert array here: (%lld,%lld) - (%lld,%lld) vs (%lld,%lld)\n", - (long long) x, - (long long) y, - (long long) x + array[0].Naxis[0], - (long long) y + array[0].Naxis[1], - (long long) matrix[0].Naxis[0], - (long long) matrix[0].Naxis[1]); + fprintf (stderr, "can't insert array here: ("OFF_T_FMT","OFF_T_FMT") - ("OFF_T_FMT","OFF_T_FMT") vs ("OFF_T_FMT","OFF_T_FMT")\n", + x, + y, + x + array[0].Naxis[0], + y + array[0].Naxis[1], + matrix[0].Naxis[0], + matrix[0].Naxis[1]); return; } Index: /branches/pap/Ohana/src/libfits/matrix/F_load_M.c =================================================================== --- /branches/pap/Ohana/src/libfits/matrix/F_load_M.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/matrix/F_load_M.c (revision 28484) @@ -95,5 +95,5 @@ } if (nbytes != Nbytes) { /* this is a FITS error, but often the image is OK */ - fprintf (stderr, "incomplete block in FITS file: (%lld, %lld)\n", (long long) nbytes, (long long) Nbytes); + fprintf (stderr, "incomplete block in FITS file: ("OFF_T_FMT", "OFF_T_FMT")\n", nbytes, Nbytes); return (TRUE); } Index: /branches/pap/Ohana/src/libfits/matrix/F_read_portion.c =================================================================== --- /branches/pap/Ohana/src/libfits/matrix/F_read_portion.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/matrix/F_read_portion.c (revision 28484) @@ -108,9 +108,9 @@ if (nbytes < Nbytes - 2880) { /* this is a bad FITS error: image is not OK */ - fprintf (stderr, "error reading in matrix data from FITS file %s (%lld < %lld - 2880)\n", filename, (long long) nbytes, (long long) Nbytes); + fprintf (stderr, "error reading in matrix data from FITS file %s ("OFF_T_FMT" < "OFF_T_FMT" - 2880)\n", filename, nbytes, Nbytes); return (FALSE); } if (nbytes != Nbytes) { /* this is a FITS error, but often the image is OK */ - fprintf (stderr, "incomplete block in %s: (%lld, %lld)\n", filename, (long long) nbytes, (long long) Nbytes); + fprintf (stderr, "incomplete block in %s: ("OFF_T_FMT", "OFF_T_FMT")\n", filename, nbytes, Nbytes); return (TRUE); } Index: /branches/pap/Ohana/src/libfits/matrix/F_read_segment.c =================================================================== --- /branches/pap/Ohana/src/libfits/matrix/F_read_segment.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/matrix/F_read_segment.c (revision 28484) @@ -148,9 +148,9 @@ if (nbytes < Nbytes - 2880) { /* this is a bad FITS error: image is not OK */ - fprintf (stderr, "error reading in matrix data from FITS file (%lld < %lld - 2880)\n", (long long) nbytes, (long long) Nbytes); + fprintf (stderr, "error reading in matrix data from FITS file ("OFF_T_FMT" < "OFF_T_FMT" - 2880)\n", nbytes, Nbytes); return (FALSE); } if (nbytes != Nbytes) { /* this is a FITS error, but often the image is OK */ - fprintf (stderr, "incomplete block in FITS file: (%lld, %lld)\n", (long long) nbytes, (long long) Nbytes); + fprintf (stderr, "incomplete block in FITS file: ("OFF_T_FMT", "OFF_T_FMT")\n", nbytes, Nbytes); return (TRUE); } Index: /branches/pap/Ohana/src/libfits/table/F_create_TH.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_create_TH.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_create_TH.c (revision 28484) @@ -27,5 +27,5 @@ for (i = 0; i < header[0].Naxes; i++) { sprintf (axis, "NAXIS%d", i + 1); - gfits_modify (header, axis, "%lld", 1, (long long) header[0].Naxis[i]); + gfits_modify (header, axis, OFF_T_FMT, 1, header[0].Naxis[i]); } Index: /branches/pap/Ohana/src/libfits/table/F_define_column.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_define_column.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_define_column.c (revision 28484) @@ -13,5 +13,5 @@ Nfields = 0; gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Naxis1); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Naxis1); Nfields ++; Naxis1 += Nbytes*Nval; @@ -35,5 +35,5 @@ /* update TFIELDS & NAXIS1 */ gfits_modify (header, "TFIELDS", "%d", 1, Nfields); - gfits_modify (header, "NAXIS1", "%lld", 1, (long long) Naxis1); + gfits_modify (header, "NAXIS1", OFF_T_FMT, 1, Naxis1); header[0].Naxis[0] = Naxis1; @@ -53,5 +53,5 @@ Nfields = 0; gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Naxis1); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Naxis1); Nstart = Naxis1 + 1; Nfields ++; @@ -70,5 +70,5 @@ gfits_modify (header, "TFIELDS", "%d", 1, Nfields); - gfits_modify (header, "NAXIS1", "%lld", 1, (long long) Naxis1); + gfits_modify (header, "NAXIS1", OFF_T_FMT, 1, Naxis1); header[0].Naxis[0] = Naxis1; Index: /branches/pap/Ohana/src/libfits/table/F_get_column.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_get_column.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_get_column.c (revision 28484) @@ -51,6 +51,6 @@ /* check existing table dimensions */ - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); /* scan columns to find insert point */ @@ -187,6 +187,6 @@ /* check existing table dimensions */ - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); /* scan columns to find insert point */ @@ -336,6 +336,6 @@ /* check existing table dimensions */ - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); /* scan columns to find insert point */ Index: /branches/pap/Ohana/src/libfits/table/F_read_T.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_read_T.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_read_T.c (revision 28484) @@ -64,5 +64,5 @@ perror (string); if (Nread < gfits_data_min_size (table[0].header)) { - fprintf (stderr, "error: fits read error in %s, read %lld, need %lld\n", __func__, (long long) Nread, (long long) gfits_data_min_size (table[0].header)); + fprintf (stderr, "error: fits read error in %s, read "OFF_T_FMT", need "OFF_T_FMT"\n", __func__, Nread, gfits_data_min_size (table[0].header)); gfits_free_table (table); return (FALSE); @@ -109,5 +109,5 @@ /* modify structure and header to match actual read rows Ny */ table[0].header[0].Naxis[1] = Nrows; - gfits_modify (table[0].header, "NAXIS2", "%lld", 1, (long long) Nrows); + gfits_modify (table[0].header, "NAXIS2", OFF_T_FMT, 1, Nrows); table[0].datasize = gfits_data_size (table[0].header); @@ -185,6 +185,8 @@ /* file pointer is at beginning of desired table data */ start = ftello (f); - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); + for (i = 0; i < Nrow; i++) { if (row[i] > Ny) { return (FALSE); } Index: /branches/pap/Ohana/src/libfits/table/F_read_TH.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_read_TH.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_read_TH.c (revision 28484) @@ -70,14 +70,14 @@ if (!status) return (FALSE); - gfits_scan (Theader, "NAXIS1", "%lld", 1, (long long *) &Theader[0].Naxis[0]); - gfits_scan (Theader, "NAXIS2", "%lld", 1, (long long *) &Theader[0].Naxis[1]); - gfits_scan (Theader, "NAXIS3", "%lld", 1, (long long *) &Theader[0].Naxis[2]); - gfits_scan (Theader, "NAXIS4", "%lld", 1, (long long *) &Theader[0].Naxis[3]); - gfits_scan (Theader, "NAXIS5", "%lld", 1, (long long *) &Theader[0].Naxis[4]); - gfits_scan (Theader, "NAXIS6", "%lld", 1, (long long *) &Theader[0].Naxis[5]); - gfits_scan (Theader, "NAXIS7", "%lld", 1, (long long *) &Theader[0].Naxis[6]); - gfits_scan (Theader, "NAXIS8", "%lld", 1, (long long *) &Theader[0].Naxis[7]); - gfits_scan (Theader, "NAXIS9", "%lld", 1, (long long *) &Theader[0].Naxis[8]); - gfits_scan (Theader, "NAXIS10", "%lld", 1, (long long *) &Theader[0].Naxis[9]); + gfits_scan (Theader, "NAXIS1", OFF_T_FMT, 1, &Theader[0].Naxis[0]); + gfits_scan (Theader, "NAXIS2", OFF_T_FMT, 1, &Theader[0].Naxis[1]); + gfits_scan (Theader, "NAXIS3", OFF_T_FMT, 1, &Theader[0].Naxis[2]); + gfits_scan (Theader, "NAXIS4", OFF_T_FMT, 1, &Theader[0].Naxis[3]); + gfits_scan (Theader, "NAXIS5", OFF_T_FMT, 1, &Theader[0].Naxis[4]); + gfits_scan (Theader, "NAXIS6", OFF_T_FMT, 1, &Theader[0].Naxis[5]); + gfits_scan (Theader, "NAXIS7", OFF_T_FMT, 1, &Theader[0].Naxis[6]); + gfits_scan (Theader, "NAXIS8", OFF_T_FMT, 1, &Theader[0].Naxis[7]); + gfits_scan (Theader, "NAXIS9", OFF_T_FMT, 1, &Theader[0].Naxis[8]); + gfits_scan (Theader, "NAXIS10", OFF_T_FMT, 1, &Theader[0].Naxis[9]); return (TRUE); Index: /branches/pap/Ohana/src/libfits/table/F_set_column.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_set_column.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_set_column.c (revision 28484) @@ -49,10 +49,10 @@ /* check existing table dimensions */ - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); if (Ny == 0) { Ny = Nrow; header[0].Naxis[1] = Ny; - gfits_modify (header, "NAXIS2", "%lld", 1, (long long) Ny); + gfits_modify (header, "NAXIS2", OFF_T_FMT, 1, Ny); nbytes = gfits_data_size (header); @@ -181,10 +181,10 @@ /* check existing table dimensions */ - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); if (Ny == 0) { Ny = Nrow; header[0].Naxis[1] = Ny; - gfits_modify (header, "NAXIS2", "%lld", 1, (long long) Ny); + gfits_modify (header, "NAXIS2", OFF_T_FMT, 1, Ny); nbytes = gfits_data_size (header); Index: /branches/pap/Ohana/src/libfits/table/F_table_format.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_table_format.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_table_format.c (revision 28484) @@ -130,6 +130,6 @@ off_t i, Nx, Ny; - gfits_scan (ftable[0].header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (ftable[0].header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (ftable[0].header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (ftable[0].header, "NAXIS2", OFF_T_FMT, 1, &Ny); if (start + Nkeep > Ny) return (FALSE); @@ -158,6 +158,6 @@ off_t i, N, Nx, Ny; - gfits_scan (ftable[0].header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (ftable[0].header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (ftable[0].header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (ftable[0].header, "NAXIS2", OFF_T_FMT, 1, &Ny); /* make empty vtable from table */ @@ -191,5 +191,5 @@ va_start (argp, table); - gfits_scan (table[0].header, "NAXIS1", "%lld", 1, (long long *) &Nx); + gfits_scan (table[0].header, "NAXIS1", OFF_T_FMT, 1, &Nx); gfits_scan (table[0].header, "TFIELDS", "%d", 1, &Nfields); @@ -236,6 +236,6 @@ off = 0; - gfits_scan (ftable[0].header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (ftable[0].header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (ftable[0].header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (ftable[0].header, "NAXIS2", OFF_T_FMT, 1, &Ny); gfits_scan (ftable[0].header, "TFIELDS", "%d", 1, &Nfields); @@ -320,6 +320,6 @@ off = 0; - gfits_scan (ftable[0].header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (ftable[0].header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (ftable[0].header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (ftable[0].header, "NAXIS2", OFF_T_FMT, 1, &Ny); gfits_scan (ftable[0].header, "TFIELDS", "%d", 1, &Nfields); Index: /branches/pap/Ohana/src/libfits/table/F_table_row.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_table_row.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_table_row.c (revision 28484) @@ -11,6 +11,6 @@ header = table[0].header; - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); if (header[0].Naxis[1] != Ny) return (FALSE); @@ -23,5 +23,5 @@ Ny += Nrow; header[0].Naxis[1] = Ny; - gfits_modify (header, "NAXIS2", "%lld", 1, (long long) Ny); + gfits_modify (header, "NAXIS2", OFF_T_FMT, 1, Ny); nbytes = gfits_data_size (header); @@ -43,6 +43,6 @@ header = table[0].header; - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); if (header[0].Naxis[1] != Ny) return (FALSE); @@ -64,5 +64,5 @@ Ny += Nrow; header[0].Naxis[1] = Ny; - gfits_modify (header, "NAXIS2", "%lld", 1, (long long) Ny); + gfits_modify (header, "NAXIS2", OFF_T_FMT, 1, Ny); table[0].datasize = gfits_data_size (table[0].header); @@ -81,6 +81,6 @@ header = table[0].header; - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); if (header[0].Naxis[1] != Ny) return (FALSE); @@ -97,5 +97,5 @@ Ny -= Nrow; header[0].Naxis[1] = Ny; - gfits_modify (header, "NAXIS2", "%lld", 1, (long long) Ny); + gfits_modify (header, "NAXIS2", OFF_T_FMT, 1, Ny); nbytes = gfits_data_size (header); Index: /branches/pap/Ohana/src/libfits/table/F_table_varlength.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_table_varlength.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_table_varlength.c (revision 28484) @@ -64,5 +64,5 @@ // heap_start must be long long so file may be very large - if (!gfits_scan (ftable->header, "THEAP", "%lld", 1, (long long *) &def->heap_start)) { + if (!gfits_scan (ftable->header, "THEAP", OFF_T_FMT, 1, &def->heap_start)) { def->heap_start = ftable->header->Naxis[0]*ftable->header->Naxis[1]; } Index: /branches/pap/Ohana/src/libfits/table/F_write_T.c =================================================================== --- /branches/pap/Ohana/src/libfits/table/F_write_T.c (revision 28483) +++ /branches/pap/Ohana/src/libfits/table/F_write_T.c (revision 28484) @@ -40,6 +40,6 @@ Nrow = table[0].Nrow; row = table[0].row; - gfits_scan (table[0].header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (table[0].header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (table[0].header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (table[0].header, "NAXIS2", OFF_T_FMT, 1, &Ny); /* file pointer is at beginning of desired table data */ @@ -79,5 +79,5 @@ /* modify vtable to represent full disk table */ - gfits_modify (ftable[0].header, "NAXIS2", "%lld", 1, (long long) Ntotal); + gfits_modify (ftable[0].header, "NAXIS2", OFF_T_FMT, 1, Ntotal); ftable[0].header[0].Naxis[1] = Ntotal; Index: /branches/pap/Ohana/src/libohana/Makefile =================================================================== --- /branches/pap/Ohana/src/libohana/Makefile (revision 28483) +++ /branches/pap/Ohana/src/libohana/Makefile (revision 28484) @@ -1,5 +1,5 @@ default: install help: - @echo "make options: install libohana clean dist" + @echo "make options: install libohana clean dist test typetest" include ../../Makefile.System @@ -19,5 +19,5 @@ TFLAGS = $(FULL_CFLAGS) $(FULL_CPPFLAGS) $(FULL_LDFLAGS) -lohana -ltap_ohana -install: $(DESTLIB)/libohana.a $(DESTLIB)/libohana.$(DLLTYPE) +install: $(DESTLIB)/libohana.a $(DESTLIB)/libohana.$(DLLTYPE) typetest libohana: $(LIB)/libohana.$(ARCH).a $(LIB)/libohana.$(ARCH).$(DLLTYPE) @@ -40,4 +40,12 @@ $(SRC)/CommOps.$(ARCH).o \ $(SRC)/version.$(ARCH).o + +TYPETEST = \ +$(TESTDIR)/typetest.$(ARCH) + +$(TYPETEST) : $(LIB)/libohana.$(ARCH).a + +typetest: $(TYPETEST) + for i in $(TYPETEST); do $$i || exit 1; done TEST = \ Index: /branches/pap/Ohana/src/libohana/include/ohana.h =================================================================== --- /branches/pap/Ohana/src/libohana/include/ohana.h (revision 28483) +++ /branches/pap/Ohana/src/libohana/include/ohana.h (revision 28484) @@ -94,4 +94,31 @@ }; +/* note: in the Ohana tree, the byte order is determined by the ARCH variable + if you have a small endian machine that is not listed here, the test + program 'typestest' should fail */ +# ifndef BYTE_SWAP +# ifdef linux +# define BYTE_SWAP +# endif + +# ifdef lin64 +# define BYTE_SWAP +# endif + +# ifdef sid +# define BYTE_SWAP +# endif + +# if defined(darwin_x86) || defined(_DARWIN_C_SOURCE) +# define BYTE_SWAP +# endif + +# ifdef dec +# define BYTE_SWAP +# endif +# else +# define NOT_BYTE_SWAP +# endif /* BYTE_SWAP */ + # ifndef NAN # ifndef BYTE_SWAP @@ -103,4 +130,22 @@ __attribute_used__ = { __nan_bytes }; # define NAN (__nan_union.__d) +# endif + +/* if your build crashes on OFF_T_MODE, you probably need to add your 64bit hardware to this list */ +# ifdef _LARGEFILE_SOURCE +# define OFF_T_FMT "%jd" +# endif +# ifdef lin64 +# define OFF_T_FMT "%jd" +# endif +# ifdef _DARWIN_C_SOURCE +# define OFF_T_FMT "%jd" +#endif +# ifndef OFF_T_FMT +# define OFF_T_FMT "%ld" +# endif + +# ifndef isfinite +# define isfinite(A) (!isnan(A)) # endif Index: /branches/pap/Ohana/src/libohana/test/typetest.c =================================================================== --- /branches/pap/Ohana/src/libohana/test/typetest.c (revision 28484) +++ /branches/pap/Ohana/src/libohana/test/typetest.c (revision 28484) @@ -0,0 +1,73 @@ +# include "ohana.h" + +enum +{ + OHANA_LITTLE_ENDIAN = 0x03020100ul, + OHANA_BIG_ENDIAN = 0x00010203ul, +}; + +static const union { + unsigned char bytes[4]; + int value; +} ohana_host_order = { { 0, 1, 2, 3 } }; + +# define OHANA_HOST_ORDER (ohana_host_order.value) + +// we need to verify some type-related issues: +int main (int argc, char **argv) { + + int status; + + status = 0; + + // 1) have we defined BYTE_SWAP correctly? + +# ifdef BYTE_SWAP + if (OHANA_HOST_ORDER == OHANA_BIG_ENDIAN) { + fprintf (stderr, "ERROR: BYTE_SWAP is defined on a big endian machine\n"); + fprintf (stderr, "(see libohana/include/ohana.h)\n"); + status = 1; + } else { + fprintf (stderr, "BYTE_SWAP is small endian\n"); + } +# else + if (OHANA_HOST_ORDER == OHANA_BIG_ENDIAN) { + fprintf (stderr, "NOT_BYTE_SWAP is big endian\n"); + } else { + fprintf (stderr, "ERROR: BYTE_SWAP not defined on a small endian machine\n"); + fprintf (stderr, "(see libohana/include/ohana.h)\n"); + status = 1; + } +# endif + + // 2) have we defined NAN correctly? + { + float fvalue; + double dvalue; + + fvalue = NAN; + dvalue = NAN; + + if (isfinite(fvalue)) { + fprintf (stderr, "ERROR: NAN definition fails for float\n"); + fprintf (stderr, "fvalue: %e\n", fvalue); + status = 1; + } + if (isfinite(dvalue)) { + fprintf (stderr, "ERROR: NAN definition fails for double\n"); + fprintf (stderr, "dvalue: %e\n", dvalue); + status = 1; + } + + fprintf (stderr, "fvalue: %e (isfinite: %d)\n", fvalue, isfinite(fvalue)); + fprintf (stderr, "dvalue: %e (isfinite: %d)\n", dvalue, isfinite(dvalue)); + +# ifdef __STDC_VERSION__ + fprintf (stderr, "STDC_VERSION: %ld\n", __STDC_VERSION__); +# else + fprintf (stderr, "STDC_VERSION is not set\n"); +# endif + + } + exit (status); +} Index: /branches/pap/Ohana/src/opihi/cmd.data/create.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/create.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/cmd.data/create.c (revision 28484) @@ -4,5 +4,5 @@ int i, N, INT; - float start, end, delta; + opihi_flt start, end, delta; Vector *vec; Index: /branches/pap/Ohana/src/opihi/cmd.data/extract.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/extract.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/cmd.data/extract.c (revision 28484) @@ -70,6 +70,6 @@ if ((out[0].header.Naxis[1] != Ny) || (out[0].header.Naxis[0] != Nx)) { gprint (GP_ERR, "matrix sizes mis-matched\n"); - gprint (GP_ERR, "%d x %d vs %lld x %lld\n", Nx, Ny, - (long long) out[0].header.Naxis[0], (long long) out[0].header.Naxis[1]); + gprint (GP_ERR, "%d x %d vs "OFF_T_FMT" x "OFF_T_FMT"\n", Nx, Ny, + out[0].header.Naxis[0], out[0].header.Naxis[1]); return (FALSE); } Index: /branches/pap/Ohana/src/opihi/cmd.data/help/histogram =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/help/histogram (revision 28483) +++ /branches/pap/Ohana/src/opihi/cmd.data/help/histogram (revision 28484) @@ -1,8 +1,19 @@ - histogram [-region sx sy nx ny] [-range min max] + histogram [] [-range ] - calculate a histogram of the image pixel values in the given - buffer, optionally constrained to the given region, with optional - max and min values. the results are placed in the vectors x and y, - which contain the pixel values and the number of occurences. + calculate a histogram of the values and store the + occurrences count in the buffer. Optionally constrained to + the given - region with step value (default + step is 1). The optional '-range ' parameter allows storing + the range - values with increment + Sample code usage: + + # create a vector ('x') containing arbitrary values [0.:1.] range + create y 0 100 1; set x = sin(y) + + # build histogram from x from 0. to 1. with 0.1 delta step + histogram x xhist 0. 1. .1 -range dx + + # plot corresponding histogram + limits dx xhist; clear; box; plot dx xhist -x 1 Index: /branches/pap/Ohana/src/opihi/cmd.data/help/imhist =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/help/imhist (revision 28484) +++ /branches/pap/Ohana/src/opihi/cmd.data/help/imhist (revision 28484) @@ -0,0 +1,8 @@ + + imhist [-region sx sy nx ny] [-range min max] + + calculate a histogram of the image pixel values in the given + buffer, optionally constrained to the given region, with optional + max and min values. the results are placed in the vectors x and y, + which contain the pixel values and the number of occurences. + Index: /branches/pap/Ohana/src/opihi/cmd.data/matrix.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/matrix.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/cmd.data/matrix.c (revision 28484) @@ -23,7 +23,7 @@ if (bufB[0].matrix.Naxis[0] != bufC[0].matrix.Naxis[1]) { - gprint (GP_ERR, "size mis-match in matrices: (%lld x %lld) * (%lld x %lld)\n", - (long long) bufB[0].matrix.Naxis[0], (long long) bufB[0].matrix.Naxis[1], - (long long) bufC[0].matrix.Naxis[0], (long long) bufC[0].matrix.Naxis[1]); + gprint (GP_ERR, "size mis-match in matrices: ("OFF_T_FMT" x "OFF_T_FMT") * ("OFF_T_FMT" x "OFF_T_FMT")\n", + bufB[0].matrix.Naxis[0], bufB[0].matrix.Naxis[1], + bufC[0].matrix.Naxis[0], bufC[0].matrix.Naxis[1]); return (FALSE); } Index: /branches/pap/Ohana/src/opihi/cmd.data/rd.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/rd.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/cmd.data/rd.c (revision 28484) @@ -222,6 +222,6 @@ buf[0].unsign = buf[0].header.unsign; - gprint (GP_LOG, "read %lld bytes from %s into buffer %s\n", - (long long) buf[0].header.datasize + buf[0].matrix.datasize, argv[2], argv[1]); + gprint (GP_LOG, "read "OFF_T_FMT" bytes from %s into buffer %s\n", + buf[0].header.datasize + buf[0].matrix.datasize, argv[2], argv[1]); blank = 0xffff; Index: /branches/pap/Ohana/src/opihi/cmd.data/rdseg.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/rdseg.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/cmd.data/rdseg.c (revision 28484) @@ -62,5 +62,5 @@ buf[0].bzero = buf[0].header.bzero; /* store the original values */ buf[0].unsign = buf[0].header.unsign; - gprint (GP_LOG, "read %lld bytes from %s into buffer %s\n", (long long) buf[0].header.datasize + buf[0].matrix.datasize, argv[2], argv[1]); + gprint (GP_LOG, "read "OFF_T_FMT" bytes from %s into buffer %s\n", buf[0].header.datasize + buf[0].matrix.datasize, argv[2], argv[1]); gfits_scan (&buf[0].header, "BLANK", "%d", 1, &blank); Index: /branches/pap/Ohana/src/opihi/cmd.data/rebin.c =================================================================== --- /branches/pap/Ohana/src/opihi/cmd.data/rebin.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/cmd.data/rebin.c (revision 28484) @@ -63,5 +63,5 @@ } } - if (VERBOSE) gprint (GP_LOG, "rebin %s to %s (%lld,%lld to %d,%d)\n", argv[1], argv[2], (long long) in[0].header.Naxis[0], (long long) in[0].header.Naxis[1], nx, ny); + if (VERBOSE) gprint (GP_LOG, "rebin %s to %s ("OFF_T_FMT","OFF_T_FMT" to %d,%d)\n", argv[1], argv[2], in[0].header.Naxis[0], in[0].header.Naxis[1], nx, ny); Nx = in[0].header.Naxis[0]; Index: /branches/pap/Ohana/src/opihi/dvo/avextract.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/avextract.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/avextract.c (revision 28484) @@ -103,5 +103,5 @@ catalog.Nsecfilt = 0; - if (VERBOSE) gprint (GP_ERR, "trying %s (%lld of %lld)\n", catalog.filename, (long long) i, (long long) skylist[0].Nregions); + if (VERBOSE) gprint (GP_ERR, "trying %s ("OFF_T_FMT" of "OFF_T_FMT")\n", catalog.filename, i, skylist[0].Nregions); // an error exit status here is a significant error Index: /branches/pap/Ohana/src/opihi/dvo/avmatch.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/avmatch.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/avmatch.c (revision 28484) @@ -102,5 +102,5 @@ catalog.Nsecfilt = 0; - if (VERBOSE) gprint (GP_ERR, "trying %s (%lld of %lld)\n", catalog.filename, (long long) i, (long long) skylist[0].Nregions); + if (VERBOSE) gprint (GP_ERR, "trying %s ("OFF_T_FMT" of "OFF_T_FMT")\n", catalog.filename, i, skylist[0].Nregions); // an error exit status here is a significant error Index: /branches/pap/Ohana/src/opihi/dvo/cmatch.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/cmatch.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/cmatch.c (revision 28484) @@ -37,5 +37,5 @@ } dvo_catalog_unlock (&catalog1); - gprint (GP_ERR, "read %lld stars from phot catalog file %s\n", (long long) catalog1.Naverage, filename); + gprint (GP_ERR, "read "OFF_T_FMT" stars from phot catalog file %s\n", catalog1.Naverage, filename); /* this is for loading from a text file, presumably hstgsc or usno Index: /branches/pap/Ohana/src/opihi/dvo/cmpread.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/cmpread.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/cmpread.c (revision 28484) @@ -65,5 +65,5 @@ /* find expected number of stars */ - if (!gfits_scan (&header, "NSTARS", "%lld", 1, (long long *) &Nstars)) { + if (!gfits_scan (&header, "NSTARS", OFF_T_FMT, 1, &Nstars)) { gprint (GP_ERR, "ERROR: can't get NSTARS from header\n"); gfits_free_header (&header); @@ -77,5 +77,5 @@ /* allocate space for stars */ gprint (GP_ERR, "reading from TEXT cmp file %s\n", argv[2]); - if (!gfits_scan (&header, "NSTARS", "%lld", 1, (long long *) &Nstars)) { + if (!gfits_scan (&header, "NSTARS", OFF_T_FMT, 1, &Nstars)) { gprint (GP_ERR, "ERROR: failed to find NSTARS\n"); exit (1); @@ -139,5 +139,5 @@ free (stars); gfits_free_header (&header); - gprint (GP_ERR, "loaded %lld objects\n", (long long) Nstars); + gprint (GP_ERR, "loaded "OFF_T_FMT" objects\n", Nstars); return (TRUE); } Index: /branches/pap/Ohana/src/opihi/dvo/dbExtractMeasures.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/dbExtractMeasures.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/dbExtractMeasures.c (revision 28484) @@ -303,4 +303,7 @@ case MEAS_AZ: /* OK */ value.Flt = measure[0].az; + break; + case MEAS_ALT: /* OK */ + value.Flt = 90.0 - DEG_RAD*acos(1.0/measure[0].airmass); break; case MEAS_EXPTIME: /* OK */ Index: /branches/pap/Ohana/src/opihi/dvo/detrend.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/detrend.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/detrend.c (revision 28484) @@ -137,5 +137,5 @@ /* load existing data from database */ - gfits_scan (&header, "NIMAGES", "%lld", 1, (long long *) &Nimage); + gfits_scan (&header, "NIMAGES", OFF_T_FMT, 1, &Nimage); ALLOCATE (pimage, RegImage, Nimage); status = fread (pimage, sizeof(RegImage), Nimage, f); @@ -143,5 +143,5 @@ if (status != Nimage) { - gprint (GP_ERR, "ERROR: header and data in dB don't match (%lld vs %lld)\n", (long long) Nimage, (long long) status); + gprint (GP_ERR, "ERROR: header and data in dB don't match ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nimage, status); gfits_free_header (&header); free (pimage); Index: /branches/pap/Ohana/src/opihi/dvo/fitcolors.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/fitcolors.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/fitcolors.c (revision 28484) @@ -174,5 +174,5 @@ // the selection criteria } - gprint (GP_ERR, "using %lld possible regions\n", (long long) skylist[0].Nregions); + gprint (GP_ERR, "using "OFF_T_FMT" possible regions\n", skylist[0].Nregions); /* vectors to save data */ Index: /branches/pap/Ohana/src/opihi/dvo/fitsed.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/fitsed.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/fitsed.c (revision 28484) @@ -185,5 +185,5 @@ /* loop over regions, extract data for each region */ // XXX add interrupt checks - gprint (GP_ERR, "using %lld possible regions\n", (long long) skylist[0].Nregions); + gprint (GP_ERR, "using "OFF_T_FMT" possible regions\n", skylist[0].Nregions); for (k = 0; k < skylist[0].Nregions; k++) { /* lock, load, unlock catalog */ Index: /branches/pap/Ohana/src/opihi/dvo/gstar.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/gstar.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/gstar.c (revision 28484) @@ -159,5 +159,5 @@ k = N1[i]; if (!QUIET) { - gprint (GP_LOG, "star: %lld\n", (long long) k); + gprint (GP_LOG, "star: "OFF_T_FMT"\n", i); gprint (GP_LOG, "%11.7f ", catalog.average[k].R); gprint (GP_LOG, "%11.7f ", catalog.average[k].D); @@ -165,5 +165,7 @@ gprint (GP_LOG, "%3d ", catalog.average[k].Nmeasure); gprint (GP_LOG, "%4.1f ", 0.01*catalog.average[k].Xp); - gprint (GP_LOG, "%5d ", catalog.average[k].flags); + gprint (GP_LOG, "%5x ", catalog.average[k].flags); + gprint (GP_LOG, "%x ", catalog.average[k].objID); + gprint (GP_LOG, "%x ", catalog.average[k].catID); if (FULL_OUTPUT) { @@ -176,6 +178,4 @@ gprint (GP_LOG, "%f ", catalog.average[k].P); gprint (GP_LOG, "%f ", catalog.average[k].dP); - gprint (GP_LOG, "%x ", catalog.average[k].objID); - gprint (GP_LOG, "%x ", catalog.average[k].catID); } @@ -244,6 +244,4 @@ gprint (GP_LOG, "%f ", catalog.measure[m].crNsigma); gprint (GP_LOG, "%f ", catalog.measure[m].extNsigma); - gprint (GP_LOG, "%f ", 0.01*catalog.measure[m].FWx); - gprint (GP_LOG, "%f ", 0.01*catalog.measure[m].FWy); gprint (GP_LOG, "%f ", (360.0/(float)0xffff)*catalog.measure[m].theta); } Index: /branches/pap/Ohana/src/opihi/dvo/imrough.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/imrough.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/imrough.c (revision 28484) @@ -267,5 +267,5 @@ /* load existing data from database */ - gfits_scan (&header, "NIMAGES", "%lld", 1, (long long *) &Nimage); + gfits_scan (&header, "NIMAGES", OFF_T_FMT, 1, &Nimage); ALLOCATE (image, RegImage, Nimage); status = fread (image, sizeof(RegImage), Nimage, f); @@ -273,5 +273,5 @@ if (status != Nimage) { - gprint (GP_ERR, "ERROR: header and data in dB don't match (%lld vs %d)\n", (long long) Nimage, status); + gprint (GP_ERR, "ERROR: header and data in dB don't match ("OFF_T_FMT" vs %d)\n", Nimage, status); gfits_free_header (&header); free (image); @@ -304,5 +304,5 @@ /* convert to internal format */ image = (RegImage *) table.buffer; - gfits_scan (table.header, "NAXIS2", "%lld", 1, (long long *) &Nimage); + gfits_scan (table.header, "NAXIS2", OFF_T_FMT, 1, &Nimage); gfits_convert_RegImage (image, sizeof (RegImage), Nimage); Index: /branches/pap/Ohana/src/opihi/dvo/imsearch.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/imsearch.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/imsearch.c (revision 28484) @@ -106,10 +106,10 @@ /* load existing data from database */ - gfits_scan (&header, "NIMAGES", "%lld", 1, (long long *) &Nimage); + gfits_scan (&header, "NIMAGES", OFF_T_FMT, 1, &Nimage); ALLOCATE (pimage, RegImage, Nimage); status = fread (pimage, sizeof(RegImage), Nimage, f); fclose (f); if (status != Nimage) { - gprint (GP_ERR, "ERROR: header and data in dB don't match (%lld vs %lld)\n", (long long) Nimage, (long long) status); + gprint (GP_ERR, "ERROR: header and data in dB don't match ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nimage, status); gfits_free_header (&header); free (pimage); @@ -131,5 +131,5 @@ obstime[strlen(obstime)-1] = 0; - gprint (GP_LOG, "%5lld %6s %6s %2d %2d ", (long long) i, get_type_name(pimage[i].type), get_mode_name(pimage[i].mode), pimage[i].ccd, pimage[i].type); + gprint (GP_LOG, OFF_T_FMT" %6s %6s %2d %2d ", i, get_type_name(pimage[i].type), get_mode_name(pimage[i].mode), pimage[i].ccd, pimage[i].type); gprint (GP_LOG, "%s %s ", pimage[i].pathname, pimage[i].filename); gprint (GP_LOG, "%s %s %f %s\n", pimage[i].filter, pimage[i].instrument, pimage[i].exptime, obstime); Index: /branches/pap/Ohana/src/opihi/dvo/mextract.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/mextract.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/mextract.c (revision 28484) @@ -120,5 +120,5 @@ catalog.Nsecfilt = Nsecfilt; - if (VERBOSE) gprint (GP_ERR, "trying %s (%lld of %lld)\n", catalog.filename, (long long) i, (long long) skylist[0].Nregions); + if (VERBOSE) gprint (GP_ERR, "trying %s ("OFF_T_FMT" of "OFF_T_FMT")\n", catalog.filename, i, skylist[0].Nregions); // an error exit status here is a significant error Index: /branches/pap/Ohana/src/opihi/dvo/mmextract.c =================================================================== --- /branches/pap/Ohana/src/opihi/dvo/mmextract.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/dvo/mmextract.c (revision 28484) @@ -186,5 +186,5 @@ catalog.Nsecfilt = Nsecfilt; - if (VERBOSE) gprint (GP_ERR, "trying %s (%lld of %lld)\n", catalog.filename, (long long) i, (long long) skylist[0].Nregions); + if (VERBOSE) gprint (GP_ERR, "trying %s ("OFF_T_FMT" of "OFF_T_FMT")\n", catalog.filename, i, skylist[0].Nregions); // an error exit status here is a significant error Index: /branches/pap/Ohana/src/opihi/include/pcontrol.h =================================================================== --- /branches/pap/Ohana/src/opihi/include/pcontrol.h (revision 28483) +++ /branches/pap/Ohana/src/opihi/include/pcontrol.h (revision 28484) @@ -120,6 +120,6 @@ JobStat state; JobStat stack; - JobOutput stdout; - JobOutput stderr; + JobOutput stdout_buf; + JobOutput stderr_buf; Ptime start; Ptime stop; Index: /branches/pap/Ohana/src/opihi/pcontrol/CheckBusyJob.c =================================================================== --- /branches/pap/Ohana/src/opihi/pcontrol/CheckBusyJob.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/pcontrol/CheckBusyJob.c (revision 28484) @@ -94,11 +94,11 @@ sscanf (p, "%*s %d", &job[0].exit_status); p = memstr (buffer[0].buffer, "STDOUT", buffer[0].Nbuffer); - sscanf (p, "%*s %d", &job[0].stdout.size); + sscanf (p, "%*s %d", &job[0].stdout_buf.size); p = memstr (buffer[0].buffer, "STDERR", buffer[0].Nbuffer); - sscanf (p, "%*s %d", &job[0].stderr.size); + sscanf (p, "%*s %d", &job[0].stderr_buf.size); // XXX runaway job if output too large? - if (job[0].stdout.size > 0x1000000) abort(); - if (job[0].stderr.size > 0x1000000) abort(); + if (job[0].stdout_buf.size > 0x1000000) abort(); + if (job[0].stderr_buf.size > 0x1000000) abort(); // job has exited : move to DONE stack Index: /branches/pap/Ohana/src/opihi/pcontrol/CheckDoneJob.c =================================================================== --- /branches/pap/Ohana/src/opihi/pcontrol/CheckDoneJob.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/pcontrol/CheckDoneJob.c (revision 28484) @@ -19,10 +19,10 @@ // we can always call this for stdout (if it is done, this is a NOP) - status1 = GetJobOutput ("stdout", host, &job[0].stdout); + status1 = GetJobOutput ("stdout", host, &job[0].stdout_buf); // we cannot try stderr until stdout is completed status2 = PCLIENT_HUNG; - if (job[0].stdout.completed) { - status2 = GetJobOutput ("stderr", host, &job[0].stderr); + if (job[0].stdout_buf.completed) { + status2 = GetJobOutput ("stderr", host, &job[0].stderr_buf); } Index: /branches/pap/Ohana/src/opihi/pcontrol/JobOps.c =================================================================== --- /branches/pap/Ohana/src/opihi/pcontrol/JobOps.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/pcontrol/JobOps.c (revision 28484) @@ -216,6 +216,6 @@ job[0].Reset = FALSE; - InitJobOutput (&job[0].stdout); - InitJobOutput (&job[0].stderr); + InitJobOutput (&job[0].stdout_buf); + InitJobOutput (&job[0].stderr_buf); job[0].mode = mode; @@ -257,6 +257,6 @@ FREE (job[0].argv); - FreeIOBuffer (&job[0].stdout.buffer); - FreeIOBuffer (&job[0].stderr.buffer); + FreeIOBuffer (&job[0].stdout_buf.buffer); + FreeIOBuffer (&job[0].stderr_buf.buffer); FREE (job); Index: /branches/pap/Ohana/src/opihi/pcontrol/StartHost.c =================================================================== --- /branches/pap/Ohana/src/opihi/pcontrol/StartHost.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/pcontrol/StartHost.c (revision 28484) @@ -14,7 +14,6 @@ if (VarConfig ("SHELL", "%s", shell) == NULL) strcpy (shell, "pclient"); -#ifndef __APPLE__ - if (VerboseMode()) gprint (GP_ERR, "starting host within thread %lld\n", (long long) pthread_self()); -#endif + if (VerboseMode()) gprint (GP_ERR, "starting host within thread\n"); + pid = rconnect (command, host[0].hostname, shell, stdio); if (!pid) { Index: /branches/pap/Ohana/src/opihi/pcontrol/StartJob.c =================================================================== --- /branches/pap/Ohana/src/opihi/pcontrol/StartJob.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/pcontrol/StartJob.c (revision 28484) @@ -13,6 +13,6 @@ ASSERT (job == (Job *) host[0].job, "invalid job"); - ResetJobOutput (&job[0].stdout); - ResetJobOutput (&job[0].stderr); + ResetJobOutput (&job[0].stdout_buf); + ResetJobOutput (&job[0].stderr_buf); /* construct command line : job arg0 arg1 ... argN\n */ @@ -36,5 +36,5 @@ } - fprintf (stderr, "command: %s\n", line); + // fprintf (stderr, "command: %s\n", line); status = PclientCommand (host, line, PCLIENT_PROMPT, PCONTROL_RESP_START_JOB); Index: /branches/pap/Ohana/src/opihi/pcontrol/StopHosts.c =================================================================== --- /branches/pap/Ohana/src/opihi/pcontrol/StopHosts.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/pcontrol/StopHosts.c (revision 28484) @@ -128,7 +128,5 @@ int i, result, waitstatus; -#ifndef __APPLE__ - if (VerboseMode()) gprint (GP_ERR, "harvesting within thread %lld\n", (long long) pthread_self()); -#endif + if (VerboseMode()) gprint (GP_ERR, "harvesting within thread\n"); if (VerboseMode()) gprint (GP_ERR, "child process %d is down, wait for exit status\n", pid); Index: /branches/pap/Ohana/src/opihi/pcontrol/check.c =================================================================== --- /branches/pap/Ohana/src/opihi/pcontrol/check.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/pcontrol/check.c (revision 28484) @@ -38,6 +38,6 @@ gprint (GP_LOG, "STATUS %s\n", GetJobStackName(job[0].stack)); gprint (GP_LOG, "EXITST %d\n", job[0].exit_status); - gprint (GP_LOG, "STDOUT %d\n", job[0].stdout.size); - gprint (GP_LOG, "STDERR %d\n", job[0].stderr.size); + gprint (GP_LOG, "STDOUT %d\n", job[0].stdout_buf.size); + gprint (GP_LOG, "STDERR %d\n", job[0].stderr_buf.size); gprint (GP_LOG, "DTIME %lf\n", job[0].dtime); if (job[0].realhost) { @@ -50,6 +50,6 @@ set_str_variable ("JOB_STATUS", GetJobStackName(job[0].stack)); set_int_variable ("JOB_EXITST", job[0].exit_status); - set_int_variable ("JOB_STDOUT_SIZE", job[0].stdout.size); - set_int_variable ("JOB_STDERR_SIZE", job[0].stderr.size); + set_int_variable ("JOB_STDOUT_SIZE", job[0].stdout_buf.size); + set_int_variable ("JOB_STDERR_SIZE", job[0].stderr_buf.size); set_variable ("JOB_DTIME", job[0].dtime); set_str_variable ("JOB_HOSTNAME", job[0].hostname); Index: /branches/pap/Ohana/src/opihi/pcontrol/stdout.c =================================================================== --- /branches/pap/Ohana/src/opihi/pcontrol/stdout.c (revision 28483) +++ /branches/pap/Ohana/src/opihi/pcontrol/stdout.c (revision 28484) @@ -43,5 +43,5 @@ found_stdout: - buffer = &job[0].stdout.buffer; + buffer = &job[0].stdout_buf.buffer; if (varName == NULL) { fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout); @@ -97,5 +97,5 @@ found_stderr: - buffer = &job[0].stderr.buffer; + buffer = &job[0].stderr_buf.buffer; if (varName == NULL) { fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout); Index: /branches/pap/Ohana/src/photdbc/include/photdbc.h =================================================================== --- /branches/pap/Ohana/src/photdbc/include/photdbc.h (revision 28483) +++ /branches/pap/Ohana/src/photdbc/include/photdbc.h (revision 28484) @@ -52,4 +52,5 @@ double CHISQ_MAX; double SIGMA_MAX; +double AVE_SIGMA_LIM; int NMEAS_MIN; double ZERO_POINT; Index: /branches/pap/Ohana/src/photdbc/src/ConfigInit.c =================================================================== --- /branches/pap/Ohana/src/photdbc/src/ConfigInit.c (revision 28483) +++ /branches/pap/Ohana/src/photdbc/src/ConfigInit.c (revision 28484) @@ -42,4 +42,5 @@ ScanConfig (config, "SIGMA_MAX", "%lf", 0, &SIGMA_MAX); + ScanConfig (config, "AVE_SIGMA_LIM", "%lf", 0, &AVE_SIGMA_LIM); ScanConfig (config, "NMEAS_MIN", "%d", 0, &NMEAS_MIN); Index: /branches/pap/Ohana/src/photdbc/src/copy_images.c =================================================================== --- /branches/pap/Ohana/src/photdbc/src/copy_images.c (revision 28483) +++ /branches/pap/Ohana/src/photdbc/src/copy_images.c (revision 28484) @@ -3,6 +3,8 @@ int copy_images (char *outdir) { - int status, Nimage; + int status; + off_t Nimage; char *ImageOut; + unsigned int imageID; FITS_DB in; FITS_DB out; @@ -46,4 +48,12 @@ dvo_image_addrows (&out, image, Nimage); + // note that imageID is unsigned int + status = gfits_scan (&in.header, "IMAGEID", "%u", 1, &imageID); + if (!status) { + status = gfits_scan (&in.header, "NIMAGES", "%u", 1, &imageID); + imageID++; + } + status = gfits_modify (&out.header, "IMAGEID", "%u", 1, imageID); + dvo_image_update (&out, VERBOSE); dvo_image_unlock (&out); Index: /branches/pap/Ohana/src/photdbc/src/join_stars.c =================================================================== --- /branches/pap/Ohana/src/photdbc/src/join_stars.c (revision 28483) +++ /branches/pap/Ohana/src/photdbc/src/join_stars.c (revision 28484) @@ -7,5 +7,5 @@ off_t Naverage, Nmeasure, *index; double *X, *Y, dX, dY, dR, RADIUS2; - double Sr, Sd, Rmid, Dmid; + double Rmid, Dmid; int basecode, baseNsec, Nsecfilt, *found; @@ -202,5 +202,5 @@ if (Nmeas != Nmeasure) { - fprintf (stderr, "failure to match %d measures (%d of %d matched)\n", Nmeasure - Nmeas, Nmeas, Nmeasure); + fprintf (stderr, "failure to match "OFF_T_FMT" measures ("OFF_T_FMT" of "OFF_T_FMT" matched)\n", Nmeasure - Nmeas, Nmeas, Nmeasure); } Index: /branches/pap/Ohana/src/photdbc/src/make_subcatalog.c =================================================================== --- /branches/pap/Ohana/src/photdbc/src/make_subcatalog.c (revision 28483) +++ /branches/pap/Ohana/src/photdbc/src/make_subcatalog.c (revision 28484) @@ -5,8 +5,9 @@ int make_subcatalog (Catalog *subcatalog, Catalog *catalog) { - int i, j, offset; - int NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm, Nsecfilt; + off_t i, j, offset; + off_t NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm, Nsecfilt; double mag, minMag; - + int keep; + Nsecfilt = GetPhotcodeNsecfilt (); assert (catalog[0].Nsecfilt == Nsecfilt); @@ -25,4 +26,15 @@ // exclude stars with too few measurements if (NMEAS_MIN && (catalog[0].average[i].Nmeasure < NMEAS_MIN)) continue; + + if (AVE_SIGMA_LIM) { + // if all of the average magnitude errors are >AVE_SIGMA_LIM, drop the object + keep = FALSE; + for (j = 0; !keep && (j < Nsecfilt); j++) { + if (catalog[0].secfilt[Nsecfilt*i+j].dM < AVE_SIGMA_LIM) { + keep = TRUE; + } + } + if (!keep) continue; + } /* assign average and secfilt values */ @@ -105,5 +117,5 @@ if (VERBOSE) { - fprintf (stderr, "%d: using %d stars (%d measures) for catalog\n", i, + fprintf (stderr, OFF_T_FMT": using "OFF_T_FMT" stars ("OFF_T_FMT" measures) for catalog\n", i, subcatalog[0].Naverage, subcatalog[0].Nmeasure); } Index: /branches/pap/Ohana/src/photdbc/src/photdbc.c =================================================================== --- /branches/pap/Ohana/src/photdbc/src/photdbc.c (revision 28483) +++ /branches/pap/Ohana/src/photdbc/src/photdbc.c (revision 28484) @@ -21,5 +21,5 @@ skylist = SkyListByPatch (sky, -1, ®ION); for (i = 0; i < skylist[0].Nregions; i++) { - if (i % 100 == 0) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name); + if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name); // set the parameters which guide catalog open/load/create @@ -56,9 +56,17 @@ } + // the output catalog needs to have the same values for 'objID' and 'sorted' as the input + outcatalog.objID = incatalog.objID; + outcatalog.sorted = incatalog.sorted; + if (!incatalog.sorted) { + fprintf (stderr, "ERROR: input db must be sorted: %s\n", incatalog.filename); + exit (2); + } + /* limit number of measures based on selections */ make_subcatalog (&outcatalog, &incatalog); // XXX add other filters here: - join_stars (&outcatalog); + // join_stars (&outcatalog); // unique_measures (catalog); // flag_measures (&db, catalog); Index: /branches/pap/Ohana/src/relastro/Makefile =================================================================== --- /branches/pap/Ohana/src/relastro/Makefile (revision 28483) +++ /branches/pap/Ohana/src/relastro/Makefile (revision 28484) @@ -56,6 +56,8 @@ $(SRC)/save_catalogs.$(ARCH).o \ $(SRC)/write_coords.$(ARCH).o \ -$(SRC)/CoordOps.$(ARCH).o \ -$(SRC)/FixProblemImages.$(ARCH).o \ +$(SRC)/CoordOps.$(ARCH).o \ +$(SRC)/FixProblemImages.$(ARCH).o \ +$(SRC)/high_speed_catalogs.$(ARCH).o \ +$(SRC)/high_speed_objects.$(ARCH).o \ $(SRC)/relastroVisual.$(ARCH).o Index: /branches/pap/Ohana/src/relastro/doc/high-speed.txt =================================================================== --- /branches/pap/Ohana/src/relastro/doc/high-speed.txt (revision 28484) +++ /branches/pap/Ohana/src/relastro/doc/high-speed.txt (revision 28484) @@ -0,0 +1,7 @@ + +Niall Deacon's 2MASS / PS1 high-speed search uses the following set of criteria (roughly): + +2 sets of detections: + +A : 2MASS only, J significant, H & K anything, set of phot_flags? +B : PS1 only, 2 y-band, 2 z-band, S/N > 5, not a galaxy Index: /branches/pap/Ohana/src/relastro/include/relastro.h =================================================================== --- /branches/pap/Ohana/src/relastro/include/relastro.h (revision 28483) +++ /branches/pap/Ohana/src/relastro/include/relastro.h (revision 28484) @@ -14,5 +14,5 @@ typedef enum {FIT_NONE, FIT_AVERAGE, FIT_PM_ONLY, FIT_PAR_ONLY, FIT_PM_AND_PAR} FitMode; -typedef enum {TARGET_NONE, TARGET_OBJECTS, TARGET_SIMPLE, TARGET_CHIPS, TARGET_MOSAICS} FitTarget; +typedef enum {TARGET_NONE, TARGET_OBJECTS, TARGET_SIMPLE, TARGET_CHIPS, TARGET_MOSAICS, TARGET_HIGH_SPEED} FitTarget; typedef struct { @@ -89,4 +89,6 @@ int SRC_MEAS_TOOFEW; //catalog objects wich fewer detections then this are ignored double MIN_ERROR; + +double RADIUS; // match radius for high-speed objects int VERBOSE; @@ -112,4 +114,8 @@ int NphotcodesKeep, NphotcodesSkip; PhotCode **photcodesKeep, **photcodesSkip; + +char *PHOTCODE_A_LIST, *PHOTCODE_B_LIST; +int NphotcodesGroupA, NphotcodesGroupB; +PhotCode **photcodesGroupA, **photcodesGroupB; int AreaSelect; @@ -321,2 +327,6 @@ int saveCoords (Coords *coords, off_t N); void resetImageRaw (Catalog *catalog, int Ncatalog, off_t im); + +int high_speed_catalogs (); +int high_speed_objects (SkyRegion *region, Catalog *catalog); +int MeasMatchesPhotcode(Measure *measure, PhotCode **photcodeSet, int Nset); Index: /branches/pap/Ohana/src/relastro/src/ImageOps.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/ImageOps.c (revision 28483) +++ /branches/pap/Ohana/src/relastro/src/ImageOps.c (revision 28484) @@ -165,5 +165,5 @@ for (i = 0; VERBOSE2 && (i < Nimage); i++) { name = GetPhotcodeNamebyCode (image[i].photcode); - fprintf (stderr, "image %lld has %lld measures (%s, %s)\n", (long long) i, (long long) Nlist[i], + fprintf (stderr, "image "OFF_T_FMT" has "OFF_T_FMT" measures (%s, %s)\n", i, Nlist[i], ohana_sec_to_date(image[i].tzero), name); } Index: /branches/pap/Ohana/src/relastro/src/UpdateChips.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/UpdateChips.c (revision 28483) +++ /branches/pap/Ohana/src/relastro/src/UpdateChips.c (revision 28484) @@ -30,7 +30,7 @@ // FitChip does iterative, clipped fitting - // fprintf (stderr, "image %lld : Nstars: %lld\n", (long long) i, (long long) Nraw); + // fprintf (stderr, "image "OFF_T_FMT" : Nstars: "OFF_T_FMT"\n", i, Nraw); if (!FitChip (raw, ref, Nraw, &image[i].coords)) { - fprintf (stderr, "reject fit for image %s (%lld) : Nstars: %lld\n", image[i].name, (long long) i, (long long) Nraw); + fprintf (stderr, "reject fit for image %s ("OFF_T_FMT") : Nstars: "OFF_T_FMT"\n", image[i].name, i, Nraw); oldCoords = getCoords (i); memcpy (&image[i].coords, oldCoords, sizeof(Coords)); Index: /branches/pap/Ohana/src/relastro/src/UpdateObjects.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/UpdateObjects.c (revision 28483) +++ /branches/pap/Ohana/src/relastro/src/UpdateObjects.c (revision 28484) @@ -80,5 +80,5 @@ for (i = 0; i < Ncatalog; i++) { - if (VERBOSE) fprintf (stderr, "astrometrize catalog %d : %lld ave, %lld meas\n", i, (long long) catalog[i].Naverage, (long long) catalog[i].Nmeasure); + if (VERBOSE) fprintf (stderr, "astrometrize catalog %d : "OFF_T_FMT" ave, "OFF_T_FMT" meas\n", i, catalog[i].Naverage, catalog[i].Nmeasure); Nave = Npar = Npm = Nskip = 0; @@ -189,5 +189,5 @@ T[k] -= Tmean; if (XVERB) { - fprintf (stderr, "%lld %f %f %f %f %f +/- %f %f\n", (long long) k, T[k], R[k], D[k], X[k], Y[k], dX[k], dY[k]); + fprintf (stderr, OFF_T_FMT" %f %f %f %f %f +/- %f %f\n", k, T[k], R[k], D[k], X[k], Y[k], dX[k], dY[k]); } } @@ -316,8 +316,8 @@ NparSum += Npar; NskipSum += Nskip; - if (VERBOSE) fprintf (stderr, "catalog %lld : %lld ave, %lld pm, %lld par : Nskip %lld\n", (long long) i, (long long) Nave, (long long) Npm, (long long) Npar, (long long) Nskip); + if (VERBOSE) fprintf (stderr, "catalog %d : "OFF_T_FMT" ave, "OFF_T_FMT" pm, "OFF_T_FMT" par : Nskip "OFF_T_FMT"\n", i, Nave, Npm, Npar, Nskip); } - fprintf (stderr, "fitted %lld objects (%lld ave, %lld pm, %lld par), skipped %lld\n", (long long) (NaveSum + NpmSum + NparSum), (long long) NaveSum, (long long) NpmSum, (long long) NparSum, (long long) NskipSum); + fprintf (stderr, "fitted "OFF_T_FMT" objects ("OFF_T_FMT" ave, "OFF_T_FMT" pm, "OFF_T_FMT" par), skipped "OFF_T_FMT"\n", (NaveSum + NpmSum + NparSum), NaveSum, NpmSum, NparSum, NskipSum); return (TRUE); } Index: /branches/pap/Ohana/src/relastro/src/args.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/args.c (revision 28483) +++ /branches/pap/Ohana/src/relastro/src/args.c (revision 28484) @@ -28,4 +28,17 @@ } } + if ((N = get_argument (argc, argv, "-high-speed"))) { + // XXX include a parallax / no-parallax option + if (N >= argc - 3) usage(); + FIT_TARGET = TARGET_HIGH_SPEED; + remove_argument (N, &argc, argv); + PHOTCODE_A_LIST = strcreate(argv[N]); + remove_argument (N, &argc, argv); + PHOTCODE_B_LIST = strcreate(argv[N]); + remove_argument (N, &argc, argv); + RADIUS = atof(argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-update-simple"))) { remove_argument (N, &argc, argv); @@ -271,6 +284,6 @@ void usage () { fprintf (stderr, "ERROR: USAGE: relastro -region RA RA DEC DEC\n"); - fprintf (stderr, " OR: relastro -catalog (ra) (dec)\n"); - fprintf (stderr, " working options: \n"); + fprintf (stderr, " OR: relastro -catalog (ra) (dec)\n\n"); + fprintf (stderr, " specify one of the following modes: \n"); fprintf (stderr, " -update-objects\n"); fprintf (stderr, " -pm\n"); @@ -280,4 +293,6 @@ fprintf (stderr, " -update-chips\n"); fprintf (stderr, " -update-mosaics\n"); + fprintf (stderr, " -high-speed (code[,code,code]) (code[,code,code]) (radius)\n\n"); + fprintf (stderr, " additional options: \n"); fprintf (stderr, " -time (start)(stop)\n"); fprintf (stderr, " +photcode (code)[,code,code...]\n"); Index: /branches/pap/Ohana/src/relastro/src/bcatalog.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/bcatalog.c (revision 28483) +++ /branches/pap/Ohana/src/relastro/src/bcatalog.c (revision 28484) @@ -104,5 +104,5 @@ if (VERBOSE) { - fprintf (stderr, "%lld: using %lld stars (%lld measures) for catalog\n", (long long) i, (long long) subcatalog[0].Naverage, (long long) subcatalog[0].Nmeasure); + fprintf (stderr, OFF_T_FMT": using "OFF_T_FMT" stars ("OFF_T_FMT" measures) for catalog\n", i, subcatalog[0].Naverage, subcatalog[0].Nmeasure); } return (TRUE); Index: /branches/pap/Ohana/src/relastro/src/high_speed_catalogs.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/high_speed_catalogs.c (revision 28484) +++ /branches/pap/Ohana/src/relastro/src/high_speed_catalogs.c (revision 28484) @@ -0,0 +1,55 @@ +# include "relastro.h" + +int high_speed_catalogs () { + + int i; + + SkyTable *sky = NULL; + SkyList *skylist = NULL; + Catalog catalog; + + // load the current sky table (layout of all SkyRegions) + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // determine the populated SkyRegions overlapping the requested area (default depth) + if (UserCatalog) { + skylist = SkyRegionByPoint (sky, -1, UserCatalogRA, UserCatalogDEC); + } else { + skylist = SkyListByPatch (sky, -1, &UserPatch); + } + + // load data from each region file, only use bright stars + for (i = 0; i < skylist[0].Nregions; i++) { + + // set up the basic catalog info + catalog.filename = skylist[0].filename[i]; + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naves_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + high_speed_objects (skylist[0].regions[i], &catalog); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + save_catalogs (&catalog, 1); + } + + return (TRUE); +} Index: /branches/pap/Ohana/src/relastro/src/high_speed_objects.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/high_speed_objects.c (revision 28484) +++ /branches/pap/Ohana/src/relastro/src/high_speed_objects.c (revision 28484) @@ -0,0 +1,338 @@ +# include "relastro.h" + +# define NEXT_I { i++; continue; } +# define NEXT_J { j++; continue; } + +// XXX these were used to spot-check specific objects +# define REF_RA1 192.455781 +# define REF_DEC1 2.563102 +# define REF_RA2 192.444302113 +# define REF_DEC2 2.5786559112 +int high_speed_objects (SkyRegion *region, Catalog *catalog) { + + // XXX seems silly to require region here just to find the catalog bounds / center + + off_t i, j, m, J, ni, nj, *N1, Nslow, NgroupA, NgroupB; + int *slowMoving, *groupA, *groupB, status, foundA, foundB, Nmatch, valid; + double *X1, *Y1; + double dX, dY, dR, RADIUS2, yJ; + Coords tcoords; + + int zcode, zNsec, ycode, yNsec, jcode, jNsec, hcode, hNsec, kcode, kNsec, USNO_R, USNO_N, Nsecfilt; + + Nsecfilt = GetPhotcodeNsecfilt(); + + ycode = GetPhotcodeCodebyName("y"); + yNsec = GetPhotcodeNsec(ycode); + + zcode = GetPhotcodeCodebyName("z"); + zNsec = GetPhotcodeNsec(zcode); + + jcode = GetPhotcodeCodebyName("J"); + jNsec = GetPhotcodeNsec(jcode); + + hcode = GetPhotcodeCodebyName("H"); + hNsec = GetPhotcodeNsec(hcode); + + kcode = GetPhotcodeCodebyName("K"); + kNsec = GetPhotcodeNsec(kcode); + + USNO_R = GetPhotcodeCodebyName("USNO_RED"); + USNO_N = GetPhotcodeCodebyName("USNO_N"); + + // high-speed between different surveys (easier case): + // we have two sets of photcodes (A) and (B) and are looking for objects + // with detections in only (A) and separately only (B). + + // we need at least 2 objects if we are going to match anything... + if (catalog[0].Naverage < 2) return (TRUE); + + // mask with which to mark objects to be ignored + ALLOCATE (slowMoving, int, catalog[0].Naverage); + memset (slowMoving, 0, catalog[0].Naverage*sizeof(int)); + + // record to which photcode group the object belongs: + NgroupA = 0; + ALLOCATE (groupA, int, catalog[0].Naverage); + memset (groupA, 0, catalog[0].Naverage*sizeof(int)); + + NgroupB = 0; + ALLOCATE (groupB, int, catalog[0].Naverage); + memset (groupB, 0, catalog[0].Naverage*sizeof(int)); + + fprintf (stderr, "checking "OFF_T_FMT" objects\n", catalog[0].Naverage); + Nslow = 0; + + fprintf (stdout, "# RA_A DEC_A RA_B DEC_B : pmx pmy dR : z dz y dy J dJ H dH K dK\n"); + //................270.2346670 20.2471540 270.2170434 20.2717396 : -59.11 88.78 106.66 : 0.000 0.000 19.582 0.112 16.041 0.110 15.388 0.098 14.858 0.001 + + // mark (exclude) objects with both sets of target photcodes + for (i = 0; i < catalog[0].Naverage; i++) { + + // if (i % 100 == 0) fprintf (stderr, "."); + + // radius = hypot((REF_DEC1 - catalog[0].average[i].D), (REF_RA1 - catalog[0].average[i].R)); + // if (radius < 0.0001) { + // fprintf (stderr, "found it\n"); + // } + // radius = hypot((REF_DEC2 - catalog[0].average[i].D), (REF_RA2 - catalog[0].average[i].R)); + // if (radius < 0.0001) { + // fprintf (stderr, "found it\n"); + // } + + // do any of the measures for this object match group A? + m = catalog[0].average[i].measureOffset; + foundA = FALSE; + for (j = 0; !foundA && (j < catalog[0].average[i].Nmeasure); j++, m++) { + if (MeasMatchesPhotcode(&catalog[0].measure[m], photcodesGroupA, NphotcodesGroupA)) { + foundA = TRUE; + } + } + + // do any of the measures for this object match group B? + m = catalog[0].average[i].measureOffset; + foundB = FALSE; + for (j = 0; !foundB && (j < catalog[0].average[i].Nmeasure); j++, m++) { + if (MeasMatchesPhotcode(&catalog[0].measure[m], photcodesGroupB, NphotcodesGroupB)) { + foundB = TRUE; + } + } + + if (foundA && foundB) { + slowMoving[i] = TRUE; + Nslow ++; + continue; + } + + // additional constraints: + // * group A : require + + // 2MASS detections: + if (foundA && !foundB) { + // average-based tests: + valid = TRUE; + valid &= ((catalog[0].average[i].flags & 0x40000) == 0); + valid &= ((catalog[0].average[i].flags & 0x80000) > 0); + valid &= (catalog[0].secfilt[i*Nsecfilt + yNsec].M < 1.0); + valid &= (catalog[0].secfilt[i*Nsecfilt + zNsec].M < 1.0); + // XXX color restrictions are applied in the pair-wise matching below + + // measure-based tests: + m = catalog[0].average[i].measureOffset; + for (j = 0; valid && (j < catalog[0].average[i].Nmeasure); j++, m++) { + if (catalog[0].measure[m].photcode == USNO_R) { + valid &= ((catalog[0].measure[m].M > 20.0) || isnan(catalog[0].measure[m].M)); + } + if (catalog[0].measure[m].photcode == USNO_N) { + valid &= ((catalog[0].measure[m].M > 18.0) || isnan(catalog[0].measure[m].M)); + } + } + + // ((objflags & 524288) == 524288) && + // ((objflags & 262144) == 0) && + // (z<1.0) && + // (y<1.0) && + // ((yp[$i]-J)>1.25) && ((yp[$i]-J)<5.0) && + // ((USNO_RED<1.0)||(USNO_RED>20.0)) && + // ((USNO_N>18.0)||(USNO_N<1.0)) + + if (valid) { + groupA[i] = TRUE; + NgroupA ++; + continue; + } else { + // fprintf (stderr, "skip "OFF_T_FMT" (group A)\n", i); + } + } + + // PS1 detections: + if (foundB && !foundA) { + + // average-based tests: + valid = TRUE; + valid &= ((catalog[0].average[i].flags & 0x10000) == 0); + valid &= ((catalog[0].average[i].flags & 0x20000) > 0); + valid &= (catalog[0].secfilt[i*Nsecfilt + jNsec].M < 1.0); + valid &= (catalog[0].secfilt[i*Nsecfilt + yNsec].Nused > 1); + valid &= (catalog[0].secfilt[i*Nsecfilt + yNsec].dM < 0.2); + + if ((catalog[0].secfilt[i*Nsecfilt + zNsec].M < 1.0) || (catalog[0].secfilt[i*Nsecfilt + zNsec].Nused == 1)) { + valid &= TRUE; + } else { + valid &= ((catalog[0].secfilt[i*Nsecfilt + zNsec].M - catalog[0].secfilt[i*Nsecfilt + yNsec].M) > 0.2); + } + + // measure-based tests: + m = catalog[0].average[i].measureOffset; + for (j = 0; valid && (j < catalog[0].average[i].Nmeasure); j++, m++) { + if (catalog[0].measure[m].photcode == USNO_R) { + valid &= ((catalog[0].measure[m].M > 20.0) || isnan(catalog[0].measure[m].M)); + } + if (catalog[0].measure[m].photcode == USNO_N) { + valid &= ((catalog[0].measure[m].M > 18.5) || isnan(catalog[0].measure[m].M)); + } + } + + // (abs(glat)>15.0) && + // ((objflags & 65536) == 0) && + // ((objflags & 131072) == 131072) && + // (J<1.0) && + // (y:nphot>1) && + // (y:err<0.2) && + // ((USNO_RED>20.0)||(USNO_RED<1.0)) && + // ((USNO_N>18.5)||(USNO_N<1.0)) && + // ((z<1.0)||(z:nphot == 1)||((z:nphot>1)&&((z-y)>0.2))) + + if (valid) { + groupB[i] = TRUE; + NgroupB ++; + continue; + } else { + // fprintf (stderr, "skip "OFF_T_FMT" (group B)\n", i); + } + } + + // this object does not have a detection from either groupA or groupB -- skip as if slow + slowMoving[i] = TRUE; + Nslow ++; + } + fprintf (stderr, OFF_T_FMT" of "OFF_T_FMT" are slow or invalid; "OFF_T_FMT" in group A, "OFF_T_FMT" in group B\n", Nslow, catalog[0].Naverage, NgroupA, NgroupB); + + // double loop over unmarked objects (sorted in RA / X) + // record all pairs within the desired match distance + + // create tmp local positional index + ALLOCATE (X1, double, catalog[0].Naverage); + ALLOCATE (Y1, double, catalog[0].Naverage); + ALLOCATE (N1, off_t, catalog[0].Naverage); + + // define a local projection + tcoords.crval1 = 0.5*(region[0].Rmin + region[0].Rmax); + if (region[0].Dmax < 90) { + tcoords.crval2 = 0.5*(region[0].Dmin + region[0].Dmax); + } else { + tcoords.crval2 = 90.0; + } + tcoords.crpix1 = 0; + tcoords.crpix2 = 0; + tcoords.cdelt1 = tcoords.cdelt2 = 1.0 / 3600.0; + tcoords.pc1_1 = tcoords.pc2_2 = 1.0; + tcoords.pc1_2 = tcoords.pc2_1 = 0.0; + tcoords.Npolyterms = 1; + strcpy (tcoords.ctype, "RA---ARC"); + + /* build spatial index (RA sort) referencing input array sequence */ + for (i = 0; i < catalog[0].Naverage; i++) { + status = RD_to_XY (&X1[i], &Y1[i], catalog[0].average[i].R, catalog[0].average[i].D, &tcoords); + N1[i] = i; + assert (status); + } + sort_coords_index (X1, Y1, N1, catalog[0].Naverage); + + Nmatch = 0; + RADIUS2 = SQ(RADIUS); + + // mark (exclude) objects with both sets of target photcodes + for (i = j = 0; (i < catalog[0].Naverage) && (j < catalog[0].Naverage);) { + + ni = N1[i]; + nj = N1[j]; + + // if (ni == 131030) { + // fprintf (stderr, "got 2mass\n"); + // } + // if (nj == 53581) { + // fprintf (stderr, "got ps1\n"); + // } + + if (slowMoving[ni]) NEXT_I; + if (slowMoving[nj]) NEXT_J; + + // i => groupA, j => groupB + if (!groupA[ni]) NEXT_I; + if (!groupB[nj]) NEXT_J; + + if (!finite(X1[i]) || !finite(Y1[i])) NEXT_I; + if (!finite(X1[j]) || !finite(Y1[j])) NEXT_J; + + // look for pairs that are within the maximum separation + dX = X1[i] - X1[j]; + + if (dX <= -1.02*RADIUS) NEXT_I; // negative dX: i is too small + if (dX >= +1.02*RADIUS) NEXT_J; // positive dX: j is too small + + // within match range; look for valid matches + for (J = j; (dX > -1.02*RADIUS) && (J < catalog[0].Naverage); J++) { + if (J == i) continue; // avoid auto-matches + + dX = X1[i] - X1[J]; + + nj = N1[J]; + if (!groupB[nj]) continue; + + dY = Y1[i] - Y1[J]; + dR = dX*dX + dY*dY; + if (dR > RADIUS2) continue; + + // y_groupA - J_groupB + yJ = catalog[0].secfilt[nj*Nsecfilt + yNsec].M - catalog[0].secfilt[ni*Nsecfilt + jNsec].M; + if (yJ < 1.25) continue; + if (yJ > 5.0) continue; + + /*** a match is found (just print it for the moment) ***/ + Nmatch ++; + + // for the moment, just write the matches to stdout: + float pmx = -dX; // proper-motion displacement in ra direction (B - A) [arcsec] + float pmy = -dY; // proper-motion displacement in dec direction (B - A) [arcsec] + fprintf (stdout, "%11.7f %11.7f %11.7f %11.7f : %7.2f %7.2f %7.2f : %7.3f %6.3f %7.3f %6.3f %7.3f %6.3f %7.3f %6.3f %7.3f %6.3f\n", + catalog[0].average[ni].R, catalog[0].average[ni].D, + catalog[0].average[nj].R, catalog[0].average[nj].D, + pmx, pmy, sqrt(dR), + catalog[0].secfilt[nj*Nsecfilt + zNsec].M, + catalog[0].secfilt[nj*Nsecfilt + zNsec].dM, + catalog[0].secfilt[nj*Nsecfilt + yNsec].M, + catalog[0].secfilt[nj*Nsecfilt + yNsec].dM, + catalog[0].secfilt[ni*Nsecfilt + jNsec].M, + catalog[0].secfilt[ni*Nsecfilt + jNsec].dM, + catalog[0].secfilt[ni*Nsecfilt + hNsec].M, + catalog[0].secfilt[ni*Nsecfilt + hNsec].dM, + catalog[0].secfilt[ni*Nsecfilt + kNsec].M, + catalog[0].secfilt[ni*Nsecfilt + kNsec].dM); + } + i++; + } + fprintf (stderr, "found %d matches\n", Nmatch); + + free (slowMoving); + free (groupA); + free (groupB); + free (X1); + free (Y1); + free (N1); + + return TRUE; +} + +// return TRUE if any measure->photcodes match the photcodeSet +int MeasMatchesPhotcode(Measure *measure, PhotCode **photcodeSet, int Nset) { + + int found, k; + + assert (Nset > 0); + + if (!finite(measure[0].dR) || !finite(measure[0].dD)) return FALSE; + if (!finite(measure[0].M)) return FALSE; + + /* select measurements by photcode, or equiv photcode, if specified */ + if (Nset > 0) { + found = FALSE; + for (k = 0; (k < Nset) && !found; k++) { + if (photcodeSet[k][0].code == measure[0].photcode) found = TRUE; + if (photcodeSet[k][0].code == GetPhotcodeEquivCodebyCode(measure[0].photcode)) found = TRUE; + } + if (!found) return FALSE; + } + + return TRUE; +} Index: /branches/pap/Ohana/src/relastro/src/initialize.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/initialize.c (revision 28483) +++ /branches/pap/Ohana/src/relastro/src/initialize.c (revision 28484) @@ -56,4 +56,46 @@ } + NphotcodesGroupA = 0; + photcodesGroupA = NULL; + if (PHOTCODE_A_LIST != NULL) { + NPHOTCODES = 10; + ALLOCATE (photcodesGroupA, PhotCode *, NPHOTCODES); + + /* parse the comma-separated list of photcodesGroupA */ + list = PHOTCODE_A_LIST; + while ((codename = strtok_r (list, ",", &ptr)) != NULL) { + list = NULL; // pass NULL on successive strtok_r calls + fprintf (stderr, "PHOTCODE_A_LIST: %s\n", PHOTCODE_A_LIST); + fprintf (stderr, "codename: %s\n", codename); + if ((photcodesGroupA[NphotcodesGroupA] = GetPhotcodebyName (codename)) == NULL) { + fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); + exit (1); + } + NphotcodesGroupA ++; + CHECK_REALLOCATE (photcodesGroupA, PhotCode *, NPHOTCODES, NphotcodesGroupA, 10); + } + } + + NphotcodesGroupB = 0; + photcodesGroupB = NULL; + if (PHOTCODE_B_LIST != NULL) { + NPHOTCODES = 10; + ALLOCATE (photcodesGroupB, PhotCode *, NPHOTCODES); + + /* parse the comma-separated list of photcodesGroupB */ + list = PHOTCODE_B_LIST; + while ((codename = strtok_r (list, ",", &ptr)) != NULL) { + list = NULL; // pass NULL on successive strtok_r calls + fprintf (stderr, "PHOTCODE_B_LIST: %s\n", PHOTCODE_B_LIST); + fprintf (stderr, "codename: %s\n", codename); + if ((photcodesGroupB[NphotcodesGroupB] = GetPhotcodebyName (codename)) == NULL) { + fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); + exit (1); + } + NphotcodesGroupB ++; + CHECK_REALLOCATE (photcodesGroupB, PhotCode *, NPHOTCODES, NphotcodesGroupB, 10); + } + } + initstats (STATMODE); Index: /branches/pap/Ohana/src/relastro/src/relastro.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/relastro.c (revision 28483) +++ /branches/pap/Ohana/src/relastro/src/relastro.c (revision 28484) @@ -23,4 +23,10 @@ if (FIT_TARGET == TARGET_OBJECTS) { relastro_objects (); + exit (0); + } + + /* the object analysis is a separate process iterating over catalogs */ + if (FIT_TARGET == TARGET_HIGH_SPEED) { + high_speed_catalogs (); exit (0); } Index: /branches/pap/Ohana/src/relastro/src/select_images.c =================================================================== --- /branches/pap/Ohana/src/relastro/src/select_images.c (revision 28483) +++ /branches/pap/Ohana/src/relastro/src/select_images.c (revision 28484) @@ -115,5 +115,5 @@ if (!FindMosaicForImage (timage, Ntimage, i)) { - fprintf (stderr, "cannot find mosaic for %lld\n", (long long) i); + fprintf (stderr, "cannot find mosaic for "OFF_T_FMT"\n", i); continue; } @@ -190,5 +190,5 @@ } - if (VERBOSE) fprintf (stderr, "found %lld images\n", (long long) nimage); + if (VERBOSE) fprintf (stderr, "found "OFF_T_FMT" images\n", nimage); REALLOCATE (image, Image, MAX (nimage, 1)); Index: /branches/pap/Ohana/src/relphot/src/GridOps.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/GridOps.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/GridOps.c (revision 28484) @@ -703,5 +703,5 @@ gfits_create_header (&header); gfits_create_matrix (&header, &matrix); - gfits_modify (&header, "NEXTEND", "%lld", 1, (long long) Nimage + 3); + gfits_modify (&header, "NEXTEND", OFF_T_FMT, 1, Nimage + 3); gfits_modify (&header, "FILTER", "%s", 1, photcode[0].name); gfits_modify_alt (&header, "COMMENT", "%S", 1, "Mosaic Photometry Grid Analysis"); Index: /branches/pap/Ohana/src/relphot/src/ImageOps.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/ImageOps.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/ImageOps.c (revision 28484) @@ -200,5 +200,5 @@ status = findCCD (idx, meas, cat, measure); if (!status) { - if (VERBOSE2) fprintf (stderr, "failed to determine CCD for %lld, %d\n", (long long) meas, cat); + if (VERBOSE2) fprintf (stderr, "failed to determine CCD for "OFF_T_FMT", %d\n", meas, cat); return; } Index: /branches/pap/Ohana/src/relphot/src/MosaicOps.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/MosaicOps.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/MosaicOps.c (revision 28484) @@ -400,5 +400,5 @@ mark = (N < IMAGE_TOOFEW) || (N < IMAGE_GOOD_FRACTION*Nlist[i]); if (mark) { - fprintf (stderr, "marked image %s (%lld), (%lld < %d) || (%lld < %f*%lld)\n", image[imlist[i][0]].name, (long long) i, (long long) N, IMAGE_TOOFEW, (long long) N, IMAGE_GOOD_FRACTION, (long long) Nlist[i]); + fprintf (stderr, "marked mosaic %s ("OFF_T_FMT"), ("OFF_T_FMT" < %d) || ("OFF_T_FMT" < %f*"OFF_T_FMT")\n", image[imlist[i][0]].name, i, N, IMAGE_TOOFEW, N, IMAGE_GOOD_FRACTION, Nlist[i]); mosaic[i].flags |= ID_IMAGE_FEW; Nfew ++; @@ -408,5 +408,5 @@ } liststats (list, dlist, N, &stats); - if (PoorImages) fprintf (stderr, "Mmos: %f %f %d %lld\n", stats.mean, stats.sigma, stats.Nmeas, (long long) N); + if (PoorImages) fprintf (stderr, "Mmos: %f %f %d "OFF_T_FMT"\n", stats.mean, stats.sigma, stats.Nmeas, N); mosaic[i].Mcal = stats.mean; mosaic[i].dMcal = stats.sigma; @@ -519,5 +519,5 @@ n++; } - fprintf (stderr, "Nmosaic: %lld, n: %lld\n", (long long) Nmosaic, (long long) n); + fprintf (stderr, "Nmosaic: "OFF_T_FMT", n: "OFF_T_FMT"\n", Nmosaic, n); liststats (list, dlist, n, &stats); @@ -599,5 +599,5 @@ } - fprintf (stderr, "%lld mosaics marked poor\n", (long long) Nmark); + fprintf (stderr, OFF_T_FMT" mosaics marked poor\n", Nmark); initstats (STATMODE); free (mlist); @@ -643,5 +643,5 @@ } - sprintf (string, "Mosaic %lld", (long long) i); + sprintf (string, "Mosaic "OFF_T_FMT, i); plot_defaults (&graphdata); plot_list (&graphdata, xlist, ylist, N, string, NULL); Index: /branches/pap/Ohana/src/relphot/src/StarOps.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/StarOps.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/StarOps.c (revision 28484) @@ -391,4 +391,6 @@ } +# define NSIGMA_CLIP 3.0 +# define NSIGMA_REJECT 5.0 void clean_measures (Catalog *catalog, int Ncatalog, int final) { @@ -396,7 +398,8 @@ int i, N, image_bad, TOOFEW; off_t *ilist; - double *tlist, *list, *dlist, Ns; + double *tlist, *list, *dlist; float Msys, Mcal, Mmos, Mgrid; StatType stats; + int Ncal, Nmos, Ngrid, Nfew; if (VERBOSE) fprintf (stderr, "marking poor measures\n"); @@ -416,6 +419,6 @@ TOOFEW = MAX (5, STAR_TOOFEW); - Ns = 3; Ndel = Nave = 0; + Ncal = Nmos = Ngrid = Nfew = 0; for (i = 0; i < Ncatalog; i++) { for (j = 0; j < catalog[i].Naverage; j++) { @@ -433,9 +436,9 @@ /* if (catalog[i].measure[m].dbFlags & MEAS_BAD) continue; */ Mcal = getMcal (m, i); - if (isnan(Mcal)) continue; + if (isnan(Mcal)) { Ncal ++; continue; } Mmos = getMmos (m, i); - if (isnan(Mmos)) continue; + if (isnan(Mmos)) { Nmos ++; continue; } Mgrid = getMgrid (m, i); - if (isnan(Mgrid)) continue; + if (isnan(Mgrid)) { Ngrid ++; continue; } Msys = PhotSys (&catalog[i].measure[m], &catalog[i].average[j], &catalog[i].secfilt[j*PhotNsec]); @@ -444,16 +447,21 @@ N++; } - if (N <= TOOFEW) continue; + if (N <= TOOFEW) { Nfew ++; continue; } /* 3-sigma clip based on stats of inner 50% */ + + // calculated mean of inner 50% initstats ("INNER_MEAN"); liststats (list, dlist, N, &stats); stats.sigma = MAX (MIN_ERROR, stats.sigma); /* if measurements agree too well, sigma -> 0.0 */ + + // ignore entries > 3sigma from inner mean for (k = m = 0; k < N; k++) { - if (fabs (list[k] - stats.median) < Ns*stats.sigma) { + if (fabs (list[k] - stats.median) < NSIGMA_CLIP*stats.sigma) { list[m] = list[k]; m++; } } + // recalculate the mean & sigma of the accepted measurements initstats ("MEAN"); liststats (list, dlist, m, &stats); @@ -483,7 +491,7 @@ if (N < TOOFEW) continue; - /* mark bad measures */ + /* mark bad measures (> 3 sigma deviant) */ for (k = 0; k < N; k++) { - if (fabs (list[k] - stats.median) > Ns*stats.sigma) { + if (fabs (list[k] - stats.median) > NSIGMA_REJECT*stats.sigma) { catalog[i].measure[ilist[k]].dbFlags |= ID_MEAS_POOR_PHOTOM; Ndel ++; @@ -494,5 +502,5 @@ } initstats (STATMODE); - if (VERBOSE) fprintf (stderr, "%lld measures marked poor, %lld total\n", (long long) Ndel, (long long) Nave); + if (VERBOSE) fprintf (stderr, OFF_T_FMT" measures marked poor, "OFF_T_FMT" total\n", Ndel, Nave); free (ilist); free (tlist); Index: /branches/pap/Ohana/src/relphot/src/bcatalog.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/bcatalog.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/bcatalog.c (revision 28484) @@ -7,4 +7,5 @@ off_t NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm; float mag; + int Ncode, Ntime, Ndophot, Nmag, Nsigma, Nimag, Nfew; // XXX PhotNsec as a global is a bad idea; either get it from catalog @@ -20,4 +21,6 @@ ALLOCATE (subcatalog[0].measure, Measure, NMEASURE); Nmeasure = Naverage = 0; + + Ncode = Ntime = Ndophot = Nmag = Nsigma = Nimag = Nfew = 0; /* exclude stars not in range or with too few measurements */ @@ -46,10 +49,10 @@ /* select measurements by photcode */ ecode = GetPhotcodeEquivCodebyCode (catalog[0].measure[offset].photcode); - if (ecode != photcode[0].code) continue; + if (ecode != photcode[0].code) { Ncode ++; continue; } /* select measurements by time */ if (TimeSelect) { - if (catalog[0].measure[offset].t < TSTART) continue; - if (catalog[0].measure[offset].t > TSTOP) continue; + if (catalog[0].measure[offset].t < TSTART) { Ntime ++; continue; } + if (catalog[0].measure[offset].t > TSTOP) { Ntime ++; continue; } } @@ -57,18 +60,18 @@ // XXX ignore this criterion for REF measurements? // XXX chnage this to select by bitflags - if (DophotSelect && ((catalog[0].measure[offset].photFlags >> 16) != DophotValue)) continue; + if (DophotSelect && ((catalog[0].measure[offset].photFlags >> 16) != DophotValue)) { Ndophot ++; continue; } /* select measurements by mag limit */ mag = PhotCat (&catalog[0].measure[offset]); - if (mag > MAG_LIM) continue; + if (mag > MAG_LIM) { Nmag ++; continue; } /* select measurements by measurement error */ - if ((SIGMA_LIM > 0) && (catalog[0].measure[offset].dM > SIGMA_LIM)) continue; + if ((SIGMA_LIM > 0) && (catalog[0].measure[offset].dM > SIGMA_LIM)) { Nsigma ++; continue; } /* select measurements by mag limit */ if (ImagSelect) { mag = PhotInst (&catalog[0].measure[offset]); - if (mag < ImagMin) continue; - if (mag > ImagMax) continue; + if (mag < ImagMin) { Nimag ++; continue; } + if (mag > ImagMax) { Nimag ++; continue; } } @@ -94,4 +97,5 @@ if (Nm <= STAR_TOOFEW) { /* enough measurements in band? */ Nmeasure -= Nm; + Nfew ++; continue; } @@ -114,6 +118,8 @@ if (VERBOSE) { - fprintf (stderr, "%lld: using %lld stars (%lld measures) for catalog\n", (long long) i, - (long long) subcatalog[0].Naverage, (long long) subcatalog[0].Nmeasure); + fprintf (stderr, "using "OFF_T_FMT" stars ("OFF_T_FMT" measures) of "OFF_T_FMT" for catalog\n", + subcatalog[0].Naverage, subcatalog[0].Nmeasure, i); + fprintf (stderr, "rejections: %d code, %d time, %d dophot, %d mag, %d sigma, %d imag, %d few\n", + Ncode, Ntime, Ndophot, Nmag, Nsigma, Nimag, Nfew); } return (TRUE); Index: /branches/pap/Ohana/src/relphot/src/load_catalogs.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/load_catalogs.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/load_catalogs.c (revision 28484) @@ -47,4 +47,7 @@ fprintf (stderr, "using %d of %d stars (%d of %d measurements)\n", Nstar, Nstar_total, Nmeas, Nmeas_total); + if (Nstar < 1) { + Shutdown ("ERROR: no stars match the minimum requirements; exiting\n"); + } // XXX consider only returning the populated catalogs Index: /branches/pap/Ohana/src/relphot/src/load_images.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/load_images.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/load_images.c (revision 28484) @@ -47,5 +47,5 @@ vtable = &db[0].vtable; - gfits_scan (vtable[0].header, "NAXIS1", "%lld", 1, (long long *) &Nx); + gfits_scan (vtable[0].header, "NAXIS1", OFF_T_FMT, 1, &Nx); for (i = 0; i < Nimage; i++) { memcpy (vtable[0].buffer[i], &image[i], Nx); Index: /branches/pap/Ohana/src/relphot/src/relphot_objects.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/relphot_objects.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/relphot_objects.c (revision 28484) @@ -55,4 +55,5 @@ for (k = 0; k < catalog.average[j].Nmeasure; k++) { catalog.measure[m+k].dbFlags = 0; + catalog.measure[m+k].Mcal = 0; } } Index: /branches/pap/Ohana/src/relphot/src/select_images.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/select_images.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/select_images.c (revision 28484) @@ -101,5 +101,5 @@ if (!FindMosaicForImage (timage, Ntimage, i)) { - fprintf (stderr, "cannot find mosaic for %lld\n", (long long) i); + fprintf (stderr, "cannot find mosaic for "OFF_T_FMT"\n", i); continue; } @@ -175,5 +175,5 @@ } - if (VERBOSE) fprintf (stderr, "found %lld images\n", (long long) nimage); + if (VERBOSE) fprintf (stderr, "found "OFF_T_FMT" images\n", nimage); REALLOCATE (image, Image, MAX (nimage, 1)); Index: /branches/pap/Ohana/src/relphot/src/setExclusions.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/setExclusions.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/setExclusions.c (revision 28484) @@ -50,7 +50,7 @@ } } - if (VERBOSE) fprintf (stderr, "%lld measurements marked by area\n", (long long) Narea); - if (VERBOSE) fprintf (stderr, "%lld measurements marked nocal\n", (long long) Nnocal); - if (VERBOSE) fprintf (stderr, "%lld measurements kept for analysis\n", (long long) Ngood); + if (VERBOSE) fprintf (stderr, OFF_T_FMT" measurements marked by area\n", Narea); + if (VERBOSE) fprintf (stderr, OFF_T_FMT" measurements marked nocal\n", Nnocal); + if (VERBOSE) fprintf (stderr, OFF_T_FMT" measurements kept for analysis\n", Ngood); return (TRUE); } Index: /branches/pap/Ohana/src/relphot/src/setMrelFinal.c =================================================================== --- /branches/pap/Ohana/src/relphot/src/setMrelFinal.c (revision 28483) +++ /branches/pap/Ohana/src/relphot/src/setMrelFinal.c (revision 28484) @@ -138,6 +138,6 @@ } } - if (VERBOSE) fprintf (stderr, "pass %d, Ntot: %lld, Ntry: %lld, Nskip: %lld, Nkeep: %lld\n", - pass, (long long) Ntot, (long long) Ntry, (long long) Nskip, (long long) Nkeep); + if (VERBOSE) fprintf (stderr, "pass %d, Ntot: "OFF_T_FMT", Ntry: "OFF_T_FMT", Nskip: "OFF_T_FMT", Nkeep: "OFF_T_FMT"\n", + pass, Ntot, Ntry, Nskip, Nkeep); } Index: /branches/pap/Ohana/src/tools/src/ftable.c =================================================================== --- /branches/pap/Ohana/src/tools/src/ftable.c (revision 28483) +++ /branches/pap/Ohana/src/tools/src/ftable.c (revision 28484) @@ -1,4 +1,5 @@ # include # include +# include "inttypes.h" char *print_table_row (char *row, Header *header); @@ -95,6 +96,6 @@ table.datasize = Nbytes; - gfits_scan (table.header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (table.header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (table.header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (table.header, "NAXIS2", OFF_T_FMT, 1, &Ny); /* print a column */ @@ -125,5 +126,5 @@ char field[16], type[16], format[16], *line; - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); @@ -209,9 +210,9 @@ for (i = 0; i < Naxis; i++) { sprintf (axisname, "NAXIS%d", i+1); - status = gfits_scan (&header, axisname, "%lld", 1, (long long *) &Nelem); + status = gfits_scan (&header, axisname, OFF_T_FMT, 1, &Nelem); if (!status) { fprintf (stderr, "missing %s\n", axisname); } - fprintf (stdout, " %7lld", (long long) Nelem); + fprintf (stdout, " "OFF_T_FMT, Nelem); } fprintf (stdout, "\n"); @@ -294,6 +295,6 @@ gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); - gfits_scan (header, "NAXIS1", "%lld", 1, (long long *) &Nx); - gfits_scan (header, "NAXIS2", "%lld", 1, (long long *) &Ny); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Nx); + gfits_scan (header, "NAXIS2", OFF_T_FMT, 1, &Ny); if (Colname != (char *) NULL) { @@ -358,5 +359,5 @@ if (!strcmp (type, "int64_t")) { memcpy (line, &data[i*Nv*Nb + Nb*j], Nb); - fprintf (stdout, "%lld ", (long long) *(int64_t*)line); + fprintf (stdout, "%" PRId64" ", *(int64_t*)line); } if (!strcmp (type, "float")) { Index: /branches/pap/Ohana/src/uniphot/src/load_images.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/load_images.c (revision 28483) +++ /branches/pap/Ohana/src/uniphot/src/load_images.c (revision 28484) @@ -23,5 +23,5 @@ image = gfits_table_get_Image (&db[0].ftable, Nimage, &db[0].swapped); - fprintf (stderr, "loaded %lld images\n", (long long) *Nimage); + fprintf (stderr, "loaded "OFF_T_FMT" images\n", *Nimage); return (image); Index: /branches/pap/Ohana/src/uniphot/src/update_catalog_setphot.c =================================================================== --- /branches/pap/Ohana/src/uniphot/src/update_catalog_setphot.c (revision 28483) +++ /branches/pap/Ohana/src/uniphot/src/update_catalog_setphot.c (revision 28484) @@ -93,5 +93,5 @@ if (found) { - fprintf (stderr, "found %lld matches\n", (long long) found); + fprintf (stderr, "found "OFF_T_FMT" matches\n", found); } } Index: /branches/pap/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /branches/pap/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 28483) +++ /branches/pap/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 28484) @@ -475,5 +475,5 @@ my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE; - my $command = "$difftool -dbname $imagedb"; + my $command = "$difftool -dbname $imagedb -pstamp_order"; my $listrun = 0; @@ -482,6 +482,9 @@ $command .= " -diffskyfile -diff_id $id -skycell_id $skycell_id"; } else { + $listrun = 1; $command .= " -listrun -diff_id $id"; - $listrun = 1; + # the following is a work around for the problem reported in ticket #1394 + # 'difftool -listrun returns multiple rows for stack-stack diffs' + $command .= " -limit 1"; } } else { @@ -492,5 +495,5 @@ my $n = $output ? scalar @$output : 0; - if (!$listrun && ($n > 1)) { + if ($n > 1) { die ("difftool returned an unexpected number of diffskyfiles: $n"); } elsif ($n == 0) { Index: /branches/pap/dbconfig/add.md =================================================================== --- /branches/pap/dbconfig/add.md (revision 28483) +++ /branches/pap/dbconfig/add.md (revision 28484) @@ -12,4 +12,7 @@ note STR 255 image_only BOOL f + minidvodb BOOL f + minidvodb_group STR 64 + minidvodb_name STR 64 END @@ -18,4 +21,5 @@ dtime_addstar F32 0.0 path_base STR 255 + dvodb_path STR 255 fault S16 0 # Key NOT NULL END @@ -24,3 +28,5 @@ label STR 64 # Primary Key END - + + + Index: /branches/pap/dbconfig/changes.txt =================================================================== --- /branches/pap/dbconfig/changes.txt (revision 28483) +++ /branches/pap/dbconfig/changes.txt (revision 28484) @@ -1634,4 +1634,5 @@ ALTER TABLE diffSkyfile ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + ALTER TABLE chipRun CHANGE maskfrac_npix maskfrac_npix FLOAT; ALTER TABLE camRun CHANGE maskfrac_npix maskfrac_npix FLOAT; @@ -1811,2 +1812,126 @@ FOREIGN KEY(warp_bg_id) REFERENCES warpBackgroundRun(warp_bg_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; + + +-- Version 1.1.68 addition of minidvodb tables and changes to addTables + +-- this modifies addTables + +ALTER TABLE addRun ADD COLUMN minidvodb tinyint NOT NULL AFTER image_only; +ALTER TABLE addRun ADD COLUMN minidvodb_group varchar(64) AFTER minidvodb; +ALTER TABLE addRun ADD COLUMN minidvodb_name varchar(64) AFTER minidvodb_group; + +ALTER TABLE addProcessedExp ADD COLUMN dvodb_path varchar(255) AFTER path_base; + + +-- this adds the new tables + +CREATE TABLE minidvodbRun ( +minidvodb_id BIGINT AUTO_INCREMENT, + minidvodb_name VARCHAR(64), + minidvodb_group VARCHAR(64) NOT NULL, + minidvodb_path VARCHAR(255) NOT NULL, + mergedvodb_path VARCHAR(255) NOT NULL, + state VARCHAR(64) NOT NULL, + creation_date TIMESTAMP, + PRIMARY KEY(minidvodb_id), KEY(minidvodb_name), INDEX(minidvodb_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE minidvodbProcessed ( + minidvodb_id BIGINT, + merge_order BIGINT, +dtime_resort FLOAT, +dtime_relphot FLOAT, +dtime_merge FLOAT, +epoch TIMESTAMP, +mergedvodb_path VARCHAR(255) NOT NULL, +fault SMALLINT NOT NULL, + KEY(minidvodb_id), CONSTRAINT UNIQUE(minidvodb_id), FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun (minidvodb_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +ALTER TABLE pstampRequest ADD COLUMN timestamp TIMESTAMP AFTER outdir; + + +-- Tables to support (re-)photometry of a diff +CREATE TABLE diffPhotRun ( + diff_phot_id BIGINT AUTO_INCREMENT, -- Identifier for diffPhotRun + diff_id BIGINT NOT NULL, -- Identifier for diffRun + state VARCHAR(64) NOT NULL, -- State of run + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + reduction VARCHAR(64), -- reduction class (for altering recipe) + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + note VARCHAR(255), -- note + PRIMARY KEY(diff_phot_id), + KEY(diff_id), + KEY(state), + KEY(label), + KEY(data_group), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE diffPhotSkyfile ( + diff_phot_id BIGINT AUTO_INCREMENT, -- Identifier for diffPhotRun + skycell_id VARCHAR(64) NOT NULL, -- Skycell identifier + path_base VARCHAR(255) NOT NULL, -- Base of path for output + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + fault SMALLINT NOT NULL, -- fault code + quality SMALLINT NOT NULL, -- bad quality flag + software_ver VARCHAR(16), -- software version for run + magicked BIGINT NOT NULL DEFAULT 0, -- magic mask applied + PRIMARY KEY(diff_phot_id, skycell_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(diff_phot_id) REFERENCES diffPhotRun(diff_phot_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +-- Version 1.1.69 warp/stack/diff summary and stack associations. + +CREATE TABLE warpSummary ( + warp_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(warp_id), + KEY(projection_cell), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffSummary ( + diff_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(diff_id), + KEY(projection_cell), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociation ( + sass_id BIGINT AUTO_INCREMENT, + data_group VARCHAR(64) NOT NULL, + projection_cell VARCHAR(64) NOT NULL, + tess_id VARCHAR(64) NOT NULL, + filter VARCHAR(64) NOT NULL, + PRIMARY KEY(sass_id), + KEY(data_group), + KEY(projection_cell), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociationMap ( + sass_id BIGINT, + stack_id BIGINT, + PRIMARY KEY(sass_id, stack_id), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackSummary ( + sass_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(sass_id), + KEY(projection_cell), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/pap/dbconfig/diff.md =================================================================== --- /branches/pap/dbconfig/diff.md (revision 28483) +++ /branches/pap/dbconfig/diff.md (revision 28484) @@ -75,2 +75,8 @@ maskfrac_advisory F32 0.0 END + +diffSummary METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + projection_cell STR 32 # Primary Key + path_base STR 255 +END Index: /branches/pap/dbconfig/diffphot.md =================================================================== --- /branches/pap/dbconfig/diffphot.md (revision 28484) +++ /branches/pap/dbconfig/diffphot.md (revision 28484) @@ -0,0 +1,24 @@ + +diffPhotRun METADATA + diff_phot_id S64 0 # Primary Key AUTO_INCREMENT + diff_id S64 0 # Key + state STR 64 # Key + workdir STR 255 + label STR 64 # Key + data_group STR 64 # Key + reduction STR 64 + registered TAI NULL + note STR 255 +END + +diffPhotSkyfile METADATA + diff_phot_id S64 0 # Primary Key + skycell_id STR 64 # Primary Key + path_base STR 255 + dtime_script F32 0.0 + hostname STR 64 + fault S16 0 # Key + quality S16 0 # Key + software_ver STR 16 + magicked S64 0 +END Index: /branches/pap/dbconfig/ipp.m4 =================================================================== --- /branches/pap/dbconfig/ipp.m4 (revision 28483) +++ /branches/pap/dbconfig/ipp.m4 (revision 28484) @@ -24,4 +24,5 @@ include(label.md) include(magic.md) +include(minidvodb.md) include(calibration.md) include(flatcorr.md) @@ -33,2 +34,3 @@ include(publish.md) include(background.md) +include(diffphot.md) Index: /branches/pap/dbconfig/minidvodb.md =================================================================== --- /branches/pap/dbconfig/minidvodb.md (revision 28484) +++ /branches/pap/dbconfig/minidvodb.md (revision 28484) @@ -0,0 +1,20 @@ +minidvodbRun METADATA + minidvodb_id S64 0 + minidvodb_name STR 64 + minidvodb_group STR 64 + minidvodb_path STR 255 + mergedvodb_path STR 255 + state STR 64 + creation_date UTC 0001-01-01T00:00:00Z +END + +minidvodbProcessed METADATA + minidvodb_id S64 0 + merge_order S64 0 + dtime_resort F32 0.0 + dtime_relphot F32 0.0 + dtime_merge F32 0.0 + epoch UTC 0001-01-01T00:00:00Z + mergedvodb_path STR 255 + fault S16 0 +END Index: /branches/pap/dbconfig/pstamp.md =================================================================== --- /branches/pap/dbconfig/pstamp.md (revision 28483) +++ /branches/pap/dbconfig/pstamp.md (revision 28484) @@ -31,4 +31,5 @@ uri STR 255 outdir STR 255 + timestamp UTC 0001-01-01T00:00:00Z fault S32 0 END Index: /branches/pap/dbconfig/stack.md =================================================================== --- /branches/pap/dbconfig/stack.md (revision 28483) +++ /branches/pap/dbconfig/stack.md (revision 28484) @@ -53,2 +53,21 @@ quality S16 0 END + +stackSummary METADATA + sass_id S64 0 # Primary Key fkey(sass_id) ref stackAssociation(sass_id) + projection_cell STR 32 # Primary Key + path_base STR 255 +END + +stackAssociation METADATA + sass_id S64 0 # Primary Key AUTO_INCREMENT + data_group STR 64 # Key + projection_cell STR 64 # Key + tess_id STR 64 # Key + filter STR 64 +END + +stackAssociationMap METADATA + sass_id S64 0 # Primary Key fkey(sass_id) ref stackAssociation(sass_id) + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) +END Index: /branches/pap/dbconfig/warp.md =================================================================== --- /branches/pap/dbconfig/warp.md (revision 28483) +++ /branches/pap/dbconfig/warp.md (revision 28484) @@ -79,2 +79,8 @@ label STR 64 # Primary Key END + +warpSummary METADATA + warp_id S64 0 # Primary Key fkey(warp_id) ref warpRun(warp_id) + projection_cell STR 32 # Primary Key + path_base STR 255 +END Index: /branches/pap/ippMonitor/Makefile.in =================================================================== --- /branches/pap/ippMonitor/Makefile.in (revision 28483) +++ /branches/pap/ippMonitor/Makefile.in (revision 28484) @@ -16,5 +16,7 @@ $(DESTBIN)/czartool_getLabels.pl \ $(DESTBIN)/czartool_checkServer.pl \ -$(DESTBIN)/czartool_getServerStatus.pl +$(DESTBIN)/czartool_getServerStatus.pl \ +$(DESTBIN)/build_histogram.dvo \ + RAWSRC = \ @@ -50,4 +52,6 @@ $(DESTWWW)/czartool_labels.php \ $(DESTWWW)/czartool_servers.php \ +$(DESTWWW)/histogram.php \ +$(DESTWWW)/show_and_delete_image.php \ @@ -121,4 +125,5 @@ $(DESTWWW)/rawScienceExp.php \ $(DESTWWW)/rawExp.php \ +$(DESTWWW)/rawSummary.php \ $(DESTWWW)/rawExpStats.php \ $(DESTWWW)/summitExp.php \ @@ -171,5 +176,6 @@ $(DESTWWW)/flatcorrRun.php \ $(DESTWWW)/flatcorrChip.php \ -$(DESTWWW)/flatcorrCamera.php +$(DESTWWW)/flatcorrCamera.php \ +$(DESTWWW)/maskStats.php PICTURES = \ Index: /branches/pap/ippMonitor/def/maskStats.d =================================================================== --- /branches/pap/ippMonitor/def/maskStats.d (revision 28484) +++ /branches/pap/ippMonitor/def/maskStats.d (revision 28484) @@ -0,0 +1,50 @@ +TABLE rawExp, chipRun, camRun, fakeRun, warpRun +TITLE Mask Stats +FILE maskStats.php +MENU ipp.imfiles.dat + +WHERE rawExp.exp_id = chipRun.exp_id +WHERE chipRun.chip_id = camRun.chip_id +WHERE camRun.cam_id = fakeRun.cam_id +WHERE fakeRun.fake_id = warpRun.fake_id + +OP OP1 $ra * 57.295783 +OP OP2 $decl * 57.295783 + +# field size format name show link to extras +FIELD rawExp.exp_name, 10, %s, Exp Name +FIELD rawExp.exp_id, 5, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD rawExp.exp_type, 8, %s, Type +FIELD rawExp.state, 5, %s, State +FIELD rawExp.obs_mode, 8, %s, obs mode +FIELD rawExp.dateobs, 19, %T, Date/Time +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 +FIELD rawExp.filter, 10, %s, FILTER +FIELD rawExp.airmass, 5, %.4f, airmass +FIELD rawExp.exp_time, 5, %.2f, exp_time + +FIELD chipRun.maskfrac_npix, 8, %.3e, chip npix +FIELD camRun.maskfrac_ref_npix, 8, %.3e, cam ref npix +FIELD camRun.maskfrac_max_npix, 8, %.3e, cam max npix +FIELD warpRun.maskfrac_npix, 8, %.3e, warp npix + +FIELD chipRun.maskfrac_static, 8, %.3f, chip static +FIELD camRun.maskfrac_ref_static, 8, %.3f, cam ref static +FIELD camRun.maskfrac_max_static, 8, %.3f, cam max static +FIELD warpRun.maskfrac_static, 8, %.3f, warp static + +FIELD chipRun.maskfrac_dynamic, 8, %.3f, chip dynamic +FIELD camRun.maskfrac_ref_dynamic, 8, %.3f, cam ref dynamic +FIELD camRun.maskfrac_max_dynamic, 8, %.3f, cam max dynamic +FIELD warpRun.maskfrac_dynamic, 8, %.3f, warp dynamic + +FIELD chipRun.maskfrac_magic, 8, %.3f, chip magic +FIELD camRun.maskfrac_ref_magic, 8, %.3f, cam ref magic +FIELD camRun.maskfrac_max_magic, 8, %.3f, cam max magic +FIELD warpRun.maskfrac_magic, 8, %.3f, warp magic + +FIELD chipRun.maskfrac_advisory, 8, %.3f, chip advisory +FIELD camRun.maskfrac_ref_advisory, 8, %.3f, cam ref advisory +FIELD camRun.maskfrac_max_advisory, 8, %.3f, cam max advisory +FIELD warpRun.maskfrac_advisory, 8, %.3f, warp advisory Index: /branches/pap/ippMonitor/def/rawExp.d =================================================================== --- /branches/pap/ippMonitor/def/rawExp.d (revision 28483) +++ /branches/pap/ippMonitor/def/rawExp.d (revision 28484) @@ -15,4 +15,5 @@ FIELD camera, 10, %s, Camera FIELD exp_type, 8, %s, Type +FIELD obs_mode, 8, %s, obs mode FIELD dateobs, 19, %T, Date/Time FIELD ra, 8, %C, RA, op=OP1 Index: /branches/pap/ippMonitor/def/rawSummary.d =================================================================== --- /branches/pap/ippMonitor/def/rawSummary.d (revision 28484) +++ /branches/pap/ippMonitor/def/rawSummary.d (revision 28484) @@ -0,0 +1,17 @@ +TABLE rawExp +TITLE Raw Summary +FILE rawSummary.php +MENU ipp.load.dat + +# field size format name show link to extras +FIELD exp_type, 15, %s, OBSTYPE +FIELD obs_mode, 15, %s, OBS_MODE +FIELD state, 7, %s, state +FIELD MIN(dateobs), 19, %T, min date +FIELD MAX(dateobs), 19, %T, max date +FIELD count(exp_id) as n, 7, %d, count + +MODE summary +GROUP state +GROUP exp_type +GROUP obs_mode Index: /branches/pap/ippMonitor/raw/czartool_labels.php =================================================================== --- /branches/pap/ippMonitor/raw/czartool_labels.php (revision 28483) +++ /branches/pap/ippMonitor/raw/czartool_labels.php (revision 28484) @@ -95,9 +95,9 @@ $str = ""; $anyFaults = false; - $link = "chipProcessedImfile_failure.php?menu=ipp.science.dat&pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel; + $link = "chipProcessedImfile_failure.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $label . "&chipRun.state=new"; getStateAndFaults($db, $label,"chipRun", $state, "chip", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel; + $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&camRun.label=" . $label . "&camRun.state=new"; getStateAndFaults($db, $label,"camRun", $state, "cam", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); @@ -107,13 +107,13 @@ write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj; + $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.label=" . $label . "&warpRun.state=new"; getStateAndFaults($db, $label,"warpRun", $state, "warp", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj; + $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.label=" . $label . "&stackRun.state=new"; getStateAndFaults($db, $label,"stackRun", $state, "stack", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj; + $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.label=" . $label . "&diffRun.state=new"; getStateAndFaults($db, $label,"diffRun", $state, "diff", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); @@ -189,9 +189,9 @@ $anyFaults = false; - $link = "chipProcessedImfile_failure.php?menu=ipp.science.dat&pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel; + $link = "chipProcessedImfile_failure.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $stdsLabel . "&chipRun.state=new"; getStateAndFaults($db, $stdsLabel,"chipRun", $selectedState, "chip", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel; + $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&camRun.label=" . $stdsLabel . "&camRun.state=new"; getStateAndFaults($db, $stdsLabel,"camRun", $selectedState, "cam", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); @@ -201,13 +201,13 @@ write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj; + $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.label=" . $stdsLabel . "&warpRun.state=new"; getStateAndFaults($db, $stdsLabel,"warpRun", $selectedState, "warp", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj; + $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.label=" . $stdsLabel . "&stackRun.state=new"; getStateAndFaults($db, $stdsLabel,"stackRun", $selectedState, "stack", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj; + $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.label=" . $stdsLabel . "&diffRun.state=new"; getStateAndFaults($db, $stdsLabel,"diffRun", $selectedState, "diff", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); Index: /branches/pap/ippMonitor/raw/histogram.php =================================================================== --- /branches/pap/ippMonitor/raw/histogram.php (revision 28484) +++ /branches/pap/ippMonitor/raw/histogram.php (revision 28484) @@ -0,0 +1,472 @@ + Histogram

"; + +// set up the form +echo "
\n"; + +$restricted = 0; + +// define restrictiosn to the queries +// ** TABLE RESTRICTIONS ** +$WHERE = "WHERE camRun.state = 'full' AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id AND camProcessedExp.cam_id = camRun.cam_id AND camProcessedExp.fault = 0"; +$WHERE = check_restrict ('rawExp.exp_name', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('chipRun.label', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('chipRun.data_group', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('chipRun.dist_group', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('rawExp.filter', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_uq', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_uq', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_uq', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_lq', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_lq', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_major_lq', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_uq', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_uq', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_uq', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_lq', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_lq', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.fwhm_minor_lq', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_major', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_major', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_major', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_minor', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_minor', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_fwhm_minor', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m2', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m2', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m2', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m3', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m3', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m3', $WHERE, 'max', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m4', $WHERE, 'string', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m4', $WHERE, 'min', 1.0); +$WHERE = check_restrict ('camProcessedExp.iq_m4', $WHERE, 'max', 1.0); +$WHERE = check_ordering ('', $WHERE); + +if ($restricted == 0) { + if ("none" != "none") { + $WHERE = "$WHERE none"; + } +} + +// get the result table count +if (basic == "basic") { + $sql = "SELECT count(*) FROM camRun, chipRun, rawExp, camProcessedExp $WHERE"; +} +if (basic == "summary") { + $sql = "SELECT count(*) FROM (SELECT rawExp.exp_name,chipRun.label,chipRun.data_group,chipRun.dist_group,rawExp.dateobs,rawExp.filter,rawExp.exp_time,rawExp.airmass,camProcessedExp.fwhm_major,camProcessedExp.fwhm_minor,camProcessedExp.fwhm_major_uq,camProcessedExp.fwhm_major_lq,camProcessedExp.fwhm_minor_uq,camProcessedExp.fwhm_minor_lq,camProcessedExp.iq_fwhm_major,camProcessedExp.iq_fwhm_minor,camProcessedExp.iq_m2,camProcessedExp.iq_m3,camProcessedExp.iq_m4 FROM camRun, chipRun, rawExp, camProcessedExp $WHERE) as TEMP"; +} + +$qry = $db->query($sql); +if (dberror($qry)) { + echo "error reading camRun, chipRun, rawExp, camProcessedExp table count
\n"; + echo "
count query : $sql
\n"; + menu_end(); +} +if (!$qry->fetchInto($row)) { + echo "error reading camRun, chipRun, rawExp, camProcessedExp table count
\n"; + echo "
count query : $sql
\n"; + menu_end(); +} +// set up the row counter variables +if ($ID['from']) { + $rowStart = $ID['from']; +} else { + $rowStart = 0; +} +$rowLast = $rowStart + $dTABLE; +$rowTotal = $row[0]; +if ($rowLast > $rowTotal) { $rowLast = $rowTotal; } +echo " $rowStart to $rowLast of $rowTotal items
\n"; + +// query the database +if (basic == "basic") { + $sql = "SELECT rawExp.exp_name,chipRun.label,chipRun.data_group,chipRun.dist_group,rawExp.dateobs,rawExp.filter,rawExp.exp_time,rawExp.airmass,camProcessedExp.fwhm_major,camProcessedExp.fwhm_minor,camProcessedExp.fwhm_major_uq,camProcessedExp.fwhm_major_lq,camProcessedExp.fwhm_minor_uq,camProcessedExp.fwhm_minor_lq,camProcessedExp.iq_fwhm_major,camProcessedExp.iq_fwhm_minor,camProcessedExp.iq_m2,camProcessedExp.iq_m3,camProcessedExp.iq_m4 FROM camRun, chipRun, rawExp, camProcessedExp $WHERE"; +} +if (basic == "summary") { + $sql = "SELECT rawExp.exp_name,chipRun.label,chipRun.data_group,chipRun.dist_group,rawExp.dateobs,rawExp.filter,rawExp.exp_time,rawExp.airmass,camProcessedExp.fwhm_major,camProcessedExp.fwhm_minor,camProcessedExp.fwhm_major_uq,camProcessedExp.fwhm_major_lq,camProcessedExp.fwhm_minor_uq,camProcessedExp.fwhm_minor_lq,camProcessedExp.iq_fwhm_major,camProcessedExp.iq_fwhm_minor,camProcessedExp.iq_m2,camProcessedExp.iq_m3,camProcessedExp.iq_m4 FROM camRun, chipRun, rawExp, camProcessedExp $WHERE"; +} + +$qry = $db->query($sql); +if (dberror($qry)) { + echo "error reading camRun, chipRun, rawExp, camProcessedExp table
\n"; + echo "
table query : $sql
\n"; + menu_end(); +} + +// ** HEAD CODE ** + +// ** BUTTON RESTRICTIONS ** +$buttonLink = button_restrict_string ('rawExp.exp_name', $buttonLink); +$buttonLink = button_restrict_string ('chipRun.label', $buttonLink); +$buttonLink = button_restrict_string ('chipRun.data_group', $buttonLink); +$buttonLink = button_restrict_string ('chipRun.dist_group', $buttonLink); +$buttonLink = button_restrict_string ('rawExp.dateobs', $buttonLink); +$buttonLink = button_restrict_min ('rawExp.dateobs', $buttonLink); +$buttonLink = button_restrict_max ('rawExp.dateobs', $buttonLink); +$buttonLink = button_restrict_string ('rawExp.filter', $buttonLink); +$buttonLink = button_restrict_string ('rawExp.exp_time', $buttonLink); +$buttonLink = button_restrict_min ('rawExp.exp_time', $buttonLink); +$buttonLink = button_restrict_max ('rawExp.exp_time', $buttonLink); +$buttonLink = button_restrict_string ('rawExp.airmass', $buttonLink); +$buttonLink = button_restrict_min ('rawExp.airmass', $buttonLink); +$buttonLink = button_restrict_max ('rawExp.airmass', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_major', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_major', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_major', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_minor', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_minor', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_minor', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_major_uq', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_major_uq', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_major_uq', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_major_lq', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_major_lq', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_major_lq', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_minor_uq', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_minor_uq', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_minor_uq', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.fwhm_minor_lq', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.fwhm_minor_lq', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.fwhm_minor_lq', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_fwhm_major', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_fwhm_major', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_fwhm_major', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_fwhm_minor', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_fwhm_minor', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_fwhm_minor', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_m2', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_m2', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_m2', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_m3', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_m3', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_m3', $buttonLink); +$buttonLink = button_restrict_string ('camProcessedExp.iq_m4', $buttonLink); +$buttonLink = button_restrict_min ('camProcessedExp.iq_m4', $buttonLink); +$buttonLink = button_restrict_max ('camProcessedExp.iq_m4', $buttonLink); + +$buttonLink = button_restrict_checkbox ('hist.exp_time', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.airmass', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_major_psf', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_minor_psf', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_uq', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_lq', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_major_moments', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_minor_moments', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_m2_moment', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_m3_moment', $buttonLink); +$buttonLink = button_restrict_checkbox ('hist.fwhm_m4_moment', $buttonLink); + +$buttonLink = button_restrict_string ('hist_delta.exp_time', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.airmass', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_major_psf', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_minor_psf', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_uq', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_lq', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_major_moments', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_minor_moments', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_m2_moment', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_m3_moment', $buttonLink); +$buttonLink = button_restrict_string ('hist_delta.fwhm_m4_moment', $buttonLink); + +navigate_buttons ($rowStart, $rowLast, $dTABLE, $rowTotal, $buttonLink, $ID, 'histogram.php'); + +echo "  :   or enter start row: \n"; + +// set up the table +echo "\n"; + +// echo "\n"; // first field is a label set below for the query rows only +// ** TABLE HEADER ** +echo "\n"; +write_header_cell ("list", "Exp Name"); +write_header_cell ("list", "Label"); +write_header_cell ("list", "data grp"); +write_header_cell ("list", "dist grp"); +write_header_cell ("list", "Date/Time"); +write_header_cell ("list", "FILTER"); +write_header_cell ("list", "exp_time "); +write_header_cell ("list", "airmass "); +write_header_cell ("list", "fwhm major psf"); +write_header_cell ("list", "fwhm minor psf"); +write_header_cell ("list", "fwhm UQ"); +write_header_cell ("list", "fwhm LQ"); +write_header_cell ("list", "fwhm major moments"); +write_header_cell ("list", "fwhm minor moments"); +write_header_cell ("list", "m2 moment"); +write_header_cell ("list", "m3 moment"); +write_header_cell ("list", "m4 moment"); +echo "\n"; + +// query restriction form +// echo "\n"; +// echo "\n"; +// ** TABLE QUERY ** +echo "\n"; +write_query_row ('rawExp.exp_name', 5, 'string'); +write_query_row ('chipRun.label', 7, 'string'); +write_query_row ('chipRun.data_group', 7, 'string'); +write_query_row ('chipRun.dist_group', 7, 'string'); +write_query_row ('rawExp.dateobs', 19, 'min'); +write_query_row ('rawExp.filter', 10, 'string'); +write_query_row ('rawExp.exp_time', 5, 'min'); +write_query_row ('rawExp.airmass', 5, 'min'); +write_query_row ('camProcessedExp.fwhm_major', 5, 'min'); +write_query_row ('camProcessedExp.fwhm_minor', 5, 'min'); +write_query_row ('camProcessedExp.fwhm_major_uq', 5, 'min'); +write_query_row ('camProcessedExp.fwhm_major_lq', 5, 'min'); +write_query_row ('camProcessedExp.iq_fwhm_major', 5, 'min'); +write_query_row ('camProcessedExp.iq_fwhm_minor', 5, 'min'); +write_query_row ('camProcessedExp.iq_m2', 5, 'min'); +write_query_row ('camProcessedExp.iq_m3', 5, 'min'); +write_query_row ('camProcessedExp.iq_m4', 5, 'min'); +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +write_query_row ('rawExp.dateobs', 19, 'max'); +echo "\n"; +write_query_row ('rawExp.exp_time', 5, 'max'); +write_query_row ('rawExp.airmass', 5, 'max'); +write_query_row ('camProcessedExp.fwhm_major', 5, 'max'); +write_query_row ('camProcessedExp.fwhm_minor', 5, 'max'); +write_query_row ('camProcessedExp.fwhm_major_uq', 5, 'max'); +write_query_row ('camProcessedExp.fwhm_major_lq', 5, 'max'); +write_query_row ('camProcessedExp.iq_fwhm_major', 5, 'max'); +write_query_row ('camProcessedExp.iq_fwhm_minor', 5, 'max'); +write_query_row ('camProcessedExp.iq_m2', 5, 'max'); +write_query_row ('camProcessedExp.iq_m3', 5, 'max'); +write_query_row ('camProcessedExp.iq_m4', 5, 'max'); +echo "\n"; +// echo "\n"; + +//checkboxes +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n";//dateobs +echo "\n"; +write_query_checkbox ('hist.exp_time', 'Histogram?'); +write_query_checkbox ('hist.airmass', 'Histogram?'); +write_query_checkbox ('hist.fwhm_major_psf', 'Histogram?'); +write_query_checkbox ('hist.fwhm_minor_psf', 'Histogram?'); +write_query_checkbox ('hist.fwhm_uq', 'Histogram?'); +write_query_checkbox ('hist.fwhm_lq', 'Histogram?'); +write_query_checkbox ('hist.fwhm_major_moments', 'Histogram?'); +write_query_checkbox ('hist.fwhm_minor_moments', 'Histogram?'); +write_query_checkbox ('hist.fwhm_m2_moment', 'Histogram?'); +write_query_checkbox ('hist.fwhm_m3_moment', 'Histogram?'); +write_query_checkbox ('hist.fwhm_m4_moment', 'Histogram?'); +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +write_query_row ('hist.exp_time_delta', 5, 'string'); +write_query_row ('hist.airmass_delta', 5, 'string'); +write_query_row ('hist.fwhm_major_psf_delta', 5, 'string'); +write_query_row ('hist.fwhm_minor_psf_delta', 5, 'string'); +write_query_row ('hist.fwhm_uq_delta', 5, 'string'); +write_query_row ('hist.fwhm_lq_delta', 5, 'string'); +write_query_row ('hist.fwhm_major_moments_delta', 5, 'string'); +write_query_row ('hist.fwhm_minor_moments_delta', 5, 'string'); +write_query_row ('hist.fwhm_m2_moment_delta', 5, 'string'); +write_query_row ('hist.fwhm_m3_moment_delta', 5, 'string'); +write_query_row ('hist.fwhm_m4_moment_delta', 5, 'string'); +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +write_query_row ('hist.exp_time_count', 5, 'string'); +write_query_row ('hist.airmass_count', 5, 'string'); +write_query_row ('hist.fwhm_major_psf_count', 5, 'string'); +write_query_row ('hist.fwhm_minor_psf_count', 5, 'string'); +write_query_row ('hist.fwhm_uq_count', 5, 'string'); +write_query_row ('hist.fwhm_lq_count', 5, 'string'); +write_query_row ('hist.fwhm_major_moments_count', 5, 'string'); +write_query_row ('hist.fwhm_minor_moments_count', 5, 'string'); +write_query_row ('hist.fwhm_m2_moment_count', 5, 'string'); +write_query_row ('hist.fwhm_m3_moment_count', 5, 'string'); +write_query_row ('hist.fwhm_m4_moment_count', 5, 'string'); +echo "\n"; + +// close the table and form +echo "
≥
≤          
           
          Bin size (delta)
          Bins number (default: 100)
\n"; +echo "About the \"good\" number of bins

\n"; +echo "\n"; +$pass = $ID['pass']; +$proj = $ID['proj']; +$menu = $ID['menu']; +echo "\n"; +echo "\n"; +echo "\n"; +echo "
\n"; + +$keysToRowIndex = array( + 'hist.exp_time' => 6, + 'hist.airmass' => 7, + 'hist.fwhm_major_psf' => 8, + 'hist.fwhm_minor_psf' => 9, + 'hist.fwhm_uq'=> 10, + 'hist.fwhm_lq' => 11, + 'hist.fwhm_major_moments' => 12, + 'hist.fwhm_minor_moments' => 13, + 'hist.fwhm_m2_moment' => 14, + 'hist.fwhm_m3_moment' => 15, + 'hist.fwhm_m4_moment' =>16); +$handles = array(); +$filenames = array(); +foreach ($keysToRowIndex as $i => $value) { + if (get_value_from_key($i, TRUE) == "on") { + $filenames[$i] = tempnam("/tmp/serge", "histodat."); + if ($debug) { + echo "Using $filenames[$i] for $i\n"; + } + $handles[$i] = fopen($filenames[$i], "w"); + } +} +// ** TAIL CODE ** +while ($qry->fetchInto($row)) { + foreach ($keysToRowIndex as $i => $value) { + if (get_value_from_key($i, TRUE) == "on") { + fwrite($handles[$i], + sprintf("%.3f\n", $row[$value])); + } + } +} +foreach ($keysToRowIndex as $i => $value) { + if ($handles[$i] != NULL) { + fclose($handles[$i]); + chmod($filenames[$i], 0777); + } +} + +if ($debug) { + $show = get_value_from_key('hist.exp_time', TRUE); + $show .= " && ".get_value_from_key('hist.airmass', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_major_psf', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_minor_psf', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_uq', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_lq', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_major_moments', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_minor_moments', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_m2_moment', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_m3_moment', TRUE); + $show .= " && ".get_value_from_key('hist.fwhm_m4_moment', TRUE); + echo "==> ($show)
\n"; +} + +//Get environment +$PATH = getenv("PATH"); +putenv("PATH=$BINDIR:$PATH"); +$LD_LIBRARY_PATH = getenv("LD_LIBRARY_PATH"); +putenv("LD_LIBRARY_PATH=$LIBDIR:$LD_LIBRARY_PATH"); + +echo "
\n"; +foreach ($filenames as $i => $value) { + $outfile = $value.".png"; + $deltaValue = get_value_from_key($i."_delta", TRUE); + $binValue = get_value_from_key($i."_count", TRUE); + if ($binValue == "") { + $binValue = "-1"; + if ($deltaValue == "") { + $binValue = "100"; + $deltaValue = "-1"; + } + } else { + if ($deltaValue != "") { + echo "Warning! Both 'Bins number' and 'delta value' are specified => 'delta value' will be ignored
\n"; + $deltaValue = "-1"; + } else { + $deltaValue = "-1"; + } + } + $title = preg_replace ('|hist\.|', '', $i); + $title = preg_replace ('|_|', '-', $title); + + echo "Showing histogram for [$title] / delta = $deltaValue / bins = $binValue
\n"; + exec("build_histogram.dvo $value $outfile $deltaValue $binValue $title", + $output, $status); + if ($debug) { + echo "    Status = $status
\n"; + for ($i = 0; $i < count($output); $i++) { + echo "    output $i: $output[$i]
"; + } + } + $blah=count($output); + if ($debug) { + echo "    end: $output / $blah
\n"; + } + echo "

\n"; + unset($output); + $filesToRemove[$value] = ''; +} + +echo " WHERE: $WHERE

\n"; +echo " SQL: $sql

\n"; + +if ($debug) { + echo "Cleaning
\n"; +} +foreach ($filesToRemove as $index => $value) { + if ($debug) { + echo "Deleting $index
\n"; + } + unlink($index); +} +if ($debug) { + echo "Cleaning end
\n"; +} + +menu_end(); + +?> Index: /branches/pap/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- /branches/pap/ippMonitor/raw/ipp.imfiles.dat (revision 28483) +++ /branches/pap/ippMonitor/raw/ipp.imfiles.dat (revision 28484) @@ -22,3 +22,4 @@ menutop | menutop | link | simple plot - cam | simplePlotcam.php menutop | menutop | link | czartool | czartool_labels.php - +menutop | menutop | link | mask stats | maskStats.php +menutop | menutop | link | histogram | histogram.php Index: /branches/pap/ippMonitor/raw/ipp.load.dat =================================================================== --- /branches/pap/ippMonitor/raw/ipp.load.dat (revision 28483) +++ /branches/pap/ippMonitor/raw/ipp.load.dat (revision 28484) @@ -2,4 +2,5 @@ menutop | menutop | plain |   | +menulink | menuselect | link | Raw Exp Summary | rawSummary.php menulink | menuselect | link | Raw Exposures | rawExp.php menulink | menuselect | link | Raw Detrend Exp | rawDetrendExp.php Index: /branches/pap/ippMonitor/raw/ipp.php =================================================================== --- /branches/pap/ippMonitor/raw/ipp.php (revision 28483) +++ /branches/pap/ippMonitor/raw/ipp.php (revision 28484) @@ -564,6 +564,41 @@ } +// +function button_restrict_checkbox ($key, $line) { + $htmlkey = preg_replace ('|\.|', '_', $key); + if ($_SERVER[REQUEST_METHOD] == 'GET') { + $value = $_GET[$htmlkey]; + } else { + $value = $_POST[$htmlkey]; + } + if ($value != "") { + if ($line) { + $line = $line . "&$htmlkey=$value"; + } else { + $line = "$htmlkey=$value"; + } + } + return $line; +} + +// +function button_restrict_radio ($key, $line) { + $htmlkey = preg_replace ('|\.|', '_', $key); + if ($_SERVER[REQUEST_METHOD] == 'GET') { + $value = $_GET[$htmlkey]; + } else { + $value = $_POST[$htmlkey]; + } + if ($value != "") { + if ($line) { + $line = $line . "&$htmlkey=$value"; + } else { + $line = "$htmlkey=$value"; + } + } + return $line; +} + function write_header_cell ($class, $name) { - echo "$name\n"; } @@ -637,6 +672,45 @@ if ($value != "") { echo "value=\"$value\">"; - } + } else { + echo ">"; + } echo "\n"; +} + +// checkbox +function write_query_checkbox ($key, $comment) { + $htmlkey = preg_replace ('|\.|', '_', $key); + + if ($_SERVER[REQUEST_METHOD] == 'GET') { + $value = $_GET[$htmlkey]; + } else { + $value = $_POST[$htmlkey]; + } + echo " "; + } else { + echo ">"; + } + echo "$comment ($htmlkey/$value)\n"; +} + +// checkbox +function write_query_radio ($value, $key) { + $htmlkey = preg_replace ('| |', '_', $key); + $htmlvalue = preg_replace ('|\.|', '_', $value); + + if ($_SERVER[REQUEST_METHOD] == 'GET') { + $value = $_GET[$htmlkey]; + } else { + $value = $_POST[$htmlkey]; + } + if ($htmlvalue == get_value_from_key($htmlkey, TRUE)) { + $checked = "checked"; + } else { + $checked = ""; + } + echo " "; + echo "$key ($htmlkey/$htmlvalue)\n"; } @@ -778,3 +852,22 @@ // $myPage = $_SERVER[SCRIPT_NAME] . "?pass=$pass"; +////////////////////////////////////////////////////////////////////////// +// Return the value associated to the key (GET or POST method) +// If convertKeyToHtmlkey is true, the key is converted to a +// so-called HTML, i.e. the key where '.' are replaced by '_' +// (e.g. the conversion of 'f.o.o.b_a_r' is 'f_o_o_b_a_r'. +// +function get_value_from_key ($key, $convertKeyToHtmlkey) { + if ($convertKeyToHtmlkey) { + $htmlkey = preg_replace ('|\.|', '_', $key); + } else { + $htmlkey = $key; + } + if ($_SERVER[REQUEST_METHOD] == 'GET') { + return $_GET[$htmlkey]; + } else { + return $_POST[$htmlkey]; + } +} + ?> Index: /branches/pap/ippMonitor/raw/show_and_delete_image.php =================================================================== --- /branches/pap/ippMonitor/raw/show_and_delete_image.php (revision 28484) +++ /branches/pap/ippMonitor/raw/show_and_delete_image.php (revision 28484) @@ -0,0 +1,18 @@ + Index: /branches/pap/ippMonitor/scripts/build_histogram.dvo =================================================================== --- /branches/pap/ippMonitor/scripts/build_histogram.dvo (revision 28484) +++ /branches/pap/ippMonitor/scripts/build_histogram.dvo (revision 28484) @@ -0,0 +1,78 @@ +#!/usr/bin/env dvo + +$debug = 0 + +$bins = 100 +$delta = -1 + +macro build_histogram + myecho "Loading data from = $1" + data $1 + read v 1 + minimum v + maximum v + myecho "minvalue = $minvalue" + myecho "maxvalue = $maxvalue" + + if ($bins >= 2) + $delta = ($maxvalue-$minvalue)/$bins + myecho "delta = $delta" + else + if ($delta <= 0) + $bins = 100 + $delta = ($maxvalue-$minvalue)/$bins + myecho "delta = $delta" + end + end + + histo v vhist $minvalue $maxvalue $delta -range dx + myecho "bins = dx[]" + myecho " Setting limits" + limits dx vhist + plot dx vhist -x 1 + clear + myecho " Showing box" + box + myecho " Plotting" + plot dx vhist -x 1 -c blue + $binsCount = dx[] + myecho " Labeling" + myecho " -x $3" + myecho " -y Occurrences " + myecho " +x 'Histogram of $3 " + myecho " interval = $minvalue : $maxvalue " + myecho " / $binsCount bins" + myecho " / bin width: $delta)'" + labels -x $3 + labels -y Occurrences + labels +x "Histogram of $3 / interval = $minvalue : $maxvalue / $binsCount bins / bin width: $delta" + + myecho "Saving graphics to [$2]" + png -name $2 +end + +if ($SCRIPT) + $KAPA = kapa -noX + resize 1000 1000 + if ($argv:n != 5) + echo "USAGE: build_histogram (input) (output) (delta) (bins) (x-axis label)" + echo " If (bins) >= 2" + echo " build histogram with (bins) bins ((delta) is ignored)" + echo " Otherwise" + echo " if delta > 0 + echo " build histogram with (delta)-wide bins" + echo " otherwise" + echo " build histogram with 100 bins" + exit 1 + end + + echo "Loading dependencies" + $helpersFilename = `which helpers.dvo` + input $helpersFilename + echo "Dependencies loaded" + + $bins = $argv:3 + 1 + $delta = $argv:2 + build_histogram $argv:0 $argv:1 $argv:4 + exit 0 +end Index: /branches/pap/ippMonitor/scripts/czartool_checkServer.pl =================================================================== --- /branches/pap/ippMonitor/scripts/czartool_checkServer.pl (revision 28483) +++ /branches/pap/ippMonitor/scripts/czartool_checkServer.pl (revision 28484) @@ -17,5 +17,5 @@ if ($line =~ m/Scheduler is stopped/) {$isRunning = 0; $foundStatus=1;last;} if ($line =~ m/Scheduler is running/) {$isRunning = 1; $foundStatus=1;last;} - if ($line =~ m/Task Staus/) {last;} + if ($line =~ m/Task Status/) {last;} } Index: /branches/pap/ippScripts/Build.PL =================================================================== --- /branches/pap/ippScripts/Build.PL (revision 28483) +++ /branches/pap/ippScripts/Build.PL (revision 28484) @@ -62,4 +62,6 @@ scripts/magic_destreak_cleanup.pl scripts/magic_destreak_defineruns.pl + scripts/minidvodb_createdb.pl + scripts/minidvodb_merge.pl scripts/ippdb.pl scripts/ipp_cleanup.pl @@ -109,4 +111,6 @@ scripts/background_chip.pl scripts/background_warp.pl + scripts/skycell_jpeg.pl + scripts/diffphot.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: /branches/pap/ippScripts/MANIFEST =================================================================== --- /branches/pap/ippScripts/MANIFEST (revision 28483) +++ /branches/pap/ippScripts/MANIFEST (revision 28484) @@ -42,3 +42,4 @@ scripts/bundle_detrends.pl scripts/ipp_cluster_load_monitor.pl +scripts/skycell_jpeg.pl t/00_distribution.t Index: /branches/pap/ippScripts/scripts/addstar_run.pl =================================================================== --- /branches/pap/ippScripts/scripts/addstar_run.pl (revision 28483) +++ /branches/pap/ippScripts/scripts/addstar_run.pl (revision 28484) @@ -36,6 +36,6 @@ exit($PS_EXIT_CONFIG_ERROR); } - -my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $image_only, $verbose, $no_update, +my $minidvodb_path; +my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( @@ -48,4 +48,7 @@ 'reduction=s' => \$reduction, # Reduction class 'dvodb|w=s' => \$dvodb, # output DVO database + 'minidvodb_name|w=s'=> \$minidvodb_name, # miniDVO database name + 'minidvodb_group|w=s' => \$minidvodb_group, # miniDVO database group + 'minidvodb' => \$minidvodb, # use minidvodb? 'image-only' => \$image_only, # Print to stdout 'verbose' => \$verbose, # Print to stdout @@ -58,5 +61,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --exp_tag --add_id --camera --outroot", + -msg => "Required options: --exp_tag --add_id --camera --outroot --dvodb --camroot", -exitval => 3, ) unless @@ -65,6 +68,9 @@ defined $outroot and defined $camroot and + defined $dvodb and defined $camera; - +if ($minidvodb && !defined($minidvodb_group)) { + my_die( "missing minidvodb_group", $add_id, 3 ); + } my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -98,11 +104,49 @@ if (defined $dvodb) { $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO - $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); -} + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR); +} + + + my $dtime_addstar = 0; +if (defined $dvodbReal) { + if ($minidvodb) { + + if (!defined($minidvodb_name)) { + #take the active one, if it's not defined on the command line + #reverts would have this already set, for example. + my $command = "addtool -listminidvodbrun "; + $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; + $command .= " -state 'active' -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + $minidvodb_path = $comp->{minidvodb_path}; + $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_path)) { + &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); + } + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); + } + } + # tack on the minidvodb part to the db. +# $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/'; +#we don't need this now that I fixed the paths + + } unless ($no_op) { - if (defined $dvodbReal) { + print $dvodbReal; ## addstar can either save the full set of detections, or just @@ -120,5 +164,5 @@ my $command = "$addstar -update"; # XXX optionally set -update? $command .= " -D CAMERA $camdir"; - $command .= " -D CATDIR $dvodbReal"; + $command .= " -D CATDIR $minidvodb_path"; $command .= " $realFile"; $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference @@ -142,6 +186,8 @@ $fpaCommand .= " -path_base $outroot"; $fpaCommand .= " -dtime_addstar $dtime_addstar"; +$fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; +$fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; $fpaCommand .= " -dbname $dbname" if defined $dbname; - +print $fpaCommand; # Add the result into the database unless ($no_update) { @@ -171,6 +217,8 @@ $command .= " -addprocessedexp"; $command .= " -fault $exit_code"; + $command .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; $command .= " -path_base $outroot" if defined $outroot; $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; $command .= " -dbname $dbname" if defined $dbname; system ($command); Index: /branches/pap/ippScripts/scripts/automate_stacks.pl =================================================================== --- /branches/pap/ippScripts/scripts/automate_stacks.pl (revision 28483) +++ /branches/pap/ippScripts/scripts/automate_stacks.pl (revision 28484) @@ -17,7 +17,9 @@ my $missing_tools = 0; my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1); my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1); @@ -27,22 +29,4 @@ exit($PS_EXIT_CONFIG_ERROR); } - -# my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000'); -# my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10', -# 'STS','SVS','SweetSpot','ThreePi'); -# my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02', -# 'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06', -# 'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10', -# 'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0'); -# my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%', -# 'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%', -# 'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%', -# 'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%'); -# my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1, -# 'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1, -# 'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1, -# 'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0); -# my $retention_time = 9000; # days. - my $db; @@ -59,4 +43,5 @@ my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old); +my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats); GetOptions( @@ -81,4 +66,8 @@ 'check_stacks' => \$check_stacks, 'queue_stacks' => \$queue_stacks, + 'check_detrends' => \$check_detrends, + 'queue_detrends' => \$queue_detrends, + 'check_dqstats' => \$check_dqstats, + 'queue_dqstats' => \$queue_dqstats, 'check_diffs' => \$check_diffs, 'queue_diffs' => \$queue_diffs, @@ -105,4 +94,8 @@ --check_stacks Confirm that stacks can be built. --queue_stacks Issue stacktool commands to queue stacks. + --check_detrends Confirm that detrend verify runs can be built. + --queue_detrends Issue dettool commands to queue detrend verify runs. + --check_dqstats Confirm that dqstats tables can be built. + --queue_dqstats Issue dqstatstool commands to queue dqstat tables. --check_diffs Confirm that diffs can be done. --queue_diffs Issue difftool commands to queue diffs. @@ -115,6 +108,6 @@ ) unless defined $check_registration or defined $define_burntool or defined $queue_burntool or - defined $queue_chips or defined $queue_stacks or - defined $check_chips or defined $check_stacks or + defined $queue_chips or defined $queue_stacks or $queue_detrends or $queue_dqstats or + defined $check_chips or defined $check_stacks or $check_detrends or $check_dqstats or defined $test_mode or defined $clean_old or defined $check_mode; @@ -126,9 +119,18 @@ my %object_list = (); my %comment_list= (); +my %cleanmods_list = (); my %stackable_list = (); my %reduction_class = (); +my @detrend_list = (); +my %dettype_list = (); +my %exptype_list = (); +my %refID_list = (); +my %refIter_list = (); +my %detfilter_list = (); +my %detmax_list = (); my %clean_commands = (); my %clean_retention = (); my %noclean_list = (); +my %clean_alternate = (); # Grab the configuration data. my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; @@ -137,5 +139,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR); } @@ -156,9 +158,9 @@ $clean_retention{$this_mode} = ${ $mentry }{value}; } + elsif (${ $mentry }{name} eq 'ALTERNATE_CMD') { + $clean_alternate{$this_mode} = ${ $mentry }{value}; + } } } -# if (${ $entry }{name} eq 'RETENTION_TIME') { -# $retention_time = ${ $entry }{value}; -# } elsif (${ $entry }{name} eq 'FILTERS') { push @filter_list, ${ $entry }{value}; @@ -193,10 +195,40 @@ $noclean_list{$this_target} = ${ $tentry }{value}; } - } - } -} - - - + else { + if (exists($clean_commands{ ${ $tentry }{name} })) { + $cleanmods_list{$this_target}{${ $tentry }{name} } = ${ $tentry }{value}; + } + } + } + } + elsif (${ $entry }{name} eq 'DETRENDS') { + my @detrend_data = @{ ${ $entry }{value} }; + my $this_detrend = ''; + foreach my $dentry (@detrend_data) { + if (${ $dentry }{name} eq 'NAME') { + $this_detrend = ${ $dentry }{value}; + push @detrend_list, $this_detrend; + } + elsif (${ $dentry }{name} eq 'DETTYPE') { + $dettype_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'EXPTYPE') { + $exptype_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'REF_ID') { + $refID_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'REF_ITER') { + $refIter_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'FILTER') { + $detfilter_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'MAX') { + $detmax_list{$this_detrend} = ${ $dentry }{value}; + } + } + } +} unless(defined($date)) { @@ -257,4 +289,28 @@ unless (defined($test_mode) || defined($check_mode)) { exit(0); } } +if (defined($check_dqstats) || defined($test_mode)) { + $metadata_out{nsState} = 'CHECKDQSTATS'; + &execute_dqstats($date,"pretend"); + return_metadata($date); + unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($queue_dqstats) || defined($test_mode)) { + $metadata_out{nsState} = 'QUEUEDQSTATS'; + &execute_dqstats($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) { + $metadata_out{nsState} = 'CHECKDETRENDS'; + &execute_detrends($date,"pretend"); + return_metadata($date); + unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($queue_detrends)) { + $metadata_out{nsState} = 'QUEUEDETRENDS'; + &execute_detrends($date); + return_metadata($date); + exit(0); +} if (defined($define_burntool) || defined($test_mode)) { $metadata_out{nsState} = 'QUEUEBURNING'; @@ -347,5 +403,5 @@ if ($summit_fault) { print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault"; - if ($exp_type ne 'OBJECT') { + if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { print STDERR " (but I don't care).\n"; } @@ -357,5 +413,5 @@ elsif (!$download_state or $download_state eq 'run') { print STDERR "check_summit_copy: $date $exp_name has download_state $download_state"; - if ($exp_type ne 'OBJECT') { + if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { print STDERR " (but I don't care).\n"; } @@ -367,5 +423,5 @@ elsif (!$new_state or $new_state eq 'run' ) { print STDERR "check_summit_copy: $date $exp_name has new_state $new_state"; - if ($exp_type ne 'OBJECT') { + if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { print STDERR " (but I don't care).\n"; } @@ -498,5 +554,5 @@ my $cmd = "$chiptool"; - $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp '; + $cmd .= " -simple -dbname $dbname -definebyquery -set_end_stage warp "; $cmd .= " -set_label $label "; $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; @@ -612,4 +668,180 @@ # +# DQstats +################################################################################ + +sub construct_dqstats_cmd { + my $date = shift; + + my $select = "-dateobs_end ${date}T23:59:59 "; + + my $cmd = "$dqstatstool"; + $cmd .= " -simple -dbname $dbname -definebyquery "; + $cmd .= " $select "; + $cmd .= " -label %.nightlyscience "; + $cmd .= " -set_label dqstats.nightlyscience "; + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub pre_dqstats_queue { + my $date = shift; + +# my $db = init_gpc_db(); +# my $trunc_date = $date; $trunc_date =~ s/-//g; + +# my $where = " label LIKE '%.nightlyscience' AND data_group' "; +# my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new' OR state = 'cleaned') AND $where "; +# my $cam_sth = "SELECT * from camRun WHERE state = 'full' AND $where "; + +# my $chip_ref = $db->selectall_arrayref( $chip_sth ); +# my $cam_ref = $db->selectall_arrayref( $cam_sth ); + + my $command = construct_dqstats_cmd($date) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR); + } + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + + return($#input_exposures + 1,1,1); # $#{ $chip_ref } + 1, $#{ $cam_ref } + 1); +} + +sub dqstats_queue { + my $date = shift; + + my $command = construct_dqstats_cmd($date); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + return(0); + +} +sub execute_dqstats { + my $date = shift; + my $pretend = shift; + my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date); + if ($Nexposures == 0) { + print STDERR "execute_dqstats: No exposures on date $date.\n"; + } + elsif ($Ncams != $Nchips) { + print STDERR "execute_dqstats: Not done processing data through camera stage.\n"; + } + else { + unless(defined($pretend)) { + dqstats_queue($date); + } + } +} +# +# Detrend verification +################################################################################ + +sub construct_dettool_cmd { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + + my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; + $date =~ s/-//g; + + my $cmd = "$dettool"; +# $cmd .= " -pretend "; + $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type "; + $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter "; + $cmd .= " $select "; + $cmd .= " -inst $camera "; + $cmd .= " -select_exp_type $exp_type "; + $cmd .= " -select_filter $filter " if defined($filter); + $cmd .= " -workdir $workdir "; + $cmd .= " -label $label "; + if ($maxN > 0) { + $cmd .= " -random_subset -random_limit $maxN "; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub verify_uniqueness_detverify { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + + my $db = init_gpc_db(); + my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub pre_detrend_queue { + my $date = shift; + my $target = shift; + + my $command = construct_dettool_cmd($date,$target) . ' -pretend '; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); + } + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + return($#input_exposures + 1); +} + +sub detrend_queue { + my $date = shift; + my $target = shift; + + my $command = construct_dettool_cmd($date,$target); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $metadata_out{nsState} = 'DETREND_QUEUED'; + return(0); + +} +sub execute_detrends { + my $date = shift; + my $pretend = shift; + my $exposures = 0; + foreach my $target (@detrend_list) { + my ($Nexposures) = pre_detrend_queue($date,$target); + if ($Nexposures == 0) { + print STDERR "execute_detrends: Target $target on $date had no exposures.\n"; + next; + } + $exposures++; + unless(defined($pretend)) { + detrend_queue($date,$target); + } + } + if ($exposures == 0) { + $metadata_out{nsState} = 'DETREND_DROP'; + } + if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) { + $metadata_out{nsState} eq 'QUEUE_DETRENDS'; + } +} + +# # Stacking ################################################################################ @@ -627,5 +859,5 @@ my $cmd = "$stacktool"; # $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 '; # Probably silly, but I want to be safe and not overwrite - $cmd .= ' -simple -dbname gpc1 -definebyquery '; + $cmd .= " -simple -dbname $dbname -definebyquery "; $cmd .= " -set_label $label -select_label $label "; $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; @@ -768,5 +1000,14 @@ my $command = $clean_commands{$mode}; - my $retention_time = $clean_retention{$mode}; + my $retention_time; + if (exists($cleanmods_list{$target}{$mode})) { + $retention_time = $cleanmods_list{$target}{$mode}; + } + else { + $retention_time = $clean_retention{$mode}; + } + if ($retention_time <= 0) { + return("no clean","true"); + } my ($year,$month,$day) = split /-/,$date; @@ -774,4 +1015,5 @@ hour => 0, minute => 0, second => 0, nanosecond => 0, time_zone => 'Pacific/Honolulu'); + $dt->subtract(days => $retention_time); my $cleaning_date = $dt->ymd; @@ -779,5 +1021,10 @@ my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); my $args = $command; - $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; + if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) { + $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date "; + } + else { + $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; + } if ($debug == 1) { $args .= ' -pretend '; @@ -790,11 +1037,10 @@ my $pretend = shift; - foreach my $mode (keys (%clean_commands)) { - foreach my $target (@target_list) { - if (exists($noclean_list{$target})) { + foreach my $mode (sort (keys (%clean_commands))) { + if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) { + my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode); + if ($cleaning_date eq 'no clean') { next; } - my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); - print STDERR "$command\n"; if (!(defined($pretend) || $debug == 1)) { @@ -808,4 +1054,25 @@ } } + else { + foreach my $target (@target_list) { + if (exists($noclean_list{$target})) { + next; + } + my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); + if ($cleaning_date eq 'no clean') { + next; + } + print STDERR "$command\n"; + if (!(defined($pretend) || $debug == 1)) { +# print STDERR "BEAR IS DRIVING!?\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } + } } return(0); @@ -826,5 +1093,5 @@ my $label = "${target}.nightlyscience"; - my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}"; + my $workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}"; my $obs_mode = $obsmode_list{$target}; my $object = $object_list{$target}; @@ -837,4 +1104,30 @@ } +sub get_dettool_parameters { + my $date = shift; + my $target = shift; + my $workdir_date = $date; $workdir_date =~ s%-%/%g; + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $exp_type = $exptype_list{$target}; + my $det_type = $dettype_list{$target}; + my $ref_det_id = $refID_list{$target}; + my $ref_iter = $refIter_list{$target}; + my $det_filter = $detfilter_list{$target}; + my $internal_filter; + if (defined($det_filter)) { + $internal_filter = $det_filter; $internal_filter =~ s/\..*//; + $internal_filter = '.' . $internal_filter; + } + else { + $internal_filter = ''; + } + my $maxN = $detmax_list{$target}; + + my $lc_type = lc($exp_type); + my $label = "${lc_type}${internal_filter}.$trunc_date"; + my $workdir = 'neb://@HOST@.0/' . $dbname . "/detverify.nt/${workdir_date}/${lc_type}${internal_filter}"; + return($label,$workdir,$det_filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN); +} sub init_gpc_db { Index: /branches/pap/ippScripts/scripts/diffphot.pl =================================================================== --- /branches/pap/ippScripts/scripts/diffphot.pl (revision 28484) +++ /branches/pap/ippScripts/scripts/diffphot.pl (revision 28484) @@ -0,0 +1,300 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $diffphottool = can_run('diffphottool') or (warn "Can't find diffphottool" and $missing_tools = 1); +my $psphot = can_run('psphot') or (warn "Can't find psphot" and $missing_tools = 1); +my $ppArith = can_run('ppArith') or (warn "Can't find ppArith" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($diff_phot_id, $skycell_id, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'diff_phot_id=s' => \$diff_phot_id, # Diffphot identifier + 'skycell_id=s' => \$skycell_id, # Skycell identifier + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --diff_phot_id --skycell_id --outroot", + -exitval => 3, + ) unless defined $diff_phot_id + and defined $skycell_id + and defined $outroot; + +my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $diff_phot_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +# XXX camera is not known here; cannot use filerules... +# my $logDest = $ipprc->filename("LOG.EXP", $outroot); +my $logDest = "$outroot.log"; +$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +# Get input components +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $inputPath; # Input path +my $camera; # Camera name +my $bothways; # Diff was done both ways? +my $magicked; # Magic status +{ + my $command = "$diffphottool -input -diff_phot_id $diff_phot_id -skycell_id $skycell_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -inputskyfile: $error_code", $diff_phot_id, $skycell_id, $error_code); + } + + my $files = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $diff_phot_id, $skycell_id, $PS_EXIT_PROG_ERROR); + &my_die("File list does not contain exactly one entry", $diff_phot_id, $skycell_id, $PS_EXIT_PROG_ERROR) unless scalar @$files == 1; + my $file = $$files[0]; # File of interest + $inputPath = $file->{path_base}; + $camera = $file->{camera}; + $bothways = $file->{bothways}; + $magicked = $file->{magicked}; +} + +&my_die("Unable to identify input", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless defined $inputPath; +&my_die("Unable to identify camera", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_psphot = $ipprc->reduction($reduction, 'DIFF_PSPHOT'); # Recipe to use for psphot +my $recipe_ppstats = 'DIFFSTATS'; +&my_die("Couldn't find selected reduction for DIFF_PSPHOT: $reduction\n", $diff_phot_id, $skycell_id, $PS_EXIT_CONFIG_ERROR) unless ($recipe_psphot); + +# Input filenames +my $inputImage = $ipprc->filename("PPSUB.OUTPUT", $inputPath); +my $inputMask = $ipprc->filename("PPSUB.OUTPUT.MASK", $inputPath); +my $inputVariance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $inputPath); +my $inputPSF = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $inputPath); +&my_die("Couldn't find input: $inputImage", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputImage); +&my_die("Couldn't find input: $inputMask", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); +&my_die("Couldn't find input: $inputVariance", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance); +&my_die("Couldn't find input: $inputPSF", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputPSF); + +my $quality; # Quality flag + +# Do forward photometry +{ + # Get the output filenames + my $outputName = $ipprc->filename("PSPHOT.OUT.CMF.MEF", "$outroot.pos"); + my $outputStats = $ipprc->filename("SKYCELL.STATS", "$outroot.pos"); + my $traceDest = $ipprc->filename("TRACE.EXP", "$outroot.pos"); + + my $command = "$psphot $outroot.pos"; + $command .= " -file $inputImage"; + $command .= " -mask $inputMask"; + $command .= " -variance $inputVariance"; + $command .= " -psf $inputPSF"; +# $command .= " -stats $outputStats"; + $command .= " -threads $threads" if defined $threads; + $command .= " -recipe PSPHOT $recipe_psphot"; + $command .= " -recipe PPSTATS $recipe_ppstats"; + $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psphot: $error_code", $diff_phot_id, $skycell_id, $error_code); + } + + if (0) { # psphot doesn't produce a stats file + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); + + $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $diff_phot_id, $skycell_id, $error_code); + } + my $cmdflags = ""; + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + if ($cmdflags =~ /-quality (\d+)/) { + $quality = $1; + } + } + + if (!$quality) { + &my_die("Couldn't find expected output file: $outputName", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + } + } else { + print "Not executing: $command\n"; + } +} + +# Do reverse photometry +if ($bothways) { + my ($tempFile, $tempName) = tempfile( "/tmp/diffphot.$diff_phot_id.$skycell_id.XXXX", + UNLINK => !$save_temps, SUFFIX => '.fits' ); + { + my ($tempRoot) = $tempName =~ /^(.*).fits/; + my $command = "$ppArith $tempRoot -file1 $inputImage -op \\* -const2 -1"; + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppArith: $error_code", $diff_phot_id, $skycell_id, $error_code); + } + &my_die("Couldn't find expected output file: $tempName", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($tempName); + } else { + print "Skipping command: $command\n"; + } + } + + # Get the output filenames + my $outputName = $ipprc->filename("PSPHOT.OUT.CMF.MEF", "$outroot.neg"); + my $outputStats = $ipprc->filename("SKYCELL.STATS", "$outroot.neg"); + my $traceDest = $ipprc->filename("TRACE.EXP", "$outroot.neg"); + + my $command = "$psphot $outroot.neg"; + $command .= " -file $tempName"; + $command .= " -mask $inputMask"; + $command .= " -variance $inputVariance"; + $command .= " -psf $inputPSF"; +# $command .= " -stats $outputStats"; + $command .= " -threads $threads" if defined $threads; + $command .= " -recipe PSPHOT $recipe_psphot"; + $command .= " -recipe PPSTATS $recipe_ppstats"; + $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psphot: $error_code", $diff_phot_id, $skycell_id, $error_code); + } + + if (0) { # psphot doesn't produce a stats file + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); + + $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $diff_phot_id, $skycell_id, $error_code); + } + my $cmdflags = ""; + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + if ($cmdflags =~ /-quality (\d+)/) { + $quality = $1; + } + } + + if (!$quality) { + &my_die("Couldn't find expected output file: $outputName", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + } + } else { + print "Not executing: $command\n"; + } +} + +unless ($no_update) { + my $command = "$diffphottool -diff_phot_id $diff_phot_id -skycell_id $skycell_id"; + $command .= " -done -path_base $outroot"; + $command .= " -magicked $magicked" if $magicked; + $command .= " -quality $quality" if defined $quality; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Command failed: $error_code", $diff_phot_id, $skycell_id, $error_code); + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $diff_phot_id = shift; # Diff identifier + my $skycell_id = shift; # Skycell identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $diff_phot_id and defined $skycell_id and not $no_update) { + my $command = "$diffphottool -done"; + $command .= " -diff_phot_id $diff_phot_id -skycell_id $skycell_id -fault $exit_code"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/pap/ippScripts/scripts/minidvodb_createdb.pl =================================================================== --- /branches/pap/ippScripts/scripts/minidvodb_createdb.pl (revision 28484) +++ /branches/pap/ippScripts/scripts/minidvodb_createdb.pl (revision 28484) @@ -0,0 +1,266 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DateTime; + +use DateTime::Format::Strptime; +use DateTime::Duration; +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + + +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $outroot, $dbname, $dvodb, $minidvodb,$minidvodb_interval, $minidvodb_group, $camera, $verbose, $no_update, + $no_op, $redirect, $save_temps); +GetOptions( + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'minidvodb_group|w=s' => \$minidvodb_group, # minidvodb_group. + 'outroot|w=s' => \$outroot, # output file base name + 'dvodb|w=s' => \$dvodb, # output DVO database + 'minidvodb|w=s' => \$minidvodb, # output miniDVODB + 'interval|w=s' => \$minidvodb_interval, #interval between creation of minidvodbs (default = 1day) + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --camera --dvodb --minidvodb_group --outroot", + -exitval => 3, + ) unless + defined $minidvodb_group and + defined $camera and + defined $outroot and + defined $dvodb; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $minidvodb_group , $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $minidvodb_group, $PS_EXIT_CONFIG_ERROR); + +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $minidvodb_group, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +# Recipes to use based on reduction class + +# XXX This is now not used: do we still need it? + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs + my $create_new = 0; +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} else { + warn("dvodb undefined:\n"); + exit(4); +} +my $minidvodbReal; +if (defined $minidvodb) { + $minidvodbReal = $ipprc->dvo_catdir( $minidvodb ); # catdir for DVO + $minidvodbReal = $ipprc->convert_filename_absolute( $minidvodbReal ); +} else { + warn("minidvodb undefined:\n"); + exit(4); +} + + +if (!defined $minidvodb_interval) { + $minidvodb_interval = 1; +} + + +unless ($no_op) { + + + +#see if there is already one in new state + my $fpaCommand1 = "$addtool -listminidvodbrun"; + $fpaCommand1 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand1 .= " -state 'new'"; + $fpaCommand1 .= " -dbname $dbname" if defined $dbname; + + +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand1, verbose => $verbose); + + + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to list minidvodb database: $error_code\n"); + exit($error_code); + } + + if (scalar(@{$stdout_buf})) { + $error_code = 3; + warn("Unwilling to create minidvodb, one already exists in new state: $error_code\n"); + exit($error_code); + } + + my $creation_date; + my $addRun_count; + my $minidvodb_name; +#find the active one's date, and find out if it has more than 1 addRun in it + + my $fpaCommand2 = "$addtool -listminidvodbrun"; + $fpaCommand2 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand2 .= " -state 'active'"; + $fpaCommand2 .= " -limit 1"; + $fpaCommand2 .= " -dbname $dbname" if defined $dbname; + +#print $fpaCommand2; + + + my ( $success2, $error_code2, $full_buf2, $stdout_buf2, $stderr_buf2 ) = + run(command => $fpaCommand2, verbose => $verbose); + &my_die( "Unable to get listminidvodbrun",$minidvodb_group, $PS_EXIT_SYS_ERROR) unless $success2; + # if it didn't list something in active state (what?) then we definitely need to create a new one + if (defined(@$stdout_buf2)) { + my $metadata2 = $mdcParser->parse(join "", @$stdout_buf2) or + &my_die("Unable to parse metadata config", $minidvodb_group, $PS_EXIT_PROG_ERROR); + + my $components2 = parse_md_list($metadata2) or + &my_die("Unable to parse metadata list", $minidvodb_group, $PS_EXIT_PROG_ERROR); + my $comp2 = $$components2[0]; + $minidvodb_name = $comp2->{minidvodb_name}; + $creation_date = $comp2->{creation_date}; + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $minidvodb_group, $PS_EXIT_PROG_ERROR); + } + if (!defined($creation_date)) { + &my_die("Unable to parse creation_date", $minidvodb_group, $PS_EXIT_PROG_ERROR); + } + } else { + $create_new = 1; #this is to force it to make a new one + } + + #find the number of add_ids that have been proccessed + my $fpaCommand3 = "$addtool -checkminidvodbrunaddrun"; + $fpaCommand3 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand3 .= " -state 'active'"; + $fpaCommand3 .= " -minidvodb_name '$minidvodb_name'" if defined $minidvodb_name; + $fpaCommand3 .= " -limit 1"; + $fpaCommand3 .= " -dbname $dbname" if defined $dbname; + + +my ( $success3, $error_code3, $full_buf3, $stdout_buf3, $stderr_buf3 ) = + run(command => $fpaCommand3, verbose => $verbose); + &my_die( "Unable to get checkminidvodbunaddrun", $minidvodb_group, $PS_EXIT_SYS_ERROR) unless $success3; + + if (defined(@$stdout_buf3)) { #checkminidvodb returns nothing IF there have been no addruns added to the db yet + my $metadata3 = $mdcParser->parse(join "", @$stdout_buf3) or + &my_die("Unable to parse metadata config", $minidvodb_group, $PS_EXIT_PROG_ERROR); + + my $components3 = parse_md_list($metadata3) or + &my_die("Unable to parse metadata list", $minidvodb_group, $PS_EXIT_PROG_ERROR); + my $comp = $$components3[0]; + $addRun_count = $comp->{addRun_count}; + } + if (!defined($addRun_count)) { + ## there's nothing to parse if there's nothing + $addRun_count = 0; + } + + + if ($addRun_count > 500) { + #it's too big, create_new + $create_new = 1; + } + if ($create_new == 0) { + my $parser = DateTime::Format::Strptime->new( pattern => '%Y-%m-%dT%H:%M:%S', time_zone => "HST" ); + my $creation_dt = $parser->parse_datetime( $creation_date )->mjd; + if ($mjd_start- $creation_dt > $minidvodb_interval && $addRun_count > 0 ) { + #db is old and has stuff in it, want to create_new + $create_new = 1; + } + } + + +} +#create the minidvodb entry (well, the command for it) + my $fpaCommand = "$addtool -addminidvodbrun"; + $fpaCommand .= " -set_minidvodb_group $minidvodb_group"; + $fpaCommand .= " -set_minidvodb_path $minidvodbReal" if defined $minidvodbReal; + $fpaCommand .= " -set_mergedvodb_path $dvodbReal"; + $fpaCommand .= " -dbname $dbname" if defined $dbname; + + unless ($no_update or !$create_new) { + + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } + } else { + print "skipping command: $fpaCommand\n"; + } +} + +sub my_die +{#complain if it doesn't work + my $msg = shift; # Warning message on die + my $minidvodb_group = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/pap/ippScripts/scripts/minidvodb_merge.pl =================================================================== --- /branches/pap/ippScripts/scripts/minidvodb_merge.pl (revision 28484) +++ /branches/pap/ippScripts/scripts/minidvodb_merge.pl (revision 28484) @@ -0,0 +1,254 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_resort; +my $dtime_relphot; +my $dtime_merge; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $mergedvodb, $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodb|w=s' => \$mergedvodb, # output DVO database + 'minidvodb|w=s' => \$minidvodb, #minidvodb database + 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id + 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_id + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodb --minidvodb --minidvodb_id -- minidvodb_group --camera", + -exitval => 3, + ) unless + defined $mergedvodb and + defined $minidvodb and + defined $minidvodb_id and + defined $minidvodb_group and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $mergedvodb, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $mergedvodb, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# convert supplied DVO database name to UNIX filename +my $mergedvodbReal; +if (defined $mergedvodb) { + $mergedvodbReal = $ipprc->dvo_catdir( $mergedvodb ); # catdir for DVO + $mergedvodbReal = $ipprc->convert_filename_absolute( $mergedvodbReal ); +} + +my $dtime_addstar = 0; +#my $dtime_relphot = 0; + +unless ($no_op) { + if (defined $mergedvodbReal) { + { + my $command = "$addstar -resort"; + $command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $minidvodb"; + + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + print "\n$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $mergedvodb, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; + # MJD of starting script + $dtime_resort = $dtime_addstar; + print "addstar -resort time $dtime_addstar\n"; + } + + { + # relphot only takes lower case gpc1 + my $relphot_camera = lc($camera); + my $command = "$relphot -averages -update"; + $command .= " -D CAMERA $relphot_camera"; + $command .= " -D CATDIR $minidvodb"; + print "$command\n"; + my $mjd_relphot_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform relphot: $error_code", $mergedvodb, $error_code); + } + $dtime_relphot = 86400.0*(DateTime->now->mjd - $mjd_relphot_start); # MJD of starting script + print "relphot time $dtime_relphot\n"; + } + + my $this_is_the_first; + { + + my $mdcParser = PS::IPP::Metadata::Config->new; + + my $command = "$addtool -listminidvodbrun -state merged -minidvodb_group " . $minidvodb_group; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of minidvodbruns", $minidvodb_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + $this_is_the_first =1; + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $minidvodb_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $minidvodb_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $minidvodb_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found at least 1 minidvodbrun in merged state\n"; + $this_is_the_first = 0; + } + } + print "$this_is_the_first $mergedvodb/Image.dat\n"; + + + if (-e "$mergedvodb/Image.dat") { + if ($this_is_the_first == 1) { + &my_die("refusing to merge, this is the first, but files already exist in dir", $minidvodb_id, 4); + } + $this_is_the_first =0; + } + + print "$this_is_the_first $mergedvodb/Image.dat\n"; + { + my $merge_command; + my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script + if ($this_is_the_first) { + + $merge_command = "rsync -rvat $minidvodb/* $mergedvodb"; + } else { + $merge_command = "$dvomerge $minidvodb into $mergedvodb"; + } + + print "\n$merge_command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $merge_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to merge: $error_code", $mergedvodb, $error_code); + } + + $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script + print "merge time $dtime_merge\n"; +# } + } + + { + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to minidvodbprocessed: $error_code", $mergedvodb, $error_code); + } + } + + } else { + &my_die("dvodb: $mergedvodb not found", $mergedvodb, $PS_EXIT_UNKNOWN_ERROR); + } +} else { + print "skipping processing for $mergedvodbReal\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $minidbvodb_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodb\n"; + +if (defined $minidvodb_id ) { + + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -fault $exit_code"; + $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dbname $dbname" if defined $dbname; + + + + #print $command; + system ($command); + } + + + + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + exit $exit_code; +} + +__END__ Index: /branches/pap/ippScripts/scripts/publish_file.pl =================================================================== --- /branches/pap/ippScripts/scripts/publish_file.pl (revision 28483) +++ /branches/pap/ippScripts/scripts/publish_file.pl (revision 28484) @@ -140,6 +140,11 @@ print $dsFile "$file|||$product|$name|\n"; -} elsif ($stage eq 'diff') { - my $command = "difftool -diffskyfile -diff_id $stage_id"; +} elsif ($stage eq 'diff' or $stage eq 'diffphot') { + my $command; # Command to run + if ($stage eq 'diff') { + $command = "difftool -diffskyfile -diff_id $stage_id"; + } elsif ($stage eq 'diffphot') { + $command = "diffphottool -data -diff_phot_id $stage_id"; + } $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -147,12 +152,10 @@ &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $success; - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + my $components = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); - my $components = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); - - my ($mopsPositiveFile, $mopsPositiveFileName) = tempfile("/tmp/publish.$pub_id.mops.pos.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS'; - my ($mopsNegativeFile, $mopsNegativeFileName) = tempfile("/tmp/publish.$pub_id.mops.neg.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS'; + my $mops = ($product =~ /^IPP-MOPS/ ? 1 : 0); # Format for MOPS? + my ($mopsPositiveFile, $mopsPositiveFileName) = tempfile("/tmp/publish.$pub_id.mops.pos.XXXX", UNLINK => !$save_temps ) if $mops; + my ($mopsNegativeFile, $mopsNegativeFileName) = tempfile("/tmp/publish.$pub_id.mops.neg.XXXX", UNLINK => !$save_temps ) if $mops; my %positive; # Data for positive diff detections @@ -169,5 +172,10 @@ my $skycell_id = $comp->{skycell_id}; - my $filename = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base ); + my $filename; + if ($stage eq 'diff') { + $filename = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base ); + } elsif ($stage eq 'diffphot') { + $filename = $ipprc->filename( "PSPHOT.OUT.CMF.MEF", "$path_base.pos" ); + } $filename = $ipprc->file_resolve($filename); @@ -187,5 +195,5 @@ diff_check(\%positive, $data, "positive"); - if ($product eq 'IPP-MOPS') { + if ($mops) { print $mopsPositiveFile "$filename\n"; } else { @@ -195,5 +203,11 @@ # Negative direction if (defined $comp->{bothways} and $comp->{bothways}) { - my $filename = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base ); + my $filename; + if ($stage eq 'diff') { + $filename = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base ); + } elsif ($stage eq 'diffphot') { + $filename = $ipprc->filename( "PSPHOT.OUT.CMF.MEF", "$path_base.neg" ); + } + $filename = $ipprc->file_resolve($filename); @@ -213,5 +227,5 @@ diff_check(\%negative, $data, "negative"); - if ($product eq 'IPP-MOPS') { + if ($mops) { print $mopsNegativeFile "$filename\n"; } else { @@ -221,8 +235,8 @@ } - close $mopsPositiveFile if $product eq 'IPP-MOPS'; - close $mopsNegativeFile if $product eq 'IPP-MOPS'; - - if ($product eq 'IPP-MOPS') { + close $mopsPositiveFile if $mops; + close $mopsNegativeFile if $mops; + + if ($mops) { if (scalar keys %positive > 0) { my $output = mops_combine(\%positive, "$outroot.pos.mops", $mopsPositiveFileName); @@ -239,5 +253,5 @@ unless ($no_update) { - my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $dsFileName"; + my $command = "$dsreg --add pub.$pub_id.$stage.$stage_id --copy --abspath --product $product --type $product --list $dsFileName"; $command .= " --ps0 \"$comment\"" if defined $comment; Index: /branches/pap/ippScripts/scripts/skycell_jpeg.pl =================================================================== --- /branches/pap/ippScripts/scripts/skycell_jpeg.pl (revision 28484) +++ /branches/pap/ippScripts/scripts/skycell_jpeg.pl (revision 28484) @@ -0,0 +1,360 @@ +#! /usr/bin/env perl + +use warnings; +use strict; +use Carp; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Temp qw( tempfile ); + +my $db; + +my ($stage, $stage_id, $outroot, $camera, $dbname, $logfile, $verbose, $save_temps); +my $missing_tools; +my ($help,$masks); +my ($tempFile,$tempName,$maskFile,$maskName); +my $ppSkycell = can_run('ppSkycell') or (warn "Can't find ppSkycell" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +GetOptions( + 'help|h' => \$help, + 'stage=s' => \$stage, + 'stage_id=s' => \$stage_id, + 'outroot=s' => \$outroot, + 'dbname=s' => \$dbname, + 'camera=s' => \$camera, + 'logfile=s' => \$logfile, + 'verbose' => \$verbose, + 'save_temps' => \$save_temps, + 'masks' => \$masks, +# 'no-op' => \$no_op, +# 'no-update' => \$no_update, + ) or pod2usage ( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV; +pod2usage( + -msg => "Required options: --stage --stage_id --path_base\nHelpful options: --camera --dbname", + -exitval => 3, + ) unless + defined $stage and defined $stage_id and defined $outroot; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up", $stage_id, $PS_EXIT_CONFIG_ERROR); # this is used for PATH, NEB filename conversions + +my %stages = ('warp' => 1, 'stack' => 1, 'diff' => 1); + +unless (exists($stages{$stage})) { + my_die("Unknown stage '$stage'",$stage_id,$PS_EXIT_UNKNOWN_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +if ($stage eq 'warp') { + my $imfiles; + my $command = "$warptool -warped -warp_id $stage_id"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform warptool: $error_code", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + # no results found; + } + + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata config doc", $stage_id, $error_code); + + my %tangents = (); + + foreach my $imfile (@$imfiles) { + my $skycell_id = $imfile->{skycell_id}; + my $path_base = $imfile->{path_base}; + my $quality = $imfile->{quality}; + my $state = $imfile->{data_state}; + my $projection_cell = $skycell_id; + if ($quality != 0 or $state ne 'full') { + next; + } + + $projection_cell =~ s/^(.*)\..*$/$1/; + + unless (exists($tangents{$projection_cell})) { + # Make a temp file and fill, but be sure to save + ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{FILE} = $tempFile; + $tangents{$projection_cell}{NAME} = $tempName; + if ($masks) { + my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{MFILE} = $maskFile; + $tangents{$projection_cell}{MNAME} = $maskName; + } + } + print "$skycell_id $projection_cell\n"; + my $file = $ipprc->filename("PSWARP.OUTPUT", $path_base, $skycell_id); + print "$file $state $quality\n"; + my $f_fh = $tangents{$projection_cell}{FILE}; + print $f_fh "$file\n"; + if ($masks) { + my $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + print "$mask\n"; + my $m_fh = $tangents{$projection_cell}{MFILE}; + print $m_fh "$mask\n"; + } + } + foreach my $projection_cell (keys %tangents) { + $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}"; + if ($masks) { + $command .= " -masks $tangents{$projection_cell}{MNAME} "; + } + $command .= " ${outroot}.${projection_cell} "; + print "$command\n"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code); + } + # Update database: + $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update warp summary: $error_code", $stage_id, $error_code); + } + } + if (scalar (keys %tangents) == 0) { + my $projection_cell = 'NULL'; + $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update diff summary: $error_code", $stage_id, $error_code); + } + } +} #end warp stage +if ($stage eq 'diff') { + my $imfiles; + my $command = "$difftool -diffskyfile -diff_id $stage_id"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform difftool: $error_code", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + # no results found; + } + + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata config doc", $stage_id, $error_code); + + my %tangents = (); + + foreach my $imfile (@$imfiles) { + my $skycell_id = $imfile->{skycell_id}; + my $path_base = $imfile->{path_base}; + my $quality = $imfile->{quality}; + my $state = $imfile->{data_state}; + my $projection_cell = $skycell_id; + if ($quality != 0 or $state ne 'full') { + next; + } + + $projection_cell =~ s/^(.*)\..*$/$1/; + + unless (exists($tangents{$projection_cell})) { + # Make a temp file and fill, but be sure to save + ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{FILE} = $tempFile; + $tangents{$projection_cell}{NAME} = $tempName; + if ($masks) { + my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{MFILE} = $maskFile; + $tangents{$projection_cell}{MNAME} = $maskName; + } + } + print "$skycell_id $projection_cell\n"; + my $file = $ipprc->filename("PPSUB.OUTPUT", $path_base, $skycell_id); + print "$file $state $quality\n"; + my $f_fh = $tangents{$projection_cell}{FILE}; + print $f_fh "$file\n"; + if ($masks) { + my $mask = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base, $skycell_id); + print "$mask\n"; + my $m_fh = $tangents{$projection_cell}{MFILE}; + print $m_fh "$mask\n"; + } + } + foreach my $projection_cell (keys %tangents) { + $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}"; + if ($masks) { + $command .= " -masks $tangents{$projection_cell}{MNAME} "; + } + $command .= " ${outroot}.${projection_cell} "; + print "$command\n"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code); + } + # Update database: + $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update diff summary: $error_code", $stage_id, $error_code); + } + } + if (scalar (keys %tangents) == 0) { + my $projection_cell = 'NULL'; + $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update diff summary: $error_code", $stage_id, $error_code); + } + } +} #end diff stage +if ($stage eq 'stack') { + my $imfiles; + my $command = "$stacktool -sassskyfile -sass_id $stage_id"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform stacktool: $error_code", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + # no results found; + } + + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata config doc", $stage_id, $error_code); + + my %tangents = (); + + foreach my $imfile (@$imfiles) { + my $skycell_id = $imfile->{skycell_id}; + my $path_base = $imfile->{path_base}; + my $quality = $imfile->{quality}; + my $state = $imfile->{state}; + my $fault = $imfile->{fault}; + my $projection_cell = $skycell_id; + if ($quality != 0 or $state ne 'full') { + next; + } + + $projection_cell =~ s/^(.*)\..*$/$1/; + + unless (exists($tangents{$projection_cell})) { + # Make a temp file and fill, but be sure to save + ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{FILE} = $tempFile; + $tangents{$projection_cell}{NAME} = $tempName; + if ($masks) { + my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{MFILE} = $maskFile; + $tangents{$projection_cell}{MNAME} = $maskName; + } + } + print "$skycell_id $projection_cell\n"; + my $file = $ipprc->filename("PPSTACK.OUTPUT", $path_base, $skycell_id); + print "$file $state $quality\n"; + my $f_fh = $tangents{$projection_cell}{FILE}; + print $f_fh "$file\n"; + if ($masks) { + my $mask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base, $skycell_id); + print "$mask\n"; + my $m_fh = $tangents{$projection_cell}{MFILE}; + print $m_fh "$mask\n"; + } + } + foreach my $projection_cell (keys %tangents) { + $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}"; + if ($masks) { + $command .= " -masks $tangents{$projection_cell}{MNAME} "; + } + $command .= " ${outroot}.${projection_cell} "; + print "$command\n"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code); + } + # Update database: + $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update stack summary: $error_code", $stage_id, $error_code); + } + } + if (scalar (keys %tangents) == 0) { + my $projection_cell = 'NULL'; + $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } +# print "$command\n"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update stack summary: $error_code", $stage_id, $error_code); + } + } + +} #end stack stage + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $stage = shift; # stage name + my $stage_id = shift; # identifier + my $exit_code = shift; # Exit code + # outputImage and path_base are globals + + carp($msg); + exit $exit_code; +} Index: /branches/pap/ippTasks/Makefile.am =================================================================== --- /branches/pap/ippTasks/Makefile.am (revision 28483) +++ /branches/pap/ippTasks/Makefile.am (revision 28484) @@ -32,7 +32,9 @@ dqstats.pro \ science.cleanup.pro \ + minidvodb.pro \ nightly_stacks.pro \ lossy_compress.pro \ - background.pro + background.pro \ + diffphot.pro other_files = \ Index: /branches/pap/ippTasks/addstar.pro =================================================================== --- /branches/pap/ippTasks/addstar.pro (revision 28483) +++ /branches/pap/ippTasks/addstar.pro (revision 28484) @@ -45,7 +45,9 @@ end + # this variable will cycle through the known database names $addstar_DB = 0 $addstar_revert_DB = 0 + # select images ready for addstar analysis @@ -64,5 +66,5 @@ task.exec - if ($LABEL:n == 0) break + # if ($LABEL:n == 0) break $run = addtool -pendingexp if ($DB:n == 0) @@ -134,4 +136,7 @@ book getword addPendingExp $pageName reduction -var REDUCTION book getword addPendingExp $pageName dvodb -var DVODB + book getword addPendingExp $pageName minidvodb -var MINIDVODB + book getword addPendingExp $pageName minidvodb_name -var MINIDVODB_NAME + book getword addPendingExp $pageName minidvodb_group -var MINIDVODB_GROUP book getword addPendingExp $pageName image_only -var IMAGE_ONLY book getword addPendingExp $pageName dbname -var DBNAME @@ -164,4 +169,11 @@ $run = $run --image-only end + if ("$MINIDVODB" == "T") + $run = $run --minidvodb + $run = $run --minidvodb_group $MINIDVODB_GROUP + if (("$MINIDVODB_NAME" != "NULL") && ("$MINIDVODB_NAME" != "(null)")) + $run = $run --minidvodb_name $MINIDVODB_NAME #have addstar_run.pl grab the 'active' name if it is NULL + end + end add_standard_args run Index: /branches/pap/ippTasks/diff.pro =================================================================== --- /branches/pap/ippTasks/diff.pro (revision 28483) +++ /branches/pap/ippTasks/diff.pro (revision 28484) @@ -11,4 +11,5 @@ book init diffSkyfile #book init diffCleanup +book init diffPendingSummary ### Database lists @@ -16,4 +17,5 @@ $diffAdvance_DB = 0 $diff_revert_DB = 0 +$diffSummary_DB = 0 #$diffCleanup_DB = 0 @@ -27,4 +29,5 @@ macro diff.reset book init diffSkyfile + book init diffPendingSummary # book init diffCleanup end @@ -44,4 +47,10 @@ active false end + task diff.summary.load + active true + end + task diff.summary.run + active true + end end @@ -58,4 +67,10 @@ end task diff.revert + active false + end + task diff.summary.load + active false + end + task diff.summary.run active false end @@ -193,5 +208,4 @@ # host anyhost # $WORKDIR = $WORKDIR_TEMPLATE - if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) $DIFF_TAG = "" @@ -477,2 +491,150 @@ end end + +### Load tasks for doing the diffs +### Tasks are loaded into diffPendingSkyCell. +task diff.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + npending 1 + + stdout NULL + stderr $LOGDIR/diff.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = difftool -tosummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffSummary_DB + $run = $run -dbname $DB:$diffSummary_DB + $diffSummary_DB ++ + if ($diffSummary_DB >= $DB:n) set diffSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + # XXX change tess_id to tess_dir when db is updated + ipptool2book stdout diffPendingSummary -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffPendingSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffPendingSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffPendingSkyCell. +task diff.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages diffPendingSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in diffPendingSkyCell (pantaskState == INIT) + book getpage diffPendingSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffPendingSummary $pageName pantaskState RUN + book getword diffPendingSummary $pageName diff_id -var DIFF_ID + book getword diffPendingSummary $pageName camera -var CAMERA + book getword diffPendingSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword diffPendingSummary $pageName dbname -var DBNAME + book getword diffPendingSummary $pageName tess_id -var TESS_DIR + book getword diffPendingSummary $pageName diff_mode -var DIFF_MODE + book getword diffPendingSummary $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + +# if ("$PATH_BASE" == "NULL") + + if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) + $DIFF_TAG = "" + end + if ($DIFF_MODE == 2) + $DIFF_TAG = "WS." + end + if ($DIFF_MODE == 3) + $DIFF_TAG = "SW." + end + if ($DIFF_MODE == 4) + $DIFF_TAG = "SS." + end + + basename $TESS_DIR -var TESS_ID + + ## generate outroot specific to this exposure + sprintf outroot "%s/%s/%s.%sdif.%s.summary" $WORKDIR $TESS_ID $TESS_ID $DIFF_TAG $DIFF_ID + + + stdout $LOGDIR/diff.summary.log + stderr $LOGDIR/diff.summary.log + + $run = skycell_jpeg.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit diffPendingSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword diffPendingSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffPendingSummary $options:0 pantaskState TIMEOUT + end +end Index: /branches/pap/ippTasks/diffphot.pro =================================================================== --- /branches/pap/ippTasks/diffphot.pro (revision 28484) +++ /branches/pap/ippTasks/diffphot.pro (revision 28484) @@ -0,0 +1,296 @@ +## diffphot.pro: (re-)photometry of diffs : -*- sh -*- + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init diffphotRuns + +### Database lists +$diffphotLoad_DB = 0 +$diffphotAdvance_DB = 0 +$diffphotRevert_DB = 0 + +### Check status of diffing tasks +macro diffphot.status + book listbook diffphotRuns +end + +### Reset diffing tasks +macro diffphot.reset + book init diffphotRuns +end + +### Turn diffing tasks on +macro diffphot.on + task diffphot.load + active true + end + task diffphot.run + active true + end + task diffphot.advance + active true + end + task diffphot.revert + active false + end +end + +### Turn diffing tasks off +macro diffphot.off + task diffphot.load + active false + end + task diffphot.run + active false + end + task diffphot.advance + active false + end + task diffphot.revert + active false + end +end + +macro diffphot.revert.on + task diffphot.revert + active true + end +end + +macro diffphot.revert.off + task diffphot.revert + active false + end +end + + +# Load processing jobs +task diffphot.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/diffphot.load.log + + task.exec + if ($LABEL:n == 0) break + $run = diffphottool -pending + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffphotLoad_DB + $run = $run -dbname $DB:$diffphotLoad_DB + $diffphotLoad_DB ++ + if ($diffphotLoad_DB >= $DB:n) set diffphotLoad_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffphotRuns -key diff_phot_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffphotRuns + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffphotRuns + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffSkyfile. +task diffphot.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages diffphotRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + book getpage diffphotRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffphotRuns $pageName pantaskState RUN + book getword diffphotRuns $pageName diff_phot_id -var DIFF_PHOT_ID + book getword diffphotRuns $pageName skycell_id -var SKYCELL_ID + book getword diffphotRuns $pageName tess_id -var TESS_DIR + book getword diffphotRuns $pageName workdir -var WORKDIR_TEMPLATE + book getword diffphotRuns $pageName state -var RUN_STATE + book getword diffphotRuns $pageName dbname -var DBNAME + book getword diffphotRuns $pageName reduction -var REDUCTION + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.dp.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $DIFF_PHOT_ID + + stdout $LOGDIR/diffphot.run.log + stderr $LOGDIR/diffphot.run.log + + $run = diffphot.pl --threads @MAX_THREADS@ --diff_phot_id $DIFF_PHOT_ID --skycell_id $SKYCELL_ID --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit diffphotRuns $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword diffphotRuns $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffphotRuns $options:0 pantaskState TIMEOUT + end +end + + +# Advance runs which have completed +task diffphot.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/diffphot.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = diffphottool -advance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffphotAdvance_DB + $run = $run -dbname $DB:$diffphotAdvance_DB + $diffphotAdvance_DB ++ + if ($diffphotAdvance_DB >= $DB:n) set diffphotAdvance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +# Revert runs that failed +task diffphot.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/diffphot.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = diffphottool -revert + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffphotRevert_DB + $run = $run -dbname $DB:$diffphotRevert_DB + $diffphotRevert_DB ++ + if ($diffphotRevert_DB >= $DB:n) set diffphotRevert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/pap/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/pap/ippTasks/ipphosts.mhpcc.config (revision 28483) +++ /branches/pap/ippTasks/ipphosts.mhpcc.config (revision 28484) @@ -1,103 +1,142 @@ ipphosts MULTI +# In this file, we assign all storage hosts to chip and skycells any +# hosts which are having problems are removed as active volumes from +# nebulous. data targeted to those machines are simply re-distributed +# randomly. the only exception is ipp004, which is the DVO db host + +# NOTE: wave 1 machines have 10TB; wave 2 & 3 machines have 20TB +# for balance, use 1 host per 10 TB in the partition ipphosts METADATA camera STR skycell - count S32 46 - sky00 STR ipp006 - sky01 STR ipp007 - sky02 STR ipp008 - sky03 STR ipp009 - sky04 STR ipp010 - sky05 STR ipp011 - sky06 STR ipp012 - sky07 STR ipp013 -# sky08 STR ipp014 - sky08 STR ipp047 - sky09 STR ipp015 -# sky10 STR ipp016 - sky10 STR ipp042 - sky11 STR ipp017 - sky12 STR ipp018 - sky13 STR ipp019 - sky14 STR ipp020 - sky15 STR ipp021 - sky16 STR ipp023 - sky17 STR ipp024 - sky18 STR ipp025 - sky19 STR ipp026 - sky20 STR ipp027 - sky21 STR ipp028 - sky22 STR ipp029 - sky23 STR ipp030 - sky24 STR ipp031 - sky25 STR ipp032 - sky26 STR ipp033 - sky27 STR ipp034 - sky28 STR ipp035 - sky29 STR ipp036 - sky30 STR ipp038 - sky31 STR ipp039 - sky32 STR ipp040 - sky33 STR ipp041 - sky34 STR ipp042 - sky35 STR ipp043 - sky36 STR ipp044 - sky37 STR ipp045 - sky38 STR ipp046 - sky39 STR ipp047 - sky40 STR ipp048 - sky41 STR ipp049 - sky42 STR ipp050 - sky43 STR ipp051 - sky44 STR ipp052 - sky45 STR ipp053 + count S32 79 + sky00 STR ipp005 + sky01 STR ipp006 + sky02 STR ipp007 + sky03 STR ipp008 + sky04 STR ipp009 + sky05 STR ipp010 + sky06 STR ipp011 + sky07 STR ipp012 + sky08 STR ipp013 + sky09 STR ipp014 + sky10 STR ipp015 + sky11 STR ipp016 + sky12 STR ipp017 + sky13 STR ipp018 + sky14 STR ipp019 + sky15 STR ipp020 + sky16 STR ipp021 + sky17 STR ipp023 + sky18 STR ipp024 + sky19 STR ipp025 + sky20 STR ipp026 + sky21 STR ipp027 + sky22 STR ipp028 + sky23 STR ipp029 + sky24 STR ipp030 + sky25 STR ipp031 + sky26 STR ipp032 + sky27 STR ipp033 + sky28 STR ipp034 + sky29 STR ipp035 + sky30 STR ipp036 + sky31 STR ipp037 + sky32 STR ipp038 + sky33 STR ipp039 + sky34 STR ipp040 + sky35 STR ipp041 + sky36 STR ipp042 + sky37 STR ipp043 + sky38 STR ipp044 + sky39 STR ipp045 + sky40 STR ipp046 + sky41 STR ipp047 + sky42 STR ipp048 + sky43 STR ipp049 + sky44 STR ipp050 + sky45 STR ipp051 + sky46 STR ipp052 + sky47 STR ipp053 + sky48 STR ipp023 + sky49 STR ipp024 + sky50 STR ipp025 + sky51 STR ipp026 + sky52 STR ipp027 + sky53 STR ipp028 + sky54 STR ipp029 + sky55 STR ipp030 + sky56 STR ipp031 + sky57 STR ipp032 + sky58 STR ipp033 + sky59 STR ipp034 + sky60 STR ipp035 + sky61 STR ipp036 + sky62 STR ipp037 + sky63 STR ipp038 + sky64 STR ipp039 + sky65 STR ipp040 + sky66 STR ipp041 + sky67 STR ipp042 + sky68 STR ipp043 + sky69 STR ipp044 + sky70 STR ipp045 + sky71 STR ipp046 + sky72 STR ipp047 + sky73 STR ipp048 + sky74 STR ipp049 + sky75 STR ipp050 + sky76 STR ipp051 + sky77 STR ipp052 + sky78 STR ipp053 END + +# we have 18 10TB nodes and 30 20TB nodes (ipp037 is used for dvo) +# 1 x W1 + 1 x W2 + 1 x W3 + 12 more nodes... ipphosts METADATA camera STR GPC1 -# XY01 STR ipp014 -# XY02 STR ipp014 - XY01 STR ipp047 - XY02 STR ipp047 - XY03 STR ipp038 + XY01 STR ipp037 + XY02 STR ipp005 + XY03 STR ipp006 XY04 STR ipp038 - XY05 STR ipp023 + XY05 STR ipp007 XY06 STR ipp023 - XY10 STR ipp039 + XY10 STR ipp008 XY11 STR ipp039 - XY12 STR ipp024 + XY12 STR ipp009 XY13 STR ipp024 - XY14 STR ipp040 + XY14 STR ipp010 XY15 STR ipp040 - XY16 STR ipp026 + XY16 STR ipp011 XY17 STR ipp026 - XY20 STR ipp041 + XY20 STR ipp012 XY21 STR ipp041 - XY22 STR ipp042 + XY22 STR ipp013 XY23 STR ipp042 - XY24 STR ipp043 + XY24 STR ipp014 XY25 STR ipp043 - XY26 STR ipp028 + XY26 STR ipp027 XY27 STR ipp028 - XY30 STR ipp044 + XY30 STR ipp016 XY31 STR ipp044 - XY32 STR ipp029 + XY32 STR ipp017 XY33 STR ipp029 - XY34 STR ipp045 + XY34 STR ipp018 XY35 STR ipp045 - XY36 STR ipp030 + XY36 STR ipp019 XY37 STR ipp030 - XY40 STR ipp046 + XY40 STR ipp020 XY41 STR ipp046 - XY42 STR ipp031 + XY42 STR ipp021 XY43 STR ipp031 XY44 STR ipp047 @@ -129,5 +168,5 @@ XY72 STR ipp052 - XY73 STR ipp015 + XY73 STR ipp053 XY74 STR ipp015 XY75 STR ipp025 @@ -138,77 +177,75 @@ camera STR gpc1 -# ota01 STR ipp014 -# ota02 STR ipp014 - ota01 STR ipp047 - ota02 STR ipp047 - ota03 STR ipp038 + ota01 STR ipp037 + ota02 STR ipp005 + ota03 STR ipp006 ota04 STR ipp038 - - ota05 STR ipp023 + + ota05 STR ipp007 ota06 STR ipp023 - ota10 STR ipp039 + ota10 STR ipp008 ota11 STR ipp039 - - ota12 STR ipp024 + + ota12 STR ipp009 ota13 STR ipp024 - ota14 STR ipp040 + ota14 STR ipp010 ota15 STR ipp040 - - ota16 STR ipp026 + + ota16 STR ipp011 ota17 STR ipp026 - ota20 STR ipp041 + ota20 STR ipp012 ota21 STR ipp041 - - ota22 STR ipp042 + + ota22 STR ipp013 ota23 STR ipp042 - ota24 STR ipp043 + ota24 STR ipp014 ota25 STR ipp043 - - ota26 STR ipp028 + + ota26 STR ipp027 ota27 STR ipp028 - ota30 STR ipp044 + ota30 STR ipp016 ota31 STR ipp044 - - ota32 STR ipp029 + + ota32 STR ipp017 ota33 STR ipp029 - ota34 STR ipp045 + ota34 STR ipp018 ota35 STR ipp045 - - ota36 STR ipp030 + + ota36 STR ipp019 ota37 STR ipp030 - ota40 STR ipp046 + ota40 STR ipp020 ota41 STR ipp046 - - ota42 STR ipp031 + + ota42 STR ipp021 ota43 STR ipp031 ota44 STR ipp047 ota45 STR ipp047 - + ota46 STR ipp032 ota47 STR ipp032 ota50 STR ipp048 ota51 STR ipp048 - + ota52 STR ipp033 ota53 STR ipp033 ota54 STR ipp049 ota55 STR ipp049 - + ota56 STR ipp034 ota57 STR ipp034 ota60 STR ipp050 ota61 STR ipp050 - + ota62 STR ipp035 ota63 STR ipp035 ota64 STR ipp051 ota65 STR ipp051 - + ota66 STR ipp036 ota67 STR ipp036 ota71 STR ipp052 ota72 STR ipp052 - - ota73 STR ipp015 + + ota73 STR ipp053 ota74 STR ipp015 ota75 STR ipp025 @@ -217,34 +254,35 @@ # this list is no longer used -ipphosts METADATA - camera STR distribution - count S32 26 - 0 STR ipp021 - 1 STR ipp023 - 2 STR ipp024 - 3 STR ipp026 - 4 STR ipp028 - 5 STR ipp029 - 6 STR ipp030 - 7 STR ipp031 - 8 STR ipp032 - 9 STR ipp033 - 10 STR ipp034 - 11 STR ipp035 - 12 STR ipp036 - 13 STR ipp038 - 14 STR ipp039 - 15 STR ipp040 - 16 STR ipp041 - 17 STR ipp043 - 18 STR ipp044 - 19 STR ipp045 - 20 STR ipp046 - 21 STR ipp047 - 22 STR ipp048 - 23 STR ipp050 - 24 STR ipp051 - 25 STR ipp052 -END +# XXX : delete if this does not cause trouble +## ipphosts METADATA +## camera STR distribution +## count S32 26 +## 0 STR ipp021 +## 1 STR ipp023 +## 2 STR ipp024 +## 3 STR ipp026 +## 4 STR ipp028 +## 5 STR ipp029 +## 6 STR ipp030 +## 7 STR ipp031 +## 8 STR ipp032 +## 9 STR ipp033 +## 10 STR ipp034 +## 11 STR ipp035 +## 12 STR ipp036 +## 13 STR ipp038 +## 14 STR ipp039 +## 15 STR ipp040 +## 16 STR ipp041 +## 17 STR ipp043 +## 18 STR ipp044 +## 19 STR ipp045 +## 20 STR ipp046 +## 21 STR ipp047 +## 22 STR ipp048 +## 23 STR ipp050 +## 24 STR ipp051 +## 25 STR ipp052 +## END ipphosts METADATA @@ -252,77 +290,75 @@ count S32 60 -# XY01 STR ipp014 -# XY02 STR ipp014 - XY01 STR ipp047 - XY02 STR ipp047 - XY03 STR ipp038 + XY01 STR ipp037 + XY02 STR ipp005 + XY03 STR ipp006 XY04 STR ipp038 - - XY05 STR ipp023 + + XY05 STR ipp007 XY06 STR ipp023 - XY10 STR ipp039 + XY10 STR ipp008 XY11 STR ipp039 - - XY12 STR ipp024 + + XY12 STR ipp009 XY13 STR ipp024 - XY14 STR ipp040 + XY14 STR ipp010 XY15 STR ipp040 - - XY16 STR ipp026 + + XY16 STR ipp011 XY17 STR ipp026 - XY20 STR ipp041 + XY20 STR ipp012 XY21 STR ipp041 - - XY22 STR ipp042 + + XY22 STR ipp013 XY23 STR ipp042 - XY24 STR ipp043 + XY24 STR ipp014 XY25 STR ipp043 - - XY26 STR ipp028 + + XY26 STR ipp027 XY27 STR ipp028 - XY30 STR ipp044 + XY30 STR ipp016 XY31 STR ipp044 - - XY32 STR ipp029 + + XY32 STR ipp017 XY33 STR ipp029 - XY34 STR ipp045 + XY34 STR ipp018 XY35 STR ipp045 - - XY36 STR ipp030 + + XY36 STR ipp019 XY37 STR ipp030 - XY40 STR ipp046 + XY40 STR ipp020 XY41 STR ipp046 - - XY42 STR ipp031 + + XY42 STR ipp021 XY43 STR ipp031 XY44 STR ipp047 XY45 STR ipp047 - + XY46 STR ipp032 XY47 STR ipp032 XY50 STR ipp048 XY51 STR ipp048 - + XY52 STR ipp033 XY53 STR ipp033 XY54 STR ipp049 XY55 STR ipp049 - + XY56 STR ipp034 XY57 STR ipp034 XY60 STR ipp050 XY61 STR ipp050 - + XY62 STR ipp035 XY63 STR ipp035 XY64 STR ipp051 XY65 STR ipp051 - + XY66 STR ipp036 XY67 STR ipp036 XY71 STR ipp052 XY72 STR ipp052 - - XY73 STR ipp015 + + XY73 STR ipp053 XY74 STR ipp015 XY75 STR ipp025 @@ -333,51 +369,82 @@ camera STR dist_skycell count S32 46 - sky00 STR ipp006 - sky01 STR ipp007 - sky02 STR ipp008 - sky03 STR ipp009 - sky04 STR ipp010 - sky05 STR ipp011 - sky06 STR ipp012 - sky07 STR ipp013 -# sky08 STR ipp014 - sky08 STR ipp047 - sky09 STR ipp015 -# sky10 STR ipp016 - sky10 STR ipp042 - sky11 STR ipp017 - sky12 STR ipp018 - sky13 STR ipp019 - sky14 STR ipp020 - sky15 STR ipp021 - sky16 STR ipp023 - sky17 STR ipp024 - sky18 STR ipp025 - sky19 STR ipp026 - sky20 STR ipp027 - sky21 STR ipp028 - sky22 STR ipp029 - sky23 STR ipp030 - sky24 STR ipp031 - sky25 STR ipp032 - sky26 STR ipp033 - sky27 STR ipp034 - sky28 STR ipp035 - sky29 STR ipp036 - sky30 STR ipp038 - sky31 STR ipp039 - sky32 STR ipp040 - sky33 STR ipp041 - sky34 STR ipp042 - sky35 STR ipp043 - sky36 STR ipp044 - sky37 STR ipp045 - sky38 STR ipp046 - sky39 STR ipp047 - sky40 STR ipp048 - sky41 STR ipp049 - sky42 STR ipp050 - sky43 STR ipp051 - sky44 STR ipp052 - sky45 STR ipp053 + sky00 STR ipp005 + sky01 STR ipp006 + sky02 STR ipp007 + sky03 STR ipp008 + sky04 STR ipp009 + sky05 STR ipp010 + sky06 STR ipp011 + sky07 STR ipp012 + sky08 STR ipp013 + sky09 STR ipp014 + sky10 STR ipp015 + sky11 STR ipp016 + sky12 STR ipp017 + sky13 STR ipp018 + sky14 STR ipp019 + sky15 STR ipp020 + sky16 STR ipp021 + sky17 STR ipp023 + sky18 STR ipp024 + sky19 STR ipp025 + sky20 STR ipp026 + sky21 STR ipp027 + sky22 STR ipp028 + sky23 STR ipp029 + sky24 STR ipp030 + sky25 STR ipp031 + sky26 STR ipp032 + sky27 STR ipp033 + sky28 STR ipp034 + sky29 STR ipp035 + sky30 STR ipp036 + sky31 STR ipp037 + sky32 STR ipp038 + sky33 STR ipp039 + sky34 STR ipp040 + sky35 STR ipp041 + sky36 STR ipp042 + sky37 STR ipp043 + sky38 STR ipp044 + sky39 STR ipp045 + sky40 STR ipp046 + sky41 STR ipp047 + sky42 STR ipp048 + sky43 STR ipp049 + sky44 STR ipp050 + sky45 STR ipp051 + sky46 STR ipp052 + sky47 STR ipp053 + sky48 STR ipp023 + sky49 STR ipp024 + sky50 STR ipp025 + sky51 STR ipp026 + sky52 STR ipp027 + sky53 STR ipp028 + sky54 STR ipp029 + sky55 STR ipp030 + sky56 STR ipp031 + sky57 STR ipp032 + sky58 STR ipp033 + sky59 STR ipp034 + sky60 STR ipp035 + sky61 STR ipp036 + sky62 STR ipp037 + sky63 STR ipp038 + sky64 STR ipp039 + sky65 STR ipp040 + sky66 STR ipp041 + sky67 STR ipp042 + sky68 STR ipp043 + sky69 STR ipp044 + sky70 STR ipp045 + sky71 STR ipp046 + sky72 STR ipp047 + sky73 STR ipp048 + sky74 STR ipp049 + sky75 STR ipp050 + sky76 STR ipp051 + sky77 STR ipp052 + sky78 STR ipp053 END Index: /branches/pap/ippTasks/minidvodb.pro =================================================================== --- /branches/pap/ippTasks/minidvodb.pro (revision 28484) +++ /branches/pap/ippTasks/minidvodb.pro (revision 28484) @@ -0,0 +1,734 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + + +#There is a book for each task, because I don't use labels. + +if (not($?haveBooks)) + book create MINIDVODB + book create MINIDVODB_MERGE + book create MINIDVODB_CREATE + book create MINIDVODB_ACTIVE + $haveBooks = TRUE +end + +$MINIDVODB_DB = 0 + +$MINIDVODB_WAIT_DB = 0 +$MINIDVODB_MERGE_DB = 0 +$MINIDVODB_CREATE_DB = 0 +$MINIDVODB_ACTIVE_DB = 0 + +book init minidvodbWaitlist +book init minidvodbMergelist +book init minidvodbCreatelist +book init minidvodbActivelist + +macro minidvodb.create.status + book listbook minidvodbCreatelist +end + +macro minidvodb.create.reset + book init minidvodbCreatelist +end + +macro minidvodb.active.status + book listbook minidvodbActivelist +end + +macro minidvodb.active.reset + book init minidvodbActivelist +end + + +macro minidvodb.wait.status + book listbook minidvodbWaitlist +end + +macro minidvodb.wait.reset + book init minidvodbWaitlist +end + +macro minidvodb.merge.status + book listbook minidvodbMergelist +end + +macro minidvodb.merge.reset + book init minidvodbMergelist +end + +#this is the create task, there is no load, it just runs +macro minidvodb.create.on + task minidvodb.create + active true + end +end +macro minidvodb.create.off + task minidvodb.create + active false + end +end +##these are the tasks that flip from new -> active +macro minidvodb.active.on + task minidvodb.active.load + active true + end + task minidvodb.active.run + active true + end +end +macro minidvodb.active.off + task minidvodb.active.load + active false + end + task minidvodb.active.run + active false + end +end + +##these are the tasks that check to see if addRun processing is finished +##for a minidvodb in state wait +macro minidvodb.wait.on + task minidvodb.wait.load + active true + end + task minidvodb.wait.run + active true + end +end +macro minidvodb.wait.off + task minidvodb.wait.load + active false + end + task minidvodb.wait.run + active false + end +end + +##these merge the dbs + +macro minidvodb.merge.on + task minidvodb.merge.load + active true + end + task minidvodb.merge.run + active true + end +end +macro minidvodb.merge.off + task minidvodb.merge.load + active false + end + task minidvodb.merge.run + active false + end +end + +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. +macro add.minidvodb + if ($0 != 6) + echo "USAGE: add.minidvodb (minidvodb_group) (minidvodb) (dvodb) (interval days) (camera)" + break + end + + #wait - shoudl be renamed MINIDVODB_WAIT + book newpage MINIDVODB $1 + book setword MINIDVODB $1 MINIDVODB_GROUP $1 + book setword MINIDVODB $1 DVODB $3 + book setword MINIDVODB $1 STATE PENDING + #merge + book newpage MINIDVODB_MERGE $1 + book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_MERGE $1 DVODB $3 + book setword MINIDVODB_MERGE $1 STATE PENDING + #active + book newpage MINIDVODB_ACTIVE $1 + book setword MINIDVODB_ACTIVE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_ACTIVE $1 DVODB $3 + book setword MINIDVODB_ACTIVE $1 STATE PENDING + + #create note that camera should be GPC1 for it to work. I couldn't figure out how to easily get this out. + book newpage MINIDVODB_CREATE $1 + book setword MINIDVODB_CREATE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_CREATE $1 MINIDVODB $2 + book setword MINIDVODB_CREATE $1 DVODB $3 + book setword MINIDVODB_CREATE $1 DVODB_DAYS $4 + book setword MINIDVODB_CREATE $1 CAMERA $5 + book setword MINIDVODB_CREATE $1 STATE PENDING +end + +macro del.minidvodb + if ($0 != 2) + echo "USAGE: del.minidvodb (minidvodb)" + break + end + book delpage MINIDVODB $1 + book delpage MINIDVODB_MERGE $1 + book delpage MINIDVODB_CREATE $1 + book delpage MINIDVODB_ACTIVE $1 +end + +macro show.minidvodb + if ($0 != 1) + echo "USAGE: show.minidvodb" + break + end + echo "minidvodb wait" + book listbook MINIDVODB + echo "minidvodb merge" + book listbook MINIDVODB_MERGE + echo "minidvodb create" + book listbook MINIDVODB_CREATE + echo "minidvodb_active" + book listbook MINIDVODB_ACTIVE +end + + + + + + + + +task minidvodb.wait.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.wait.load.log + + task.exec + book npages MINIDVODB -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB $i -var minidvodb_group + book setword MINIDVODB $minidvodb_group STATE NEW + end + book getpage MINIDVODB 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end +#using check as opposed to list because it sees if it is done with the addRun state yet. + book setword MINIDVODB $minidvodb_group STATE DONE + $run = addtool -checkminidvodbrunaddrun -state waiting + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbWaitlist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbWaitlist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbWaitlist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.wait.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbWaitlist -var N + if ($N == 0) break + + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage minidvodbWaitlist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbWaitlist $pageName pantaskState RUN + book getword minidvodbWaitlist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbWaitlist $pageName state -var STATE + stdout $LOGDIR/minidvodb.wait.run.log + stderr $LOGDIR/minidvodb.wait.run.log + + $run = addtool -updateminidvodbrun -minidvodb_id $MINIDVODB_ID -set_state to_be_merged + + +if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit minidvodbWaitlist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + echo "hostname: $JOB_HOSTNAME" + process_exit minidvodbWaitlist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbWaitlist $options:0 pantaskState TIMEOUT + end +end + + + +task minidvodb.merge.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.merge.load.log + + task.exec + book npages MINIDVODB_MERGE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_MERGE $i -var minidvodb_group + book setword MINIDVODB_MERGE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + #finds the minidvodbs in a state of 'to_be_merged' + book setword MINIDVODB_MERGE $minidvodb_group STATE DONE + $run = addtool -listminidvodbrun -state to_be_merged -limit 1 + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbMergelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbMergelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.merge.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60000 + + #we only want one running at a time + + host local + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbMergelist -var N + if ($N == 0) break + + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage minidvodbMergelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbMergelist $pageName pantaskState RUN + book getword minidvodbMergelist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbMergelist $pageName minidvodb_group -var MINIDVODB_GROUP + book getword minidvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH + book getword minidvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword minidvodbMergelist $pageName state -var STATE + stdout $LOGDIR/minidvodb.merge.run.log + stderr $LOGDIR/minidvodb.merge.run.log + + $run = minidvodb_merge.pl --camera GPC1 --mergedvodb $MERGEDVODB_PATH --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run --dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit minidvodbMergelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + + echo "hostname: $JOB_HOSTNAME" + + # Set a fault code in the database + + process_exit minidvodbMergelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbMergelist $options:0 pantaskState TIMEOUT + end +end + + +#this is the complicated script - it creates the minidvodbRuns +#since none may exist by default, I decided not to have a load.task +#it calls a perl script, which creates a new one if: +#there is none in an active state (true if there is none at all) +#the current one is older than the dvodb_age +#there are none in new (we have a task that moves it to 'active') +#the current active one has > 30000 add_ids in it + +#also confusing: it succeeds if it doesn't create it (if it doesn't want to create one) +#and it succeeds if it creates one. (otherwise there would be a lot of false failures, +# like in replication) +#it fails if it has problems with one of the addtool commands in the script. + +task minidvodb.create + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 240 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.create.log + + task.exec + + periods -poll 60 +#wait a bit before trying again + + book npages MINIDVODB_CREATE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_CREATE $i -var minidvodb_group + book setword MINIDVODB_CREATE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + + book setword MINIDVODB_CREATE $minidvodb_group STATE DONE + book getword MINIDVODB_CREATE $minidvodb_group MINIDVODB -var minidvodb + book getword MINIDVODB_CREATE $minidvodb_group DVODB -var dvodb + book getword MINIDVODB_CREATE $minidvodb_group DVODB_DAYS -var dvodb_days + book getword MINIDVODB_CREATE $minidvodb_group CAMERA -var camera + $run = minidvodb_createdb.pl --camera $camera --outroot $dvodb --dvodb $dvodb --minidvodb $minidvodb --minidvodb_group $minidvodb_group --interval $dvodb_days + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run --dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + command $run + end + + # success + task.exit 0 + if ($VERBOSE > 2) + showcommand + end + end + + # locked list + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + +task minidvodb.active.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.active.load.log + + task.exec + book npages MINIDVODB_ACTIVE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_ACTIVE $i -var minidvodb_group + book setword MINIDVODB_ACTIVE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + book setword MINIDVODB_ACTIVE $minidvodb_group STATE DONE + + $run = addtool -listminidvodbrun -state new + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbActivelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbActivelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbActivelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.active.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbActivelist -var N + if ($N == 0) break + + # look for new images in minidvodbActivelist (pantaskState == INIT) + book getpage minidvodbActivelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbActivelist $pageName pantaskState RUN + book getword minidvodbActivelist $pageName minidvodb_group -var MINIDVODB_GROUP + stdout $LOGDIR/minidvodb.active.run.log + stderr $LOGDIR/minidvodb.active.run.log + + $run = addtool -flipminidvodbrun -minidvodb_group $MINIDVODB_GROUP + + +if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end +# save the pageName for future reference below + options $pageName + + + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end +# default exit status + task.exit default + process_exit minidvodbActivelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + #showcommand crash + echo "hostname: $JOB_HOSTNAME" + # Set a fault code in the database + + process_exit minidvodbActivelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbActivelist $options:0 pantaskState TIMEOUT + end +end + Index: /branches/pap/ippTasks/nightly_stacks.pro =================================================================== --- /branches/pap/ippTasks/nightly_stacks.pro (revision 28483) +++ /branches/pap/ippTasks/nightly_stacks.pro (revision 28484) @@ -5,4 +5,6 @@ macro nightly.stacks.on ns.initday.on + ns.detrends.off + ns.dqstats.on ns.registration.on ns.burntool.on @@ -13,4 +15,5 @@ macro nightly.stacks.off ns.initday.off + ns.detrends.off ns.registration.off ns.burntool.off @@ -21,4 +24,5 @@ macro ns.on ns.initday.on + ns.detrends.off ns.registration.on ns.burntool.on @@ -29,4 +33,5 @@ macro ns.off ns.initday.off + ns.detrends.off ns.registration.off ns.burntool.off @@ -41,4 +46,16 @@ end +macro ns.detrends.on + task ns.detrends.load + active true + end +end + +macro ns.dqstats.on + task ns.dqstats.load + active true + end +end + macro ns.registration.on task ns.registration.load @@ -76,4 +93,16 @@ macro ns.initday.off task ns.initday.load + active false + end +end + +macro ns.detrends.off + task ns.detrends.load + active false + end +end + +macro ns.dqstats.off + task ns.dqstats.load active false end @@ -186,4 +215,75 @@ #operation times out? task.exit timeout + showcommand timeout + end +end + +# +# Queue off any possible detrend verification runs +# +task ns.detrends.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + trange 23:00:00 23:59:59 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.detrends.log + stdout $LOGDIR/ns.detrends.log + $today = `date +%Y-%m-%d` + + command automate_stacks.pl --queue_detrends --date $today + end + + task.exit 0 + # nothign to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Queue dqstats runs +# +task ns.dqstats.load + host local + periods -poll 3600 + periods -exec $LOADEXEC + periods -timeout 30 + trange 22:00:00 23:59:59 + trange 00:00:00 10:00:00 + npending 1 + + task.exec + stdout $LOGDIR/ns.dqstats.log + stderr $LOGDIR/ns.dqstats.log + $today = `date +%Y-%m-%d` + + command automate_stacks.pl --queue_dqstats --date $today + end + + task.exit 0 + # nothing to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout showcommand timeout end Index: /branches/pap/ippTasks/stack.pro =================================================================== --- /branches/pap/ippTasks/stack.pro (revision 28483) +++ /branches/pap/ippTasks/stack.pro (revision 28484) @@ -10,4 +10,5 @@ ### Initialise the books containing the tasks to do book init stackSumSkyfile +book init stackPendingSummary #book init stackCleanup @@ -16,4 +17,5 @@ $stack_revert_DB = 0 #$stackCleanup_DB = 0 +$stackSummary_DB = 0 ### Check status of stacking tasks @@ -26,4 +28,5 @@ macro stack.reset book init stackSumSkyfile + book init stackPendingSummary # book init stackCleanup end @@ -40,4 +43,10 @@ active false end + task stack.summary.load + active true + end + task stack.summary.run + active true + end end @@ -51,4 +60,10 @@ end task stack.revert + active false + end + task stack.summary.load + active false + end + task stack.summary.run active false end @@ -349,4 +364,7 @@ # end + + + task stack.revert host local @@ -400,2 +418,132 @@ end +### Load tasks for doing the stack summary +### Tasks are loaded into stackPendingSummary +task stack.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + trange 07:00:00 08:00:00 -nmax 1 + npending 1 + + stdout NULL + stderr $LOGDIR/stack.summary.load + + task.exec + if ($LABEL:n == 0) break + $run = stacktool -tosummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$stackSummary_DB + $run = $run -dbname $DB:$stackSummary_DB + $stackSummary_DB ++ + if ($stackSummary_DB >= $DB:n) set stackSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackPendingSummary -key sass_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackPendingSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackPendingSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for doing the stack summary +task stack.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages stackPendingSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in stackPendingSummary (pantaskState == INIT) + book getpage stackPendingSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackPendingSummary $pageName pantaskState RUN + book getword stackPendingSummary $pageName sass_id -var SASS_ID + book getword stackPendingSummary $pageName camera -var CAMERA + book getword stackPendingSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword stackPendingSummary $pageName dbname -var DBNAME + book getword stackPendingSummary $pageName tess_id -var TESS_DIR + book getword stackPendingSummary $pageName state -var RUN_STATE + book getword stackPendingSummary $pageName projection_cell -var projection_cell + + # set the host and workdir + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR + + basename $TESS_DIR -var TESS_ID + + ## generate outroot specific to this association + sprintf outroot "%s/%s/%s.stk.%s.summary" $WORKDIR $TESS_ID $TESS_ID $SASS_ID + + stdout $LOGDIR/stack.summary.log + stderr $LOGDIR/stack.summary.log + + $run = skycell_jpeg.pl --stage stack --stage_id $SASS_ID --camera $CAMERA --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit stackPendingSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword stackPendingSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackPendingSummary $options:0 pantaskState TIMEOUT + end +end + Index: /branches/pap/ippTasks/survey.pro =================================================================== --- /branches/pap/ippTasks/survey.pro (revision 28483) +++ /branches/pap/ippTasks/survey.pro (revision 28484) @@ -207,10 +207,11 @@ macro survey.add.addstar - if ($0 != 3) - echo "USAGE: survey.add.addstar (label) (dvodb)" + if ($0 != 4) + echo "USAGE: survey.add.addstar (label) (dvodb) (minidvodb_group)" break end book newpage SURVEY_ADDSTAR $1 book setword SURVEY_ADDSTAR $1 DVODB $2 + book setword SURVEY_ADDSTAR $1 MINIDVODB_GROUP $3 book setword SURVEY_ADDSTAR $1 STATE PENDING end @@ -638,4 +639,5 @@ book setword SURVEY_ADDSTAR $label STATE DONE book getword SURVEY_ADDSTAR $label DVODB -var dvodb + book getword SURVEY_ADDSTAR $label MINIDVODB_GROUP -var minidvodb_group # $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb @@ -648,5 +650,5 @@ end - $run = $run -definebyquery -destreaked -label $label -set_dvodb $dvodb + $run = $run -definebyquery -destreaked -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group -set_minidvodb -set_label $minidvodb_group echo $run command $run Index: /branches/pap/ippTasks/warp.pro =================================================================== --- /branches/pap/ippTasks/warp.pro (revision 28483) +++ /branches/pap/ippTasks/warp.pro (revision 28484) @@ -13,4 +13,5 @@ book init warpPendingSkyCell book init warpPendingCleanup +book init warpPendingSummary ### Database lists @@ -19,4 +20,5 @@ $warp_revert_overlap_DB = 0 $warp_revert_warped_DB = 0 +$warpSummary_DB = 0 ### Check status of warping tasks @@ -30,4 +32,5 @@ book init warpInputExp book init warpPendingSkyCell + book init warpPendingSummary end @@ -55,4 +58,10 @@ active true end + task warp.summary.load + active true + end + task warp.summary.run + active true + end end @@ -78,4 +87,10 @@ end task warp.revert.warped + active false + end + task warp.summary.load + active false + end + task warp.summary.run active false end @@ -525,2 +540,136 @@ end end + +### Load tasks for doing the warps +### Tasks are loaded into warpPendingSkyCell. +task warp.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = warptool -tosummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warpSummary_DB + $run = $run -dbname $DB:$warpSummary_DB + $warpSummary_DB ++ + if ($warpSummary_DB >= $DB:n) set warpSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + # XXX change tess_id to tess_dir when db is updated + ipptool2book stdout warpPendingSummary -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpPendingSkyCell. +task warp.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages warpPendingSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpPendingSkyCell (pantaskState == INIT) + book getpage warpPendingSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingSummary $pageName pantaskState RUN + book getword warpPendingSummary $pageName warp_id -var WARP_ID + book getword warpPendingSummary $pageName camera -var CAMERA + book getword warpPendingSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword warpPendingSummary $pageName dbname -var DBNAME + book getword warpPendingSummary $pageName tess_id -var TESS_DIR + book getword warpPendingSummary $pageName exp_tag -var EXP_TAG + book getword warpPendingSummary $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + +# if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure + basename $TESS_DIR -var TESS_ID + + sprintf outroot "%s/%s/%s.wrp.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID $TESS_ID + + + stdout $LOGDIR/warp.summary.log + stderr $LOGDIR/warp.summary.log + + $run = skycell_jpeg.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpPendingSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword warpPendingSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingSummary $options:0 pantaskState TIMEOUT + end +end Index: /branches/pap/ippToPsps/config/init/data.xml =================================================================== --- /branches/pap/ippToPsps/config/init/data.xml (revision 28483) +++ /branches/pap/ippToPsps/config/init/data.xml (revision 28484) @@ -101463,5 +101463,61 @@ - + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Index: /branches/pap/ippToPsps/config/init/tables.xml =================================================================== --- /branches/pap/ippToPsps/config/init/tables.xml (revision 28483) +++ /branches/pap/ippToPsps/config/init/tables.xml (revision 28484) @@ -112,3 +112,13 @@
+ + + + +
+ + + + +
Index: /branches/pap/ippToPsps/scripts/createDb.pl =================================================================== --- /branches/pap/ippToPsps/scripts/createDb.pl (revision 28484) +++ /branches/pap/ippToPsps/scripts/createDb.pl (revision 28484) @@ -0,0 +1,105 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use DBI; +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); + +my $dbname = 'ippToPsps'; +my $dbserver = 'ippdb01'; +my $dbuser = 'ipp'; +my $dbpass = 'ipp'; +GetOptions( 'dbname|d=s' => \$dbname); + +# connect to database +my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + +print "\n*******************************************************************************\n*\n"; + +print "* Connected to '$dbname' on 'dbserver'\n"; + +if (!doesTableExist("revision")) { + createRevision_1(); +} + + +$db->disconnect(); +print "* Disconnected from '$dbname' on 'dbserver'\n"; +print "*\n*******************************************************************************\n\n"; + +####################################################################################### +# +# Create revision 1 of the database +# +####################################################################################### +sub createRevision_1 { + + print "* Creating revision 1 of '$dbname'\n"; + + my $query = $db->prepare(<execute; + + $query = $db->prepare(<execute; + + $query = $db->prepare(<execute; + +} + +####################################################################################### +# +# Gets current revision of ippToPsps database +# +####################################################################################### +sub doesTableExist { + my ($table) = @_; + + my $query = $db->prepare(<execute; + + my $count = $query->fetchrow_array(); + + printf( "* Table '$table' %s\n", $count ? "exists" : "does not exist"); + + return $count; +} + Index: /branches/pap/ippToPsps/scripts/ippToPsps_run.pl =================================================================== --- /branches/pap/ippToPsps/scripts/ippToPsps_run.pl (revision 28483) +++ /branches/pap/ippToPsps/scripts/ippToPsps_run.pl (revision 28484) @@ -12,5 +12,5 @@ # globals -my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $label, $singleExpId, $no_publish, $force); +my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $survey, $singleExpId, $no_publish, $force, $initBatch); # default values for certain globals @@ -25,4 +25,5 @@ $datastoreProduct = "PSPS_test"; $force = 0; +$initBatch = 0; # get user args @@ -31,5 +32,5 @@ 'batch|b=s' => \$batchType, 'dvodb|d=s' => \$dvodb, - 'label|l=s' => \$label, + 'survey|s=s' => \$survey, 'expid|e=s' => \$singleExpId, 'product|p=s' => \$datastoreProduct, @@ -49,5 +50,5 @@ "--batch \n". "--output \n". - "--label