Index: /branches/eam_branches/ipp-20110906/Nebulous-Server/Build.PL =================================================================== --- /branches/eam_branches/ipp-20110906/Nebulous-Server/Build.PL (revision 32629) +++ /branches/eam_branches/ipp-20110906/Nebulous-Server/Build.PL (revision 32630) @@ -49,4 +49,5 @@ bin/neb-voladd bin/neb-voladm + bin/neb-insedit bin/neb-host bin/nebdiskd Index: /branches/eam_branches/ipp-20110906/Nebulous-Server/MANIFEST =================================================================== --- /branches/eam_branches/ipp-20110906/Nebulous-Server/MANIFEST (revision 32629) +++ /branches/eam_branches/ipp-20110906/Nebulous-Server/MANIFEST (revision 32630) @@ -10,4 +10,5 @@ bin/neb-cabadd bin/neb-voladm +bin/neb-insedit bin/neb-host bin/nebdiskd Index: /branches/eam_branches/ipp-20110906/Nebulous-Server/bin/neb-admin =================================================================== --- /branches/eam_branches/ipp-20110906/Nebulous-Server/bin/neb-admin (revision 32629) +++ /branches/eam_branches/ipp-20110906/Nebulous-Server/bin/neb-admin (revision 32630) @@ -29,4 +29,5 @@ $pending, $balance, + $retarget, $balance_N_sources, $balance_M_destinations, @@ -49,4 +50,5 @@ 'pendingreplicate|r' => \$pending, 'pendingbalance|b' => \$balance, + 'pendingtarget|t' => \$retarget, 'balancesources|N=s' => \$balance_N_sources, 'balancedestinations|M=s' => \$balance_M_destinations, @@ -64,7 +66,7 @@ pod2usage( -msg => "--limit is meaningless without --pendingreplicate", -exitval => 2 ) - if defined $limit and not (defined $pending or defined $balance); + if defined $limit and not (defined $pending or defined $balance or defined $retarget); pod2usage( -msg => "no operation specified", -exitval => 2 ) - unless defined $pending or defined $balance; + unless defined $pending or defined $balance or defined $retarget; pod2usage( -msg => "--pendingbalance needs --balancesources and --balancedestinations options", -exitval => 2) @@ -81,4 +83,7 @@ elsif (defined $balance) { $pidfile = '/var/tmp/neb-admin.balance'; +} +elsif (defined $retarget) { + $pidfile = '/var/tmp/neb-admin.retarget'; } else { @@ -107,4 +112,6 @@ } elsif ($balance) { balance(); +} elsif ($retarget) { + retarget(); } else { die "THIS SHOULD NOT HAPPEN"; @@ -524,4 +531,159 @@ } +sub retarget +{ + my %destination; + %{ $destination{VOL} } = ('ota01' => 7, 'ota02' => 8, 'ota03' => 3, 'ota04' => 3, 'ota05' => 4, 'ota06' => 4, + 'ota10' => 50, 'ota11' => 50, 'ota12' => 6, 'ota13' => 6, 'ota14' => 7, 'ota15' => 8, 'ota16' => 15, 'ota17' => 15, + 'ota20' => 16, 'ota21' => 18, 'ota22' => 18, 'ota23' => 17, 'ota24' => 9, 'ota25' => 9, 'ota26' => 10, 'ota27' => 11, + 'ota30' => 12, 'ota31' => 52, 'ota32' => 14, 'ota33' => 14, 'ota34' => 19, 'ota35' => 16, 'ota36' => 20, 'ota37' => 12, + 'ota40' => 32, 'ota41' => 11, 'ota42' => 10, 'ota43' => 17, 'ota44' => 49, 'ota45' => 23, 'ota46' => 24, 'ota47' => 25, + 'ota50' => 26, 'ota51' => 27, 'ota52' => 28, 'ota53' => 29, 'ota54' => 30, 'ota55' => 31, 'ota56' => 51, 'ota57' => 33, + 'ota60' => 34, 'ota61' => 35, 'ota62' => 36, 'ota63' => 37, 'ota64' => 38, 'ota65' => 39, 'ota66' => 41, 'ota67' => 42, + 'ota71' => 43, 'ota72' => 44, 'ota73' => 45, 'ota74' => 46, 'ota75' => 47, 'ota76' => 48); + %{ $destination{HOST} } = ('ota01' => 'ipp010','ota02' => 'ipp011','ota03' => 'ipp006','ota04' => 'ipp006','ota05' => 'ipp007','ota06' => 'ipp007', + 'ota10' => 'ipp008','ota11' => 'ipp008','ota12' => 'ipp009','ota13' => 'ipp009','ota14' => 'ipp010','ota15' => 'ipp011','ota16' => 'ipp012','ota17' => 'ipp012', + 'ota20' => 'ipp013','ota21' => 'ipp014','ota22' => 'ipp014','ota23' => 'ipp015','ota24' => 'ipp016','ota25' => 'ipp016','ota26' => 'ipp017','ota27' => 'ipp018', + 'ota30' => 'ipp019','ota31' => 'ipp020','ota32' => 'ipp021','ota33' => 'ipp021','ota34' => 'ipp023','ota35' => 'ipp013','ota36' => 'ipp024','ota37' => 'ipp019', + 'ota40' => 'ipp025','ota41' => 'ipp018','ota42' => 'ipp017','ota43' => 'ipp015','ota44' => 'ipp027','ota45' => 'ipp028','ota46' => 'ipp029','ota47' => 'ipp030', + 'ota50' => 'ipp031','ota51' => 'ipp032','ota52' => 'ipp033','ota53' => 'ipp034','ota54' => 'ipp035','ota55' => 'ipp036','ota56' => 'ipp037','ota57' => 'ipp038', + 'ota60' => 'ipp039','ota61' => 'ipp040','ota62' => 'ipp041','ota63' => 'ipp042','ota64' => 'ipp043','ota65' => 'ipp044','ota66' => 'ipp046','ota67' => 'ipp047', + 'ota71' => 'ipp048','ota72' => 'ipp049','ota73' => 'ipp050','ota74' => 'ipp051','ota75' => 'ipp052','ota76' => 'ipp053'); + %{ $destination{NAME} } = ('ota01' => 'ipp010.0','ota02' => 'ipp011.0','ota03' => 'ipp006.0','ota04' => 'ipp006.0','ota05' => 'ipp007.0','ota06' => 'ipp007.0', + 'ota10' => 'ipp008.0','ota11' => 'ipp008.0','ota12' => 'ipp009.0','ota13' => 'ipp009.0','ota14' => 'ipp010.0','ota15' => 'ipp011.0','ota16' => 'ipp012.0','ota17' => 'ipp012.0', + 'ota20' => 'ipp013.0','ota21' => 'ipp014.0','ota22' => 'ipp014.0','ota23' => 'ipp015.0','ota24' => 'ipp016.0','ota25' => 'ipp016.0','ota26' => 'ipp017.0','ota27' => 'ipp018.0', + 'ota30' => 'ipp019.0','ota31' => 'ipp020.0','ota32' => 'ipp021.0','ota33' => 'ipp021.0','ota34' => 'ipp023.0','ota35' => 'ipp013.0','ota36' => 'ipp024.0','ota37' => 'ipp019.0', + 'ota40' => 'ipp025.0','ota41' => 'ipp018.0','ota42' => 'ipp017.0','ota43' => 'ipp015.0','ota44' => 'ipp027.0','ota45' => 'ipp028.0','ota46' => 'ipp029.0','ota47' => 'ipp030.0', + 'ota50' => 'ipp031.0','ota51' => 'ipp032.0','ota52' => 'ipp033.0','ota53' => 'ipp034.0','ota54' => 'ipp035.0','ota55' => 'ipp036.0','ota56' => 'ipp037.0','ota57' => 'ipp038.0', + 'ota60' => 'ipp039.0','ota61' => 'ipp040.0','ota62' => 'ipp041.0','ota63' => 'ipp042.0','ota64' => 'ipp043.0','ota65' => 'ipp044.0','ota66' => 'ipp046.0','ota67' => 'ipp047.0', + 'ota71' => 'ipp048.0','ota72' => 'ipp049.0','ota73' => 'ipp050.0','ota74' => 'ipp051.0','ota75' => 'ipp052.0','ota76' => 'ipp053.0'); + my %blocked_volume = ('ipp027' => 1, + 'ipp037' => 1, + 'ipp043' => 1, + 'ipp044' => 1, + 'ipp053' => 1, + 'ipp023' => 1,'ipp024' => 1,'ipp025' => 1,'ipp026' => 1,'ipp028' => 1,'ipp029' => 1,'ipp030' => 1,'ipp031' => 1,'ipp032' => 1,'ipp033' => 1, + 'ipp034' => 1,'ipp035' => 1,'ipp036' => 1,'ipp038' => 1,'ipp039' => 1,'ipp040' => 1,'ipp041' => 1,'ipp042' => 1,'ipp045' => 1,'ipp046' => 1, + 'ipp047' => 1,'ipp048' => 1,'ipp049' => 1,'ipp050' => 1,'ipp051' => 1,'ipp052' => 1); + if (not defined $so_id_start) { $so_id_start = 0; } + if (not defined $so_id_range) { $so_id_range = 100000; } + my $so_id_end = $so_id_start + $so_id_range; + + # XXX check if so_id_start is beyond MAX(so_id): if so, exit with exit status 10 + { + my $query = $dbh->prepare("SELECT MAX(so_id) from storage_object"); + $query->execute; + my $answer = $query->fetchrow_arrayref; + my $max_so_id = $$answer[0]; + $query->finish; + + if ($so_id_start > $max_so_id) { + print STDERR "at end of so_id range, reset please\n"; + exit 10; + } + } + + # Determine a list of first instances + my $query = $dbh->prepare( + " +SELECT S.so_id,ext_id,ins_id,uri,substr(uri,-10,5) AS class_id,vol_id,name AS vol_name,host AS vol_host FROM ( + select * FROM ( + select MAX(ins_id) AS MAXins_id, MIN(ins_id) AS MINins_id, so_id,ext_id,value AS user_copies FROM storage_object + JOIN storage_object_xattr USING(so_id) + RIGHT JOIN instance USING(so_id) + WHERE name = 'user.copies' + AND value >= 2 + AND ext_id LIKE '%ota%fits' + AND so_id >= $so_id_start + AND so_id < $so_id_end + GROUP BY so_id + ) AS V + WHERE MAXins_id != MINins_id + ) AS S + JOIN instance ON (S.MINins_id = instance.ins_id AND S.so_id = instance.so_id) JOIN volume USING(vol_id) WHERE volume.available = 1 + LIMIT $limit + OFFSET $offset +"); + $query->execute(); + + my @rows; + while (my $row = $query->fetchrow_hashref) { + push @rows, $row; + } + $query->finish; + + print STDERR "No rows found\n" unless scalar @rows; + exit unless scalar @rows; + + # compare number of responses to limit below + my $Npending = @rows; + + print "targetPending MULTI\n\n"; + + foreach my $obj (@rows) { + if (defined $verbose) { + require Data::Dumper; + print Data::Dumper::Dumper($obj); + } + + my $so_id = $obj->{so_id}; + my $key = $obj->{ext_id}; + + my $ins_id = $obj->{ins_id}; + my $uri = $obj->{uri}; + my $class_id = $obj->{class_id}; + + my $source_vol_id = $obj->{vol_id}; + my $source_name = $obj->{vol_name}; + my $source_host = $obj->{vol_host}; + + + my $destination_vol_id = $destination{VOL}{$class_id}; + my $destination_name = $destination{NAME}{$class_id}; + my $destination_host = $destination{HOST}{$class_id}; + my $destination_uri = $uri; + unless ((defined($destination_uri))&& + (defined($source_name))&& + (defined($destination_name))) { + next; + } + if (defined($blocked_volume{$destination_host})) { + next; + } + $destination_uri =~ s/$source_name/$destination_name/; + + my %md = ( + so_id => $so_id, + key => $key, + ins_id => $ins_id, + uri => $uri, + class_id => $class_id, + + source_vol_id => $source_vol_id, + source_name => $source_name, + source_host => $source_host, + + destination_vol_id => $destination_vol_id, + destination_name => $destination_name, + destination_host => $destination_host, + destination_uri => $destination_uri, + ); + if ($source_vol_id != $destination_vol_id) { + print_metadata("targetPending", \%md); + print "\n"; + } + } + + # use a different exit status if we hit the limit (likely more files pending) + print STDERR "Which exit: $Npending $limit\n"; + # This is a better way to do this. This should be even better-er. + if ($Npending < $limit) { + exit 0; + } + else { + exit 1; + } +} + sub removal { Index: /branches/eam_branches/ipp-20110906/Nebulous-Server/bin/neb-insedit =================================================================== --- /branches/eam_branches/ipp-20110906/Nebulous-Server/bin/neb-insedit (revision 32630) +++ /branches/eam_branches/ipp-20110906/Nebulous-Server/bin/neb-insedit (revision 32630) @@ -0,0 +1,292 @@ +#!/usr/bin/env perl + +# Copyright (C) 2005-2008 Joshua Hoblitt +# +# $Id: neb-insedit,v 1.2 2008-10-01 21:05:57 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.02'; + +use DBI; +use Nebulous::Server::SQL; +use Nebulous::Client; +use URI::file; +use URI; + +use Digest::MD5; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +my ($db, $dbhost, $dbuser, $dbpass, $key, $ins_id, $new_uri, $new_volume,$nebhost); + +$db = $ENV{'NEB_DB'} unless $db; +$dbhost = $ENV{'NEB_DBHOST'} || 'localhost'; +$dbuser = $ENV{'NEB_USER'} unless $dbuser; +$dbpass = $ENV{'NEB_PASS'} unless $dbpass; +$nebhost= $ENV{'NEB_SERVER'} unless $nebhost; + +GetOptions( + 'db|d=s' => \$db, + 'host=s' => \$dbhost, + 'pass|p=s' => \$dbpass, + 'user|u=s' => \$dbuser, + 'neb_host=s' => \$nebhost, + 'key=s' => \$key, + 'ins_id=s' => \$ins_id, + 'uri=s' => \$new_uri, + 'volume=s' => \$new_volume, +) || pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --db --user --pass --key --ins_id --uri --volume", -exitval => 2 ) + unless $db && $dbuser && $dbpass && $ins_id && $new_uri && $key && $new_volume; + +my $dbh = DBI->connect( + "DBI:mysql:database=$db:host=$dbhost", + $dbuser, + $dbpass, + { + RaiseError => 1, + PrintError => 0, + AutoCommit => 1, + }, +); + +# Connect and retrieve instance information via stat: +my $neb = Nebulous::Client->new( + proxy => "$nebhost", + ); + +die "can't connected to Nebulous Server: $nebhost" + unless defined $neb; + + +# Pre-emptively attempt repair of file. This will ~double our execution time, but should help the efficiency. +system("neb-repair $key"); +if ($? >> 8 != 0) { + die "Failure to repair $key.\n"; +} + +# Do checks now. + +my $stat = $neb->stat($key); +die "nebulous key: $key not found" unless $stat; +my $instances = $neb->find_instances($key, 'any'); +die "no instances found" unless $instances; + +my $user_copies; +# eval { +# $user_copies = $neb->getxattr($key, "user.copies"); +# }; +$user_copies = 1; +unless(defined($user_copies)) { + $user_copies = 1; +} +my @validation; +my %md5sum_uniq; +my $existing_copies = 0; +my @existance; +my @md5sums; +my @files = map {URI->new($_)->file if $_} @$instances; + +for (my $i = 0; $i <= $#files; $i++) { + if (-e $files[$i]) { + $existance[$i] = 1; + $existing_copies++; + my $fh; + open($fh,$files[$i]); + $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest; + $md5sum_uniq{$md5sums[$i]} = 1; + close($fh); + } + else { + $existance[$i] = 0; + $md5sums[$i] = 'NON-EXISTANT'; + $md5sum_uniq{$md5sums[$i]} = 1; + } + $validation[$i] = sprintf("% 3d %32s %s", + $existance[$i], + $md5sums[$i], + $files[$i]); +} +# Confirm everything is in a valid state +my $N_md5sums = scalar(keys %md5sum_uniq); +if ($N_md5sums != 1) { + die "Cannot modify this instance! storage_object $key in inconsistent state!"; +} +if ($#files < 1) { + die "Insufficient instances! $key!"; +} +my $test_md5sum = $md5sums[0]; + +# Copy instance to new uri +my $destination_file = URI->new($new_uri)->file; +my $source_file = $files[0]; +system("cp $source_file $destination_file"); +system("glockfile $destination_file xcld 0"); + +my $fh; +open($fh,$destination_file); +my $new_md5sum = Digest::MD5->new->addfile($fh)->hexdigest; +close($fh); + +if ($new_md5sum ne $test_md5sum) { + die "md5sum mismatch with new copy. $destination_file ($new_md5sum) != $source_file ($test_md5sum) Exiting!"; +} + +# Update database table +my $sql = Nebulous::Server::SQL->new(); + +my $query = $dbh->prepare( $sql->get_volume_by_name ); +$query->execute( $new_volume ); +my ($vol_id, $vol_name, $vol_host, $vol_path) = $query->fetchrow_array; +$query->finish; + +print "$new_volume $vol_id $vol_name $vol_host $vol_path $new_uri $ins_id\n"; +unless (defined($vol_id)) { + die "Invalid destination volume."; +} + +$query = $dbh->prepare( $sql->update_instance_uri ); +$query->execute( $vol_id, $new_uri, $ins_id ); +$query->finish; + +unlink($source_file); +print " OK\n"; + +__END__ + +=pod + +=head1 NAME + +neb-insedit - edit the instance information + +=head1 SYNOPSIS + + neb-insedit --key --ins_id --uri --volume + [--db ] [--user ] [--pass ] [--host + +Nebulous key for the storage object associated with the instance to move. + +=item * --ins_id + +Instance identifier that will be moved + +=item * --uri + +Final location desired for this instance. Should be of the form "file://data/ipp0XX.0/nebulous/aa/aa/ins_id.nebulous:directory:tree:file". + +=item * --volume + +Volume name that this instance will be moved to. + +=item * --db|-d + +Name of database (C) to create tables in. + +Optional if the appropriate environment variable is set. + +=item * --user|-u + +Username to authenticate with. + +Optional if the appropriate environment variable is set. + +=item * --pass|-p + +Password to authenticate with. + +Optional if the appropriate environment variable is set. + +=item * --host + +Name of host on which the database resides. + +=item * --neb_host + +Location of the client side server to connect to. + +Optional. Defaults to C. + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --db|-d + +=item * C + +Equivalent to --host + +=item * C + +Equivalent to --user|-u + +=item * C + +Equivalent to --pass|-p + +=item * C + +Equivalent to --neb_host + +=back + +=head1 CREDITS + +Josh Hoblitt and Chris Waters + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Joshua Hoblitt + +=head1 COPYRIGHT + +Copyright (C) 2005-2008 Joshua Hoblitt. All rights reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L + +=cut Index: /branches/eam_branches/ipp-20110906/Nebulous-Server/bin/neb-voladm =================================================================== --- /branches/eam_branches/ipp-20110906/Nebulous-Server/bin/neb-voladm (revision 32629) +++ /branches/eam_branches/ipp-20110906/Nebulous-Server/bin/neb-voladm (revision 32630) @@ -68,5 +68,5 @@ if (defined $allocate and $allocate !~ m/^[01]$/) or (defined $available and $available !~ m/^[01]$/) - or (defined $xattr and $xattr !~ m/^[01]$/); + or (defined $xattr and $xattr !~ m/^[012]$/); } Index: /branches/eam_branches/ipp-20110906/Nebulous/Build.PL =================================================================== --- /branches/eam_branches/ipp-20110906/Nebulous/Build.PL (revision 32629) +++ /branches/eam_branches/ipp-20110906/Nebulous/Build.PL (revision 32630) @@ -116,4 +116,5 @@ bin/neb-mv bin/neb-replicate + bin/neb-repair bin/neb-rm bin/neb-shift Index: /branches/eam_branches/ipp-20110906/Nebulous/MANIFEST =================================================================== --- /branches/eam_branches/ipp-20110906/Nebulous/MANIFEST (revision 32629) +++ /branches/eam_branches/ipp-20110906/Nebulous/MANIFEST (revision 32630) @@ -17,4 +17,5 @@ bin/neb-mv bin/neb-replicate +bin/neb-repair bin/neb-rm bin/neb-shift Index: /branches/eam_branches/ipp-20110906/Nebulous/bin/neb-repair =================================================================== --- /branches/eam_branches/ipp-20110906/Nebulous/bin/neb-repair (revision 32630) +++ /branches/eam_branches/ipp-20110906/Nebulous/bin/neb-repair (revision 32630) @@ -0,0 +1,241 @@ +#!/usr/bin/env perl + +# Copyright (C) 2007-2008 Chris Waters +# + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Nebulous::Client; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +use Digest::MD5; +use URI; + +my ($validate,$server,$nouris); + +$server = $ENV{'NEB_SERVER'} unless $server; + +$validate = 1; +$nouris = 1; + +GetOptions( + 'server|s=s' => \$server, +) || pod2usage( 2 ); + +my $key = shift; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --server", -exitval => 2 ) + unless $server; +pod2usage( -msg => "missing key", exitval => 2 ) + unless defined $key; + +my $neb = Nebulous::Client->new( + proxy => "$server", +); + +die "can't connected to Nebulous Server: $server" + unless defined $neb; + +my $stat = $neb->stat($key); +die "nebulous key: $key not found" unless $stat; +my $instances = $neb->find_instances($key, 'any'); +die "no instances found" unless $instances; + +my $user_copies; +eval { + $user_copies = $neb->getxattr($key, "user.copies"); +}; +unless(defined($user_copies)) { + $user_copies = 2; +} + +my @validation; +my %md5sum_uniq; +my %md5sum_valid; +my $last_f = ''; +my $existing_copies = 0; +my @existance; +my @md5sums; +my @files = map {URI->new($_)->file if $_} @$instances; + +for (my $i = 0; $i <= $#files; $i++) { + if (-e $files[$i]) { + $existance[$i] = 1; + $existing_copies++; + my $fh; + open($fh,$files[$i]); + $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest; + close($fh); + + $md5sum_uniq{$md5sums[$i]} = 1; + + if ($md5sums[$i] eq 'd41d8cd98f00b204e9800998ecf8427e') { + $existance[$i] = 0; + push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i]; + next; + } + + push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i]; + $last_f = $files[$i]; + } + else { + $existance[$i] = 0; + $md5sums[$i] = 'NON-EXISTANT'; + $md5sum_uniq{$md5sums[$i]} = 1; + push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i]; + } +} + +my $N_mds = scalar(keys(%md5sum_uniq)); +my $N_files = scalar(@files); +if ($N_files < $user_copies) { + print "Insufficient copies ($N_files) of $key, replicating!\n"; + system("neb-replicate $key"); +} +if ($N_mds == 1) { + print "Nebulous key $key is consistent.\n"; +} +else { + my $valid_sum = ''; + my $valid_file = ''; + my $m; + foreach $m (keys %md5sum_valid) { + if (($m ne 'd41d8cd98f00b204e9800998ecf8427e')&&($m ne 'NON-EXISTANT')) { + if ($valid_sum ne '') { + die "Too many acceptable files for key $key\n"; + } + $valid_sum = $m; + $valid_file = ${ $md5sum_valid{$m} }[0]; + } + } + if ($valid_sum eq '') { + die "No acceptable file for key $key\n"; + } + else { + print "Repairing instances\n"; + foreach $m (keys %md5sum_valid) { + if ($m ne $valid_sum) { + foreach my $f (@{ $md5sum_valid{$m} }) { + print "\tcp $valid_file $f\n"; + system("cp $valid_file $f"); + if ($? >> 8 != 0) { + if ($existance[0] == 1) { + # We can fix this with culls and replications. + print "Attempting to fix this.\n"; + print("neb-replicate $key\n"); + system("neb-replicate $key"); + if ($? >> 8 == 0) { + my $bad_volume = (split /\//, $f)[2]; + print("neb-cull --volume $bad_volume $key\n"); + system("neb-cull --volume $bad_volume $key"); + } + else { + die "Failed to copy file $valid_file to $f, and subsequent replication also failed!\n"; + } + } + else { + die "Failed to copy file $valid_file to $f!\n"; + } + } + } + } + } + } +} + +# if ($N_files > $user_copies) { +# my @split = split /\//, $last_f; +# my $volume = $split[2]; +# print "Culling extra copy: neb-cull --volume $volume $key\n"; +# system("neb-cull --volume $volume $key"); +# } + + + +__END__ + +=pod + +=head1 NAME + +neb-repair - tool to check and repair instances + +=head1 SYNOPSIS + + neb-repair [--server ] + +=head1 DESCRIPTION + +This program repairs instances and corrects instance counts for a nebulous key. + +=head1 OPTIONS + +=over 4 + +=item * --server|-s + +URL of the Nebulous server to connect to. + +Optional if the appropriate environment variable is set. + +=back + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --server|-s + +=back + +=head1 CREDITS + + + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Chris Waters + +=head1 COPYRIGHT + +Copyright (C) 2007-2008 Chris Waters + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L, L, +L, C + +=cut Index: /branches/eam_branches/ipp-20110906/PS-IPP-Config/lib/PS/IPP/Config.pm =================================================================== --- /branches/eam_branches/ipp-20110906/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 32629) +++ /branches/eam_branches/ipp-20110906/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 32630) @@ -20,4 +20,5 @@ use Getopt::Long 2.33 qw( GetOptions :config gnu_getopt pass_through ); # Set pass_through so we don't kill @ARGV use IPC::Cmd 0.36 qw( can_run run ); +use POSIX qw( strftime ); use base qw( Exporter Class::Accessor::Fast ); @@ -692,4 +693,6 @@ } + + # redirect stderr and stdout streams to a file sub redirect_output @@ -722,4 +725,43 @@ return 1; +} + +# redirect stderr and stdout streams to log file. If file already exists rename it and record +# the name in the new log file. +sub redirect_to_logfile { + my $self = shift; + my $name = shift; + + my $tstring = strftime "%Y%m%dT%H%M%S", gmtime; + + my $old_name; + my $scheme = file_scheme($name); + if ($scheme and $scheme eq 'neb') { + my $neb = $self->nebulous; + if ($neb->storage_object_exists($name)) { + eval { + $old_name = "$name.$tstring"; + $neb->move($name, $old_name); + }; + if ($@) { + carp "nebulous move failed for $name"; + return undef; + } + } + } else { + if ($self->file_exists($name)) { + my $resolved = $self->file_resolve($name); + $old_name = "$resolved.$tstring"; + if (!rename($resolved, $old_name)) { + carp "Failed to rename $resolved $old_name"; + return undef; + } + } + } + my $result = $self->redirect_output($name); + if ($result and $old_name) { + print "\nPrevious log file moved to $old_name\n"; + } + return $result; } Index: /branches/eam_branches/ipp-20110906/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/ipp-20110906/dbconfig/changes.txt (revision 32629) +++ /branches/eam_branches/ipp-20110906/dbconfig/changes.txt (revision 32630) @@ -2135,2 +2135,44 @@ alter table addRun add key(stage), add key (stage_id); + +CREATE TABLE vpRun ( + vp_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + workdir VARCHAR(255), + note VARCHAR(255), + dest_id BIGINT, + outroot VARCHAR(255), + dtime_script FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL DEFAULT 0, + PRIMARY KEY(vp_id), + KEY(exp_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(fault), + INDEX(vp_id, exp_id), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE vpProcessedCell ( + vp_id BIGINT, + class_id VARCHAR(64), + cell_id VARCHAR(64), + dtime_photom FLOAT, + quality SMALLINT NOT NULL DEFAULT 0, + path_base VARCHAR(255), + fault SMALLINT NOT NULL DEFAULT 0, + KEY(quality), + KEY(fault), + FOREIGN KEY(vp_id) REFERENCES vpRun(vp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- changes to prevent repeatedly queueing diffRuns for warps with no overlaps +ALTER TABLE diffInputSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY(diff_skyfile_id); +ALTER TABLE diffInputSkyfile CHANGE COLUMN skycell_id skycell_id VARCHAR(64) DEFAULT NULL; + +UPDATE dbversion set schema_version = '1.1.71', updated= CURRENT_TIMESTAMP(); Index: /branches/eam_branches/ipp-20110906/dbconfig/ipp.m4 =================================================================== --- /branches/eam_branches/ipp-20110906/dbconfig/ipp.m4 (revision 32629) +++ /branches/eam_branches/ipp-20110906/dbconfig/ipp.m4 (revision 32630) @@ -36,2 +36,3 @@ include(diffphot.md) include(lap.md) +include(vp.md) Index: /branches/eam_branches/ipp-20110906/dbconfig/vp.md =================================================================== --- /branches/eam_branches/ipp-20110906/dbconfig/vp.md (revision 32630) +++ /branches/eam_branches/ipp-20110906/dbconfig/vp.md (revision 32630) @@ -0,0 +1,24 @@ +vpRun METADATA + vp_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 64 # Key INDEX(vp_id, exp_id) fkey (exp_id) ref rawExp(exp_id) + state STR 64 # Key + label STR 64 # Key + data_group STR 64 # Key + workdir STR 255 + note STR 255 + dest_id S64 0 + outroot STR 255 + dtime_script F32 0.0 + hostname STR 64 + fault S16 0 # Key NOT NULL +END + +vpProcessedCell METADATA + vp_id S64 0 # Primary Key fkey (vp_id) ref vpRun(vp_id) + class_id STR 64 # Primary Key + cell_id STR 64 # Primary Key + dtime_photom F32 0.0 + quality S16 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END Index: /branches/eam_branches/ipp-20110906/ippMonitor/INSTALL =================================================================== --- /branches/eam_branches/ipp-20110906/ippMonitor/INSTALL (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippMonitor/INSTALL (revision 32630) @@ -14,5 +14,5 @@ # note that we have a special version for ippmonitor without $PSCONFIG/$PSCONFDIR values: -ipp@ippdb01 / ippdb01 example: +ipp@ippdb03 / ippdb03 example: psconfigure --dbhost ippdb01 --dbuser ipp --dbpass ipp --htdocs /var/www/localhost/htdocs --site ~/ippconfig/ippmonitor.config Index: /branches/eam_branches/ipp-20110906/ippMonitor/czartool/czartool/CzarDb.pm =================================================================== --- /branches/eam_branches/ipp-20110906/ippMonitor/czartool/czartool/CzarDb.pm (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippMonitor/czartool/czartool/CzarDb.pm (revision 32630) @@ -682,5 +682,5 @@ # added, and store value to subtract from linear value below. If > 1000 # processed per hour, then it must be wrong - if (($thisProcessed - $lastProcessed)/($timeSep/3600) > 1000) { + if (($thisProcessed - $lastProcessed)/(($timeSep/3600)+1) > 1000) { $positiveJump = $thisProcessed - $lastProcessed; } @@ -702,7 +702,7 @@ # calculate first derivative in units of images per hour - $derivPending = $diffPending/($timeSep/3600); - $derivFaults = $diffFaults/($timeSep/3600); - $derivProcessed = $diffProcessed/($timeSep/3600); + $derivPending = $diffPending/($timeSep/3600 +1); + $derivFaults = $diffFaults/($timeSep/3600 +1); + $derivProcessed = $diffProcessed/($timeSep/3600 +1); } Index: /branches/eam_branches/ipp-20110906/ippMonitor/czartool/czartool/Gpc1Db.pm =================================================================== --- /branches/eam_branches/ipp-20110906/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 32630) @@ -233,4 +233,14 @@ } + # 'new' stuff really includes 'update' stuff, at least for LAP. This is a hack, but there you go + elsif ($state eq "new") { + + $query = $self->{_db}->prepare(<Documentationhere"; echo "Cluster loadhere"; -echo "Who's using the cluster?here"; +echo "Who's using the cluster?here"; echo "IPP metricshere"; echo "Czar log pageshere"; @@ -200,5 +200,5 @@ echo ""; createTableColumnHeader("Database"); - createTableColumnHeader("Status"); + createTableColumnHeader("Status (sec behind master)"); showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1); showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS); @@ -903,5 +903,5 @@ //die("MySQL DB connection error: Check the configuration for $replDatabaseName"); //die(""); - return; + return -1; } @@ -910,5 +910,6 @@ # Have a look for Last_Errno in http://dev.mysql.com/doc/refman/5.0/en/show-slave-status.html $errorStatusInMySql = $row[18]; - $replStatus = ($errorStatusInMySql==0?"OK":"PROBLEM"); + $sec_behind = $row[32]; + $replStatus = ($errorStatusInMySql==0?"OK ($sec_behind)":"PROBLEM"); echo "$replDatabaseName$replStatus"; if ($errorStatusInMySql!=0) { Index: /branches/eam_branches/ipp-20110906/ippMonitor/raw/warpProcessedExp.php =================================================================== --- /branches/eam_branches/ipp-20110906/ippMonitor/raw/warpProcessedExp.php (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippMonitor/raw/warpProcessedExp.php (revision 32630) @@ -117,5 +117,5 @@ // query the database if (basic == "basic") { - $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage') FROM fakeRun,camRun,chipRun,rawExp,warpRun LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE LIMIT $dTABLE OFFSET $rowStart"; + $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage') FROM warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) JOIN warpSummary USING(warp_id) $WHERE LIMIT $dTABLE OFFSET $rowStart"; } Index: /branches/eam_branches/ipp-20110906/ippMonitor/raw/warpProcessedExp_Images.php =================================================================== --- /branches/eam_branches/ipp-20110906/ippMonitor/raw/warpProcessedExp_Images.php (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippMonitor/raw/warpProcessedExp_Images.php (revision 32630) @@ -77,12 +77,10 @@ if ($restricted == 0) { - if ("none" != "none") { - $WHERE = "$WHERE none"; - } + $WHERE = "WHERE 0=1"; } // get the result table count if (basic == "basic") { - $sql = "SELECT count(*) FROM fakeRun,camRun,chipRun,rawExp,warpRun LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE"; + $sql = "SELECT count(*) FROM rawExp JOIN chipRun USING(exp_id) JOIN camRun USING (chip_id) JOIN fakeRun USING(cam_id) JOIN warpRun USING(fake_id) LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE"; } @@ -111,5 +109,5 @@ // query the database if (basic == "basic") { - $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage') FROM fakeRun,camRun,chipRun,rawExp,warpRun LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE LIMIT $dTABLE OFFSET $rowStart"; + $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage'),warpRun.tess_id FROM rawExp JOIN chipRun USING(exp_id) JOIN camRun USING (chip_id) JOIN fakeRun USING(cam_id) JOIN warpRun USING(fake_id) LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE LIMIT $dTABLE OFFSET $rowStart"; } @@ -175,4 +173,5 @@ $buttonLink = button_restrict_string ('warpSummary.path_base', $buttonLink); $buttonLink = button_restrict_string ('warpSummary.projection_cell', $buttonLink); +$buttonLink = button_restrict_string ('warpRun.tess_id', $buttonLink); navigate_buttons ($rowStart, $rowLast, $dTABLE, $rowTotal, $buttonLink, $ID, 'warpProcessedExp_Images.php'); @@ -210,4 +209,5 @@ write_header_cell ("list", "stdev "); write_header_cell ("list", "Comment"); +write_header_cell ("list", "TessId"); echo "\n"; echo "\n"; @@ -236,4 +236,5 @@ write_sort_cell ("list", "rawExp.bg_stdev", $buttonLink, $ID, 'warpProcessedExp_Images.php'); write_sort_cell ("list", "rawExp.comment", $buttonLink, $ID, 'warpProcessedExp_Images.php'); +write_sort_cell ("list", "warpRun.tess_id", $buttonLink, $ID, 'warpProcessedExp_Images.php'); echo "\n"; // echo "\n"; @@ -268,4 +269,5 @@ write_query_row ('rawExp.bg_stdev', 5, 'min'); write_query_row ('rawExp.comment', 65, 'string'); +write_query_row ('warpRun.tess_id', 65, 'string'); echo "≤\n"; echo "   \n"; @@ -292,4 +294,5 @@ write_query_row ('rawExp.bg_stdev', 5, 'max'); echo "   \n"; +echo "   \n"; echo "\n"; // echo "\n"; @@ -312,13 +315,14 @@ //Identify current warpId $warpId = $row[5]; -/* echo "Number of elements in warpIdCollection: ".count($warpIdCollection)."\n"; */ -/* echo "(warpId=$warpId)\n"; */ + $tessId = $row[24]; + /* echo "Number of elements in warpIdCollection: ".count($warpIdCollection)."\n"; */ + /* echo "(warpId=$warpId)\n"; */ if (!isSet($warpIdCollection[$warpId])) { -/* echo "$warpId not in warpIdCollection"; */ + /* echo "$warpId not in warpIdCollection"; */ $warpIdCollection[$warpId] = array(); $skyCellValue = preg_replace("|skycell[.]|","",$row[23]); $warpIdCollection[$warpId]["12"] = $skyCellValue; -/* echo "Skycell is [$skyCellValue]"; */ + /* echo "Skycell is [$skyCellValue]"; */ echo "\n"; @@ -344,10 +348,10 @@ echo "\n"; echo "\n"; -/* echo " \n"; */ + /* echo " \n"; */ echo " \n"; echo " \n"; echo " \n"; echo "\n"; -/* echo " \n"; */ + /* echo " \n"; */ echo " \n"; echo " \n"; @@ -364,10 +368,10 @@ echo "
$noImageValue$noImageValue$noImageValue$noImageValue
$noImageValue$noImageValue$noImageValue\n"; echo "\n"; -/* echo " \n"; */ + /* echo " \n"; */ echo " \n"; echo " \n"; echo " \n"; echo "\n"; -/* echo " \n"; */ + /* echo " \n"; */ echo " \n"; echo " \n"; @@ -376,9 +380,18 @@ echo "\n"; - $cellContent = " "; - echo ""; - $imageNameValue = ($row[23]=="noimage" ? - "No image" : $row[23]); - echo ""; + if ($tessId != "RINGS.V3") { + $cellContent = " "; + echo ""; + $imageNameValue = ($row[23]=="noimage" ? + "No image" : $row[23]); + echo ""; + } else { + //The cell to update is the upper right one for new tessellation + $cellContent = " "; + echo ""; + $imageNameValue = ($row[23]=="noimage" ? + "No image" : $row[23]); + echo ""; + } //resume with "regular" columns @@ -399,4 +412,5 @@ write_table_cell ($class, '%.2f', "", $row[20]); write_table_cell ($class, '%s', "", $row[21]); + write_table_cell ($class, '%s', "", $row[24]); echo "\n"; @@ -415,36 +429,77 @@ $cellContent = " "; - if ($skyCellValue == $lrValue + 1) { - $warpIdCollection[$warpId]["11"] = $skyCellValue; - echo ""; - echo ""; - } else { - if ($skyCellValue == $lrValue + 2) { - $warpIdCollection[$warpId]["10"] = $skyCellValue; - echo ""; - echo ""; + if ($tessId != "RINGS.V3") { + if ($skyCellValue == $lrValue + 1) { + $warpIdCollection[$warpId]["11"] = $skyCellValue; + echo ""; + echo ""; } else { - if (!isSet($warpIdCollection[$warpId]["02"])) { - /* echo ""; */ - $warpIdCollection[$warpId]["02"] = $skyCellValue; - echo ""; - echo ""; - } else { - $urValue = $warpIdCollection[$warpId]["02"]; - if ($urValue + 1 == $skyCellValue) { - echo ""; - echo ""; + if ($skyCellValue == $lrValue + 2) { + $warpIdCollection[$warpId]["10"] = $skyCellValue; + echo ""; + echo ""; + } else { + if (!isSet($warpIdCollection[$warpId]["02"])) { + /* echo ""; */ + $warpIdCollection[$warpId]["02"] = $skyCellValue; + echo ""; + echo ""; } else { - if ($urValue + 2 == $skyCellValue) { - echo ""; - echo ""; + $urValue = $warpIdCollection[$warpId]["02"]; + if ($urValue + 1 == $skyCellValue) { + echo ""; + echo ""; } else { - $expected = $urValue + 1; - $actual = $skyCellValue; - echo ""; - echo ""; - } - } + if ($urValue + 2 == $skyCellValue) { + echo ""; + echo ""; + } else { + $expected = $urValue + 1; + $actual = $skyCellValue; + echo ""; + echo ""; + } + } + } } + } + } else { /* New tessellation */ + + /* echo "

$lrValue - $skyCellValue

"; */ + + if ($skyCellValue == $lrValue + 1) { + $warpIdCollection[$warpId]["11"] = $skyCellValue; + echo ""; + echo ""; + } else { + if ($skyCellValue == $lrValue + 2) { + $warpIdCollection[$warpId]["10"] = $skyCellValue; + echo ""; + echo ""; + } else { + if (!isSet($warpIdCollection[$warpId]["02"])) { + /* echo ""; */ + $warpIdCollection[$warpId]["02"] = $skyCellValue; + echo ""; + echo ""; + } else { + $urValue = $warpIdCollection[$warpId]["02"]; + if ($urValue + 1 == $skyCellValue) { + echo ""; + echo ""; + } else { + if ($urValue + 2 == $skyCellValue) { + echo ""; + echo ""; + } else { + $expected = $urValue + 1; + $actual = $skyCellValue; + echo ""; + echo ""; + } + } + } + } + } } Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/camera_exp.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/camera_exp.pl (revision 32630) @@ -88,5 +88,5 @@ if ($redirect) { - $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $cam_id, $PS_EXIT_SYS_ERROR ); + $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $cam_id, $PS_EXIT_SYS_ERROR ); print "\n\n"; print "Starting script $0 on $host\n\n"; @@ -404,4 +404,18 @@ $ipprc->file_exists($file); + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $cam_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $cam_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + + my $scheme = file_scheme($file); if ($replicate and $scheme and (file_scheme($file) eq 'neb')) { Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/chip_imfile.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/chip_imfile.pl (revision 32630) @@ -34,4 +34,6 @@ my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1); + if ($missing_tools) { warn("Can't find required tools."); @@ -44,6 +46,4 @@ my ( $exp_id, $chip_id, $class_id, $chip_imfile_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose, $no_update, $save_temps, $no_op, $redirect, $magicked, $deburned ); - -my $zaplog = 0; GetOptions( @@ -65,5 +65,4 @@ 'no-op' => \$no_op, # Don't do any operations? 'redirect-output' => \$redirect, - 'zaplog' => \$zaplog, 'save-temps' => \$save_temps, # Save temporary files? ) or pod2usage( 2 ); @@ -92,15 +91,18 @@ } -my ($logDest, $traceDest); +my ($logRule, $traceDest); if ($run_state eq 'new') { - $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, $zaplog); + $logRule = "LOG.IMFILE"; $traceDest = prepare_output("TRACE.IMFILE", $outroot, $class_id, 1); } else { - $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1); + $logRule = "LOG.IMFILE.UPDATE"; $traceDest = prepare_output("TRACE.IMFILE.UPDATE", $outroot, $class_id, 1); } if ($redirect) { - $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR ); + my $logDest = $ipprc->filename($logRule, $outroot, $class_id); + + $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR ); + print STDOUT "\n\n"; print STDOUT "Starting script $0 on $host\n\n"; @@ -330,4 +332,14 @@ # get the UNIX version of the (possible) neb: or path: filename my $uriReal = $ipprc->file_resolve( $uri ); + # Catch errors here + if (!$uriReal) { + my $repair_cmd = "$nebrepair $uri"; + my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose); + unless ($repair_success) { + &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + $uriReal = $ipprc->file_resolve( $uri ); + } + &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal; @@ -336,4 +348,13 @@ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose); unless ($success) { + # Catch errors here + my $repair_cmd = "$nebrepair $uri"; + my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose); + unless ($repair_success) { + &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose); + } + unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); @@ -348,5 +369,14 @@ $burntoolTable_uri =~ s/fits$/burn.tbl/; $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri ); + if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) { + my $repair_cmd = "$nebrepair $burntoolTable_uri"; + my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose); + unless ($repair_success) { + &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri ); + } unless ($ipprc->file_exists($burntoolTable_uri)) { + # Catch errors here &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR); } @@ -451,5 +481,5 @@ $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id; $command .= " -source_id $source_id" if defined $source_id; - $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -tracedest $traceDest"; $command .= " -dbname $dbname" if defined $dbname; @@ -727,4 +757,16 @@ &my_die("Couldn't find expected output file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); } + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + } + ##### if ($replicate and $neb) { Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/diff_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/diff_skycell.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/diff_skycell.pl (revision 32630) @@ -40,5 +40,4 @@ my ($skycell_id, $diff_skyfile_id); my ($use_convolved); -my $zaplog = 0; GetOptions( @@ -57,5 +56,4 @@ 'no-op' => \$no_op, # Don't do any operations? 'redirect-output' => \$redirect, - 'zaplog' => \$zaplog, # delete any previous log file ) or pod2usage( 2 ); @@ -79,15 +77,9 @@ # XXX camera is not known here; cannot use filerules... -# bills 2011-03-07 This is unfortunate because it prevents us from using prepare_output ... -# my $logDest = $ipprc->filename("LOG.EXP", $outroot); my $logDest = "$outroot.log"; if ($run_state eq 'update') { $logDest .= '.update'; - # ... well we can just do the dirty work here - $ipprc->kill_file($logDest); -} elsif ($zaplog) { - $ipprc->kill_file($logDest); -} -$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; +} +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF); @@ -468,4 +460,18 @@ &my_die("Couldn't find expected output file: $file", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + + if ($replicate and $neb) { $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR); Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/dist_bundle.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/dist_bundle.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/dist_bundle.pl (revision 32630) @@ -636,4 +636,8 @@ $file{file_rule} = $key; $file{name} = $val; + if ($val eq "STDERR" or $val eq "STDOUT") { + print STDERR "Skipping $key because filename is $val\n"; + next; + } push @file_list, \%file; } Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/lap_science.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/lap_science.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/lap_science.pl (revision 32630) @@ -27,4 +27,5 @@ my ( $camera, $dbname); my ( $lap_id ); +my ( $queue_list ); my ( $chip_mode, $monitor_mode, $cleanup_mode); @@ -44,4 +45,5 @@ 'lap_id=s' => \$lap_id, + 'queue_list=s' => \$queue_list, 'chip_mode' => \$chip_mode, @@ -200,4 +202,122 @@ return($exposure); } + +sub remake_this_exposure_by_update { + my $exposure = shift; + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + + my $label = $exposure->{label}; + + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $chiptool_info_cmd = "chiptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} "; + $chiptool_info_cmd .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chiptool_info_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -listrun: $error_code", $exposure->{exp_id}, $data_group); + } + my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from chiptool -listrun", $exposure->{exp_id}, $data_group); + # There should be only one. + my $chip = ${ $chips }[0]; + my $chip_magicDS_id = 0; + if ($chip) { + $chip_magicDS_id = $chip->{magic_ds_id}; + } + if ($chip_magicDS_id == 0) { +# return(&remake_this_exposure($exposure)); + } + + my $warptool_info_cmd = "warptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} "; + $warptool_info_cmd .= " -dbname $dbname " if defined $dbname; + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $warptool_info_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -listrun: $error_code", $exposure->{exp_id}, $data_group); + } + my $warps = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from warptool -listrun", $exposure->{exp_id}, $data_group); + # There should be only one. + my $warp = ${ $warps }[0]; + my $warp_id = 0; + if ($warp) { + $warp_id = $warp->{warp_id}; + } + else { + $exposure->{data_state} = 'drop'; + return($exposure); + } + if ($warp_id == 0) { + # Handle this correctly. Probably want to drop the exposure here. + $exposure->{data_state} = 'drop'; + return($exposure); + } + + if (($chip->{state} eq 'goto_cleaned')|| + ($warp->{state} eq 'goto_cleaned')|| + ($chip->{dsRun_state} eq 'goto_cleaned')) { + $exposure->{data_state} = 'pending_update'; + &update_this_exposure($exposure); + return($exposure); + } + + + my $chiptool_update_cmd = "chiptool -setimfiletoupdate -chip_id $exposure->{chip_id} -set_label $label"; + $chiptool_update_cmd .= " -dbname $dbname " if defined $dbname; + my $magicDS_update_cmd = "magicdstool -setfiletoupdate -magic_ds_id $chip_magicDS_id -set_label $label"; + $magicDS_update_cmd .= " -dbname $dbname " if defined $dbname; + my $warptool_update_cmd = "warptool -setskyfiletoupdate -warp_id $warp_id -set_label $label"; + $warptool_update_cmd .= " -dbname $dbname " if defined $dbname; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chiptool_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } + if ($chip_magicDS_id != 0) { + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $magicDS_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } + } + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $warptool_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } + + $exposure->{active} = 1; + return($exposure); +} + +sub determine_if_can_update { + my $exposure = shift; + + if (S64_IS_NOT_NULL($exposure->{chip_id})) { + if (($exposure->{chip_state} eq 'cleaned')|| + ($exposure->{chip_state} eq 'goto_cleaned')|| + ($exposure->{chip_state} eq 'error_cleaned')) { + return(1); + } + } + + return(0); +} + # This is the "user level" subroutine. @@ -240,5 +360,7 @@ my $comment = $exposure->{comment}; - # This is a hack to fix old exposures that have no object. + my $updateable = determine_if_can_update($exposure); + + # This is a hack to fix old exposures that have no usable object/comment data. unless(defined($comment)) { $comment = ''; @@ -247,22 +369,41 @@ if ($comment =~ /3pi_/) { $object = $comment; - $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/; + $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/F1 $1/; } elsif ($comment =~ / ps1_/) { $object = $comment; - $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/; - } - else { - $exposure->{data_state} = 'new'; + $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/F2 $1/; + } + elsif ($comment =~ /ThreePi.*3pi_/) { + $object = $comment; + $object =~ s/^ThreePi . (\S+? 3pi_\d\d_\d\d\d\d) .*$/F3 $1/; + } + elsif ($comment =~ /ThreePi.*ps1_/) { + $object = $comment; + $object =~ s/^ThreePi . (\S+? ps1_\d\d_\d\d\d\d) .*$/F4 $1/; + } + elsif ($comment =~ /ThreePi /) { + $object = $comment; + $object =~ s/^ThreePi . (\S+? \d\d\d\d) .*$/F5 $1/; + } + elsif (($comment =~ /focus/i)|| + ($comment =~ /test/i)|| + ($comment =~ /hyster/i)|| + ($comment =~ /dither/i)|| + ($comment =~ /camera/i) + ){ + # This is junk that shouldn't exist. + $object = 'DROP'; + $exposure->{data_state} = 'drop'; $exposure->{pairwise} = 0; - $exposure->{private} = 1; - $exposure->{pair_id} = 9223372036854775807; - update_this_exposure($exposure); - $exposure = remake_this_exposure($exposure); - $counter++; + $exposure->{pair_id} = 9223372036854775807; + $exposure->{active} = 0; + &update_this_exposure($exposure); + $counter++; # To ensure everyone else is consistent next; } } + # Determine the current state of chipRuns for these exposures, and update/remake as needed. if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id if (($pairwise) && !($pair_id)) { @@ -270,18 +411,25 @@ &my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id); } + $exposure->{data_state} = 'exists'; + if ($updateable) { # We know about this run, but the data needs to be regenerated. + $exposure->{data_state} = 'update'; + $exposure = remake_this_exposure_by_update($exposure); + } + &update_this_exposure($exposure); $matching{$object}{$comment} = $exp_id; $indexing{$exp_id} = $counter; $counter++; - next; } else { # We do not already have a chip_id. # Make a chipRun, and update the exposure. + $exposure->{data_state} = 'run'; $exposure = remake_this_exposure($exposure); - # Save our information for diff pairing. + &update_this_exposure($exposure); $matching{$object}{$comment} = $exp_id; $indexing{$exp_id} = $counter; $counter++; } + if ($verbose) { print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; @@ -296,5 +444,5 @@ print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n"; } - @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; + @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; # This is effectively a sort by dateobs. if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering @@ -309,14 +457,17 @@ my $exp_B = ${ $exposures }[$indexing{$exp_id_B}]; print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n"; - $exp_A->{pairwise} = 1; - $exp_A->{private} = 0; - $exp_A->{pair_id} = $exp_B->{chip_id}; - - $exp_B->{pairwise} = 1; - $exp_B->{private} = 0; - $exp_B->{pair_id} = $exp_A->{chip_id}; - - if ($verbose) { - print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n"; + if ($exp_A->{diff_id} == $exp_B->{diff_id}) { + $exp_A->{pairwise} = 1; + $exp_A->{pair_id} = $exp_B->{chip_id}; + + $exp_B->{pairwise} = 1; + $exp_B->{pair_id} = $exp_A->{chip_id}; + if ($verbose) { + print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n"; + } + } + else { + $exp_A->{pairwise} = 0; + $exp_B->{pairwise} = 0; } } @@ -331,8 +482,5 @@ $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. } - if (!($exposure->{pairwise})) { - $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private - } - + update_this_exposure($exposure); } @@ -457,6 +605,6 @@ # Things I want to know before I'm through # my $needs_qstack = 0; - my $needs_something_remade = 0; - my $needs_something_private = 0; + my $needs_something_remade = 0; # I never need something remade, because we're committed to one-exp=one-chip + my $needs_something_private = 0; # I never need something private, as that is calculated by the difference engine. # my $can_qstack = 0; my $have_diff = 0; @@ -480,5 +628,4 @@ else { # We claimed to be pairwise, but do not have a valid pair_id. $exposure->{pairwise} = 0; - $exposure->{private} = 1; &update_this_exposure($exposure); } @@ -487,22 +634,25 @@ if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. $are_warped++; -# $can_qstack ++; $can_diff ++; $have_diff ++; $are_magicked ++; -# $can_fstack ++; + next; + } + + if ($exposure->{data_state} eq 'pending_update') { + $exposure->{data_state} = 'update'; + $exposure = &remake_this_exposure_by_update($exposure); + &update_this_exposure($exposure); next; } if ($exposure->{private}) { # I've declared this exposure private to this lapRun. -# $needs_qstack++; push @lonely_exposures, $exposure; -# $lonely_exposures++; } - if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun - $needs_something_remade = 1; - $exposure = remake_this_exposure($exposure); - } +# if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun +# $needs_something_remade = 1; +# $exposure = remake_this_exposure($exposure); +# } # Do quality checks here my $is_bad_quality = 0; @@ -536,18 +686,14 @@ ($exposure->{diffRun_state} eq 'full')) { $are_warped++; -# $needs_qstack++; - $needs_something_private = 1; if ($companion) { - $companion->{private} = 1; $companion->{pairwise} = 0; &update_this_exposure($companion); - push @lonely_exposures, $companion; -# $lonely_exposures++; - } - $exposure->{private} = 1; +# push @lonely_exposures, $companion; + } $exposure->{pairwise} = 0; } $exposure->{data_state} = 'drop'; &update_this_exposure($exposure); + next; } @@ -558,13 +704,19 @@ # $can_qstack ++; $can_diff ++; - } - if (($exposure->{magicked}&& - &S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. - $are_magicked++; -# $can_fstack ++; + $exposure->{data_state} = 'to_diff'; } if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { $have_diff ++; - } + $exposure->{data_state} = 'to_magic'; + } + if (($exposure->{magicked})&& + ($exposure->{warpRun_state})&& + ($exposure->{warpRun_state} eq 'full')&& + (&S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. + $are_magicked++; + $exposure->{data_state} = 'full'; +# $can_fstack ++; + } + &update_this_exposure($exposure); } @@ -575,13 +727,10 @@ print "STATUS: HAVE_FSTACK: $have_fstack\n"; -# print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; print "STATUS: NEEDS_REMADE: $needs_something_remade\n"; print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n"; print "STATUS: ARE WARPED: $are_warped\n"; print "STATUS: ARE MAGICKED: $are_magicked\n"; -# print "STATUS: CAN_QSTACK: $can_qstack\n"; print "STATUS: CAN_DIFF: $can_diff\n"; print "STATUS: HAVE_DIFF: $have_diff\n"; -# print "STATUS: CAN_FSTACK: $can_fstack\n"; print "STATUS: TOTAL_EXPOSURES: $total_exposures\n"; @@ -924,5 +1073,5 @@ next; } - if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # Not sure how this would happen, but still. ## This happens when we inherit a complete exposure. + if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # This happens when we inherit a complete exposure. next; } @@ -964,9 +1113,21 @@ } else { # warp-qstack + # We need to decide if we can make a warp-stack diff: + if (&can_warp_stack_diff_be_made($exposure)) { # next; # $command .= '-pretend'; - $command .= " -definewarpstack -available -good_frac 0.2 "; - $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} "; - $already_queued{$exposure->{warp_id}} = 1; + $command .= " -definewarpstack -available -good_frac 0.2 "; + $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} "; + $already_queued{$exposure->{warp_id}} = 1; + $exposure->{private} = 0; + &update_this_exposure($exposure); + } + else { + $exposure->{private} = 1; + $exposure->{data_state} = 'drop'; + $already_queued{$exposure->{warp_id}} = 1; + &update_this_exposure($exposure); + next; + } } @@ -1006,10 +1167,52 @@ } +sub can_warp_stack_diff_be_made { + my $exposure = shift; + my $lap_id = $exposure->{lap_id}; + my $warp_id = $exposure->{warp_id}; + + my $command = "$laptool -diffcheck -lap_id $lap_id -warp_id $warp_id"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -diffcheck: $error_code", $lap_id, $warp_id); + } + if (@$stdout_buf == 0) { + return(0); + } + my $skycells = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -diffcheck", $lap_id, $warp_id); + foreach my $skycell (@$skycells) { + unless (defined($skycell->{stack_state})) { + return(0); + } + if ($skycell->{stack_state} ne 'full') { + return(0); + } + if ($skycell->{warp_state} ne 'full') { + return(0); + } + unless (S64_IS_NOT_NULL($skycell->{stack_id})) { + return(0); + } + } + return(1); +} + + # Deactivate all exposures in this run. sub deactivate_exposures { my $exposures = shift; foreach my $exposure (@$exposures) { - $exposure->{active} = 0; - update_this_exposure($exposure); + if ($exposure->{private} == 1) { # This can probably be relaxed since we can update an undestreaked warp. + $exposure->{active} = 1; + } + else { + $exposure->{active} = 0; + } + update_this_exposure($exposure); } } @@ -1022,4 +1225,54 @@ sub cleanup_mode { my $lap_id = shift; + + if (defined($queue_list)) { + my $successful = 1; + open(Q,$queue_list) or ($successful = 0); + if ($successful == 1) { + while() { + chomp; + my $cmd = $_; + my ($projection_cell,$seq_id,$filter,$label); + $projection_cell = $cmd; + $projection_cell =~ s/.*?-projection_cell (skycell.\w+?) .*/$1/; + $seq_id = $cmd; + $seq_id =~ s/.*-seq_id (\d+?) .*/$1/; + $filter = $cmd; + $filter =~ s/.*-filter (\w\.00000) .*/$1/; + if ($filter =~ /\s+?/) { + $filter =~ s/.*-filter (\w\.00000)$/$1/; + } + $label = $cmd; + $label =~ s/.*-label (.+?) .*/$1/; + + my $response; + chomp($response = + `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`); + my $state = (split /\s+/, $response)[5]; + + unless (defined($state)) { + if ($verbose) { + print "Queuing: $cmd\n"; + } + system($cmd); + + my $i = 0; + do { + sleep(5); + chomp($response = + `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`); + $state = (split /\s+/, $response)[5]; + $i++; + unless(defined($state)) { + $successful = 0; + } + } while (($state ne 'run')&&($i < 20)&&($successful)); + last; + } + } + close(Q); + } + } + my $command = "$laptool -inactiveexp -lap_id $lap_id "; $command .= " -dbname $dbname " if defined $dbname; @@ -1043,7 +1296,9 @@ 'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@', 'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@', + 'magictool -dbname @DBNAME@ -updaterun -set_state full -set_label @LABEL@.old -magic_id @MAGIC_ID@', 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@', 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@', 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage diff -stage_id @DIFF_ID@'); + foreach my $exposure (@$exposures) { if ($exposure->{is_in_use}) { @@ -1072,4 +1327,13 @@ if (S64_IS_NOT_NULL($exposure->{diff_id})) { $command =~ s/\@DIFF_ID\@/$exposure->{diff_id}/; + } + else { + next; + } + } + if ($command =~ /\@MAGIC_ID\@/) { + if ((S64_IS_NOT_NULL($exposure->{magicked}))&& + ($exposure->{magicked} > 0)) { + $command =~ s/\@MAGIC_ID\@/$exposure->{magicked}/; } else { @@ -1139,5 +1403,5 @@ &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); } - + return(0); } Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/magic_destreak.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/magic_destreak.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/magic_destreak.pl (revision 32630) @@ -35,4 +35,5 @@ my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -45,4 +46,5 @@ my ($outroot, $recoveryroot, $magicked); my ($replace, $release); +my ($diff_tess_id, $mismatched_tess); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -60,4 +62,6 @@ 'stage_id=s' => \$stage_id, # exp_id, chip_id, warp_id, or diff_id 'component=s' => \$component, # the class_id or skycell_id + 'mismatched_tess'=> \$mismatched_tess, # true if tess_id of input does not match tess_id of the diff used for magic + 'diff_tess_id=s' => \$diff_tess_id, # tess_id of diffRun used to compute the streaks 'uri=s' => \$uri, # uri of the input image 'path_base=s' => \$path_base, # path_base of the input @@ -106,8 +110,8 @@ if (($stage eq "raw") or ($stage eq "chip") or $stage eq "chip_bg") { $class_id = $component; - $skycell_args = " -class_id $component"; + $skycell_args = " -class_id"; } elsif ($stage eq "warp" or $stage eq "warp_bg") { $skycell_id = $component; - $skycell_args = " -skycell_id $component"; + $skycell_args = " -skycell_id"; } elsif ($stage eq "diff") { $skycell_id = $component; @@ -215,41 +219,61 @@ my ($sfh, $skycell_list); if ($skycell_args) { - my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code); - } - - my $getskycells_output = join "", @$stdout_buf; - if ($getskycells_output) { - my $metadata = $mdcParser->parse($getskycells_output) or - &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); - - my $skycells = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); - - ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps); - - foreach my $skycell (@$skycells) { - my $skycell_uri; - if ($skycell->{data_state} eq "full" and $skycell->{quality} = 0) { - $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base}); - } else { - # diff run must have been cleaned up, need to create this skycell file on the fly - if (!defined $temp_dir ) { - $temp_dir = tempdir( CLEANUP => !$save_temps); + my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells"; + $command .= " $skycell_args"; + my @diff_components; + if ($mismatched_tess) { + # tessellation for this skycell does not match the tessellation used for the magic analysis + # used to calculate the streaks file + @diff_components = get_overlaps($path_base, $component, $diff_tess_id); + } else { + @diff_components = ($component); + } + + # hash of uris by skycell. + my %diff_skycells; + foreach my $component (@diff_components) { + my $this_command = "$command $component"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $this_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code); + } + + my $getskycells_output = join "", @$stdout_buf; + if ($getskycells_output) { + my $metadata = $mdcParser->parse($getskycells_output) or + &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); + + my $skycells = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); + + foreach my $skycell (@$skycells) { + my $skycell_uri; + if ($skycell->{data_state} eq "full" and $skycell->{quality} == 0) { + $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base}); + $diff_skycells{$skycell_id} = $skycell_uri; + } else { + # diff run must have been cleaned up, need to create this skycell file on the fly + if (!defined $temp_dir ) { + $temp_dir = tempdir( CLEANUP => !$save_temps); + } + my $skycell_id = $skycell->{skycell_id}; + if (!$diff_skycells{$skycell_id}) { + $skycell_uri = "$temp_dir/$skycell_id"; + $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or + &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); + } + $diff_skycells{$skycell_id} = $skycell_uri; } - my $skycell_id = $skycell->{skycell_id}; - $skycell_uri = "$temp_dir/$skycell_id"; - $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or - &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); } - print $sfh "$skycell_uri\n"; - } - close $sfh - } + } + } + # write the skycell list file + ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps); + foreach my $skycell_id (keys %diff_skycells) { + print $sfh $diff_skycells{$skycell_id} . "\n"; + } + close $sfh } @@ -360,10 +384,5 @@ $command .= " -chip_mask $ch_mask" if defined $ch_mask; $command .= " -weight $weight" if defined $weight; - if ((defined($sources))&&($ipprc->file_exists($sources))) { - $command .= " -sources $sources" if defined $sources; - } - else { - print "Did not add sources because they do not appear to exist. This may be an error.\n"; - } + $command .= " -sources $sources" if defined $sources; $command .= " -skycelllist $skycell_list" if defined $skycell_list; $command .= " -replace" if $replace; @@ -475,5 +494,5 @@ my $statsFlags; -if ($stage ne "camera") { +if (!$no_op and $stage ne "camera") { file_check($statsFile); { @@ -595,4 +614,35 @@ } + # @diff_components = get_overlaps($path_base, $component, $tess_id, $diff_tess_id); +sub get_overlaps { + my $path_base = shift; + my $component = shift; + my $diff_tess_id = shift; + + my $tess_dir = $ipprc->tessellation_catdir( $diff_tess_id ); + my $catdir = $ipprc->convert_filename_absolute($tess_dir); + + my $cmf = $ipprc->file_resolve("$path_base.cmf"); + &my_die("Unable to resolve $path_base.$cmf", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless $cmf; + + my $command = "$dvoImageOverlaps -D CATDIR $catdir -accept-astrom $cmf"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dvoImageOverlaps $error_code", $magic_ds_id, $component, + $error_code); + } + my @list; + my $output = join "\n", @$stdout_buf; + if ($output) { + foreach my $line (split "\n", $output) { + my (undef, undef, $skycell_id) = split " ", $line; + push @list, $skycell_id; + } + } + return @list +} + sub my_die Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/nightly_science.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/nightly_science.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/nightly_science.pl (revision 32630) @@ -1708,4 +1708,5 @@ $cmd .= "-input_label $label -template_label $label "; $cmd .= "-set_workdir $workdir -set_dist_group $new_dist_group -set_data_group $new_data_group "; + $cmd .= " -mintimediff 40000 "; $cmd .= " -simple -set_label $label -filter $filter "; $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/register_exp.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/register_exp.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/register_exp.pl (revision 32630) @@ -54,5 +54,5 @@ my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $PS_EXIT_CONFIG_ERROR ); -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $exp_id, $PS_EXIT_SYS_ERROR ) if $logfile; pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/register_imfile.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/register_imfile.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/register_imfile.pl (revision 32630) @@ -60,5 +60,5 @@ my $ipprc = PS::IPP::Config->new() or my_die_for_add( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile; pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; @@ -404,4 +404,7 @@ } } + if ($exp_name =~ /a$/) { + return(1); + } return(0); } @@ -517,5 +520,5 @@ carp($msg); - if (defined $exp_id && defined $tmp_class_id and not $no_update) { + if (defined $exp_id && defined $class_id and not $no_update) { my $command = "$regtool -updateprocessedimfile"; $command .= " -exp_id $exp_id"; Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/stack_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/stack_skycell.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/stack_skycell.pl (revision 32630) @@ -104,5 +104,5 @@ } -$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $stack_id, $PS_EXIT_SYS_ERROR ) if $redirect; +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $stack_id, $PS_EXIT_SYS_ERROR ) if $redirect; my $neb; @@ -243,4 +243,5 @@ print $listFile "END\n\n"; } +close($listFile); print "I've checked everything: $stack_id\n"; @@ -491,4 +492,17 @@ &my_die("Couldn't find expected output file: $file", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $stack_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $stack_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + if ($replicate and $neb) { $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $stack_id, $PS_EXIT_SYS_ERROR); Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/videophot_process.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/videophot_process.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/videophot_process.pl (revision 32630) @@ -8,7 +8,4 @@ my $host = hostname(); my $date = `date`; -print "\n\n"; -print "Starting script $0 on $host at $date\n\n"; - use DateTime; my $mjd_start = DateTime->now->mjd; # MJD of starting script @@ -25,24 +22,32 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); +use File::Temp qw( tempfile ); +use File::Basename qw( basename ); # Look for programs we need my $missing_tools; my $vptool = can_run('vptool') or (warn "Can't find vptool" and $missing_tools = 1); -my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); my $psvideophot = can_run('psvideophot') or (warn "Can't find psvideophot" and $missing_tools = 1); +my $dumpvideo = can_run('dumpvideo') or (warn "Can't find dumpvideo" and $missing_tools = 1); my $listvideocells = can_run('listvideocells.pl') or (warn "Can't find listvideocells.pl" and $missing_tools = 1); +my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); exit($PS_EXIT_CONFIG_ERROR); } -my ($vp_id, $exp_id, $camera, $outroot, $dbname, $verbose, $no_update, $no_op, $redirect); +my ($vp_id, $camera, $outroot, $dest_id, $dbname, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($product, $ds_dbname, $ds_dbhost); GetOptions( 'vp_id=s' => \$vp_id, - 'exp_id=s' => \$exp_id, 'camera=s' => \$camera, 'outroot=s' => \$outroot, + 'dest_id=s' => \$dest_id, + 'product=s' => \$product, + 'ds_dbname=s' => \$ds_dbname, + 'ds_dbhost=s' => \$ds_dbhost, 'dbname|d=s' => \$dbname, 'verbose' => \$verbose, + 'save-temps' => \$save_temps, 'no-update' => \$no_update, # Don't update the database? 'no-op' => \$no_op, # Don't do any operations? @@ -52,41 +57,50 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --vp_id --exp_id --outroot --camera", + -msg => "Required options: --vp_id --outroot --camera", -exitval => 3, ) unless defined $vp_id - and defined $exp_id and defined $camera and defined $outroot; +pod2usage( + -msg => " --product -ds_dbname and --dsdbhost are required if --dest_id", + -exitval => 3, + ) if ($dest_id and !( defined $product and defined $ds_dbname and defined $ds_dbhost)); + $no_update = 1 if $no_op; - -my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, 'nochip', 'nocell', $PS_EXIT_CONFIG_ERROR ); # IPP configuration +$vptool .= " -dbname $dbname" if $dbname; + +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration my $logDest = $ipprc->filename("LOG.EXP", $outroot); -$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $vp_id, 'nochip', 'nocell', $PS_EXIT_SYS_ERROR ) if $redirect; - -my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF); - -# Get list of components for subtraction +if ($redirect) { + $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $vp_id, $PS_EXIT_SYS_ERROR ); + + print "\n\n"; + print "Starting script $0 on $host at $date\n\n"; +} + +# Get list of chips with pending video cells my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files my $files; { -# my $command = "$vptool -pendingimfile -vp_id $vp_id"; -# $command .= " -dbname $dbname" if defined $dbname; - my $command = "$regtool -processedimfile -video_cells -exp_id $exp_id"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, 'nochip', 'nocell', $error_code); - } - - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR); - $files = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR); -} + my $command = "$vptool -pendingimfile -vp_id $vp_id"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code); + } + + my $output = join "", @$stdout_buf; + if ($output) { + my $metadata = $mdcParser->parse($output) or + &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR); + } +} + foreach my $file (@$files) { @@ -95,5 +109,5 @@ my $resolved = $ipprc->file_resolve($uri); - &my_die("Unable to resolve $uri", $vp_id, $class_id, 'nocell', $PS_EXIT_UNKNOWN_ERROR) unless $resolved; + &my_die("Unable to resolve $uri", $vp_id, $PS_EXIT_UNKNOWN_ERROR) unless $resolved; my $command = "$listvideocells --file $resolved"; @@ -102,17 +116,18 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command: $error_code", $vp_id, $class_id, - 'nocell', $error_code); + &my_die("Unable to perform $command: $error_code", $vp_id, $error_code); } my @video_cells = split "\n", (join "", @$stdout_buf); - &my_die("No video cells found in $uri", $vp_id, $class_id, - 'nocell', $PS_EXIT_UNKNOWN_ERROR) unless scalar @video_cells; + &my_die("No video cells found in $uri", $vp_id, $PS_EXIT_UNKNOWN_ERROR) unless scalar @video_cells; foreach my $cell_id (@video_cells) { - my $command = "$psvideophot $outroot"; + my $path_base = "$outroot.$class_id.$cell_id"; + my $output = $ipprc->filename("PSVIDEOPHOT.OUTPUT", $path_base); + my $command = "$psvideophot $output"; $command .= " -file $uri"; + $command .= " -class_id $class_id"; $command .= " -cell_id $cell_id"; - $command .= " -dbname $dbname" if defined $dbname; + my $vpstart = DateTime->now->mjd; unless ($no_op) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -120,16 +135,29 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppSub: $error_code", $vp_id, $class_id, $cell_id, $error_code); + &my_die("Unable to perform psvideophot: $error_code", $vp_id, $error_code); } - - # check_output($outputStats, 1); } else { print "Not executing: $command\n"; } + # dump the video cell and the video table to a file + unless ($no_op) { + my $output = $path_base . ".fits"; + my $extname = "$cell_id"; + my $command = "$dumpvideo $uri $extname $output -includetable"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dumpvideo: $error_code", $vp_id, $error_code); + } + } else { + print "Not executing: $command\n"; + } + unless ($no_update) { - my $command = "$vptool -addvpcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id"; - $command .= " -path_base $outroot"; - - $command .= " -dbname $dbname" if defined $dbname; + my $command = "$vptool -addprocessedcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id"; + $command .= " -path_base $path_base"; + $command .= " -hostname $host"; + $command .= " -dtime_photom " . ((DateTime->now->mjd - $vpstart) * 86400); my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -138,7 +166,70 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); my $err_message = "Unable to perform vptool -addvpcell"; - &my_die("$err_message: $error_code", $vp_id, $class_id, $cell_id, $error_code); + &my_die("$err_message: $error_code", $vp_id, $error_code); } } + } +} +if ($dest_id) { + my $command = "$vptool -processedcell -vp_id $vp_id"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code); + } + + my $output = join "", @$stdout_buf; + my $cells; + if ($output) { + my $metadata = $mdcParser->parse($output) or + &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR); + $cells = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR); + } + my ($reglist, $reglistName) = tempfile("/tmp/filelist.$vp_id.XXXX", UNLINK => !$save_temps); + + if ($redirect) { + print $reglist "$logDest|||text|\n"; + } + my @exts = qw( vpt fits ); + my @types = qw( table fits ); + foreach my $cell (@$cells) { + my $path_base = $cell->{path_base}; + for (my $i = 0; $i < scalar @exts; $i++) { + my $file = "$path_base.$exts[$i]"; + my $type = $types[$i]; + + print $reglist "$file|||$type|\n"; + } + + } + close $reglist or &my_die("failed to close $reglistName", $vp_id, $PS_EXIT_UNKNOWN_ERROR); + unless ($no_update or $no_op) { + my $fileset = basename($outroot); + my $command = "$dsreg --add $fileset --product $product --type dump --list $reglistName"; + $command .= " --abspath --link"; + $command .= " --dbname $ds_dbname --dbhost $ds_dbhost"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command error_code: $error_code", $vp_id, $error_code); + } + } +} + +unless ($no_update) { + my $command = "$vptool -updaterun -vp_id $vp_id -set_state full"; + $command .= " -set_outroot $outroot"; + $command .= " -set_hostname $host"; + $command .= " -set_dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform vptool -updaterun"; + &my_die("$err_message: $error_code", $vp_id, $error_code); } } @@ -149,7 +240,5 @@ { my $msg = shift; # Warning message on die - my $vp_id = shift; - my $class_id = shift; # chip identifier - my $cell_id = shift; # chip identifier + my $vp_id = shift; # vpRun id my $exit_code = shift; # Exit code to add @@ -157,11 +246,12 @@ warn($msg); - if (defined $vp_id and defined $class_id and not $no_update) { - my $command = "$vptool -vp_id $vp_id -class_id $class_id -cell_id -fault $exit_code"; - $command .= " -addvpcell"; - $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $command .= " -hostname $host" if defined $host; - $command .= " -path_base $outroot" if defined $outroot; - $command .= " -dbname $dbname" if defined $dbname; + if (defined $vp_id and not $no_update) { + my $command = "$vptool -vp_id $vp_id"; + $command .= " -updaterun"; + $command .= " -set_fault $exit_code"; + $command .= " -set_hostname $host" if defined $host; + $command .= " -set_outroot $outroot" if defined $outroot; + $command .= " -set_dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400); + run(command => $command, verbose => $verbose); } @@ -169,9 +259,3 @@ } -END { - my $exit = $?; - system("sync") == 0 or die "failed to execute sync: $!"; - $? = $exit; -} - __END__ Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_overlap.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_overlap.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_overlap.pl (revision 32630) @@ -57,5 +57,5 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $warp_id, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $warp_id, $PS_EXIT_SYS_ERROR ) if $logfile; &my_die("Tessellation identifier not provided: $tess_dir", $warp_id, $PS_EXIT_SYS_ERROR) unless $tess_dir ne "NULL"; Index: /branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_skycell.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_skycell.pl (revision 32630) @@ -38,5 +38,4 @@ my ($warp_id, $skycell_id, $warp_skyfile_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $run_state, $magicked, $verbose, $no_update, $no_op, $redirect, $save_temps); -my $zaplog = 0; GetOptions( @@ -55,5 +54,4 @@ 'no-update' => \$no_update, # Don't update the database? 'no-op' => \$no_op, # Don't do any operations? - 'zaplog' => \$zaplog, # delete previous logfile 'redirect-output' => \$redirect, 'save-temps' => \$save_temps, # Save temporary files? @@ -77,9 +75,9 @@ my $do_stats; if ($run_state eq 'new') { - $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, $zaplog); + $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id); $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1); $do_stats = 1; } elsif ($run_state eq 'update') { - $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 1); + $logDest = $ipprc->filename("LOG.EXP.UPDATE", $outroot, $skycell_id); $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1); } else { @@ -93,5 +91,5 @@ } -$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect; +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect; # Recipes to use based on reduction class @@ -333,5 +331,5 @@ } } - + print "Quality: $quality\n"; unless ($no_update) { # XXX change -tess_id to -tess_dir when db is updated @@ -358,4 +356,19 @@ } } else { + + my $quality = 0; + + unless ($ipprc->file_exists($outputPSF)) { + $quality = 8006; # bad data. + } +# if (!$quality) { +# check_output($outputImage, 0); +# check_output($outputMask, 0); +# check_output($outputWeight, 0); +# check_output($outputSources, 1); +# check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF') ; +# } + print "Quality: $quality\n"; + # $run_state eq 'update' unless ($no_update) { @@ -364,4 +377,5 @@ $command .= " -skycell_id $skycell_id"; $command .= " -set_magicked $magicked" if $magicked; + $command .= " -set_quality $quality" if $quality; $command .= " -dbname $dbname" if defined $dbname; @@ -407,4 +421,17 @@ &my_die("Couldn't find expected output file: $file", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + } + } + ##### if ($replicate and $neb) { Index: /branches/eam_branches/ipp-20110906/ippTasks/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20110906/ippTasks/Makefile.am (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTasks/Makefile.am (revision 32630) @@ -42,5 +42,6 @@ background.pro \ diffphot.pro \ - lap.pro + lap.pro \ + vp.pro other_files = \ Index: /branches/eam_branches/ipp-20110906/ippTasks/destreak.pro =================================================================== --- /branches/eam_branches/ipp-20110906/ippTasks/destreak.pro (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTasks/destreak.pro (revision 32630) @@ -206,5 +206,6 @@ book getword magicToDS $pageName magicked -var MAGICKED book getword magicToDS $pageName dbname -var DBNAME - + book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID + book getword magicToDS $pageName mismatched_tess -var MISMATCHED_TESS substr $COMPONENT 0 3 COMP_HEAD @@ -224,4 +225,8 @@ $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE + if ($MISMATCHED_TESS) + book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID + $run = $run --mismatched_tess --diff_tess_id $DIFF_TESS_ID + end add_standard_args run Index: /branches/eam_branches/ipp-20110906/ippTasks/diskbalance.pro =================================================================== --- /branches/eam_branches/ipp-20110906/ippTasks/diskbalance.pro (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTasks/diskbalance.pro (revision 32630) @@ -15,4 +15,8 @@ book init balanceControl + +book init targetPending +book init targetControl + macro balance.reset book init balancePending @@ -41,4 +45,30 @@ end +macro target.reset + book init targetPending +end + +macro target.status + book listbook targetPending +end + +macro target.on + task target.load + active true + end + task target.run + active true + end +end + +macro target.off + task target.load + active false + end + task target.run + active false + end +end + macro set.host.for.balance if ($0 != 2) @@ -71,4 +101,15 @@ book setword balanceControl control mDestinations $2 end + +macro set.balance.range + if ($0 != 3) + echo "USAGE: set.balance.range (SO_ID_START) (SO_ID_RANGE)" + break + end + + $BALANCE_SO_ID_START = $1 + $BALANCE_SO_ID_RANGE = $2 +end + macro show.balance.numbers @@ -90,4 +131,25 @@ end $BALANCE_TARGET = $1 +end + +macro set.target.range + if ($0 != 3) + echo "USAGE: set.target.range (SO_ID_START) (SO_ID_RANGE)" + break + end + + $TARGET_SO_ID_START = $1 + $TARGET_SO_ID_RANGE = $2 +end + + +macro show.target.numbers + echo "target so_ids: $TARGET_SO_ID_START $TARGET_SO_ID_RANGE" +end + +macro show.targetPending + book npages targetPending -var N + echo "$N pages in book" + book listbook targetPending end @@ -250,5 +312,155 @@ - - - +# This value will need to be incremented periodically. It looks like we grow by about 5e7 so_ids per month. +$TARGET_BIG_SO_ID = 6e8 +$TARGET_SO_ID_START = int($TARGET_BIG_SO_ID * rnd(0)) +$TARGET_SO_ID_RANGE = 500000 +$TARGET_LIMIT = 500 +$TARGET_OFFSET = 0 +$TARGET_TARGET = "SOURCE" +# Select Nebulous objects which should be shuffled +task target.load + host local + + periods -poll 60 + periods -exec 60 + periods -timeout 1500 + npending 1 + + # logs + stdout NULL + stderr $LOGSUBDIR/target.log + + task.exec + book npages targetPending -var N + if ($N > 2000) + process_cleanup targetPending + break + end + + command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingtarget --limit $TARGET_LIMIT --offset $TARGET_OFFSET --so_id_start $TARGET_SO_ID_START --so_id_range $TARGET_SO_ID_RANGE + end + + # success : 0 -- we did not hit the limit, advance so_id counter + task.exit 0 + # advance the so_id counter + $TARGET_SO_ID_START = $TARGET_SO_ID_START + $TARGET_SO_ID_RANGE + $TARGET_OFFSET = 0 + # convert 'stdout' to book format + ipptool2book stdout targetPending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook targetPending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup targetPending + end + + # success : 1 -- we DID hit the limit, do NOT advance so_id counter + task.exit 1 + # convert 'stdout' to book format + ipptool2book stdout targetPending -key key -uniq -setword pantaskState INIT + $TARGET_OFFSET = $TARGET_OFFSET + $TARGET_LIMIT + if ($VERBOSE > 2) + book listbook targetPending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup targetPending + end + + # out of so_id range, reset + task.exit 10 + # advance the so_id counter + $TARGET_SO_ID_START = 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# task to execute shuffles +task target.run + periods -poll 0.5 + periods -exec 5 + periods -timeout 5 + npending 200 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec 5 + + book npages targetPending -var N + if ($NETWORK == 0) break + if ($N == 0) break + + # look for new objects in targetPending + book getpage targetPending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword targetPending $pageName pantaskState RUN + + book getword targetPending $pageName key -var KEY + book getword targetPending $pageName source_name -var SOURCE + book getword targetPending $pageName source_host -var SOURCE_HOST + book getword targetPending $pageName destination_name -var DESTINATION + book getword targetPending $pageName destination_host -var DEST_HOST + book getword targetPending $pageName destination_uri -var DEST_URI + book getword targetPending $pageName ins_id -var INS_ID + + # Fix this with multihost restriction when possible. + if ("$TARGET_TARGET" == "DESTINATION") + set.host.for.target $DEST_HOST + else + set.host.for.target $SOURCE_HOST + end + + + stdout NULL + stderr $LOGSUBDIR/target.log + + $run = neb-insedit --key $KEY --ins_id $INS_ID --uri $DEST_URI --volume $DESTINATION + $run = $run --db $NEB_DB --host $NEB_HOST --user $NEB_USER --pass $NEB_PASS + $run = $run --neb_host http://ippc04/nebulous + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit targetPending $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword targetPending $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword targetPending $options:0 pantaskState TIMEOUT + end +end + + Index: /branches/eam_branches/ipp-20110906/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/eam_branches/ipp-20110906/ippTasks/ipphosts.mhpcc.config (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTasks/ipphosts.mhpcc.config (revision 32630) @@ -95,6 +95,6 @@ camera STR GPC1 - XY01 STR ipp005 - XY02 STR ipp005 + XY01 STR ipp010 + XY02 STR ipp011 XY03 STR ipp006 XY04 STR ipp006 @@ -128,13 +128,13 @@ XY33 STR ipp021 XY34 STR ipp023 - XY35 STR ipp023 + XY35 STR ipp013 XY36 STR ipp024 - XY37 STR ipp024 + XY37 STR ipp019 XY40 STR ipp025 - XY41 STR ipp025 - - XY42 STR ipp026 - XY43 STR ipp026 + XY41 STR ipp018 + + XY42 STR ipp017 + XY43 STR ipp015 XY44 STR ipp027 XY45 STR ipp028 @@ -174,6 +174,6 @@ camera STR gpc1 - ota01 STR ipp005 - ota02 STR ipp005 + ota01 STR ipp010 + ota02 STR ipp011 ota03 STR ipp006 ota04 STR ipp006 @@ -207,13 +207,13 @@ ota33 STR ipp021 ota34 STR ipp023 - ota35 STR ipp023 + ota35 STR ipp013 ota36 STR ipp024 - ota37 STR ipp024 + ota37 STR ipp019 ota40 STR ipp025 - ota41 STR ipp025 - - ota42 STR ipp026 - ota43 STR ipp026 + ota41 STR ipp018 + + ota42 STR ipp017 + ota43 STR ipp015 ota44 STR ipp027 ota45 STR ipp028 @@ -285,6 +285,6 @@ count S32 60 - XY01 STR ipp005 - XY02 STR ipp005 + XY01 STR ipp010 + XY02 STR ipp011 XY03 STR ipp006 XY04 STR ipp006 @@ -318,13 +318,13 @@ XY33 STR ipp021 XY34 STR ipp023 - XY35 STR ipp023 + XY35 STR ipp013 XY36 STR ipp024 - XY37 STR ipp024 + XY37 STR ipp019 XY40 STR ipp025 - XY41 STR ipp025 - - XY42 STR ipp026 - XY43 STR ipp026 + XY41 STR ipp018 + + XY42 STR ipp017 + XY43 STR ipp015 XY44 STR ipp027 XY45 STR ipp028 @@ -441,2 +441,81 @@ sky75 STR ipp053 END + +ipphosts METADATA + camera STR STARE + + ota01 STR ipp010 + ota02 STR ipp010 + ota03 STR ipp010 + ota04 STR ipp010 + + ota05 STR ipp011 + ota06 STR ipp011 + ota10 STR ipp011 + ota11 STR ipp011 + + ota12 STR ipp012 + ota13 STR ipp012 + ota14 STR ipp012 + ota15 STR ipp012 + + ota16 STR ipp013 + ota17 STR ipp013 + ota20 STR ipp013 + ota21 STR ipp013 + + ota22 STR ipp014 + ota23 STR ipp014 + ota24 STR ipp014 + ota25 STR ipp014 + + ota26 STR ipp015 + ota27 STR ipp015 + ota30 STR ipp015 + ota31 STR ipp015 + + ota32 STR ipp016 + ota33 STR ipp016 + ota34 STR ipp016 + ota35 STR ipp016 + + ota36 STR ipp017 + ota37 STR ipp017 + ota40 STR ipp017 + ota41 STR ipp017 + + ota42 STR ipp018 + ota43 STR ipp018 + ota44 STR ipp018 + ota45 STR ipp018 + + ota46 STR ipp019 + ota47 STR ipp019 + ota50 STR ipp019 + ota51 STR ipp019 + + ota52 STR ipp020 + ota53 STR ipp020 + ota54 STR ipp020 + ota55 STR ipp020 + + ota56 STR ipp021 + ota57 STR ipp021 + ota60 STR ipp021 + ota61 STR ipp021 + + ota62 STR ipp010 + ota63 STR ipp011 + ota64 STR ipp012 + ota65 STR ipp013 + + ota66 STR ipp014 + ota67 STR ipp015 + ota71 STR ipp016 + ota72 STR ipp017 + + ota73 STR ipp018 + ota74 STR ipp019 + ota75 STR ipp020 + ota76 STR ipp021 +END Index: /branches/eam_branches/ipp-20110906/ippTasks/lap.pro =================================================================== --- /branches/eam_branches/ipp-20110906/ippTasks/lap.pro (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTasks/lap.pro (revision 32630) @@ -10,4 +10,6 @@ $lap_RunPage = 0 $lap_DonePage = 0 + +$LAP_QUEUE = NULL book init lapNewRuns @@ -135,4 +137,8 @@ +macro lap.define.queue + $LAP_QUEUE = $1 +end + task lap.initial.load host local @@ -388,5 +394,5 @@ stderr $LOGDIR/lap.load.log - $run = laptool -pendingrun -state done + $run = laptool -pendingrun -state full if ($lapSeq:n != 0) @@ -463,4 +469,8 @@ option $LAP_ID $run = lap_science.pl --cleanup_mode --dbname $DBNAME --lap_id $LAP_ID + if ("$LAP_QUEUE" != "NULL") + $run = $run --queue_list $LAP_QUEUE + end + command $run Index: /branches/eam_branches/ipp-20110906/ippTasks/nightly_science.pro =================================================================== --- /branches/eam_branches/ipp-20110906/ippTasks/nightly_science.pro (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTasks/nightly_science.pro (revision 32630) @@ -178,5 +178,5 @@ periods -poll $LOADPOLL periods -exec $LOADEXEC - periods -timeout 30 + periods -timeout 300 trange 0:00:00 1:00:00 -nmax 1 npending 1 @@ -260,5 +260,5 @@ periods -poll 3600 periods -exec $LOADEXEC - periods -timeout 30 + periods -timeout 300 trange 22:00:00 23:59:59 trange 00:00:00 10:00:00 Index: /branches/eam_branches/ipp-20110906/ippTasks/summit.copy.pro =================================================================== --- /branches/eam_branches/ipp-20110906/ippTasks/summit.copy.pro (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTasks/summit.copy.pro (revision 32630) @@ -442,5 +442,12 @@ # this function uses workdir_template, default_host, volume_template, volume_default, # it sets workdir and volume - set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + + # Look to see if we have a stare type observation, and redirect it to an appropriate host + substr $EXP_NAME 10 1 EXPTYPE_KEY + if ("$EXPTYPE_KEY" == "a") + set.workdir.by.camera STARE $CLASS_ID $workdir_template $default_host workdir_base + else + set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + end # figure out filename Index: /branches/eam_branches/ipp-20110906/ippTasks/vp.pro =================================================================== --- /branches/eam_branches/ipp-20110906/ippTasks/vp.pro (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTasks/vp.pro (revision 32630) @@ -0,0 +1,243 @@ +## vp.pro : tasks for video photometry : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/vp +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init vpPendingRun + +### Database lists +$vp_DB = 0 +$vp_revert_DB = 0 + +### Check status of vping tasks +macro vp.status + book listbook vpPendingRun +end + +### Reset vping tasks +macro vp.reset + book init vpPendingRun +end + +### Turn vp tasks on +macro vp.on + task vp.load + active true + end + task vp.run + active true + end + task vp.revert + active true + end +end + +### Turn vp tasks off +macro vp.off + task vp.load + active false + end + task vp.run + active false + end + task vp.revert + active false + end +end + +macro vp.revert.on + task vp.revert + active true + end +end + +macro vp.revert.off + task vp.revert + active false + end +end + + +### Load tasks for doing video photometry +### Tasks are loaded into vpPendingRun. +task vp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/vp.load.log + + task.exec + if ($LABEL:n == 0) break + $run = vptool -pendingrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$vp_DB + $run = $run -dbname $DB:$vp_DB + $vp_DB ++ + if ($vp_DB >= $DB:n) set vp_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout vpPendingRun -key vp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook vpPendingRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup vpPendingRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +### Run video photometery tasks +### Tasks are taken from vpPendingRun. +task vp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + + task.exec + book npages vpPendingRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new runs in vpPendingRun (pantaskState == INIT) + book getpage vpPendingRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword vpPendingRun $pageName pantaskState RUN + book getword vpPendingRun $pageName vp_id -var VP_ID + book getword vpPendingRun $pageName camera -var CAMERA + book getword vpPendingRun $pageName exp_tag -var EXP_TAG + book getword vpPendingRun $pageName workdir -var WORKDIR + book getword vpPendingRun $pageName dest_id -var DEST_ID + book getword vpPendingRun $pageName dest_name -var DEST_NAME + book getword vpPendingRun $pageName ds_dbname -var DS_DBNAME + book getword vpPendingRun $pageName ds_dbhost -var DS_DBHOST + book getword vpPendingRun $pageName dbname -var DBNAME + + # set the host and workdir based on the skycell hash +# set.host.for.skycell $SKYCELL_ID +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf outroot "%s/%s.vp.%d" $WORKDIR $EXP_TAG $VP_ID + + stdout $LOGSUBDIR/vp.log + stderr $LOGSUBDIR/vp.log + + $run = videophot_process.pl --vp_id $VP_ID --outroot $outroot --redirect-output --camera $CAMERA + if ($DEST_ID) + $run = $run --dest_id $DEST_ID --product $DEST_NAME --ds_dbname $DS_DBNAME --ds_dbhost $DS_DBHOST + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit vpPendingRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword vpPendingRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword vpPendingRun $options:0 pantaskState TIMEOUT + end +end + + +task vp.revert + host local + + periods -poll 5 + periods -exec 600 + periods -exec 30 + periods -timeout 120 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/vp.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = vptool -revertrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$vp_revert_DB + $run = $run -dbname $DB:$vp_revert_DB + $vp_revert_DB ++ + if ($vp_revert_DB >= $DB:n) set vp_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/eam_branches/ipp-20110906/ippToPsps/config/IN/tables.vot =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/config/IN/tables.vot (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/config/IN/tables.vot (revision 32630) @@ -22,5 +22,4 @@ - Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/batch.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/batch.py (revision 32630) @@ -39,4 +39,5 @@ ippToPspsDb, id, + batchID, batchType, fits): @@ -53,15 +54,26 @@ self.logger.debug("Batch class constructor") - # check FITS file is ok - TODO could be neater - if self.fits: - self.header = self.fits.getPrimaryHeader() - if not self.header: return - # set up class variables self.id = id + self.batchID = batchID self.gpc1Db = gpc1Db self.ippToPspsDb = ippToPspsDb self.batchType = batchType; self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot" + + # check FITS file is ok + if self.fits: + self.header = self.fits.getPrimaryHeader() + + # now check that the fits header is readable + if not self.header: + logger.errorPair("Could not read FITS for id", "%d" % id) + return + + # if no fits file, and this is not an IN batch, then get out of here + else: + if batchType != "IN": + logger.errorPair("Could not read FITS for id", "%d" % id) + return # get info from config @@ -88,13 +100,5 @@ # create datastore object - self.datastore = Datastore(self.logger, self.doc) - - # create a new batch - self.batchID = self.ippToPspsDb.createNewBatch( - self.batchType, - self.id, - self.survey, - self.dvoGpc1Label, - self.datastore.product) + self.datastore = Datastore(self.logger, self.doc, self.ippToPspsDb) # get local storage location from config @@ -247,5 +251,5 @@ outPath = self.localOutPath + "/BatchManifest.xml" - tmpPath = "./tmp.xml" + tmpPath = self.localOutPath + "/tmp.xml" self.logger.infoPair("Creating manifest", outPath) root = Element('manifest') @@ -263,19 +267,8 @@ else: root.attrib['maxObjId'] = str(self.maxObjID) - # get md5sum - #p = Popen("md5sum " + self.outputFitsPath, shell=True, stdout=PIPE) - #p.wait() - #out = p.stdout.read() - #md5sum = out[0:out.rfind(" ")] - - # get file size - #fileSize = os.path.getsize(self.outputFitsPath) - # file information child = Element('file') root.append(child) child.attrib['name'] = self.outputFitsFile - #child.attrib['bytes'] = str(fileSize) - #child.attrib['md5'] = md5sum # now create doc and write to file @@ -316,5 +309,5 @@ # now publish to the datastore - Batch.publishToDatastore(self.datastore, self.ippToPspsDb, self.batchID, self.batchName, self.subDir, tarballFile) + Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile) ''' @@ -322,11 +315,6 @@ ''' @staticmethod - def publishToDatastore(datastore, ippToPspsDb, batchID, batchName, subDir, tarballFile): - - if datastore.publish(batchName, subDir, tarballFile, "tgz"): - ippToPspsDb.updateLoadedToDatastore(batchID, 1) - else: - ippToPspsDb.updateLoadedToDatastore(batchID, -1) - + def publishToDatastore(datastore, batchID, batchName, subDir, tarballFile): + datastore.publish(batchName, subDir, tarballFile, "tgz") ''' Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/cleanup.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/cleanup.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/cleanup.py (revision 32630) @@ -6,5 +6,5 @@ import re import sys - +import time import logging.config @@ -13,4 +13,5 @@ from batch import Batch from datastore import Datastore +from dxlayer import DXLayer ''' @@ -27,8 +28,8 @@ logger.infoTitle("Cleaning up batch type %s" % batchType) - localIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromLocalDisk(batchType, EPOCH, DVOLABEL) + localIDs = ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, EPOCH, DVOLABEL) datastoreIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, EPOCH, DVOLABEL) dxlayerIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, EPOCH, DVOLABEL) - logger.infoPair("Loaded to ODM but still on local disk", "%d" % len(localIDs)) + logger.infoPair("Merged but still on local disk", "%d" % len(localIDs)) logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(datastoreIDs)) logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(dxlayerIDs)) @@ -53,9 +54,19 @@ if datastore.remove(batchName): - ippToPspsDb.updateDeletedDatastore(id, 1) count = count + 1 logger.infoPair("Removed from datastore", "%d" % count) logger.infoPair("Remaining on datastore", "%d" % (len(datastoreIDs) - count)) + + if DELETEDXLAYER: + count = 0 + for id in dxlayerIDs: + + if dxlayer.deleteBatch(id): + ippToPspsDb.updateDeletedDxlayer(id, 1) + count = count + 1 + + logger.infoPair("Removed from DXLayer", "%d" % count) + logger.infoPair("Remaining on DXLayer", "%d" % (len(dxlayerIDs) - count)) @@ -66,6 +77,12 @@ if len(sys.argv) > 1: CONFIG = sys.argv[1] else: - print "\n** Usage: " + sys.argv[0] + " \n" + print "\n** Usage: " + sys.argv[0] + " [hours]\n" sys.exit(1) + +if len(sys.argv) > 2: + HOURS = float(sys.argv[2]) + SECONDS = HOURS * 60.0 * 60.0 +else: + SECONDS = None # open config file @@ -78,5 +95,6 @@ ippToPspsDb = IppToPspsDb(logger, configDoc) -datastore = Datastore(logger, configDoc) +datastore = Datastore(logger, configDoc, ippToPspsDb) +dxlayer = DXLayer(logger) DVOLABEL = configDoc.find("dvo/gpc1Label").text @@ -88,8 +106,17 @@ logger.infoTitle("ippToPsps clean-up") +if SECONDS: logger.infoPair("Performing clean-up every", "%f hours" % (SECONDS/60.0/60.0)) logger.infoBool("Deleting from local disk?", DELETELOCAL) logger.infoBool("Deleting from datastore?", DELETEDATASTORE) logger.infoBool("Deleting from DXLayer?", DELETEDXLAYER) -clean("P2") -clean("ST") +while True: + + clean("P2") + clean("ST") + + if SECONDS: + logger.infoPair("Waiting for", "%f hours" % HOURS) + time.sleep(SECONDS) + else: break + Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/init.xml =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/init.xml (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/init.xml (revision 32630) @@ -0,0 +1,75 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + BOGUS + BOGUS + + + + + 0 + BOGUS + BOGUS + + + + /data/ipp005.0/rhenders + + + 0 + + + + 2011-07-01-15:34 + 3PI + 3PI + 0 + 0 + 0 + 0 + 0 + 0 + + + + + + Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/oldthreepi.xml =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/oldthreepi.xml (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/oldthreepi.xml (revision 32630) @@ -0,0 +1,86 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + ThreePi.V3 + /data/ipp045.0/eugene/3pi.20110819/catdir.20110819.v1 + 135 + 150 + -100 + 100 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-10-27 + 3PI + OLD + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/sas12.xml =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/sas12.xml (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/sas12.xml (revision 32630) @@ -51,5 +51,5 @@ - 0 + 1 1 1 Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/sas123.xml =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/sas123.xml (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/sas123.xml (revision 32630) @@ -51,5 +51,5 @@ - 0 + 1 1 1 Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/sasFootprint.xml =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/sasFootprint.xml (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/configs/sasFootprint.xml (revision 32630) @@ -51,5 +51,5 @@ - 0 + 1 1 1 Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/datastore.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/datastore.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/datastore.py (revision 32630) @@ -7,4 +7,6 @@ import re from xml.etree.ElementTree import ElementTree, Element, tostring + +from ipptopspsdb import IppToPspsDb ''' @@ -17,9 +19,10 @@ ''' - def __init__(self, logger, doc): + def __init__(self, logger, doc, ippToPspsDb): # setup logger self.logger = logger self.doc = doc + self.ippToPspsDb = ippToPspsDb self.logger.debug("Datastore constructor") @@ -35,4 +38,7 @@ def publish(self, name, path, file, fileType): + # TODO should handle this better + batchID = int(name[1:]) + self.logger.infoPair("Attempting to publish", path + "/" + file) tempFile = tempfile.NamedTemporaryFile(mode='w+b') @@ -40,5 +46,5 @@ tempFile.seek(0) - # build dsreg command command + # build dsreg command command = "dsreg --add " + name + "\ --copy --datapath " + path + "\ @@ -53,9 +59,12 @@ self.logger.errorPair("Datastore publish", "failed") ret = False + self.ippToPspsDb.updateLoadedToDatastore(batchID, -1) else: self.logger.infoPair("Datastore publish", "successful") ret = True + self.ippToPspsDb.updateLoadedToDatastore(batchID, 1) tempFile.close() + return ret @@ -66,4 +75,7 @@ ''' def remove(self, name): + + # TODO should handle this better + batchID = int(name[1:]) command = "dsreg \ @@ -80,4 +92,5 @@ self.logger.infoPair("Datastore batch not found", name) ret = True + self.ippToPspsDb.updateDeletedDatastore(batchID, 1) else: self.logger.infoPair("Datastore removal FAILED", name) @@ -86,4 +99,5 @@ self.logger.infoPair("Datastore removal successful", name) ret = True + self.ippToPspsDb.updateDeletedDatastore(batchID, 1) return ret Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/detectionbatch.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/detectionbatch.py (revision 32630) @@ -22,4 +22,7 @@ ''' DetectionBatch class + +This class, a sub-class of Batch, processes single exposures in the form of IPP smf files + ''' class DetectionBatch(Batch): @@ -34,5 +37,6 @@ gpc1Db, ippToPspsDb, - camID): + camID, + batchID): super(DetectionBatch, self).__init__( @@ -43,4 +47,5 @@ ippToPspsDb, camID, + batchID, "P2", gpc1Db.getCameraStageSmf(camID)) Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/dxlayer.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/dxlayer.py (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/dxlayer.py (revision 32630) @@ -0,0 +1,73 @@ +#!/usr/bin/env jython + +import re +import sys +import tempfile +from xml.etree.ElementTree import ElementTree, Element, tostring +from subprocess import call, PIPE, Popen + +from batch import Batch + +''' +Class for using DXLayer webservice +''' +class DXLayer(object): + + ''' + Constructor + ''' + def __init__(self, logger): + + self.logger = logger + + ''' + Delete's a batch from the DXLayer + ''' + def deleteBatch(self, batchID): + + batch = Batch.getNameFromID(batchID) + + tempFile = tempfile.NamedTemporaryFile(mode='w+b') + + p = Popen("../perl/deleteFromDXLayer.pl -b " + batch + " -f " + tempFile.name, + shell=True, + stdout=PIPE) + p.wait() + + if p.returncode != 0: + self.logger.debugPair("DXLayer webservice", "failed") + return 0 + else: + self.logger.debugPair("DXLayer webservice", "successful") + + ret = self.decodeDXLayerResponse(tempFile.name) + + tempFile.close() + p.stdout.close() + + if ret: self.logger.infoPair("Deleted batch from DXLayer", batch) + else: self.logger.errorPair("Unable to delete batch from DXLayer", batch) + + return ret + + ''' + Parses the XML returned from the DXLayer webservice call + ''' + def decodeDXLayerResponse(self, dxLayerResponseFile): + + doc = ElementTree(file=dxLayerResponseFile) + + try: + MESSAGE = doc.find("Rows/Row/Value").text + + #print "MESSAGE " + MESSAGE + if re.search("Path not found", MESSAGE): + ret = True + elif re.search("Deleted", MESSAGE): + ret = True + + except: + self.logger.error("Cannot read DXLayer response file") + ret = False + + return ret Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/fits.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/fits.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/fits.py (revision 32630) @@ -31,11 +31,12 @@ # ok, we have a file, now copy it locally to save on NFS overhead self.logger.debugPair("FITS file", self.originalPath) - self.localCopyPath = self.localDir + "/temp.fits" - shutil.copy2(self.originalPath, self.localCopyPath) + #self.localCopyPath = self.localDir + "/temp.fits" + #shutil.copy2(self.originalPath, self.localCopyPath) # open the local copy and parse the ridiculou plain-text header - self.file = open(self.localCopyPath) + #self.file = open(self.localCopyPath) + self.file = open(self.originalPath) self.header = self.parseHeader() - self.logger.debugPair("FITS local copy", self.localCopyPath) + #self.logger.debugPair("FITS local copy", self.localCopyPath) self.logger.debugPair("FITS primary header", "%d cards found" % len(self.header)) @@ -53,5 +54,6 @@ ''' def getPath(self): - return self.localCopyPath + #return self.localCopyPath + return self.originalPath ''' Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/gpc1db.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/gpc1db.py (revision 32630) @@ -60,7 +60,7 @@ ''' - Gets a list of ids in this DVO database for this stage, could be cam or staticsky (so far) - ''' - def getIDsInThisDVODbForThisStage(self, dvoDb, batchType): + Gets a list of ids in this DVO database for this stage, could be cam or staticsky, and a box can be defined with equatorial coordinates + ''' + def getIDsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999): if batchType == "P2": # TODO define these someplace @@ -71,15 +71,19 @@ JOIN minidvodbRun USING(minidvodb_name) \ JOIN minidvodbProcessed USING(minidvodb_id) \ + JOIN camRun ON (stage_id = cam_id) \ + JOIN chipRun USING (chip_id) \ + JOIN rawExp USING (exp_id) \ WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \ AND minidvodbRun.state = 'merged' \ AND minidvodbProcessed.fault = 0 \ AND addRun.stage = '" + stage + "' \ - AND addRun.state = 'full'" + AND addRun.state = 'full' \ + AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \ + AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")" elif batchType == "ST": stage = "staticsky" - sql = "SELECT DISTINCT stack_id \ - FROM staticskyInput \ + sql = "SELECT DISTINCT stack_id FROM staticskyInput \ JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \ JOIN minidvodbRun USING(minidvodb_name) \ @@ -90,5 +94,5 @@ AND addRun.stage = '" + stage + "' \ AND addRun.state = 'full'" - + try: rs = self.executeQuery(sql) Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/initbatch.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/initbatch.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/initbatch.py (revision 32630) @@ -30,5 +30,6 @@ configDoc, gpc1Db, - ippToPspsDb): + ippToPspsDb, + batchID): super(InitBatch, self).__init__(logger, configPath, @@ -37,4 +38,5 @@ ippToPspsDb, 0, + batchID, "IN", None) @@ -49,4 +51,9 @@ self.logger.debug("importIppTables method here to satidfy base-class") + ''' + Here to satidfy the base-class + ''' + def alterPspsTables(self): + return True ''' Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/ipptopspsdb.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/ipptopspsdb.py (revision 32630) @@ -20,42 +20,35 @@ super(IppToPspsDb, self).__init__(logger, doc, "ipptopspsdatabase") - ''' - Creates a new batch - ''' - def createNewBatch(self, batchType, stageID, survey, dvoDb, datastoreProduct): - - sql = "INSERT INTO batch ( \ - batch_type, \ - stage_id, \ - survey, \ - dvo_db, \ - datastore_product \ - ) VALUES ( \ - '" + batchType + "', \ - " + str(stageID) + ", \ - '" + survey + "', \ - '" + dvoDb + "', \ - '" + datastoreProduct + "' \ - )" - - self.execute(sql) - - sql = "SELECT MAX(batch_id) FROM batch" - - batchID = -1; - - try: - rs = self.executeQuery(sql) - rs.first() - batchID = rs.getInt(1) - except: - self.logger.exception("Unable to get batch ID") - - self.logger.debug("Created new batch in ippToPsps database with batchID = %d" % batchID) - - return batchID; - - ''' - Returns a list of processed batch IDs that are merged but not yet deleted + self.MAX_FAILS = 4 + + ''' + Returns a list of merged batch IDs that are merged but not yet deleted + ''' + def getMergedButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column): + + sql = "SELECT DISTINCT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND merged = 1 \ + AND " + column + " = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + + return ids + + ''' + Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted ''' def getLoadedToODMButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column): @@ -66,5 +59,5 @@ AND batch_type = '" + batchType + "' \ AND dvo_db = '" + dvoGpc1Label + "' \ - AND loaded_to_ODM != 0 \ + AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ AND " + column + " = 0" @@ -84,5 +77,11 @@ ''' - Returns a list of processed batch IDs that are merged but not deleted from local disk + Returns a list of merged batch IDs that are not deleted from local disk + ''' + def getMergedButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label): + return self.getMergedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_local") + + ''' + Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from local disk ''' def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label): @@ -90,5 +89,5 @@ ''' - Returns a list of processed batch IDs that are merged but not deleted from datastore + Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from datastore ''' def getLoadedToODMButNotDeletedFromDatastore(self, batchType, epoch, dvoGpc1Label): @@ -96,5 +95,5 @@ ''' - Returns a list of processed batch IDs that are merged but not deleted from DXLayer + Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from DXLayer ''' def getLoadedToODMButNotDeletedFromDXLayer(self, batchType, epoch, dvoGpc1Label): @@ -130,7 +129,124 @@ ''' - Returns a list of processed batch IDs that have not yet been loaded to the ODM - ''' - def getBatchIDsUnloadedToODM(self, epoch, dvoGpc1Label): + Returns a list of processed batch IDs that either not yet loaded to the ODM or not merge_worthy or not merged + NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch + ''' + def getAllUnfinishedBatchIDs(self, epoch, dvoGpc1Label, batchType): + + sql = "SELECT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND loaded_to_datastore = 1 \ + AND batch_type = '" + batchType + "' \ + AND deleted_datastore = 0 \ + AND loaded_to_ODM != -1 \ + AND merge_worthy != -1 \ + AND merged = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for unloaded batch ids in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d unloaded items" % len(ids)) + + return ids + + ''' + Returns a list of processed batch IDs that are not yet loaded to the ODM + NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch + ''' + def getUnloadedBatchIDs(self, epoch, dvoGpc1Label, batchType): + + sql = "SELECT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND loaded_to_datastore = 1 \ + AND batch_type = '" + batchType + "' \ + AND loaded_to_ODM = 0 \ + AND deleted_datastore = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for unloaded batch ids in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d unloaded items" % len(ids)) + + return ids + + ''' + Returns a list of processed batch IDs that are not yet merge_worthy and that have not failed to load + NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch + ''' + def getUnmergeWorthyBatchIDs(self, epoch, dvoGpc1Label, batchType): + + sql = "SELECT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND loaded_to_datastore = 1 \ + AND batch_type = '" + batchType + "' \ + AND merge_worthy = 0 \ + AND loaded_to_ODM != -1 \ + AND deleted_datastore = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for un-mergeworthy batch ids in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d unloaded items" % len(ids)) + + return ids + + ''' + Returns a list of processed batch IDs that are merge_worthy, but not yet merged, for this epoch and dvo label + ''' + def getUnmergedBatchIDs(self, epoch, dvoGpc1Label, batchType): + + sql = "SELECT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND merge_worthy = 1 \ + AND batch_type = '" + batchType + "' \ + AND merged != 1" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for unmerged batch ids in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d un-merged items" % len(ids)) + + return ids + + ''' + Returns a list of batch IDs for this epoch and dvo label for which the specified column is set to value + ''' + def getBatchIDs(self, epoch, dvoGpc1Label, column, value): sql = "SELECT DISTINCT batch_id \ @@ -138,59 +254,4 @@ WHERE timestamp > '" + epoch + "' \ AND dvo_db = '" + dvoGpc1Label + "' \ - AND loaded_to_datastore = 1 \ - AND loaded_to_ODM = 0" - - - ids = [] - try: - rs = self.executeQuery(sql) - while (rs.next()): - ids.append(rs.getInt(1)) - except: - self.logger.exception("Can't query for unloaded batch ids in ipptopsps Db") - - rs.close() - - self.logger.debug("Found %d unloaded items" % len(ids)) - - return ids - - ''' - Returns a list of processed batch IDs that have not failed to load and are not yet merged, for this epoch and dvo label - ''' - def getUnfinishedBatchIDs(self, epoch, dvoGpc1Label): - - sql = "SELECT DISTINCT batch_id \ - FROM batch \ - WHERE timestamp > '" + epoch + "' \ - AND dvo_db = '" + dvoGpc1Label + "' \ - AND loaded_to_datastore = 1 \ - AND merged != 1 \ - AND loaded_to_ODM != -1" - - - ids = [] - try: - rs = self.executeQuery(sql) - while (rs.next()): - ids.append(rs.getInt(1)) - except: - self.logger.exception("Can't query for unfinished batch ids in ipptopsps Db") - - rs.close() - - self.logger.debug("Found %d unfinished items" % len(ids)) - - return ids - - ''' - Returns a list of batch IDs for this epoch and dvo label for which the specified column is set to value - ''' - def getBatchIDs(self, epoch, dvoGpc1Label, column, value): - - sql = "SELECT DISTINCT batch_id \ - FROM batch \ - WHERE timestamp > '" + epoch + "' \ - AND dvo_db = '" + dvoGpc1Label + "' \ AND " + column + " = " + str(value) @@ -208,4 +269,31 @@ return ids + + ''' + Returns the total detections published to the datastore for this epoch, dvo label and batch type + ''' + def getTotalDetectionsPublished(self, batchType, epoch, dvoGpc1Label): + + sql = "SELECT SUM(total_detections) \ + FROM batch \ + WHERE batch_type = '" + batchType + "' \ + AND timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND processed = 1 \ + AND loaded_to_datastore = 1" + + total = -1 + try: + rs = self.executeQuery(sql) + rs.first() + total = rs.getLong(1) + except: + self.logger.exception("Can't query for total detections published") + + rs.close() + + self.logger.debug("Found %d detections" % total) + + return total ''' @@ -273,12 +361,12 @@ ''' def getStages(self): - return ['processed', 'loaded_to_datastore', 'loaded_to_ODM', 'merge_worthy', 'merged', 'deleted_datastore', 'deleted_dxlayer', 'deleted_local'] - - ''' - Returns batches per hour. Simply goes back in time for the specified number of hours, so will return 0.0 if not loading - ''' - def getBatchesPerHour(self, batchType, epoch, dvoGpc1Label, hours): - - sql = "SELECT COUNT(batch_id)/ " + str(hours) + " \ + return ['processed', 'loaded_to_datastore', 'loaded_to_ODM', 'merge_worthy', 'deleted_datastore', 'deleted_dxlayer', 'merged', 'deleted_local' ] + + ''' + Returns a count of batches processed in the specified time period. + ''' + def countBatchesInLastPeriod(self, batchType, epoch, dvoGpc1Label, interval): + + sql = "SELECT COUNT(batch_id) \ FROM batch \ WHERE batch_type = '" + batchType + "' \ @@ -286,12 +374,12 @@ AND timestamp > '" + epoch + "' \ AND dvo_db = '" + dvoGpc1Label + "' \ - AND timestamp > (now() - INTERVAL " + str(hours) + " HOUR)" - - try: - rs = self.executeQuery(sql) - if rs.first(): return rs.getFloat(1) - except: - self.logger.exception("Unable to get batches per hour") - return 0.0 + AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()" + + try: + rs = self.executeQuery(sql) + if rs.first(): return rs.getInt(1) + except: + self.logger.exception("Unable to count batches in interval") + return 0 ''' @@ -301,7 +389,7 @@ def getTimeOfLastBatchPublished(self, batchType, epoch, dvoGpc1Label): - minutes = None - - sql = "SELECT TIMESTAMPDIFF(MINUTE, timestamp, now()) \ + seconds = None + + sql = "SELECT TIMESTAMPDIFF(SECOND, timestamp, now()) \ FROM batch \ WHERE batch_type = '" + batchType + "' \ @@ -313,15 +401,17 @@ try: rs = self.executeQuery(sql) - if rs.first(): minutes = rs.getFloat(1) + if rs.first(): seconds = rs.getFloat(1) except: self.logger.exception("Unable to get last batch published") - if not minutes: return "Never" - + if not seconds: return "Never" + + minutes = seconds/60.0 hours = minutes/60.0 days = hours/24.0 weeks = days/7.0 - if minutes < 60: return "%.1f mins ago" % minutes + if seconds < 60: return "%.1f secs ago" % seconds + elif minutes < 60: return "%.1f mins ago" % minutes elif hours < 48: return "%.1f hours ago" % hours elif days < 7: return "%.1f days ago" % days @@ -398,4 +488,46 @@ ''' + Is someone processing this item right now? Checks whether this was attempted within the last 2 HOUR + ''' + def processingNow(self, batchType, stage_id, epoch, dvoGpc1Label): + + sql = "SELECT COUNT(*) \ + FROM batch \ + WHERE stage_id = " + str(stage_id) + " \ + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND timestamp BETWEEN now() - INTERVAL 2 HOUR AND now()" + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: + self.logger.errorPair(str(stage_id) + " is already being processed", "skipping") + return True + else: + return False + except: + self.logger.exception("Unable to check whether this batch is being processed") + + ''' + Gets the batch type of this batch + ''' + def getBatchType(self, batch_id): + + sql = "SELECT batch_type \ + FROM batch \ + WHERE batch_id = " + str(batch_id) + + batch_type = "NA" + try: + rs = self.executeQuery(sql) + rs.first() + batch_type = rs.getString(1) + except: + self.logger.exception("Unable to check whether this batch has already been processed") + + return batch_type + + ''' Have we already processed and published this batch? ''' @@ -425,5 +557,5 @@ Has this stage_id consistently failed to process? ''' - def consistentlyFailed(self, batchType, stage_id, epoch, dvoGpc1Label, count): + def consistentlyFailed(self, batchType, stage_id, epoch, dvoGpc1Label): sql = "SELECT COUNT(*) \ @@ -438,6 +570,6 @@ rs = self.executeQuery(sql) rs.first() - if rs.getInt(1) >= count: - self.logger.errorPair(str(stage_id) + " has failed %d times" % count, "skipping") + if rs.getInt(1) >= self.MAX_FAILS: + self.logger.errorPair(str(stage_id) + " has failed %d times" % self.MAX_FAILS, "skipping") return True else: @@ -446,4 +578,77 @@ self.logger.exception("Unable to check whether this batch has consistently failed") + ''' + Returns a list of stage_ids that have failed the max number of times + ''' + def getConsistentlyFailedIDs(self, batchType, epoch, dvoGpc1Label): + + sql = "SELECT DISTINCT stage_id, COUNT(stage_id) AS numoccur \ + FROM batch \ + WHERE batch_type = '" + batchType + "' \ + AND timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND processed = -1 \ + GROUP BY stage_id HAVING numoccur >= " + str(self.MAX_FAILS) + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for consistantly failed ids in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d consistantly failed stage_ids for batch type='%s'" % (len(ids), batchType)) + + return ids + + ''' + Locks the batch table. This will wait if the lock is held by someone else + ''' + def lockBatchTable(self): + self.lockTable("batch") + + ''' + Creates a new batch. This is done with a locked table in order that no two clients can start work on the same item + ''' + def createNewBatch(self, batchType, stageID, survey, epoch, dvoDb, datastoreProduct, force): + + batchID = -1; + + if force or \ + (not self.alreadyProcessed(batchType, stageID, epoch, dvoDb) \ + and not self.processingNow(batchType, stageID, epoch, dvoDb) \ + and not self.consistentlyFailed(batchType, stageID, epoch, dvoDb)): + + sql = "INSERT INTO batch ( \ + batch_type, \ + stage_id, \ + survey, \ + dvo_db, \ + datastore_product \ + ) VALUES ( \ + '" + batchType + "', \ + " + str(stageID) + ", \ + '" + survey + "', \ + '" + dvoDb + "', \ + '" + datastoreProduct + "' \ + )" + + self.execute(sql) + + sql = "SELECT MAX(batch_id) FROM batch" + + try: + rs = self.executeQuery(sql) + rs.first() + batchID = rs.getInt(1) + self.logger.debug("Created new batch in ippToPsps database with batchID = %d" % batchID) + except: + self.logger.exception("Unable to get batch ID") + batchID = -1 + + return batchID; Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/load.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/load.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/load.py (revision 32630) @@ -7,4 +7,6 @@ import time import sys +import os +import socket import logging.config from xml.etree.ElementTree import ElementTree, Element, tostring @@ -35,6 +37,5 @@ logger.infoPair("Batch name", batchName) - Batch.publishToDatastore(datastore, ippToPspsDb, batchID, batchName, subDir, tarballFile) - + Batch.publishToDatastore(datastore, batchID, batchName, subDir, tarballFile) ''' @@ -54,9 +55,14 @@ logger.errorPair("Problem loading IDs from config under", "options/ids/id") - # no config dictated IDs, so get from DVO instead if len(allIDs) > 0: logger.infoPair("Using config IDs (not DVO)", "%d" % len(allIDs)) + # no config dictated IDs, so get from DVO instead else: - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType) + + if MINRA and MAXRA and MINDEC and MAXDEC: + allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType, MINRA, MAXRA, MINDEC, MAXDEC) + else: + allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType) + logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs)) @@ -67,15 +73,28 @@ else: processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL) - logger.infoPair("Processed %s items" % batchType, "%d" % len(processedIDs)) - ids = list(set(allIDs) - set(processedIDs)) - logger.infoPair("Unprocessed %s items" % batchType, "%d" % len(ids)) + consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, EPOCH, DVOGPC1LABEL) + ids = list(set(allIDs) - set(processedIDs) - set(consistaentlyFailedIDs)) + logger.infoPair("Processed items", "%d" % len(processedIDs)) + logger.infoPair("Consistently failing items", "%d" % len(consistaentlyFailedIDs)) + logger.infoPair("Unprocessed items", "%d" % len(ids)) logger.infoPair("%s items queued" % batchType, "%d" % len(ids)) # loop round IDs of all items to be processed + ippToPspsDb.lockBatchTable() for id in ids: - if not FORCE and ippToPspsDb.alreadyProcessed(batchType, id, EPOCH, DVOGPC1LABEL): continue - if not FORCE and ippToPspsDb.consistentlyFailed(batchType, id, EPOCH, DVOGPC1LABEL, 3): continue + batchID = ippToPspsDb.createNewBatch( + batchType, + id, + SURVEY, + EPOCH, + DVOGPC1LABEL, + datastore.product, + FORCE) + + if batchID < 0: continue; + + ippToPspsDb.unlockTables() if batchType == "P2": @@ -85,5 +104,6 @@ gpc1Db, ippToPspsDb, - id) + id, + batchID) elif batchType == "ST": batch = StackBatch(logger, @@ -92,11 +112,17 @@ gpc1Db, ippToPspsDb, - id) + id, + batchID) batch.run() + + ippToPspsDb.lockBatchTable() + + logger.infoSeparator() # if in TEST mode, then quit after one batch if TEST: break + ippToPspsDb.unlockTables() ''' @@ -122,11 +148,15 @@ logger = logging.getLogger("ippToPspsLog") +# get hostnamee and PID for unique log naming +HOST = socket.gethostname() +PID = os.getpid() + # if in test mode, print log to screen, otherwise, only to file -logger.setup(configDoc, "ippToPsps", TEST, not TEST) +logger.setup(configDoc, "ippToPsps", TEST, not TEST, HOST, PID) # create database objects gpc1Db = Gpc1Db(logger, configDoc) ippToPspsDb = IppToPspsDb(logger, configDoc) -datastore = Datastore(logger, configDoc) +datastore = Datastore(logger, configDoc, ippToPspsDb) # check we connected ok @@ -143,4 +173,18 @@ QUEUE_P2 = int(configDoc.find("options/queueP2").text) QUEUE_ST = int(configDoc.find("options/queueST").text) +SURVEY = configDoc.find("options/survey").text + +# get equatorial coord limits, if any +MINRA = None +MAXRA = None +MINDEC = None +MAXDEC = None +try: + MINRA = float(configDoc.find("dvo/minRA").text) + MAXRA = float(configDoc.find("dvo/maxRA").text) + MINDEC = float(configDoc.find("dvo/minDec").text) + MAXDEC = float(configDoc.find("dvo/maxDec").text) +except: + pass # prompt user: FORCE and PUBLISH is a dangerous combination @@ -180,6 +224,15 @@ # queue all items for all batch types, sequentially if QUEUE_IN: - batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb) - batch.run() + batchID = ippToPspsDb.createNewBatch( + "IN", + 0, + SURVEY, + EPOCH, + DVOGPC1LABEL, + datastore.product, + 1) + if batchID > 0: + batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb, batchID) + batch.run() break; Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/metrics.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/metrics.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/metrics.py (revision 32630) @@ -52,5 +52,8 @@ # get a master list of IDs in DVO for this batch type - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType) + if MINRA and MAXRA and MINDEC and MAXDEC: + allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType, MINRA, MAXRA, MINDEC, MAXDEC) + else: + allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType) prevList = allIDs numEverything = len(allIDs) @@ -73,4 +76,7 @@ numPending = len(pending) + #if stage == 'merge_worthy' and batchType == "P2": + # for n in pending: print n + str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m " % (getIntAsString(numPending), getIntAsString(numSuccess), getIntAsString(numFail)) sys.stdout.write("|%-19s" % str) @@ -79,7 +85,11 @@ print >> DATFILE, stage, numSuccess, numFail, numPending + if stage == 'processed': numPendingProcessed = numPending + sys.stdout.write("|\n") plotMe(batchType, tempFilename) DATFILE.close() + + return numPendingProcessed ''' @@ -91,4 +101,21 @@ for i in range(0, colCount): sys.stdout.write("+-------------------") sys.stdout.write("+\n") + +''' +Prints various timing stats for this batch type +''' +def printStats(batchType): + + rate = ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "1 HOUR") / 1.0 + + logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" % + (batchType, + rate, + ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "24 HOUR"), + ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "1 WEEK") / 7.0, + ippToPspsDb.getTotalDetectionsPublished(batchType, EPOCH, DVOLABEL), + ippToPspsDb.getTimeOfLastBatchPublished(batchType, EPOCH, DVOLABEL))) + + return rate ''' @@ -112,4 +139,9 @@ gpc1Db = Gpc1Db(logger, configDoc) +MINRA = None +MAXRA = None +MINDEC = None +MAXDEC = None + DVOLABEL = configDoc.find("dvo/gpc1Label").text EPOCH = configDoc.find("options/epoch").text @@ -119,8 +151,22 @@ logger.infoPair("Loading epoch", EPOCH) logger.infoPair("DVO label", DVOLABEL) -logger.infoPair("P2 batches per hour", "%.1f" % ippToPspsDb.getBatchesPerHour("P2", EPOCH, DVOLABEL, 2)) -logger.infoPair("ST batches per hour", "%.1f" % ippToPspsDb.getBatchesPerHour("ST", EPOCH, DVOLABEL, 2)) -logger.infoPair("Last P2 batch published", ippToPspsDb.getTimeOfLastBatchPublished("P2", EPOCH, DVOLABEL)) -logger.infoPair("Last ST batch published", ippToPspsDb.getTimeOfLastBatchPublished("ST", EPOCH, DVOLABEL)) + +try: + MINRA = float(configDoc.find("dvo/minRA").text) + MAXRA = float(configDoc.find("dvo/maxRA").text) + MINDEC = float(configDoc.find("dvo/minDec").text) + MAXDEC = float(configDoc.find("dvo/maxDec").text) + logger.infoPair("Coordinates", "%.1f to %.1f RA, %.1f to %.1f Dec" % (MINRA, MAXRA, MINDEC, MAXDEC)) +except: + pass + + +logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") +logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |") +logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") +rateP2 = printStats("P2") +rateST = printStats("ST") +logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + logger.info("") @@ -135,6 +181,11 @@ sys.stdout.write("|\n") writetableSeparator(colCount) -printTableRow(stages, "P2") -printTableRow(stages, "ST") +pendP2Processed = printTableRow(stages, "P2") +pendSTProcessed = printTableRow(stages, "ST") writetableSeparator(colCount) +logger.info("") +try: logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2)) +except: pass +try: logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) +except: pass Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/mysql.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/mysql.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/mysql.py (revision 32630) @@ -47,5 +47,4 @@ self.everythingOK = True - ''' Disconnect from database @@ -61,4 +60,16 @@ self.logger.debug("MySql destructor") self.disconnect() + + ''' + Locks a table. This is wait if the locks is already held by another session + ''' + def lockTable(self, table): + self.execute("LOCK TABLES " + table + " WRITE") + + ''' + Unlocks all tables locked by this session. + ''' + def unlockTables(self): + self.execute("UNLOCK TABLES") ''' Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/odm.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/odm.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/odm.py (revision 32630) @@ -45,4 +45,5 @@ tempFile.close() + p.stdout.close() if not ret: self.logger.debugPair("Cannot read ODM file for batch", "%d" % batchID) @@ -67,12 +68,20 @@ MESSAGE = doc.find("{%s}OdmBatchState/{%s}Message" % (NAMESPACE, NAMESPACE)).text - if re.search("failed", STATE): + # we assume that batches 'OnHold' are failed as it is very rare for such + # batches to have there state changed + if re.search("failed", STATE) or re.search("OnHold", MESSAGE): + loadedToODM = 1 loadFailed = 1 DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text results['DETAILS'] = DETAILS - if re.search("MergeWorthy", MESSAGE): mergeWorthy = 1 - if re.search("Merge[1-9]Completed", MESSAGE): merged = 1 - if merged: mergeWorthy = 1 - if mergeWorthy or merged or loadFailed: loadedToODM = 1 + if re.search("processing", STATE): + loadedToODM = 1 + if re.search("MergeWorthy", MESSAGE): + loadedToODM = 1 + mergeWorthy = 1 + if re.search("Merge[1-9]Completed", MESSAGE): + loadedToODM = 1 + mergeWorthy = 1 + merged = 1 except: return 0 Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/pollOdm.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/pollOdm.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/pollOdm.py (revision 32630) @@ -6,4 +6,5 @@ import re import sys +import time import logging.config @@ -13,9 +14,88 @@ from batch import Batch +''' +Method to actually poll ODM +''' +def poll(batchType): + -if len(sys.argv) > 1: CONFIG = sys.argv[1] + lookingForUnloaded = 0 + lookingForUnmergeworthy = 0 + lookingForUnmerged = 0 + lookingForAll = 0 + + if TYPE == "unloaded": + lookingForUnloaded = 1 + ids = ippToPspsDb.getUnloadedBatchIDs(EPOCH, DVOLABEL, batchType) + elif TYPE == "unmergeworthy": + lookingForUnmergeworthy = 1 + ids = ippToPspsDb.getUnmergeWorthyBatchIDs(EPOCH, DVOLABEL, batchType) + elif TYPE == "unmerged": + lookingForUnmerged = 1 + ids = ippToPspsDb.getUnmergedBatchIDs(EPOCH, DVOLABEL, batchType) + elif TYPE == "unfinished": + lookingForAll = 1 + ids = ippToPspsDb.getAllUnfinishedBatchIDs(EPOCH, DVOLABEL, batchType) + + logger.infoTitle("Checking %d %s %s batches" % (len(ids), TYPE, batchType)) + + results = {} + newLoaded = 0 + newMergeWorthy = 0 + newMerged = 0 + + logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+") + logger.info("| Batch | Loaded to ODM | Load failed | Merge worthy | Merged | New loaded |New merge worthy| New merged |") + logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+") + for id in ids: + + if odm.checkBatch(id, results): + ippToPspsDb.updateOdmStatus(id, results) + + if lookingForUnloaded: + if results['LOADEDTOODM'] == 1: newLoaded = newLoaded + 1 + if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 + if results['MERGED'] == 1: newMerged = newMerged + 1 + + elif lookingForUnmergeworthy: + if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 + if results['MERGED'] == 1: newMerged = newMerged + 1 + + elif lookingForUnmerged: + if results['MERGED'] == 1: newMerged = newMerged + 1 + + logger.info("| %9s | %14d | %14d | %14d | %14d | %14d | %14d | %14d |" % + (Batch.getNameFromID(id), + results['LOADEDTOODM'], + results['LOADFAILED'], + results['MERGEWORTHY'], + results['MERGED'], + newLoaded, + newMergeWorthy, + newMerged)) + + + else: + logger.info("| %9s | failed to poll ODM |" % Batch.getNameFromID(id)) + + logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+") + + +''' +Program starts here +''' +if len(sys.argv) > 3: + CONFIG = sys.argv[1] + BATCHTYPE = sys.argv[2] + TYPE = sys.argv[3] else: - print "\n** Usage: " + sys.argv[0] + " \n" + print "\n** Usage: " + sys.argv[0] + " [hours]\n" sys.exit(1) + +if len(sys.argv) > 4: + HOURS = float(sys.argv[4]) + SECONDS = HOURS * 60.0 * 60.0 +else: + SECONDS = None # open config file @@ -33,30 +113,16 @@ EPOCH = configDoc.find("options/epoch").text -if len(sys.argv) > 2 and sys.argv[2] == "all": - ids = ippToPspsDb.getUnfinishedBatchIDs(EPOCH, DVOLABEL) -else: - ids = ippToPspsDb.getBatchIDsUnloadedToODM(EPOCH, DVOLABEL) +while True: -logger.infoPair("Checking", "%d batches" % len(ids)) + if BATCHTYPE == "all": + poll("P2") + poll("ST") + else: + poll(BATCHTYPE) + + if SECONDS: + logger.infoPair("Waiting for", "%f hours" % HOURS) + time.sleep(SECONDS) + else: break -results = {} - -logger.info("+-----------+----------------+----------------+----------------+----------------+") -logger.info("| Batch | Loaded to ODM | Load failed | Merge worthy | Merged |") -logger.info("+-----------+----------------+----------------+----------------+----------------+") -for id in ids: - - if odm.checkBatch(id, results): - ippToPspsDb.updateOdmStatus(id, results) - logger.info("| %9s | %14d | %14d | %14d | %14d |" % - (Batch.getNameFromID(id), - results['LOADEDTOODM'], - results['LOADFAILED'], - results['MERGEWORTHY'], - results['MERGED'])) - else: - logger.info("| %9s | failed to poll ODM |" % Batch.getNameFromID(id)) - -logger.info("+-----------+----------------+----------------+----------------+----------------+") - Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/pslogger.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/pslogger.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/pslogger.py (revision 32630) @@ -9,5 +9,5 @@ Sets up logging using values from the ippToPsps config file ''' - def setup(self, doc, name, stdout=1, sendToFile=0): + def setup(self, doc, name, stdout=1, sendToFile=0, host="host", pid=1234): formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S') @@ -18,5 +18,5 @@ if not os.path.exists(PATH): os.makedirs(PATH) DVOLABEL = doc.find("dvo/gpc1Label").text - FULLPATH = PATH + "/" + name + "_" + DVOLABEL + ".log" + FULLPATH = PATH + "/" + name + "_" + DVOLABEL + "_" + host + "_" + str(pid) + ".log" # opens file to be appended Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/removeFromDatastore.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/removeFromDatastore.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/removeFromDatastore.py (revision 32630) @@ -5,7 +5,10 @@ import sys import getopt -from pslogger import PSLogger from xml.etree.ElementTree import ElementTree, Element, tostring + +from pslogger import PSLogger +from ipptopspsdb import IppToPspsDb + @@ -21,5 +24,6 @@ logger.setup(configDoc, "removeFromDatastore") -datastore = Datastore(logger, configDoc) +ippToPspsDb = IppToPspsDb(logger, configDoc) +datastore = Datastore(logger, configDoc, ippToPspsDb) if len(sys.argv) < 4: Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/republishbatch.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/republishbatch.py (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/republishbatch.py (revision 32630) @@ -0,0 +1,56 @@ +import sys + +from pslogger import PSLogger +import logging +from xml.etree.ElementTree import ElementTree, Element, tostring + +from ipptopspsdb import IppToPspsDb +from datastore import Datastore +from batch import Batch + + +''' + +''' +if len(sys.argv) > 2: + CONFIG = sys.argv[1] + BATCHID = int(sys.argv[2]) +else: + print "** Usage: " + sys.argv[0] + " " + sys.exit(1) + +# open config file +configDoc = ElementTree(file=CONFIG) + +logging.setLoggerClass(PSLogger) +logger = logging.getLogger(sys.argv[0]) +logger.setup(configDoc, sys.argv[0]) + +ippToPspsDb = IppToPspsDb(logger, configDoc) +datastore = Datastore(logger, configDoc, ippToPspsDb) + +DVOLABEL = configDoc.find("dvo/gpc1Label").text +EPOCH = configDoc.find("options/epoch").text +BASEPATH = configDoc.find("localOutPath").text +DATASTOREPRODUCT = configDoc.find("datastore/product").text + +logger.infoTitle("Batch republisher") +logger.infoPair("Loading epoch", EPOCH) +logger.infoPair("DVO label", DVOLABEL) + + +batchName = Batch.getNameFromID(BATCHID) +BATCHTYPE = ippToPspsDb.getBatchType(BATCHID) +subDir = Batch.getSubDir(BASEPATH, BATCHTYPE, DVOLABEL) +tarballFile = Batch.getTarballFile(BATCHID) + +logger.infoPair("batch ID", "%d" % BATCHID) +logger.infoPair("batchname", batchName) +logger.infoPair("batch type", BATCHTYPE) +logger.infoPair("subdir", subDir) +logger.infoPair("tarballfile", tarballFile) +logger.infoPair("publishing to", DATASTOREPRODUCT) +Batch.publishToDatastore(datastore, BATCHID, batchName, subDir, tarballFile) + + + Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/scratchdb.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/scratchdb.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/scratchdb.py (revision 32630) @@ -40,12 +40,17 @@ self.logger.debug("ScratchDb destructor") - ''' Creates a stored procedure for psfLikelihood. This uses an implementation of the complimentary error function taken from Numerical Recipes in C. ''' def createPsfLikelihoodFunction(self): - sql = "DELIMITER $$ \ - DROP FUNCTION IF EXISTS `psfLikelihood` $$ \ - CREATE FUNCTION `psfLikelihood`(x REAL) RETURNS REAL \ + + sql = "DROP FUNCTION IF EXISTS `psfLikelihood`" + try: + self.execute(sql) + except: + self.logger.errorPair("Could not drop stored procedure", "psfLikelihood") + return False + + sql = "CREATE FUNCTION `psfLikelihood`(x REAL) RETURNS REAL \ DETERMINISTIC \ COMMENT 'Returns the psfLikelihood using the complimentary error function with fractional error everywhere less than 1.2 x 10-7. From Numerical Recipes in C' \ @@ -62,6 +67,5 @@ END IF; \ RETURN ans; \ - END $$ \ - DELIMITER ;" + END" try: Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/setupScratchDb.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/setupScratchDb.py (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/setupScratchDb.py (revision 32630) @@ -0,0 +1,49 @@ +#!/usr/bin/env jython + +import os.path +import sys +import time +import stilts +from java.lang import * +from java.sql import * +import logging.config + +from java.lang import Math +from org.apache.commons.math.special import Erf +from xml.etree.ElementTree import ElementTree, Element, tostring + +from pslogger import PSLogger +from scratchdb import ScratchDb + +''' +Start of program. +''' + +if len(sys.argv) > 1: CONFIG = sys.argv[1] +else: + print "** Usage: " + sys.argv[0] + " [init]" + sys.exit(1) + +# open config file +configDoc = ElementTree(file=CONFIG) + +# set up logging +logging.setLoggerClass(PSLogger) +logger = logging.getLogger("setupScratchDb") +logger.setup(configDoc, "setupScratchDb") + +scratchDb = ScratchDb(logger, configDoc) + +# install stored procedure +logger.info("Installing psfLikelihood() stored procedure") +scratchDb.createPsfLikelihoodFunction() + +# install IN tables +logger.info("Installing initialization tables") +tables = stilts.treads("../config/IN/tables.vot") +for table in tables: + logger.debug("Creating IN table: " + table.name) + table.write(scratchDb.url + '#' + table.name) + +logger.info("Done") + Index: /branches/eam_branches/ipp-20110906/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20110906/ippToPsps/jython/stackbatch.py (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippToPsps/jython/stackbatch.py (revision 32630) @@ -19,4 +19,7 @@ ''' StackBatch class + +This class, a sub-class of Batch, processes single stack skycells in the form of IPP cmf files + ''' class StackBatch(Batch): @@ -31,5 +34,6 @@ gpc1Db, ippToPspsDb, - stackID): + stackID, + batchID): super(StackBatch, self).__init__( @@ -40,4 +44,5 @@ ippToPspsDb, stackID, + batchID, "ST", gpc1Db.getStackStageCmf(stackID)) @@ -56,8 +61,18 @@ self.filter = self.filter[0:1] self.skycell = meta[1]; - # TODO HACK fix this + + # skycell is, eg "skycell.1133.081" + # we need 1133.081 for skycell, but store as an int, so instead store 1133081 + # projection cell is just the 1133 bit + # this is how it should be done + #self.projectioncell = self.skycell[8:12] + #self.skycell = self.skycell[8:] + #self.skycell = self.skycell.replace(".", "") + #self.projectioncell = self.skycell[8:12] + + # TODO this is the wrong way, but is consistent with SkyCell table in IN data self.skycell = self.skycell[8:12] - # mangling e.g. 0683.043 into 0683043 for now until we have a schema change - #self.skycell = self.skycell.replace(".", "") + self.projectioncell = self.skycell + self.analysisVer = meta[2]; @@ -81,5 +96,5 @@ self.scratchDb.dropTable("SkyChip_xsrc") - # create an output filename, which is {filterID}{skycellID}.FITS + # create an output filename, which is {stackID}.FITS self.outputFitsFile = "%08d.FITS" % self.id self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) @@ -94,5 +109,4 @@ if not self.useFullTables: self.scratchDb.insertNewDvoExternID(self.header['SOURCEID'], self.header['IMAGEID']) - # dump stuff to log @@ -100,7 +114,6 @@ self.logger.infoPair("Stack type", "%s" % self.stackType) self.logger.infoPair("Skycell", "%s" % self.skycell) + self.logger.infoPair("Projectioncell", "%s" % self.projectioncell) self.logger.infoPair("Filter", "%s" % self.filter) - - ''' @@ -251,4 +264,6 @@ ,expTime \ ,psfModelID \ + ,psfFwhm_mean \ + ,psfFwhm_max \ ,ctype1 \ ,ctype2 \ @@ -270,4 +285,6 @@ ," + str(self.expTime) + " \ ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \ + ,'" + self.safeDictionaryAccess(self.header, 'FWHM_MAJ') + "' \ + ,'" + self.safeDictionaryAccess(self.header, 'FW_MJ_UQ') + "' \ ,'" + self.header['CTYPE1'] + "' \ ,'" + self.header['CTYPE2'] + "' \ @@ -301,4 +318,5 @@ ippDetectID \ ,skyCellID \ + ,projectionCellID \ ,obsTime \ ,xPos \ @@ -315,4 +333,5 @@ ,psfWidMinor \ ,psfTheta \ + ,psfLikelihood \ ,infoFlag \ ,psfCf \ @@ -339,4 +358,5 @@ IPP_IDET \ ," + self.skycell + " \ + ," + self.projectioncell + " \ ," + self.header['MJD-OBS'] + " \ ,X_PSF \ @@ -353,4 +373,5 @@ ,PSF_MINOR \ ,PSF_THETA \ + ,psfLikelihood(EXT_NSIGMA) \ ,FLAGS << 32 | FLAGS2 \ ,PSF_QF \ @@ -531,8 +552,10 @@ objID \ ,ippObjID \ + ,projectionCellID \ ) \ SELECT \ DISTINCT objID \ ,ippObjID \ + ,projectionCellID \ FROM StackDetection" self.scratchDb.execute(sql) @@ -573,4 +596,6 @@ self.scratchDb.createIndex("StackModelFit", "ippDetectID") + return True + ''' Applies indexes to the IPP tables @@ -583,4 +608,6 @@ self.scratchDb.createIndex("SkyChip_xrad", "IPP_IDET") self.scratchDb.createIndex("SkyChip_xsrc", "IPP_IDET") + + return True ''' Index: /branches/eam_branches/ipp-20110906/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/Makefile.am (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/Makefile.am (revision 32630) @@ -76,4 +76,6 @@ chiptool_definecopy.sql \ chiptool_donecleanup.sql \ + chiptool_dropprocessedimfile_update.sql \ + chiptool_dropprocessedimfile_delete.sql \ chiptool_find_rawexp.sql \ chiptool_listrun.sql \ @@ -275,4 +277,5 @@ magicdstool_definebyquery_warp_bg.sql \ magicdstool_definebyquery_diff.sql \ + magicdstool_definecopy_camera.sql \ magicdstool_definecopy_chip.sql \ magicdstool_definecopy_warp.sql \ @@ -424,4 +427,10 @@ laptool_pendingexp.sql \ laptool_pendingrun.sql \ - laptool_stacks.sql + laptool_WSdiff_check.sql \ + laptool_stacks.sql \ + vptool_find_rawexp.sql \ + vptool_pendingimfile.sql \ + vptool_pendingrun.sql \ + vptool_processedcell.sql \ + vptool_revertrun.sql Index: /branches/eam_branches/ipp-20110906/ippTools/share/camtool_find_pendingimfile.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/camtool_find_pendingimfile.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/camtool_find_pendingimfile.sql (revision 32630) @@ -13,7 +13,9 @@ USING (chip_id) JOIN chipProcessedImfile - USING (chip_id) + USING (chip_id, exp_id) JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id + USING(exp_id) +JOIN rawImfile + USING(exp_id, class_id, exp_time) LEFT JOIN camProcessedExp USING(cam_id) @@ -22,2 +24,3 @@ WHERE camMask.label IS NULL + AND rawImfile.ignored = 0 Index: /branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_delete.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_delete.sql (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_delete.sql (revision 32630) @@ -0,0 +1,6 @@ +DELETE FROM magicDSFile + USING magicDSRun, magicDSFile, chipRun, chipProcessedImfile +WHERE chipRun.chip_id = chipProcessedImfile.chip_id + AND magicDSRun.stage ='chip' and magicDSRun.stage_id = chipRun.chip_id + AND magicDSFile.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id Index: /branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_update.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_update.sql (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/share/chiptool_dropprocessedimfile_update.sql (revision 32630) @@ -0,0 +1,5 @@ +UPDATE chipProcessedImfile + SET quality = %d, + data_state = 'full', + magicked = 0, + fault = 0 Index: /branches/eam_branches/ipp-20110906/ippTools/share/laptool_WSdiff_check.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/laptool_WSdiff_check.sql (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/share/laptool_WSdiff_check.sql (revision 32630) @@ -0,0 +1,24 @@ +SELECT warp_id,warp_state,W.skycell_id,stack_id,stack_state,S.seq_id + FROM +( SELECT warp_id,skycell_id,seq_id,warpRun.state AS warp_state + FROM warpSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + JOIN lapExp USING(exp_id,chip_id) + JOIN lapRun USING(lap_id) + WHERE 1 + @WHERE@ +) AS W +LEFT JOIN +( SELECT stack_id,skycell_id,seq_id,stackRun.state AS stack_state + FROM stackRun + JOIN stackAssociationMap USING(stack_id) + JOIN lapRun ON (stackAssociationMap.sass_id = lapRun.final_sass_id) + ) AS S +ON ( (W.seq_id = S.seq_id) + AND (W.skycell_id = S.skycell_id) ) + + Index: /branches/eam_branches/ipp-20110906/ippTools/share/laptool_definerun.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/laptool_definerun.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/laptool_definerun.sql (revision 32630) @@ -1,3 +1,8 @@ -SELECT DISTINCT want.exp_id, have.chip_id, false as private, true as active, false as pairwise +SELECT DISTINCT + want.exp_id, + have.chip_id, + COALESCE(have.private,false) as private, + true as active, + false as pairwise FROM (SELECT exp_id FROM rawExp @@ -8,15 +13,21 @@ ) AS want LEFT JOIN - (SELECT exp_id,MAX(chip_id) AS chip_id, chipRun.state + (SELECT + exp_id, + MAX(chip_id) AS chip_id, + chipRun.state, + lapExp.private FROM lapExp JOIN chipRun USING(exp_id,chip_id) - where private IS FALSE - AND chip_id IS NOT NULL + JOIN lapRun USING(lap_id) + where chip_id IS NOT NULL AND - ( +-- This is the restriction to only draw from the current lapSequence + @CHIPWHERE@ +-- ( -- (active = TRUE) OR - (chipRun.state = 'full') OR - (chipRun.state = 'new') + -- (chipRun.state = 'full') OR + -- (chipRun.state = 'new') -- when we can do updates, put that here. - ) +-- ) GROUP BY exp_id ) AS have USING(exp_id) Index: /branches/eam_branches/ipp-20110906/ippTools/share/laptool_pendingexp.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/laptool_pendingexp.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/laptool_pendingexp.sql (revision 32630) @@ -3,7 +3,9 @@ lapRun.dist_group, lapRun.registered, lapRun.fault, lapRun.quick_sass_id, lapRun.final_sass_id, exp_id, chip_id, pair_id, private, pairwise, active, lapExp.data_state, - dateobs, object, comment + dateobs, object, comment, + chipRun.state AS chip_state FROM lapRun JOIN lapExp USING(lap_id) JOIN rawExp USING(exp_id,filter) + LEFT JOIN chipRun USING(exp_id,chip_id) WHERE active IS TRUE AND lapRun.fault = 0 -- lap_id restriction here. Index: /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_definecopy_camera.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_definecopy_camera.sql (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_definecopy_camera.sql (revision 32630) @@ -0,0 +1,31 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + chipRun.exp_id, + 'camera' AS stage, + camRun.cam_id AS stage_id, + camRun.data_group, + camRun.cam_id, + camRun.label, + CAST(NULL AS SIGNED) AS inv_magic_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun AS oldWarpRun + ON (!magicRun.inverse AND diffInputSkyfile.warp1 = oldWarpRun.warp_id) + OR ( magicRun.inverse AND diffInputSkyfile.warp2 = oldWarpRun.warp_id) + -- This ends the usual JOIN down to the raw level + JOIN rawExp + ON rawExp.exp_id = magicRun.exp_id + JOIN chipRun + ON chipRun.exp_id = rawExp.exp_id + JOIN camRun + ON camRun.chip_id = chipRun.chip_id + LEFT JOIN magicDSRun + ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'camera' + AND magicDSRun.label = '%s' +WHERE magicRun.state = 'full' + -- HOOK: NEWLINE HERE TO ACTIVATE %s AND magicDSRun.magic_ds_id IS NULL + AND camRun.magicked = 0 + AND camRun.state = 'full' Index: /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_camera.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_camera.sql (revision 32630) @@ -14,4 +14,5 @@ stage_id, 'exposure' AS component, + 0 AS mismatched_tess, CAST(NULL AS CHAR(255)) AS uri, camProcessedExp.path_base, Index: /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_chip.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_chip.sql (revision 32630) @@ -14,4 +14,5 @@ stage_id, class_id AS component, + 0 AS mismatched_tess, chipProcessedImfile.uri, chipProcessedImfile.path_base, Index: /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_diff.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_diff.sql (revision 32630) @@ -15,4 +15,5 @@ diffRun.magicked, diffSkyfile.skycell_id AS component, + 0 AS mismatched_tess, CAST(NULL AS CHAR(255)) AS uri, diffSkyfile.path_base, @@ -63,4 +64,5 @@ diffRun.magicked, diffSkyfile.skycell_id AS component, + 0 AS mismatched_tess, CAST(NULL AS CHAR(255)) AS uri, diffSkyfile.path_base, Index: /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_raw.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_raw.sql (revision 32630) @@ -14,4 +14,5 @@ rawExp.magicked, class_id as component, + 0 AS mismatched_tess, rawImfile.uri AS uri, -- XXX: replace this with rawImfile.path_base once it exists Index: /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_warp.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/magicdstool_todestreak_warp.sql (revision 32630) @@ -14,4 +14,6 @@ warpRun.magicked, warpSkyfile.skycell_id as component, + diffRun.tess_id != warpRun.tess_id AS mismatched_tess, + diffRun.tess_id as diff_tess_id, warpSkyfile.uri, warpSkyfile.path_base, @@ -27,4 +29,5 @@ JOIN magicMask USING (magic_id) JOIN magicRun USING (magic_id) +JOIN diffRun USING(diff_id) JOIN warpRun ON warp_id = stage_id JOIN warpSkyfile USING(warp_id) Index: /branches/eam_branches/ipp-20110906/ippTools/share/pubtool_definerun.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/pubtool_definerun.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/pubtool_definerun.sql (revision 32630) @@ -15,4 +15,5 @@ JOIN diffRun JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile using(diff_id) JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 -- Only JOINing input, not reference! JOIN fakeRun USING(fake_id) @@ -30,4 +31,5 @@ ) ) + AND diffSkyfile.quality = 0 -- WHERE hook %s UNION Index: /branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_create_tables.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_create_tables.sql (revision 32630) @@ -554,6 +554,5 @@ KEY(workdir_state), KEY(label), - KEY(data_group), - INDEX(add_id, cam_id) + KEY(data_group) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1157,6 +1156,6 @@ tess_id VARCHAR(64), diff_skyfile_id BIGINT AUTO_INCREMENT, + PRIMARY KEY(diff_skyfile_id), KEY(diff_id, skycell_id), - KEY(diff_skyfile_id), KEY(warp1), KEY(warp2), @@ -1969,9 +1968,11 @@ dtime_verify FLOAT, dtime_merge FLOAT, - dtime_script, - epoch TIMESTAMP, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, mergedvodb_path VARCHAR(255) NOT NULL, fault SMALLINT NOT NULL, - KEY(minidvodb_id), CONSTRAINT UNIQUE(minidvodb_id), FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun (minidvodb_id) + UNIQUE KEY (minidvodb_id), + KEY(minidvodb_id), + FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1990,7 +1991,39 @@ FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; - - - +CREATE TABLE vpRun ( + vp_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + workdir VARCHAR(255), + note VARCHAR(255), + dest_id BIGINT, + outroot VARCHAR(255), + dtime_script FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL DEFAULT 0, + PRIMARY KEY(vp_id), + KEY(exp_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(fault), + INDEX(vp_id, exp_id), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE vpProcessedCell ( + vp_id BIGINT, + class_id VARCHAR(64), + cell_id VARCHAR(64), + dtime_photom FLOAT, + quality SMALLINT NOT NULL DEFAULT 0, + path_base VARCHAR(255), + fault SMALLINT NOT NULL DEFAULT 0, + KEY(quality), + KEY(fault), + FOREIGN KEY(vp_id) REFERENCES vpRun(vp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; -- This comment line is here to avoid empty query error. Index: /branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_drop_tables.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/pxadmin_drop_tables.sql (revision 32630) @@ -97,4 +97,12 @@ DROP TABLE IF EXISTS diffPhotRun; DROP TABLE IF EXISTS diffPhotSkyfile; +DROP TABLE IF EXISTS lapExp; +DROP TABLE IF EXISTS lapRun; +DROP TABLE IF EXISTS lapSequence; +DROP TABLE IF EXISTS minidvodbRun; +DROP TABLE IF EXISTS minidvodbCopy; +DROP TABLE IF EXISTS minidvodbProcessed; +DROP TABLE IF EXISTS vpProcessedCell; +DROP TABLE IF EXISTS vpRun; SET FOREIGN_KEY_CHECKS=1 Index: /branches/eam_branches/ipp-20110906/ippTools/share/vptool_find_rawexp.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/vptool_find_rawexp.sql (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/share/vptool_find_rawexp.sql (revision 32630) @@ -0,0 +1,12 @@ +-- this query is used to find potental rawExps to be queued for vptool +-- processeing +SELECT DISTINCT + rawExp.*, + newExp.label +FROM rawExp +JOIN newExp using (exp_id) +JOIN rawImfile using(exp_id) +LEFT JOIN vpRun USING (exp_id) +WHERE + rawExp.fault = 0 + AND rawImfile.video_cells Index: /branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingimfile.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingimfile.sql (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingimfile.sql (revision 32630) @@ -0,0 +1,12 @@ +SELECT + vpRun.*, + rawImfile.class_id, + rawImfile.uri +FROM vpRun + JOIN rawExp USING(exp_id) + JOIN rawImfile USING(exp_id) + LEFT JOIN vpProcessedCell USING(vp_id, class_id) +WHERE rawImfile.video_cells + AND vpRun.state = 'new' + AND vpProcessedCell.vp_id IS NULL + Index: /branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingrun.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingrun.sql (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/share/vptool_pendingrun.sql (revision 32630) @@ -0,0 +1,17 @@ +SELECT vp_id, + rawExp.camera, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + vpRun.label, + vpRun.workdir, + vpRun.dest_id, + rcDestination.name AS dest_name, + rcDestination.dbname AS ds_dbname, + rcDestination.dbhost AS ds_dbhost, + IFNULL(Label.priority, 10000) as priority +FROM vpRun + JOIN rawExp USING(exp_id) + LEFT JOIN rcDestination USING(dest_id) + LEFT JOIN Label ON vpRun.label = Label.label +WHERE vpRun.state = 'new' + AND vpRun.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: /branches/eam_branches/ipp-20110906/ippTools/share/vptool_processedcell.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/vptool_processedcell.sql (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/share/vptool_processedcell.sql (revision 32630) @@ -0,0 +1,22 @@ +SELECT + vpProcessedCell.*, + vpRun.state, + vpRun.label, + vpRun.data_group, + vpRun.outroot, + rawExp.exp_id, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.filter, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + rawExp.dateobs, + rawExp.obs_mode, + rawExp.comment, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM vpRun + JOIN vpProcessedCell USING(vp_id) + JOIN rawExp USING(exp_id) Index: /branches/eam_branches/ipp-20110906/ippTools/share/vptool_revertrun.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/vptool_revertrun.sql (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/share/vptool_revertrun.sql (revision 32630) @@ -0,0 +1,4 @@ +UPDATE vpRun + SET fault = 0 +WHERE vpRun.state = 'new' + AND fault > 0 Index: /branches/eam_branches/ipp-20110906/ippTools/share/warptool_scmap.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/warptool_scmap.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/warptool_scmap.sql (revision 32630) @@ -29,4 +29,5 @@ ON chipRun.chip_id = chipProcessedImfile.chip_id AND warpSkyCellMap.class_id = chipProcessedImfile.class_id + AND chipProcessedImfile.quality = 0 JOIN rawImfile ON chipRun.exp_id = rawImfile.exp_id Index: /branches/eam_branches/ipp-20110906/ippTools/share/warptool_towarped.sql =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/share/warptool_towarped.sql (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/share/warptool_towarped.sql (revision 32630) @@ -100,4 +100,7 @@ AND warpSkyfile.fault = 0 AND camRun.state = 'full' + AND chipProcessedImfile.quality = 0 + -- Maybe add this to solve the if any chip is bad a whole warp can't be updated problem? + -- AND (chipProcessedImfile.fault = 0 OR chipProcessedImfile.fault = 26) AND (Label.active OR Label.active IS NULL) -- where hook 2 %s Index: /branches/eam_branches/ipp-20110906/ippTools/src/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/Makefile.am (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/Makefile.am (revision 32630) @@ -29,5 +29,6 @@ diffphottool \ minidvodbtool \ - laptool + laptool \ + vptool pkginclude_HEADERS = \ @@ -78,5 +79,6 @@ diffphottool.h \ minidvodbtool.h \ - laptool.h + laptool.h \ + vptool.h lib_LTLIBRARIES = libpxtools.la @@ -298,4 +300,10 @@ laptoolConfig.c +vptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +vptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +vptool_SOURCES = \ + vptool.c \ + vptoolConfig.c + clean-local: -rm -f TAGS Index: /branches/eam_branches/ipp-20110906/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/addtool.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/addtool.c (revision 32630) @@ -280,4 +280,18 @@ } + + //if we grab a group of camRun/stackRuns that have multiple camRun/stackRun pointing to the same exp_id/stack_id, and that exp_id/stack_id has never before been added to addRun, we need to group by exp_id/stack_id to ensure that only 1 of the camRun/stackRun for that exp_id is added to the addRun stage. + + if (strcmp(stage,"cam") == 0) { + psStringAppend(&query, " GROUP BY exp_id"); + } + if (strcmp(stage,"stack") == 0) { + psStringAppend(&query, " GROUP BY stack_id"); + } + if (strcmp(stage,"staticsky") == 0) { + psStringAppend(&query, " GROUP BY stack_id"); + } + + psTrace("addtool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); Index: /branches/eam_branches/ipp-20110906/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/chiptool.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/chiptool.c (revision 32630) @@ -43,4 +43,5 @@ static bool revertprocessedimfileMode(pxConfig *config); static bool updateprocessedimfileMode(pxConfig *config); +static bool dropprocessedimfileMode(pxConfig *config); static bool advanceexpMode(pxConfig *config); static bool blockMode(pxConfig *config); @@ -88,4 +89,5 @@ MODECASE(CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); MODECASE(CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_DROPPROCESSEDIMFILE, dropprocessedimfileMode); MODECASE(CHIPTOOL_MODE_ADVANCEEXP, advanceexpMode); MODECASE(CHIPTOOL_MODE_BLOCK, blockMode); @@ -962,5 +964,5 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); if (psListLength(where->list) == 0) { psError(PS_ERR_UNKNOWN, true, "search parameters are required"); @@ -986,4 +988,76 @@ + return true; +} + +// "drop" previously processed components as though the original processing +// had poor quality. Sets quality, data_state, clears fault and magicked +// Deletes corresponding magicDSFile (if one exists) +static bool dropprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipProcessedImfile.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_S64(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", true, false); + + if (!quality) { + psError(PXTOOLS_ERR_CONFIG, true, "non-zero quality value is required"); + return false; + } + + if (psListLength(where->list) == 0) { + // won't get here + psError(PXTOOLS_ERR_CONFIG, true, "-chip_id and -class_id are required"); + return false; + } + + psString query = pxDataGet("chiptool_dropprocessedimfile_update.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString delete_query = pxDataGet("chiptool_dropprocessedimfile_delete.sql"); + if (!delete_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(where); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!p_psDBRunQueryF(config->dbh, query, quality)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error on rollback"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psStringAppend(&delete_query, " AND %s", whereClause); + psFree(whereClause); + if (!p_psDBRunQuery(config->dbh, delete_query)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error on rollback"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete_query); + return false; + } + psFree(delete_query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; } @@ -1418,5 +1492,5 @@ psMetadata *where = psMetadataAlloc(); pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); // look for completed chipPendingExp // migrate them to chipProccessedExp & camPendingExp @@ -1432,4 +1506,6 @@ psFree(whereClause); } + + psTrace("chiptool", 10, "%s\n", query); psFree(where); Index: /branches/eam_branches/ipp-20110906/ippTools/src/chiptool.h =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/chiptool.h (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/chiptool.h (revision 32630) @@ -33,4 +33,5 @@ CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, + CHIPTOOL_MODE_DROPPROCESSEDIMFILE, CHIPTOOL_MODE_SETIMFILETOUPDATE, CHIPTOOL_MODE_ADVANCEEXP, Index: /branches/eam_branches/ipp-20110906/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/chiptoolConfig.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/chiptoolConfig.c (revision 32630) @@ -237,5 +237,12 @@ psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); - // -processedExp + + // -dropprocessedimfile + psMetadata *dropprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(dropprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(dropprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID (required)", NULL); + psMetadataAddS16(dropprocessedimfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality (required)", 0); + + // -listrun psMetadata *listrunArgs = psMetadataAlloc(); pxchipSetSearchArgs(listrunArgs); @@ -372,4 +379,5 @@ PXOPT_ADD_MODE("-processedimfile", "show processed imfiles", CHIPTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs); + PXOPT_ADD_MODE("-dropprocessedimfile","drop previously procesed imfile", CHIPTOOL_MODE_DROPPROCESSEDIMFILE, dropprocessedimfileArgs); PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); PXOPT_ADD_MODE("-runstate", "list the states of chip run", CHIPTOOL_MODE_RUNSTATE, runstateArgs); Index: /branches/eam_branches/ipp-20110906/ippTools/src/difftool.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/difftool.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/difftool.c (revision 32630) @@ -1634,4 +1634,6 @@ PXOPT_COPY_F32(config->args, selectWhere, "-timediff", "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<="); + PXOPT_COPY_F32(config->args, selectWhere, "-mintimediff", + "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", ">="); // other where restrictions: @@ -1867,5 +1869,5 @@ const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id"); psString input_data_group = psMetadataLookupStr(NULL, row, "input_data_group"); - if (input_data_group) { + if (!data_group) { data_group = input_data_group; } @@ -1934,19 +1936,70 @@ return false; } + psS64 numSkycells = psDBAffectedRows(config->dbh); + if (numSkycells > 0) { + if (!setdiffRunState(config, run->diff_id, "new", false)) { + psError(PS_ERR_UNKNOWN, false, "Failed to change diffRun.state for diff_id: %" PRId64, + run->diff_id); + psFree(input); + psFree(diff); + psFree(run); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + run->state = psStringCopy("new"); + } else { + // No overlap between the warps. Insert a dummy skycell which will complete this pair of warps + // XXX: we'd like to use diffInputSkyfileRowAlloc and diffInputSkyfileInsert but there doesn't + // seem to be a way to pass in NULL for skycell_id, stack1 and stack2 + psString dummyQuery = NULL; + psStringAppend(&dummyQuery, + "INSERT INTO diffInputSkyfile VALUES(%" PRId64 ", NULL, %s, NULL, %s, NULL, '%s', 0)", + run->diff_id, + input, // warp1 + template, // warp2 + tess_id); + if (!p_psDBRunQuery(config->dbh, dummyQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(dummyQuery); + psFree(input); + psFree(diff); + psFree(run); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(dummyQuery); + psString finishQuery = NULL; + psStringAppend( &finishQuery, + "UPDATE diffRun set state ='full', note = 'dummy run - no overlap' WHERE diff_id = %" PRId64, + run->diff_id); + if (!p_psDBRunQuery(config->dbh, finishQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(finishQuery); + psFree(input); + psFree(diff); + psFree(run); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(finishQuery); + run->state = psStringCopy("full"); + } psFree(input); psFree(diff); - - if (!setdiffRunState(config, run->diff_id, "new", false)) { - psError(PS_ERR_UNKNOWN, false, "Failed to change diffRun.state for diff_id: %" PRId64, - run->diff_id); - psFree(run); - psFree(list); - psFree(insert); - psFree(results); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } psArrayAdd(list, list->n, run); Index: /branches/eam_branches/ipp-20110906/ippTools/src/difftoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/difftoolConfig.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/difftoolConfig.c (revision 32630) @@ -283,4 +283,5 @@ psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-timediff", 0, "limit time difference between input and template", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-mintimediff", 0, "limit time difference between input and template to be greater than this", NAN); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-backwards", 0, "Template comes after input?", false); psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-rotdiff", 0, "limit rotator difference between input and template", NAN); Index: /branches/eam_branches/ipp-20110906/ippTools/src/laptool.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/laptool.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/laptool.c (revision 32630) @@ -30,4 +30,6 @@ static bool updateexpMode(pxConfig *config); +static bool diffcheckMode(pxConfig *config); + static bool inactiveexpMode(pxConfig *config); @@ -62,4 +64,6 @@ MODECASE(LAPTOOL_MODE_UPDATEEXP, updateexpMode); + MODECASE(LAPTOOL_MODE_DIFFCHECK, diffcheckMode); + MODECASE(LAPTOOL_MODE_INACTIVEEXP, inactiveexpMode); default: @@ -276,4 +280,7 @@ PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + psMetadata *chipWhere = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, chipWhere, "-seq_id", "lapRun.seq_id", "=="); + // This seems unnecessarily clunky. if (!all_obsmode) { @@ -296,8 +303,13 @@ } + psString chipWhereClause = psDBGenerateWhereConditionSQL(chipWhere,NULL); if (whereClause) { psStringSubstitute(&query,whereClause,"@WHERE@"); } + if (chipWhereClause) { + psStringSubstitute(&query,chipWhereClause,"@CHIPWHERE@"); + } psFree(where); + psFree(chipWhere); // Fetch exposures @@ -755,6 +767,68 @@ } - - +static bool diffcheckMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lapRun.lap_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + + psString query = pxDataGet("laptool_WSdiff_check.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return(false); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause, "\n AND "); + psStringSubstitute(&query,whereClause,"@WHERE@"); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error %d",err); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapRunWarpStackSkycells", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return(true); +} static bool inactiveexpMode(pxConfig *config) Index: /branches/eam_branches/ipp-20110906/ippTools/src/laptool.h =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/laptool.h (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/laptool.h (revision 32630) @@ -19,4 +19,5 @@ LAPTOOL_MODE_STACKS, LAPTOOL_MODE_UPDATEEXP, + LAPTOOL_MODE_DIFFCHECK, LAPTOOL_MODE_INACTIVEEXP } laptoolMode; Index: /branches/eam_branches/ipp-20110906/ippTools/src/laptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/laptoolConfig.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/laptoolConfig.c (revision 32630) @@ -132,4 +132,14 @@ ADD_OPT(Bool,updateexpArgs, "-inactive", "set this exposure to active", 0); + // -diffcheck + psMetadata *diffcheckArgs = psMetadataAlloc(); + ADD_OPT(S64, diffcheckArgs, "-lap_id", "search by lap run ID", 0); + ADD_OPT(S64, diffcheckArgs, "-exp_id", "search by exposure ID", 0); + ADD_OPT(S64, diffcheckArgs, "-chip_id", "search by chip ID", 0); + ADD_OPT(S64, diffcheckArgs, "-warp_id", "search by warp ID", 0); + ADD_OPT(S64, diffcheckArgs, "-seq_id", "search by lap sequence ID", 0); + ADD_OPT(Str, diffcheckArgs, "-skycell_id", "search by skycell ID", 0); + ADD_OPT(Bool, diffcheckArgs, "-simple", "use simple output format", 0); + // -inactiveexp psMetadata *inactiveexpArgs = psMetadataAlloc(); @@ -152,4 +162,5 @@ PXOPT_ADD_MODE("-stacks", "", LAPTOOL_MODE_STACKS, stacksArgs); PXOPT_ADD_MODE("-updateexp", "", LAPTOOL_MODE_UPDATEEXP, updateexpArgs); + PXOPT_ADD_MODE("-diffcheck", "", LAPTOOL_MODE_DIFFCHECK, diffcheckArgs); PXOPT_ADD_MODE("-inactiveexp", "", LAPTOOL_MODE_INACTIVEEXP, inactiveexpArgs); Index: /branches/eam_branches/ipp-20110906/ippTools/src/magicdstool.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/magicdstool.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/magicdstool.c (revision 32630) @@ -350,9 +350,9 @@ PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", true, false); // Optional PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); - PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); @@ -374,5 +374,9 @@ pxAddLabelSearchArgs (config, where, "-magic_label", "magicRun.label", "=="); // define magic label psString labelName = NULL; // Name of label - psStringAppend(&labelName, "%sRun.label", stage); + if (strcmp(stage, "camera")) { + psStringAppend(&labelName, "%sRun.label", stage); + } else { + psStringAppend(&labelName, "camRun.label"); + } pxAddLabelSearchArgs (config, where, "-stage_label", labelName, "=="); // define stageRun label psFree(labelName); @@ -392,4 +396,6 @@ break; case IPP_STAGE_CAMERA: + query = pxDataGet("magicdstool_definecopy_camera.sql"); + break; case IPP_STAGE_DIFF: case IPP_STAGE_FAKE: Index: /branches/eam_branches/ipp-20110906/ippTools/src/magicdstoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/magicdstoolConfig.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/magicdstoolConfig.c (revision 32630) @@ -79,5 +79,5 @@ psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); - psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); Index: /branches/eam_branches/ipp-20110906/ippTools/src/pxtools.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/pxtools.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/pxtools.c (revision 32630) @@ -35,4 +35,5 @@ if (!strcmp(state, "drop")) return true; if (!strcmp(state, "wait")) return true; + if (!strcmp(state, "keep")) return true; if (!strcmp(state, "goto_cleaned")) return true; if (!strcmp(state, "error_cleaned")) return true; Index: /branches/eam_branches/ipp-20110906/ippTools/src/regtool.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/regtool.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/regtool.c (revision 32630) @@ -268,4 +268,8 @@ } } + char *is_stareim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'a'); + if (is_stareim) { + continue; + } if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| @@ -386,4 +390,9 @@ } } + char *is_stareim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'a'); + if (is_stareim) { + continue; + } + bool status = false; @@ -882,12 +891,18 @@ PXOPT_LOOKUP_STR(set_md5sum, config->args, "-set_md5sum", false, false); PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_BOOL(set_ignored, config->args, "-set_ignored", false); + PXOPT_LOOKUP_BOOL(clear_ignored, config->args, "-clear_ignored", false); PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state) { - psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected"); + if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state && !set_ignored && !clear_ignored) { + psError(PS_ERR_UNKNOWN, false, "at least one of -fault or -burntool_state -set_ignored -clear_ignored or -set_state must be selected"); return false; } if ((fault != INT16_MAX) && (burntool_state != INT16_MAX)) { psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected"); + return false; + } + if (set_ignored && clear_ignored) { + psError(PS_ERR_UNKNOWN, true, "only one of -set_ignored or -clear_ignored may be selected"); return false; } @@ -898,5 +913,6 @@ if (fault != INT16_MAX) { - // this is fairly dangerous : can set all if the where is not set... + // this is fairly dangerous : can set all if the where is not set... but since exp_id and class_id + // were required above this is safe if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault, 0)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); @@ -930,4 +946,16 @@ } psStringAppend(&setvalues,"rawImfile.data_state = '%s'",set_state); + } + if (set_ignored) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.ignored = 1"); + } + if (clear_ignored) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.ignored = 0"); } if (hostname) { Index: /branches/eam_branches/ipp-20110906/ippTools/src/regtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/regtoolConfig.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/regtoolConfig.c (revision 32630) @@ -194,4 +194,6 @@ ADD_OPT(Str, updateprocessedimfileArgs, "-set_md5sum", "set md5sum", NULL); ADD_OPT(Str, updateprocessedimfileArgs, "-set_state", "set data state", NULL); + psMetadataAddBool(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_ignored", 0, "set imfile to be ignored for processing", false); + psMetadataAddBool(updateprocessedimfileArgs, PS_LIST_TAIL, "-clear_ignored", 0, "set imfile to not be ignored for processing", false); Index: /branches/eam_branches/ipp-20110906/ippTools/src/vptool.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/vptool.c (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/src/vptool.c (revision 32630) @@ -0,0 +1,1885 @@ +/* + * vptool.c + * + * Copyright (C) 2011 IfA University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "pxchip.h" + +#include "vptool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool pendingrunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool revertrunMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedcellMode(pxConfig *config); +static bool processedcellMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = vptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(VPTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(VPTOOL_MODE_PENDINGRUN, pendingrunMode); + MODECASE(VPTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(VPTOOL_MODE_REVERTRUN, revertrunMode); + MODECASE(VPTOOL_MODE_LISTRUN, listrunMode); + MODECASE(VPTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(VPTOOL_MODE_ADDPROCESSEDCELL, addprocessedcellMode); + MODECASE(VPTOOL_MODE_PROCESSEDCELL, processedcellMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +// Queue exposures for videophot processing +static bool queue_exposures(pxConfig *config, // Configuration + const psArray *exps, // Exposures information + const char *label, // Label, or NULL to inherit + const char *data_group, // Data group, or NULL to inherit + const char *workdir, // Working directory, or NULL to inherit + const char *note, // Note + const psS64 dest_id + ) +{ + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + +# define GET_VALUE(PTYPE,CTYPE,VALUE,NAME) \ + PTYPE VALUE; \ + { \ + bool status; \ + VALUE = psMetadataLookup##CTYPE(&status, md, NAME); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", NAME); \ + return false; \ + } \ + } + + // loop over our list of exp_ids + for (long i = 0; i < psArrayLength(exps); i++) { + psMetadata *md = exps->data[i]; + + rawExpRow *row = rawExpObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + return false; + } + + GET_VALUE (psS64, S64, exp_id, "exp_id"); + GET_VALUE (psString, Str, raw_workdir, "workdir"); + GET_VALUE (psString, Str, raw_label, "label"); + GET_VALUE (psString, Str, raw_reduction, "reduction"); + // GET_VALUE (psString, Str, raw_expgroup, "expgroup"); + GET_VALUE (psString, Str, raw_dvodb, "dvodb"); + GET_VALUE (psString, Str, raw_tess_id, "tess_id"); + GET_VALUE (psString, Str, raw_end_stage, "end_stage"); + + if (!row->exp_id) { + psError(PS_ERR_UNKNOWN, false, "failed to find value for exp_id"); + return false; + } + + // queue the exp + if (!vpRunInsert(config->dbh, + 0, //vp_id + exp_id, + "new", + label ? label : raw_label, + data_group ? data_group : (label ? label : raw_label), + workdir ? workdir : raw_workdir, + note, + dest_id, + NULL, // outroot + 0, // dtime_script + NULL, // hostname + 0 // fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_id: %" PRId64, exp_id); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE"); // define using newExp label + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_S64(dest_id, config->args, "-set_dest_id", false, false); + + // default + PXOPT_LOOKUP_BOOL(unique, config->args, "-unique", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("vptool_find_rawexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + psFree(whereClause); + psFree(where); + + if (unique) { + psStringAppend(&query, "AND vp_id IS NULL"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!queue_exposures(config, output, label, data_group, workdir, note, dest_id)) { + psError(psErrorCodeLast(), false, "Unable to queue exposures for videophot processing."); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool pendingrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + // Do we need pxchipArgs? + pxchipGetSearchArgs (config, where); //chipRun, rawExp + PXOPT_COPY_S64(config->args, where, "-vp_id", "vpRun.vp_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "vpRun.label", "=="); + + psString query = pxDataGet("vptool_pendingrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, "\nGROUP BY vp_id HAVING"); + psStringAppend(&query, "\nORDER BY priority DESC, vp_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "vpPendingRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +#ifdef notdef +static bool definecopyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipOld.data_group", "LIKE"); + + // psListLength(where->list) is at least 1 because exp_type defaults to "object" + // so we require a list longer than 1 entry + if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + // default + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("vptool_definecopy.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!queue_exposures(config, output, workdir, label, data_group, dist_group, + note)) { + psError(psErrorCodeLast(), false, "Unable to queue exposures for chip."); + psFree(output); + return false; + } + psFree(output); + + return true; +} +#endif + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); +// pxchipGetSearchArgs (config, where); // rawExp, vpRun + PXOPT_COPY_S64(config->args, where, "-vp_id", "vpRun.vp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "vpRun.label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "vpRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "vpRun.data_group", "LIKE"); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-set_hostname", false, false); + PXOPT_LOOKUP_STR(outroot, config->args, "-set_outroot", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); + + if (!state && !label && !data_group && !note && !hostname && !fault) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "must supply at least one set parameter"); + return false; + } + + psString query = psStringCopy("UPDATE vpRun"); + + char *separator = " SET "; + char * comma = ","; + +# define addColumn(_tab, _fmt, _val) \ + do { \ + if (_val) { \ + psStringAppend(&query, "%s %s.%s =" _fmt, separator, _tab, #_val, _val); \ + separator = comma; \ + } \ + } while (0) + + addColumn("vpRun", "'\%s'", state); + addColumn("vpRun", "'\%s'", label); + addColumn("vpRun", "'\%s'", data_group); + addColumn("vpRun", "'\%s'", note); + addColumn("vpRun", "'\%s'", hostname); + addColumn("vpRun", "'\%s'", outroot); + addColumn("vpRun", "%d", fault); + addColumn("vpRun", "%f", dtime_script); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + + return true; +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + // Do we need pxchipArgs? + pxchipGetSearchArgs (config, where); //chipRun, rawExp + PXOPT_COPY_S64(config->args, where, "-vp_id", "vpRun.vp_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "vpRun.label", "=="); + + psString query = pxDataGet("vptool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "vpPendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedcellMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // vp_id, class_id, and cell_id are required + PXOPT_LOOKUP_S64(vp_id, config->args, "-vp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(cell_id, config->args, "-cell_id", true, false); + + // optional + PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!vpProcessedCellInsert(config->dbh, + vp_id, + class_id, + cell_id, + dtime_photom, + quality, + path_base, + fault + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool processedcellMode(pxConfig *config) +{ +return false; +#ifdef notyet + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(allfiles, config->args, "-allfiles", false); + if (allfiles) { + faulted = false; + } + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_imfile_id", "chipImfile.chip_imfile_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "=="); + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "chipProcessedImfile"); + // add cuts on ra and decl if supplied + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + psString query = pxDataGet("vptool_processedcell.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : ""); + psFree(whereClause); + } else if (psMetadataLookupBool(NULL, config->args, "-all") || (faulted || where2)) { + psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : ""); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); + return false; + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0"); + } else if (!allfiles) { + // don't list faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); + } + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, chip_id DESC"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +#endif +} + +static bool revertrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-vp_id", "vpRun.vp_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "vpRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "vpProcessedCell.fault", "=="); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // find vpRuns with faulted cells + psString query = pxDataGet("vptool_revertrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + int numUpdated = psDBAffectedRows(config->dbh); + psLogMsg("vptool", PS_LOG_INFO, "Reverted %d vpRuns", numUpdated); + + return true; +} + +#ifdef NOTDEF +static bool revertcleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + char* newState = NULL; + if (!state) { + state = "error_cleaned"; + } + if (!strcmp(state, "error_cleaned")) { + newState = "goto_cleaned"; + } else if (!strcmp(state, "error_purged")) { + newState = "goto_purged"; + } else if (!strcmp(state, "error_scrubbed")) { + newState = "goto_scrubbed"; + } else { + psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("vptool_revertcleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool updateprocessedcellMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL); + if (psListLength(where->list) == 0) { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + + if (!state) { + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + psFree(where); + } + else { + if (!pxchipProcessedImfileSetStateByQuery(config,where,state)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipProcessedImfile state"); + return(false); + } + } + + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!chipMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM chipMask"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// return the list of labels which are NOT blocked +static bool unmaskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + psString query = pxDataGet("vptool_unmasked.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, "chipUnmask"); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipUnmask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM chipMask WHERE label = '%s'"; + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("vptool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingcleanupimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (chip_id) { + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + } + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("vptool_pendingcleanupimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingCleanupImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("vptool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool runMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // make sure that the state string is valid + if (!pxIsValidState(state)) { + psError(PXTOOLS_ERR_CONFIG, false, "%s is not a valid state", state); + return false; + } + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + psString query = pxDataGet("vptool_run.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool advanceexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + // look for completed chipPendingExp + // migrate them to chipProccessedExp & camPendingExp + psString query = pxDataGet("vptool_completely_processed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS64 chip_id = psMetadataLookupS64(NULL,row,"chip_id"); + + psString software_ver = NULL; + psS64 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"vptool_coalesce_run.sql",chip_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + psMetadataAddStr(row, PS_LIST_TAIL, "software_ver", 0, "Software version", software_ver); + psMetadataAddS32(row, PS_LIST_TAIL, "maskfrac_npix", 0, "Number of pixels used for mask stats", (psS32) maskfrac_npix); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_static", 0, "Fraction of static masked pixels", maskfrac_static); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_dynamic", 0, "Fraction of dynamic masked pixels", maskfrac_dynamic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_magic", 0, "Fraction of magic masked pixels", maskfrac_magic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_advisory", 0, "Fraction of advisory pixels", maskfrac_advisory); +/* psWarning("ADVANCE %ld %s %d %f %f %f %f\n",chip_id,software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory); */ + chipRunRow *chipRun = chipRunObjectFromMetadata(row); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // Set chipRun.software_ver to the appropriate value + if (software_ver) { + if (!pxSetRunSoftware(config, "chipRun", "chip_id", chip_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } + // Set chipRun.maskfrac* to the appropriate values. + if (maskfrac_npix) { + if (!pxSetRunMaskfrac(config, "chipRun", "chip_id",chip_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } + // set chipRun.state to 'stop' and update the magicked state + if (!pxchipRunSetState(config, chipRun->chip_id, "full", chipRun->magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change chipRun.state for chip_id: %" PRId64, chipRun->chip_id); + psFree(chipRun); + psFree(output); + return false; + } + + // should we stop here or proceed on to the cam stage? + // NULL for end_stage means go as far as possible + if (chipRun->end_stage && psStrcasestr(chipRun->end_stage, "chip")) { + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(chipRun); + continue; + } + // else continue on... + + // camQueueChipID() can only be run after chipRun.state has been set to + // stop + if (!pxcamQueueByChipID(config, + chipRun->chip_id, + chipRun->workdir, + chipRun->label, + chipRun->data_group, + chipRun->dist_group, + chipRun->reduction, + chipRun->expgroup, + chipRun->dvodb, + chipRun->tess_id, + chipRun->end_stage, + chipRun->magicked, + NULL // note does not propagate + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp"); + psFree(chipRun); + psFree(output); + return false; + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(chipRun); + } + + psFree(output); + + return true; +} + +// update chipProcessedImfile.data_state to given value. +// afterwards, if all imfiles in the exposure have the new state, update the state for the exposure as well +// shared code for the modes -tocleanedimfile -tofullimfile -topurgedimfile + +static bool change_imfile_data_state(pxConfig *config, psString data_state, psString run_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // chip_id, class_id are required + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + psString query = pxDataGet("vptool_change_imfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString set_magicked_imfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); + if (!strcmp(data_state, "full")) { + // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will + // block warp processing until destreaking has been done + // otherwise copy magicked from the rawImfile + // Same thing for chipRun/rawExp + psStringAppend(&set_magicked_imfile, "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0" + " AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)"); + psStringAppend(&set_magicked_run, "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), " + " chipRun.magicked, rawExp.magicked)"); + + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + // if magicked is non-zero set it to -1 + // Once one imfile has been cleaned, the chipRun is no longer 'magicked' + psStringAppend(&set_magicked_imfile, "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)," + " chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + psFree(set_magicked_imfile); + psFree(set_magicked_run); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("vptool_change_exp_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) { + psFree(query); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool tocleanedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "cleaned", "goto_cleaned"); +} +static bool tofullimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "full", "update"); +} +static bool topurgedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 3; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(dummy, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + + ExportTable tables [] = { + {"chipRun", "vptool_export_run.sql"}, + {"chipImfile", "vptool_export_imfile.sql"}, + {"chipProcessedImfile", "vptool_export_processed_imfile.sql"}, + }; + + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + bool success = true; + if (!strcmp(tables[i].tableName, "chipRun")) { + success = pxSetStateCleaned("chipRun", "state", output); + } else if (!strcmp(tables[i].tableName, "chipProcessedImfile")) { + success = pxSetStateCleaned("chipProcessedImfile", "data_state", output); + } + if (!success) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + unsigned int nFail = 0; // Number of failed lines + psMetadata *input = psMetadataConfigRead(NULL, &nFail, infile, false); + if (nFail) { + psError(PS_ERR_IO, false, "%d failed lines in input", nFail); + psFree(input); + return false; + } + + psVector *identifiers = psVectorAllocEmpty(16, PS_TYPE_U64); // Identifiers inserted + + psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL); // Iterator + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + // first item is the dbversion, skip it + psMetadataItem *dbversion = psMetadataGetAndIncrement(iter); + (void) dbversion; + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psMetadataItem *item; // Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + PXMIRROR_PRIMARY(item, "chipRun", chipRunRow, chipRunObjectFromMetadata, identifiers, chip_id, + chipRunInsertObject, config->dbh, + { psFree(iter); psFree(identifiers); psFree(input); }); + + PXMIRROR_OTHER(item, "chipImfile", chipImfileRow, chipImfileObjectFromMetadata, identifiers, chip_id, + chipImfileInsertObject, config->dbh, + { psFree(iter); psFree(identifiers); psFree(input); }); + + PXMIRROR_OTHER(item, "chipProcessedImfile", chipProcessedImfileRow, + chipProcessedImfileObjectFromMetadata, identifiers, chip_id, + chipProcessedImfileInsertObject, config->dbh, + { psFree(iter); psFree(identifiers); psFree(input); }); + } + psFree(iter); + psFree(input); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psLogMsg("vptool", PS_LOG_INFO, "%ld chipRuns added", identifiers->n); + psFree(identifiers); + + return true; +} + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("vptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +// a very specfic function to queue a cleaned chipProcessedImfile to be updated +static bool setimfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("vptool_setimfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , chipRun.label = '%s'", label); + } + + if (class_id) { + psStringAppend(&query, " AND (chipProcessedImfile.class_id = '%s')", class_id); + } + + // we do not update components with the magic fault value. They are non-updateable + psStringAppend(&query, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + + if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} +#endif // NOTDEF +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "chipRun.dist_group", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "chipRun"); + // add cuts on ra and decl if supplied + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + psString query = pxDataGet("vptool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : ""); + psFree(whereClause); + } else if (psMetadataLookupBool(NULL, config->args, "-all") || where2) { + psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : ""); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); + return false; + } + psFree(where); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, chip_id DESC"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/ipp-20110906/ippTools/src/vptool.h =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/vptool.h (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/src/vptool.h (revision 32630) @@ -0,0 +1,39 @@ +/* + * vptool.h + * + * Copyright (C) 2011 University of Hawaii IFA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef VPTOOL_H +#define VPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + VPTOOL_MODE_NONE = 0x0, + VPTOOL_MODE_DEFINEBYQUERY, + VPTOOL_MODE_PENDINGRUN, + VPTOOL_MODE_UPDATERUN, + VPTOOL_MODE_REVERTRUN, + VPTOOL_MODE_LISTRUN, + VPTOOL_MODE_PENDINGIMFILE, + VPTOOL_MODE_ADDPROCESSEDCELL, + VPTOOL_MODE_PROCESSEDCELL, +} vptoolMode; + +pxConfig *vptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // VPTOOL_H Index: /branches/eam_branches/ipp-20110906/ippTools/src/vptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/vptoolConfig.c (revision 32630) +++ /branches/eam_branches/ipp-20110906/ippTools/src/vptoolConfig.c (revision 32630) @@ -0,0 +1,297 @@ +/* + * vptoolConfig.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "vptool.h" + +pxConfig *vptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + // allow selecting by all of the args used to queue chipRuns + pxchipSetSearchArgs (definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by newExp label (LIKE comparison)", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-set_dest_id", 0, "define destination", 0); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-unique", 0, "only queue exposures that have no previous vpRun", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -pendingrun + psMetadata *pendingrunArgs = psMetadataAlloc(); + psMetadataAddS64(pendingrunArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + pxchipSetSearchArgs(pendingrunArgs); + psMetadataAddU64(pendingrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); +// pxchipSetSearchArgs (updaterunArgs); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-vp_id", 0, "search by vpRun ID", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "set data group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "set note", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_hostname", 0, "set hostname", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_outroot", 0, "set outroot", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + + // -revertrun + psMetadata *revertrunArgs = psMetadataAlloc(); + psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by vpRun label (LIKE comparison)", NULL); + psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by vpRun data_group (LIKE comparison)", NULL); + // pxchipSetSearchArgs(revertrunArgs); + psMetadataAddS16(revertrunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + pxchipSetSearchArgs(pendingimfileArgs); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -addprocessedcell + psMetadata *addprocessedcellArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedcellArgs, PS_LIST_TAIL, "-vp_id", 0, "define chip ID (required)", 0); + psMetadataAddStr(addprocessedcellArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddStr(addprocessedcellArgs, PS_LIST_TAIL, "-cell_id", 0, "define class ID (required)", NULL); + + psMetadataAddStr(addprocessedcellArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location (required)", NULL); + psMetadataAddF32(addprocessedcellArgs, PS_LIST_TAIL, "-dtime_photom", 0, "define elapsed time for photometry (seconds)", NAN); + psMetadataAddStr(addprocessedcellArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + + psMetadataAddS16(addprocessedcellArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddS16(addprocessedcellArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + + + // -processedcell + psMetadata *processedcellArgs = psMetadataAlloc(); + pxchipSetSearchArgs(processedcellArgs); + psMetadataAddS64(processedcellArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(processedcellArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(processedcellArgs, PS_LIST_TAIL, "-cell_id", 0, "search by class ID", NULL); + psMetadataAddStr(processedcellArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by vpRun label (LIKE comparison)", NULL); + psMetadataAddStr(processedcellArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by vpRun data_group (LIKE comparison)", NULL); + + psMetadataAddU64(processedcellArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedcellArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + pxspaceAddArguments(processedcellArgs); + +#ifdef notdef + // -updateprocessedcell + psMetadata *updateprocessedcellArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedcellArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(updateprocessedcellArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(updateprocessedcellArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddStr(updateprocessedcellArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddS16(updateprocessedcellArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); +#endif // notdef + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + pxchipSetSearchArgs(listrunArgs); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-vp_id", 0, "search by vpRun ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by vpRun label (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by vpRun state", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by vpRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by vpRun dist_group (LIKE comparison)", NULL); + // pxmagicAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + pxspaceAddArguments(listrunArgs); + +#ifdef notdef + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddStr(maskedArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(maskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unmasked + psMetadata *unmaskedArgs = psMetadataAlloc(); + psMetadataAddStr(unmaskedArgs, PS_LIST_TAIL, "-label", 0, "restrict to specified label", NULL); + psMetadataAddBool(unmaskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(unmaskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupimfile + psMetadata *pendingcleanupimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL); + psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -revertcleanup + psMetadata *revertcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by vpRun label (LIKE comparison)", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by vpRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state", 0, "search by current state", NULL); + psMetadataAddS16(revertcleanupArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + // -run + psMetadata *runArgs = psMetadataAlloc(); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddBool(runArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(runArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); + pxchipSetSearchArgs(runArgs); + psMetadataAddS64(runArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + + // -advanceexp + psMetadata *advanceexpArgs = psMetadataAlloc(); + psMetadataAddS64(advanceexpArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(advanceexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "advance exposures for specified label", NULL); + psMetadataAddU64(advanceexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -tocleanedimfile + psMetadata *tocleanedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "chip ID to update", 0); + psMetadataAddStr(tocleanedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -tofullimfile + psMetadata *tofullimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "chip ID to update", 0); + psMetadataAddStr(tofullimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -topurgedimfile + psMetadata *topurgedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "chip ID to update", 0); + psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "chip ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + // pxchipSetSearchArgs (exportrunArgs); XXX include search terms? + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-vp_id", 0, "export this chip ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "mark tables as cleaned", false); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); +#endif //notdef + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", VPTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-pendingrun", "list runs ready to process", VPTOOL_MODE_PENDINGRUN, pendingrunArgs); + PXOPT_ADD_MODE("-updaterun", "change vpRun properties", VPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-revertrun","clear a faulted vpRun", VPTOOL_MODE_REVERTRUN, revertrunArgs); + PXOPT_ADD_MODE("-pendingimfile", "list pending imfiles", VPTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + + PXOPT_ADD_MODE("-addprocessedcell", "add a processed cell", VPTOOL_MODE_ADDPROCESSEDCELL, addprocessedcellArgs); + PXOPT_ADD_MODE("-processedcell", "show processed cells", VPTOOL_MODE_PROCESSEDCELL, processedcellArgs); + PXOPT_ADD_MODE("-listrun", "list vpRuns", VPTOOL_MODE_LISTRUN, listrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/ipp-20110906/ippTools/src/warptool.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/warptool.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/warptool.c (revision 32630) @@ -984,5 +984,5 @@ psString query = psStringCopy(originalQuery); - + psStringAppend(&query,"\nORDER BY warp_id"); // treat limit == 0 as "no limit" psString limitString = psStringCopy(""); @@ -993,4 +993,5 @@ psStringAppend(&query, "%s", limitString); } + psString thisWhere = NULL; if (whereStr) { @@ -2051,8 +2052,12 @@ // magicked is only an argument for for -tofullskyfile PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); if (magicked) { psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked); psStringAppend(&set_magicked_run, "\n, warpRun.magicked = %" PRId64, magicked); } + if (quality) { + psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.quality = %"PRId16, quality); + } } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { // if magicked is currently nonzero set it to -1 @@ -2433,4 +2438,6 @@ PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); Index: /branches/eam_branches/ipp-20110906/ippTools/src/warptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110906/ippTools/src/warptoolConfig.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippTools/src/warptoolConfig.c (revision 32630) @@ -269,7 +269,9 @@ psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); - psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); - psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); - psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure id", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", NULL); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip id", 0); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam id", 0); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag (fake_id)", 0); psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); @@ -396,5 +398,6 @@ psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-set_magicked", 0, "new value for magicked", 0); - + psMetadataAddS16(tofullskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "new value for quality", 0); + // -updateskyfile psMetadata *updateskyfileArgs = psMetadataAlloc(); Index: /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20090120.config =================================================================== --- /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20090120.config (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20090120.config (revision 32630) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -505,5 +505,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20090220.config =================================================================== --- /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20090220.config (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20090220.config (revision 32630) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -505,5 +505,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20091209.config =================================================================== --- /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20091209.config (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20091209.config (revision 32630) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -505,5 +505,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20100122.config =================================================================== --- /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20100122.config (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20100122.config (revision 32630) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -506,5 +506,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20100228.config =================================================================== --- /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20100228.config (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20100228.config (revision 32630) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -506,5 +506,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20100723.config =================================================================== --- /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20100723.config (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippconfig/gpc1/format_20100723.config (revision 32630) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -506,5 +506,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: /branches/eam_branches/ipp-20110906/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/eam_branches/ipp-20110906/ippconfig/recipes/nightly_science.config (revision 32629) +++ /branches/eam_branches/ipp-20110906/ippconfig/recipes/nightly_science.config (revision 32630) @@ -104,5 +104,5 @@ NAME STR MD03 DISTRIBUTION STR MD03 - TESS STR MD03.V2 + TESS STR MD03.V3 OBSMODE STR MD OBJECT STR MD03% @@ -115,5 +115,5 @@ NAME STR MD04 DISTRIBUTION STR MD04 - TESS STR MD04.V2 + TESS STR MD04.V3 OBSMODE STR MD OBJECT STR MD04% @@ -126,5 +126,5 @@ NAME STR MD05 DISTRIBUTION STR MD05 - TESS STR MD05.V2 + TESS STR MD05.V3 OBSMODE STR MD OBJECT STR MD05% @@ -137,5 +137,5 @@ NAME STR MD06 DISTRIBUTION STR MD06 - TESS STR MD06 + TESS STR MD06.V3 OBSMODE STR MD OBJECT STR MD06% @@ -148,5 +148,5 @@ NAME STR MD07 DISTRIBUTION STR MD07 - TESS STR MD07 + TESS STR MD07.V3 OBSMODE STR MD OBJECT STR MD07% @@ -210,7 +210,7 @@ REDUCTION STR STS_DATASET # bump the retention times for the time being - CHIP S16 700 - WARP S16 700 - DIFF S16 700 + CHIP S16 7 + WARP S16 7 + DIFF S16 7 END TARGETS METADATA Index: /branches/eam_branches/ipp-20110906/magic/remove/src/streaksremove.c =================================================================== --- /branches/eam_branches/ipp-20110906/magic/remove/src/streaksremove.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/magic/remove/src/streaksremove.c (revision 32630) @@ -981,53 +981,53 @@ } - // XXX: Handle case where numCols == 0 Can that happen? - // Previously we were writing a blank extension - - // For speed we get intimate with the table - // avoid 2 * numRows lookups of the column number by looking - // it up ahead of time and verifying that the type is what we - // expect. - int xCol = psFitsTableGetColumnNumber(table, "X_PSF"); - assert(xCol >= 0 && xCol < table->numCols); - assert(table->columns[xCol].type == PS_DATA_F32); - - int yCol = psFitsTableGetColumnNumber(table, "Y_PSF"); - assert(yCol >= 0 && yCol < table->numCols); - assert(table->columns[yCol].type == PS_DATA_F32); - - // We mark censored rows in this array. - bool *rowCensored = psAlloc(table->numRows * sizeof(bool)); - memset(rowCensored, 0, table->numRows * sizeof(bool)); int numCensored = 0; - - for (int i = 0 ; i < table->numRows; i++) { + bool *rowCensored = NULL; + if (table->numRows > 0) { + // For speed we get intimate with the table + // avoid 2 * numRows lookups of the column number by looking + // it up ahead of time and verifying that the type is what we + // expect. + int xCol = psFitsTableGetColumnNumber(table, "X_PSF"); + assert(xCol >= 0 && xCol < table->numCols); + assert(table->columns[xCol].type == PS_DATA_F32); + + int yCol = psFitsTableGetColumnNumber(table, "Y_PSF"); + assert(yCol >= 0 && yCol < table->numCols); + assert(table->columns[yCol].type == PS_DATA_F32); + + // We mark censored rows in this array. + rowCensored = psAlloc(table->numRows * sizeof(bool)); + memset(rowCensored, 0, table->numRows * sizeof(bool)); + + for (int i = 0 ; i < table->numRows; i++) { #ifndef GO_SLOWER - psF32 x = table->columns[xCol].data.F32[i]; - psF32 y = table->columns[yCol].data.F32[i]; + psF32 x = table->columns[xCol].data.F32[i]; + psF32 y = table->columns[yCol].data.F32[i]; #else - psF32 x = psFitsTableGetF32(NULL, table, i, "X_PSF"); - psF32 y = psFitsTableGetF32(NULL, table, i, "Y_PSF"); + psF32 x = psFitsTableGetF32(NULL, table, i, "X_PSF"); + psF32 y = psFitsTableGetF32(NULL, table, i, "Y_PSF"); #endif - psImageMaskType mask; - if ((x >= maskImage->numCols) || (y >= maskImage->numRows) || (x < 0) || (y < 0) || isnan(x) || isnan(y)) { - mask = maskStreak; - } else { - mask = maskImage->data.PS_TYPE_IMAGE_MASK_DATA[(int)y][(int)x]; - } - - // Key the source if the center pixel is not masked with maskStreak - if (mask & maskStreak) { - rowCensored[i] = true; - numCensored++; - } - } - - // remove the censored rows from the table - if (!psFitsTableCensor(table, rowCensored)) { - streaksExit("", PS_EXIT_PROG_ERROR); - } - - addDestreakKeyword(header, true); + psImageMaskType mask; + if ((x >= maskImage->numCols) || (y >= maskImage->numRows) || (x < 0) || (y < 0) || isnan(x) || isnan(y)) { + mask = maskStreak; + } else { + mask = maskImage->data.PS_TYPE_IMAGE_MASK_DATA[(int)y][(int)x]; + } + + // Key the source if the center pixel is not masked with maskStreak + if (mask & maskStreak) { + rowCensored[i] = true; + numCensored++; + } + } + + // remove the censored rows from the table + if (!psFitsTableCensor(table, rowCensored)) { + streaksExit("", PS_EXIT_PROG_ERROR); + } + + addDestreakKeyword(header, true); + } if (table->numCols > 0) { Index: /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMops.c =================================================================== --- /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMops.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMops.c (revision 32630) @@ -3,4 +3,24 @@ #include "ppMops.h" + +void test() +{ + psMetadata *md = psMetadataAlloc(); + + psVector *vec = psVectorAlloc(42, PS_TYPE_S32); + + psMetadataAddVector(md, PS_LIST_TAIL, "TEST", 0, NULL, vec); + + psFree(vec); + psFree(md); + + psLibFinalize(); + + fprintf (stderr, "found %d leaks at %s\n", + psMemCheckLeaks2 (0, + NULL, stdout, false, 500), "ppMops"); + + exit(0); +} /* @@ -39,4 +59,6 @@ psLibInit(NULL); + // test(); + ppMopsArguments *args = ppMopsArgumentsParse(argc, argv); // Parsed arguments if (!args) { @@ -45,4 +67,5 @@ } + psArray *detections = ppMopsRead(args); // Detections from each input if (!detections) { @@ -51,19 +74,19 @@ } - ppMopsDetections *merged = ppMopsMerge(detections); // Merged detections - psFree(detections); - if (!merged) { + if (!ppMopsPurgeDuplicates(detections)) { psErrorStackPrint(stderr, "Unable to merge detections"); exit(PS_EXIT_SYS_ERROR); } - if (!ppMopsWrite(merged, args)) { + if (!ppMopsWrite(detections, args)) { psErrorStackPrint(stderr, "Unable to write detections"); exit(PS_EXIT_SYS_ERROR); } - psFree(merged); + for (int i = 0; i < detections->n; i++) { + psFree(detections->data[i]); + } + psFree(detections); psFree(args); - psFree(detections); psLibFinalize(); Index: /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMops.h =================================================================== --- /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMops.h (revision 32629) +++ /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMops.h (revision 32630) @@ -62,4 +62,5 @@ typedef struct { + psString component; // skycell_id for these detections psString raBoresight, decBoresight; // RA,Dec of telescope boresight psString filter; // Filter for exposure @@ -70,53 +71,16 @@ double mjd; // Modified Julian Date float seeing; // Seeing of exposure + int naxis1, naxis2; // size of the image long num; // Number of detections + long numGood; // Number of "good" detections + psS64 diffSkyfileId; // unique id for input skyfile + psMetadata *table; // Columns from the input file psVector *x, *y; // Image coordinates psVector *ra, *dec; // Sky coordinates psVector *raErr, *decErr; // Error in sky coordinates - psVector *mag, *magErr; // Magnitude and associated error - psVector *chi2, *dof; // Chi^2 from fitting, with associated degrees of freedom - psVector *cr, *extended; // Measures of CR-ness and extendedness - psVector *psfMajor, *psfMinor, *psfTheta; // PSF major and minor axes, and position angle - psVector *quality, *numPix; // PSF quality factor and number of pixels - psVector *xxMoment, *xyMoment, *yyMoment; // Moments - psVector *flags; // psphot flags - psVector *diffSkyfileId; // Identifier for source image - psVector *naxis1, *naxis2; // Size of image psVector *mask; // Mask for detections - psVector *nPos; // Number of positive pixels - psVector *fPos; // Fraction of positive flux - psVector *nRatioBad; // Fraction of positive pixels to negative - psVector *nRatioMask; // Fraction of positive pixels to masked - psVector *nRatioAll; // Fraction of positive pixels to all - psVector *psfInstFlux; // PSF fit instrumental magnitude - psVector *psfInstFluxSig; // Sigma of PSF instrumental magnitude - psVector *apMag; // Magnitude in standard aperture - psVector *apMagRadius; // Radius used for aperture mags - psVector *apMagRaw; // Magnitude in real aperture - psVector *apFlux; // Instrumental flux in standard aperture - psVector *apFluxSig; // Aperture flux error - psVector *peakFluxAsMag; // Peak flux expressed as magnitude - psVector *calPsfMag; // PSF Magnitude using supplied calibration - psVector *calPsfMagSig; // Measured scatter of zero point calibration - psVector *sky; // Sky level - psVector *skySig; // Sigma of sky level - psVector *qualityPerfect; // PSF coverage/quality factor (poor) - psVector *momentsR1; // First radial moment - psVector *momentsRH; // Half radial moment - psVector *kronFlux; // Kron Flux (in 2.5 R1) - psVector *kronFluxErr; // Kron Flux Error - psVector *kronFluxInner; // Kron Flux (in 1.0 R1) - psVector *kronFluxOuter; // Kron Flux (in 4.0 R1) - psVector *diffRP; // Distance to positive match source - psVector *diffSnP; // Signal-to-noise of pos match src - psVector *diffRM; // Distance to negative match source - psVector *diffSnM; // Signal-to-noise of neg match src - psVector *flags2; // psphot flags (group 2) - psVector *ippIdet; // IPP detection identifier index - psVector *nFrames; // Number of frames overlapping source center - psVector *padding; // Padding } ppMopsDetections; -ppMopsDetections *ppMopsDetectionsAlloc(long num); +ppMopsDetections *ppMopsDetectionsAlloc(); /// Copy a detection @@ -130,8 +94,9 @@ /// Merge detections -ppMopsDetections *ppMopsMerge(const psArray *detections); +// ppMopsDetections *ppMopsMerge(const psArray *detections); +bool ppMopsPurgeDuplicates(const psArray *detections); /// Write detections -bool ppMopsWrite(const ppMopsDetections *detections, const ppMopsArguments *args); +bool ppMopsWrite(const psArray *detections, const ppMopsArguments *args); /// Get the version contained in EXTTYPE of the "SkyChip.psf" table: Index: /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsDetections.c =================================================================== --- /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsDetections.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsDetections.c (revision 32630) @@ -8,7 +8,9 @@ static void mopsDetectionsFree(ppMopsDetections *det) { + psFree(det->component); psFree(det->raBoresight); psFree(det->decBoresight); psFree(det->filter); + psFree(det->table); psFree(det->x); psFree(det->y); @@ -17,62 +19,13 @@ psFree(det->raErr); psFree(det->decErr); - psFree(det->mag); - psFree(det->magErr); - psFree(det->chi2); - psFree(det->dof); - psFree(det->cr); - psFree(det->extended); - psFree(det->psfMajor); - psFree(det->psfMinor); - psFree(det->psfTheta); - psFree(det->quality); - psFree(det->numPix); - psFree(det->xxMoment); - psFree(det->xyMoment); - psFree(det->yyMoment); - psFree(det->flags); - psFree(det->diffSkyfileId); - psFree(det->naxis1); - psFree(det->naxis2); psFree(det->mask); - psFree(det->nPos); - psFree(det->fPos); - psFree(det->nRatioBad); - psFree(det->nRatioMask); - psFree(det->nRatioAll); - psFree(det->psfInstFlux); - psFree(det->psfInstFluxSig); - psFree(det->apMag); - psFree(det->apMagRadius); - psFree(det->apMagRaw); - psFree(det->apFlux); - psFree(det->apFluxSig); - psFree(det->peakFluxAsMag); - psFree(det->calPsfMag); - psFree(det->calPsfMagSig); - psFree(det->sky); - psFree(det->skySig); - psFree(det->qualityPerfect); - psFree(det->momentsR1); - psFree(det->momentsRH); - psFree(det->kronFlux); - psFree(det->kronFluxErr); - psFree(det->kronFluxInner); - psFree(det->kronFluxOuter); - psFree(det->diffRP); - psFree(det->diffSnP); - psFree(det->diffRM); - psFree(det->diffSnM); - psFree(det->flags2); - psFree(det->ippIdet); - psFree(det->nFrames); - psFree(det->padding); return; } -ppMopsDetections *ppMopsDetectionsAlloc(long num) +ppMopsDetections *ppMopsDetectionsAlloc() { ppMopsDetections *det = psAlloc(sizeof(ppMopsDetections)); // Detections, to return psMemSetDeallocator(det, (psFreeFunc)mopsDetectionsFree); + det->component = NULL; det->raBoresight = NULL; det->decBoresight = NULL; @@ -86,396 +39,13 @@ det->seeing = NAN; det->num = 0; - det->x = psVectorAllocEmpty(num, PS_TYPE_F32); - det->y = psVectorAllocEmpty(num, PS_TYPE_F32); - det->ra = psVectorAllocEmpty(num, PS_TYPE_F64); - det->dec = psVectorAllocEmpty(num, PS_TYPE_F64); - det->raErr = psVectorAllocEmpty(num, PS_TYPE_F64); - det->decErr = psVectorAllocEmpty(num, PS_TYPE_F64); - det->mag = psVectorAllocEmpty(num, PS_TYPE_F32); - det->magErr = psVectorAllocEmpty(num, PS_TYPE_F32); - det->chi2 = psVectorAllocEmpty(num, PS_TYPE_F32); - det->dof = psVectorAllocEmpty(num, PS_TYPE_S32); - det->cr = psVectorAllocEmpty(num, PS_TYPE_F32); - det->extended = psVectorAllocEmpty(num, PS_TYPE_F32); - det->psfMajor = psVectorAllocEmpty(num, PS_TYPE_F32); - det->psfMinor = psVectorAllocEmpty(num, PS_TYPE_F32); - det->psfTheta = psVectorAllocEmpty(num, PS_TYPE_F32); - det->quality = psVectorAllocEmpty(num, PS_TYPE_F32); - det->numPix = psVectorAllocEmpty(num, PS_TYPE_S32); - det->xxMoment = psVectorAllocEmpty(num, PS_TYPE_F32); - det->xyMoment = psVectorAllocEmpty(num, PS_TYPE_F32); - det->yyMoment = psVectorAllocEmpty(num, PS_TYPE_F32); - det->flags = psVectorAllocEmpty(num, PS_TYPE_U32); - det->diffSkyfileId = psVectorAllocEmpty(num, PS_TYPE_S64); - det->naxis1 = psVectorAllocEmpty(num, PS_TYPE_S32); - det->naxis2 = psVectorAllocEmpty(num, PS_TYPE_S32); - det->mask = psVectorAllocEmpty(num, PS_TYPE_U8); - det->nPos = psVectorAllocEmpty(num, PS_TYPE_S32); - det->fPos = psVectorAllocEmpty(num, PS_TYPE_F32); - det->nRatioBad = psVectorAllocEmpty(num, PS_TYPE_F32); - det->nRatioMask = psVectorAllocEmpty(num, PS_TYPE_F32); - det->nRatioAll = psVectorAllocEmpty(num, PS_TYPE_F32); - det->psfInstFlux = psVectorAllocEmpty(num, PS_TYPE_F32); - det->psfInstFluxSig = psVectorAllocEmpty(num, PS_TYPE_F32); - det->apMag = psVectorAllocEmpty(num, PS_TYPE_F32); - det->apMagRadius = psVectorAllocEmpty(num, PS_TYPE_F32); - det->apMagRaw = psVectorAllocEmpty(num, PS_TYPE_F32); - det->apFlux = psVectorAllocEmpty(num, PS_TYPE_F32); - det->apFluxSig = psVectorAllocEmpty(num, PS_TYPE_F32); - det->peakFluxAsMag = psVectorAllocEmpty(num, PS_TYPE_F32); - det->calPsfMag = psVectorAllocEmpty(num, PS_TYPE_F32); - det->calPsfMagSig = psVectorAllocEmpty(num, PS_TYPE_F32); - det->sky = psVectorAllocEmpty(num, PS_TYPE_F32); - det->skySig = psVectorAllocEmpty(num, PS_TYPE_F32); - det->qualityPerfect = psVectorAllocEmpty(num, PS_TYPE_F32); - det->momentsR1 = psVectorAllocEmpty(num, PS_TYPE_F32); - det->momentsRH = psVectorAllocEmpty(num, PS_TYPE_F32); - det->kronFlux = psVectorAllocEmpty(num, PS_TYPE_F32); - det->kronFluxErr = psVectorAllocEmpty(num, PS_TYPE_F32); - det->kronFluxInner = psVectorAllocEmpty(num, PS_TYPE_F32); - det->kronFluxOuter = psVectorAllocEmpty(num, PS_TYPE_F32); - det->diffRP = psVectorAllocEmpty(num, PS_TYPE_F32); - det->diffSnP = psVectorAllocEmpty(num, PS_TYPE_F32); - det->diffRM = psVectorAllocEmpty(num, PS_TYPE_F32); - det->diffSnM = psVectorAllocEmpty(num, PS_TYPE_F32); - det->flags2 = psVectorAllocEmpty(num, PS_TYPE_U32); - det->ippIdet = psVectorAllocEmpty(num, PS_TYPE_U32); - det->nFrames = psVectorAllocEmpty(num, PS_TYPE_U16); - det->padding = psVectorAllocEmpty(num, PS_TYPE_S16); + det->table = NULL; + det->x = NULL; + det->y = NULL; + det->ra = NULL; + det->dec = NULL; + det->raErr = NULL; + det->decErr = NULL; + det->mask = NULL; + det->diffSkyfileId = 0; return det; } - -ppMopsDetections *ppMopsDetectionsRealloc(ppMopsDetections *det, long num) -{ - det->x = psVectorRealloc(det->x, num); - det->y = psVectorRealloc(det->y, num); - det->ra = psVectorRealloc(det->ra, num); - det->dec = psVectorRealloc(det->dec, num); - det->raErr = psVectorRealloc(det->raErr, num); - det->decErr = psVectorRealloc(det->decErr, num); - det->mag = psVectorRealloc(det->mag, num); - det->magErr = psVectorRealloc(det->magErr, num); - det->chi2 = psVectorRealloc(det->chi2, num); - det->dof = psVectorRealloc(det->dof, num); - det->cr = psVectorRealloc(det->cr, num); - det->extended = psVectorRealloc(det->extended, num); - det->psfMajor = psVectorRealloc(det->psfMajor, num); - det->psfMinor = psVectorRealloc(det->psfMinor, num); - det->psfTheta = psVectorRealloc(det->psfTheta, num); - det->quality = psVectorRealloc(det->quality, num); - det->numPix = psVectorRealloc(det->numPix, num); - det->xxMoment = psVectorRealloc(det->xxMoment, num); - det->xyMoment = psVectorRealloc(det->xyMoment, num); - det->yyMoment = psVectorRealloc(det->yyMoment, num); - det->flags = psVectorRealloc(det->flags, num); - det->diffSkyfileId = psVectorRealloc(det->diffSkyfileId, num); - det->naxis1 = psVectorRealloc(det->naxis1, num); - det->naxis2 = psVectorRealloc(det->naxis2, num); - det->mask = psVectorRealloc(det->mask, num); - det->nPos = psVectorRealloc(det->nPos, num); - det->fPos = psVectorRealloc(det->fPos, num); - det->nRatioBad = psVectorRealloc(det->nRatioBad, num); - det->nRatioMask = psVectorRealloc(det->nRatioMask, num); - det->nRatioAll = psVectorRealloc(det->nRatioAll, num); - det->psfInstFlux = psVectorRealloc(det->psfInstFlux, num); - det->psfInstFluxSig = psVectorRealloc(det->psfInstFluxSig, num); - det->apMag = psVectorRealloc(det->apMag, num); - det->apMagRadius = psVectorRealloc(det->apMagRadius, num); - det->apMagRaw = psVectorRealloc(det->apMagRadius, num); - det->apFlux = psVectorRealloc(det->apFlux, num); - det->apFluxSig = psVectorRealloc(det->apFluxSig, num); - det->peakFluxAsMag = psVectorRealloc(det->peakFluxAsMag, num); - det->calPsfMag = psVectorRealloc(det->calPsfMag, num); - det->calPsfMagSig = psVectorRealloc(det->calPsfMagSig, num); - det->sky = psVectorRealloc(det->sky, num); - det->skySig = psVectorRealloc(det->skySig, num); - det->qualityPerfect = psVectorRealloc(det->qualityPerfect, num); - det->momentsR1 = psVectorRealloc(det->momentsR1, num); - det->momentsRH = psVectorRealloc(det->momentsRH, num); - det->kronFlux = psVectorRealloc(det->kronFlux, num); - det->kronFluxErr = psVectorRealloc(det->kronFluxErr, num); - det->kronFluxInner = psVectorRealloc(det->kronFluxInner, num); - det->kronFluxOuter = psVectorRealloc(det->kronFluxOuter, num); - det->diffRP = psVectorRealloc(det->diffRP, num); - det->diffSnP = psVectorRealloc(det->diffSnP, num); - det->diffRM = psVectorRealloc(det->diffRM, num); - det->diffSnM = psVectorRealloc(det->diffSnM, num); - det->flags2 = psVectorRealloc(det->flags2, num); - det->ippIdet = psVectorRealloc(det->ippIdet, num); - det->nFrames = psVectorRealloc(det->nFrames, num); - det->padding = psVectorRealloc(det->padding, num); - return det; -} - -bool ppMopsDetectionsAdd(ppMopsDetections *det, float x, float y, double ra, double dec, - double raErr, double decErr, float mag, float magErr, - float chi2, int dof, float cr, float extended, float psfMajor, - float psfMinor, float psfTheta, float quality, int numPix, - float xxMoment, float xyMoment, float yyMoment, - psU32 flags, psS64 diffSkyfileId, int naxis1, int naxis2, - int nPos, float fPos, float nRatioBad, float nRatioMask, float nRatioAll, - float psfInstFlux, float psfInstFluxSig, - float apMag, float apMagRadius, float apMagRaw, float apFlux, float apFluxSig, - float peakFluxAsMag, float calPsfMag, float calPsfMagSig, - float sky, float skySig, float qualityPerfect, - float momentsR1, float momentsRH, - float kronFlux, float kronFluxErr, float kronFluxInner, float kronFluxOuter, - float diffRP, float diffSnP, float diffRM, float diffSnM, - psU32 flags2, psU32 ippIdet, psU16 nFrames, psS16 padding) -{ - psVectorAppend(det->x, x); - psVectorAppend(det->y, y); - psVectorAppend(det->ra, ra); - psVectorAppend(det->dec, dec); - psVectorAppend(det->raErr, raErr); - psVectorAppend(det->decErr, decErr); - psVectorAppend(det->mag, mag); - psVectorAppend(det->magErr, magErr); - psVectorAppend(det->chi2, chi2); - psVectorAppend(det->dof, dof); - psVectorAppend(det->cr, cr); - psVectorAppend(det->extended, extended); - psVectorAppend(det->psfMajor, psfMajor); - psVectorAppend(det->psfMinor, psfMinor); - psVectorAppend(det->psfTheta, psfTheta); - psVectorAppend(det->quality, quality); - psVectorAppend(det->numPix, numPix); - psVectorAppend(det->xxMoment, xxMoment); - psVectorAppend(det->xyMoment, xyMoment); - psVectorAppend(det->yyMoment, yyMoment); - psVectorAppend(det->flags, flags); - psVectorAppend(det->diffSkyfileId, diffSkyfileId); - psVectorAppend(det->naxis1, naxis1); - psVectorAppend(det->naxis2, naxis2); - psVectorAppend(det->mask, 0); - psVectorAppend(det->nPos, nPos); - psVectorAppend(det->fPos, fPos); - psVectorAppend(det->nRatioBad, nRatioBad); - psVectorAppend(det->nRatioMask, nRatioMask); - psVectorAppend(det->nRatioAll, nRatioAll); - psVectorAppend(det->psfInstFlux, psfInstFlux); - psVectorAppend(det->psfInstFluxSig, psfInstFluxSig); - psVectorAppend(det->apMag, apMag); - psVectorAppend(det->apMagRadius, apMagRadius); - psVectorAppend(det->apMagRaw, apMagRaw); - psVectorAppend(det->apFlux, apFlux); - psVectorAppend(det->apFluxSig, apFluxSig); - psVectorAppend(det->peakFluxAsMag, peakFluxAsMag); - psVectorAppend(det->calPsfMag, calPsfMag); - psVectorAppend(det->calPsfMagSig, calPsfMagSig); - psVectorAppend(det->sky, sky); - psVectorAppend(det->skySig, skySig); - psVectorAppend(det->qualityPerfect, qualityPerfect); - psVectorAppend(det->momentsR1, momentsR1); - psVectorAppend(det->momentsRH, momentsRH); - psVectorAppend(det->kronFlux, kronFlux); - psVectorAppend(det->kronFluxErr, kronFluxErr); - psVectorAppend(det->kronFluxInner, kronFluxInner); - psVectorAppend(det->kronFluxOuter, kronFluxOuter); - psVectorAppend(det->diffRP, diffRP); - psVectorAppend(det->diffSnP, diffSnP); - psVectorAppend(det->diffRM, diffRM); - psVectorAppend(det->diffSnM, diffSnM); - psVectorAppend(det->flags2, flags2); - psVectorAppend(det->ippIdet, ippIdet); - psVectorAppend(det->nFrames, nFrames); - psVectorAppend(det->padding, padding); - return true; -} - -bool ppMopsDetectionsCopySingle(ppMopsDetections *target, const ppMopsDetections *source, long index) -{ - psVectorAppend(target->x, source->x->data.F32[index]); - psVectorAppend(target->y, source->y->data.F32[index]); - psVectorAppend(target->ra, source->ra->data.F64[index]); - psVectorAppend(target->dec, source->dec->data.F64[index]); - psVectorAppend(target->raErr, source->raErr->data.F64[index]); - psVectorAppend(target->decErr, source->decErr->data.F64[index]); - psVectorAppend(target->mag, source->mag->data.F32[index]); - psVectorAppend(target->magErr, source->magErr->data.F32[index]); - psVectorAppend(target->chi2, source->chi2->data.F32[index]); - psVectorAppend(target->dof, source->dof->data.S32[index]); - psVectorAppend(target->cr, source->cr->data.F32[index]); - psVectorAppend(target->extended, source->extended->data.F32[index]); - psVectorAppend(target->psfMajor, source->psfMajor->data.F32[index]); - psVectorAppend(target->psfMinor, source->psfMinor->data.F32[index]); - psVectorAppend(target->psfTheta, source->psfTheta->data.F32[index]); - psVectorAppend(target->quality, source->quality->data.F32[index]); - psVectorAppend(target->numPix, source->numPix->data.S32[index]); - psVectorAppend(target->xxMoment, source->xxMoment->data.F32[index]); - psVectorAppend(target->xyMoment, source->xyMoment->data.F32[index]); - psVectorAppend(target->yyMoment, source->yyMoment->data.F32[index]); - psVectorAppend(target->flags, source->flags->data.U32[index]); - psVectorAppend(target->diffSkyfileId, source->diffSkyfileId->data.S64[index]); - psVectorAppend(target->naxis1, source->naxis1->data.S32[index]); - psVectorAppend(target->naxis2, source->naxis2->data.S32[index]); - psVectorAppend(target->mask, 0); - psVectorAppend(target->nPos, source->nPos->data.S32[index]); - psVectorAppend(target->fPos, source->fPos->data.F32[index]); - psVectorAppend(target->nRatioBad, source->nRatioBad->data.F32[index]); - psVectorAppend(target->nRatioMask, source->nRatioMask->data.F32[index]); - psVectorAppend(target->nRatioAll, source->nRatioAll->data.F32[index]); - psVectorAppend(target->psfInstFlux, source->psfInstFlux->data.F32[index]); - psVectorAppend(target->psfInstFluxSig, source->psfInstFluxSig->data.F32[index]); - psVectorAppend(target->apMag, source->apMag->data.F32[index]); - psVectorAppend(target->apMagRadius, source->apMagRadius->data.F32[index]); - psVectorAppend(target->apMagRaw, source->apMagRaw->data.F32[index]); - psVectorAppend(target->apFlux, source->apFlux->data.F32[index]); - psVectorAppend(target->apFluxSig, source->apFluxSig->data.F32[index]); - psVectorAppend(target->peakFluxAsMag, source->peakFluxAsMag->data.F32[index]); - psVectorAppend(target->calPsfMag, source->calPsfMag->data.F32[index]); - psVectorAppend(target->calPsfMagSig, source->calPsfMagSig->data.F32[index]); - psVectorAppend(target->sky, source->sky->data.F32[index]); - psVectorAppend(target->skySig, source->skySig->data.F32[index]); - psVectorAppend(target->qualityPerfect, source->qualityPerfect->data.F32[index]); - psVectorAppend(target->momentsR1, source->momentsR1->data.F32[index]); - psVectorAppend(target->momentsRH, source->momentsRH->data.F32[index]); - psVectorAppend(target->kronFlux, source->kronFlux->data.F32[index]); - psVectorAppend(target->kronFluxErr, source->kronFluxErr->data.F32[index]); - psVectorAppend(target->kronFluxInner, source->kronFluxInner->data.F32[index]); - psVectorAppend(target->kronFluxOuter, source->kronFluxOuter->data.F32[index]); - psVectorAppend(target->diffRP, source->diffRP->data.F32[index]); - psVectorAppend(target->diffSnP, source->diffSnP->data.F32[index]); - psVectorAppend(target->diffRM, source->diffRM->data.F32[index]); - psVectorAppend(target->diffSnM, source->diffSnM->data.F32[index]); - psVectorAppend(target->flags2, source->flags2->data.U32[index]); - psVectorAppend(target->ippIdet, source->ippIdet->data.U32[index]); - psVectorAppend(target->nFrames, source->nFrames->data.U16[index]); - psVectorAppend(target->padding, source->padding->data.S16[index]); - - target->num++; - - return true; -} - - -bool ppMopsDetectionsPurge(ppMopsDetections *det) -{ - long num = 0; - for (long i = 0; i < det->num; i++) { - if (!det->mask->data.U8[i]) { - if (i == num) { - // No need to copy - num++; - continue; - } - det->x->data.F32[num] = det->x->data.F32[i]; - det->y->data.F32[num] = det->y->data.F32[i]; - det->ra->data.F64[num] = det->ra->data.F64[i]; - det->dec->data.F64[num] = det->dec->data.F64[i]; - det->raErr->data.F64[num] = det->raErr->data.F64[i]; - det->decErr->data.F64[num] = det->decErr->data.F64[i]; - det->mag->data.F32[num] = det->mag->data.F32[i]; - det->magErr->data.F32[num] = det->magErr->data.F32[i]; - det->chi2->data.F32[num] = det->chi2->data.F32[i]; - det->dof->data.S32[num] = det->dof->data.S32[i]; - det->cr->data.F32[num] = det->cr->data.F32[i]; - det->extended->data.F32[num] = det->extended->data.F32[i]; - det->psfMajor->data.F32[num] = det->psfMajor->data.F32[i]; - det->psfMinor->data.F32[num] = det->psfMinor->data.F32[i]; - det->psfTheta->data.F32[num] = det->psfTheta->data.F32[i]; - det->quality->data.F32[num] = det->quality->data.F32[i]; - det->numPix->data.S32[num] = det->numPix->data.S32[i]; - det->xxMoment->data.F32[num] = det->xxMoment->data.F32[i]; - det->xyMoment->data.F32[num] = det->xyMoment->data.F32[i]; - det->yyMoment->data.F32[num] = det->yyMoment->data.F32[i]; - det->flags->data.U32[num] = det->flags->data.U32[i]; - det->diffSkyfileId->data.S64[num] = det->diffSkyfileId->data.S64[i]; - det->naxis1->data.S32[num] = det->naxis1->data.S32[i]; - det->naxis2->data.S32[num] = det->naxis2->data.S32[i]; - det->mask->data.U8[num] = 0; - det->nPos->data.S32[num] = det->nPos->data.S32[i]; - det->fPos->data.F32[num] = det->fPos->data.F32[i]; - det->nRatioBad->data.F32[num] = det->nRatioBad->data.F32[i]; - det->nRatioMask->data.F32[num] = det->nRatioMask->data.F32[i]; - det->nRatioAll->data.F32[num] = det->nRatioAll->data.F32[i]; - det->psfInstFlux->data.F32[num] = det->psfInstFlux->data.F32[i]; - det->psfInstFluxSig->data.F32[num] = det->psfInstFluxSig->data.F32[i]; - det->apMag->data.F32[num] = det->apMag->data.F32[i]; - det->apMagRadius->data.F32[num] = det->apMagRadius->data.F32[i]; - det->apMagRaw->data.F32[num] = det->apMagRaw->data.F32[i]; - det->apFlux->data.F32[num] = det->apFlux->data.F32[i]; - det->apFluxSig->data.F32[num] = det->apFluxSig->data.F32[i]; - det->peakFluxAsMag->data.F32[num] = det->peakFluxAsMag->data.F32[i]; - det->calPsfMag->data.F32[num] = det->calPsfMag->data.F32[i]; - det->calPsfMagSig->data.F32[num] = det->calPsfMagSig->data.F32[i]; - det->sky->data.F32[num] = det->sky->data.F32[i]; - det->skySig->data.F32[num] = det->skySig->data.F32[i]; - det->qualityPerfect->data.F32[num] = det->qualityPerfect->data.F32[i]; - det->momentsR1->data.F32[num] = det->momentsR1->data.F32[i]; - det->momentsRH->data.F32[num] = det->momentsRH->data.F32[i]; - det->kronFlux->data.F32[num] = det->kronFlux->data.F32[i]; - det->kronFluxErr->data.F32[num] = det->kronFluxErr->data.F32[i]; - det->kronFluxInner->data.F32[num] = det->kronFluxInner->data.F32[i]; - det->kronFluxOuter->data.F32[num] = det->kronFluxOuter->data.F32[i]; - det->diffRP->data.F32[num] = det->diffRP->data.F32[i]; - det->diffSnP->data.F32[num] = det->diffSnP->data.F32[i]; - det->diffRM->data.F32[num] = det->diffRM->data.F32[i]; - det->diffSnM->data.F32[num] = det->diffSnM->data.F32[i]; - det->flags2->data.U32[num] = det->flags2->data.U32[i]; - det->ippIdet->data.U32[num] = det->ippIdet->data.U32[i]; - det->nFrames->data.U16[num] = det->nFrames->data.U16[i]; - det->padding->data.S16[num] = det->padding->data.S16[i]; - num++; - } - } - det->x->n = num; - det->y->n = num; - det->ra->n = num; - det->dec->n = num; - det->raErr->n = num; - det->decErr->n = num; - det->mag->n = num; - det->magErr->n = num; - det->chi2->n = num; - det->dof->n = num; - det->cr->n = num; - det->extended->n = num; - det->psfMajor->n = num; - det->psfMinor->n = num; - det->psfTheta->n = num; - det->quality->n = num; - det->numPix->n = num; - det->xxMoment->n = num; - det->xyMoment->n = num; - det->yyMoment->n = num; - det->flags->n = num; - det->diffSkyfileId->n = num; - det->naxis1->n = num; - det->naxis2->n = num; - det->mask->n = num; - det->num = num; - det->nPos->n = num; - det->fPos->n = num; - det->nRatioBad->n = num; - det->nRatioMask->n = num; - det->nRatioAll->n = num; - det->psfInstFlux->n = num; - det->psfInstFluxSig->n = num; - det->apMag->n = num; - det->apMagRadius->n = num; - det->apMagRaw->n = num; - det->apFlux->n = num; - det->apFluxSig->n = num; - det->peakFluxAsMag->n = num; - det->calPsfMag->n = num; - det->calPsfMagSig->n = num; - det->sky->n = num; - det->skySig->n = num; - det->qualityPerfect->n = num; - det->momentsR1->n = num; - det->momentsRH->n = num; - det->kronFlux->n = num; - det->kronFluxErr->n = num; - det->kronFluxInner->n = num; - det->kronFluxOuter->n = num; - det->diffRP->n = num; - det->diffSnP->n = num; - det->diffRM->n = num; - det->diffSnM->n = num; - det->flags2->n = num; - det->ippIdet->n = num; - det->nFrames->n = num; - det->padding->n = num; - return true; -} Index: /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsMerge.c =================================================================== --- /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsMerge.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsMerge.c (revision 32630) @@ -17,4 +17,12 @@ #define AIRMASS_TOL 1.0e-3 // Tolerance for airmass matching +#if 0 +#undef psTrace +#define psTrace(facil, level, ...) \ + if (level <= 5) { \ + fprintf(stderr, __VA_ARGS__); \ + } +#endif + // Get distance from detection to centre of image static float mergeDistance(const ppMopsDetections *detections, // Detections of interest @@ -22,94 +30,148 @@ ) { - float dx = (float) (detections->x->data.F32[index] - detections->naxis1->data.S32[index] / 2.0); - float dy = (float) (detections->y->data.F32[index] - detections->naxis2->data.S32[index] / 2.0); - return PS_SQR(dx) + PS_SQR(dy); + float dx = detections->x->data.F32[index] - detections->naxis1 / 2.0; + float dy = detections->y->data.F32[index] - detections->naxis2 / 2.0; + return PS_SQR(dx) + PS_SQR(dy); } -ppMopsDetections *ppMopsMerge(const psArray *detections) +bool ppMopsPurgeDuplicates(const psArray *detections) { PS_ASSERT_ARRAY_NON_NULL(detections, NULL); - psTrace("ppMops.merge", 1, "Merging detections from %ld inputs\n", detections->n); - - ppMopsDetections *merged = NULL; // Merged list - int num = 0; // Number of merged files - psVector *coords = psVectorAlloc(2, PS_TYPE_F64); // Coordinates of interest + int numInputs = detections->n; // Number of inputs + psTrace("ppMops.merge", 1, "Checking detections from %d inputs\n", numInputs); + + long total = 0; // Total number of sources + psArray *duplicates = psArrayAlloc(numInputs); // Vector of duplicate bits for each input + psVector *dupNum = psVectorAlloc(numInputs, PS_TYPE_U32); // Number of duplicates for each input + psVectorInit(dupNum, 0); + for (int i = 0; i < numInputs; i++) { + ppMopsDetections *det = detections->data[i]; // Detections from + if (!det || det->num == 0) { + continue; + } + psVector *dupes = duplicates->data[i] = psVectorAlloc(det->num, PS_TYPE_U8); + psVectorInit(dupes, 0); + total += det->num; + } + psTrace("ppMops.merge", 2, "Total detections: %ld\n", total); + + psVector *raMerged = psVectorAlloc(total, PS_TYPE_F64); // Merged RAs + psVector *decMerged = psVectorAlloc(total, PS_TYPE_F64); // Merged Decs + psVector *sourceMerged = psVectorAlloc(total, PS_TYPE_U16); // Source image of merged sources + psVector *indexMerged = psVectorAlloc(total, PS_TYPE_U32); // Index of merged sources + long num = 0; // Number of merged sources + + const char *raBoresight = NULL, *decBoresight = NULL; // Boresight coordinates + const char *filter = NULL; // Filter name + float airmass = NAN, exptime = NAN; // Exposure details + double posangle = NAN, alt = NAN, az = NAN; // Telescope pointing + double mjd = NAN; // Time of exposure + + for (int i = 0; i < numInputs; i++) { + ppMopsDetections *det = detections->data[i]; // Detections of interest + if (!det) { + psTrace("ppMops.merge", 3, "Ignoring NULL input %d\n", i); + continue; + } else if (det->num == 0) { + psTrace("ppMops.merge", 3, "Ignoring empty input %d\n", i); + continue; + } + + // Check exposure characteristics + if (num == 0) { + raBoresight = det->raBoresight; + decBoresight = det->decBoresight; + filter = det->filter; + airmass = det->airmass; + exptime = det->exptime; + posangle = det->posangle; + alt = det->alt; + az = det->az; + } else { + if (strcmp(raBoresight, det->raBoresight) != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure RA values differ: %s vs %s", + raBoresight, det->raBoresight); + return false; + } + if (strcmp(decBoresight, det->decBoresight) != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure Dec values differ: %s vs %s", + decBoresight, det->decBoresight); + return false; + } + if (strcmp(filter, det->filter) != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure filter values differ: %s vs %s", + filter, det->filter); + return false; + } + if (fabsf(airmass - det->airmass) > AIRMASS_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure airmass values differ: %f vs %f", + airmass, det->airmass); + return false; + } + if (fabsf(exptime - det->exptime) > EXPTIME_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure exposure time values differ: %f vs %f", + exptime, det->exptime); + return false; + } + if (fabs(posangle - det->posangle) > POSANGLE_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure position angle values differ: %f vs %f", + posangle, det->posangle); + return false; + } + if (fabs(alt - det->alt) > BORESIGHT_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure altitude values differ: %lf vs %lf", + alt, det->alt); + return false; + } + if (fabs(az - det->az) > BORESIGHT_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure azimuth values differ: %lf vs %lf", + az, det->az); + return false; + } + if (fabs(mjd - det->mjd) > MJD_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure MJD values differ: %lf vs %lf", + mjd, det->mjd); + return false; + } + } + + psTrace("ppMops.merge", 3, "Accepting %ld detections from input %d\n", det->num, i); + memcpy(&raMerged->data.F64[num], det->ra->data.F64, det->num * PSELEMTYPE_SIZEOF(PS_TYPE_F64)); + memcpy(&decMerged->data.F64[num], det->dec->data.F64, det->num * PSELEMTYPE_SIZEOF(PS_TYPE_F64)); + for (long j = 0, k = num; j < det->num; j++, k++) { + sourceMerged->data.U16[k] = i; + indexMerged->data.U32[k] = j; + } + num += det->num; + } + + psTrace("ppMops.merge", 3, "Generating kd-tree from %ld sources\n", num); + psTree *tree = psTreePlant(2, LEAF_SIZE, PS_TREE_SPHERICAL, raMerged, decMerged); // kd tree + if (!tree) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate kd tree"); + return false; + } + for (int i = 0; i < detections->n; i++) { ppMopsDetections *det = detections->data[i]; // Detections of interest if (!det) { - psTrace("ppMops.merge", 3, "Ignoring NULL input %d\n", i); continue; } else if (det->num == 0) { - psTrace("ppMops.merge", 3, "Ignoring empty input %d\n", i); - continue; - } - num++; - if (!merged) { - psTrace("ppMops.merge", 3, "Accepting %ld detections from input %d\n", det->num, i); - merged = psMemIncrRefCounter(det); - continue; - } - psTrace("ppMops.merge", 3, "Merging %ld detections from input %d\n", det->num, i); - - // XXX compare exposure properties - if (strcmp(merged->raBoresight, det->raBoresight) != 0) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure RA values differ: %s vs %s", - merged->raBoresight, det->raBoresight); - return NULL; - } - if (strcmp(merged->decBoresight, det->decBoresight) != 0) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure Dec values differ: %s vs %s", - merged->decBoresight, det->decBoresight); - return NULL; - } - if (strcmp(merged->filter, det->filter) != 0) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure filter values differ: %s vs %s", - merged->filter, det->filter); - return NULL; - } - - if (fabsf(merged->airmass - det->airmass) > AIRMASS_TOL) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure airmass values differ: %f vs %f", - merged->airmass, det->airmass); - return NULL; - } - if (fabsf(merged->exptime - det->exptime) > EXPTIME_TOL) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure exposure time values differ: %f vs %f", - merged->exptime, det->exptime); - return NULL; - } - if (fabs(merged->posangle - det->posangle) > POSANGLE_TOL) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure position angle values differ: %f vs %f", - merged->posangle, det->posangle); - return NULL; - } - if (fabs(merged->alt - det->alt) > BORESIGHT_TOL) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure altitude values differ: %lf vs %lf", - merged->alt, det->alt); - return NULL; - } - if (fabs(merged->az - det->az) > BORESIGHT_TOL) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure azimuth values differ: %lf vs %lf", - merged->az, det->az); - return NULL; - } - if (fabs(merged->mjd - det->mjd) > MJD_TOL) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure MJD values differ: %lf vs %lf", - merged->mjd, det->mjd); - return NULL; - } - - merged->seeing += det->seeing; // Taking average - - psTree *tree = psTreePlant(2, LEAF_SIZE, PS_TREE_SPHERICAL, merged->ra, merged->dec); // kd tree - if (!tree) { - psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate kd tree"); - psFree(merged); - return NULL; - } - + continue; + } + psTrace("ppMops.merge", 3, "Checking %ld detections from input %d\n", det->num, i); + + psVector *dupes = duplicates->data[i]; // Duplicates list + psVector *coords = psVectorAlloc(2, PS_TYPE_F64); // Coordinates of interest for (int j = 0; j < det->num; j++) { + // XXX: added by Bill + if (det->mask->data.U8[j]) { + // we marked this source marked as bad when we read it, mark it as a duplicate + dupes->data.U8[j] = 0xFF; + continue; + } + coords->data.F64[0] = det->ra->data.F64[j]; coords->data.F64[1] = det->dec->data.F64[j]; @@ -119,14 +181,14 @@ psFree(coords); psFree(tree); - psFree(merged); - return NULL; - } - if (indices->n == 0) { - psTrace("ppMops.merge", 9, "No matches for source %d in input %d\n", j, i); + return false; + } + psTrace("ppMops.merge", 5, "%ld matches for source %d from input %d\n", indices->n, j, i); + psAssert(indices->n > 0, "Expect at least one match for source %d in input %d", j, i); + + if (indices->n == 1 && sourceMerged->data.U16[indices->data.S64[0]] == i) { + // It's myself psFree(indices); - ppMopsDetectionsCopySingle(merged, det, j); continue; } - psTrace("ppMops.merge", 5, "%ld matches for source %d from input %d\n", indices->n, j, i); // Which one do we keep? @@ -134,6 +196,16 @@ long bestIndex = -1; // Index with best distance for (int k = 0; k < indices->n; k++) { - long index = indices->data.S64[k]; // Index of point - float distance = mergeDistance(merged, index); // Distance to centre of image + long mergeIndex = indices->data.S64[k]; // Index of point in merged list + int source = sourceMerged->data.U16[mergeIndex]; // Source image + if (source == i) { + continue; + } + long index = indexMerged->data.U32[mergeIndex]; // Index in source + psVector *dupes = duplicates->data[source]; // Duplicates list + if (dupes->data.U8[index]) { + continue; + } + + float distance = mergeDistance(detections->data[source], index); // Distance to centre of image if (distance < bestDistance) { bestDistance = distance; @@ -142,36 +214,100 @@ } - float distance = mergeDistance(det, j); // Distance to centre of image - if (distance < bestDistance) { - psTrace("ppMops.merge", 6, "New source clobbers old sources\n"); - // Blow away existing sources - for (int k = 0; k < indices->n; k++) { - long index = indices->data.S64[k]; // Index of point - merged->mask->data.U8[index] = 0xFF; + if (bestIndex >= 0) { + float distance = mergeDistance(det, j); // Distance to centre of image + if (bestIndex >= 0 && distance < bestDistance) { + psTrace("ppMops.merge", 6, "New source clobbers old sources\n"); + for (int k = 0; k < indices->n; k++) { + long mergeIndex = indices->data.S64[k]; // Index of point + int source = sourceMerged->data.U16[mergeIndex]; // Source image + if (source == i) { + continue; + } + long index = indexMerged->data.U32[mergeIndex]; // Index in source + psVector *dupes = duplicates->data[source]; // Duplicates list + if (!dupes->data.U8[index]) { + dupes->data.U8[index] = 0xFF; + dupNum->data.U32[source]++; + } + } + } else { + psTrace("ppMops.merge", 6, "Old sources clobber new source\n"); + dupes->data.U8[j] = 0xFF; + dupNum->data.U32[i]++; } - ppMopsDetectionsCopySingle(merged, det, j); - } else { - psTrace("ppMops.merge", 6, "Old sources clobber new source\n"); } psFree(indices); } - - psTrace("ppMops.merge", 3, "Done merging input %d, %ld merged sources\n", i, merged->num); - - psFree(tree); - ppMopsDetectionsPurge(merged); - } - psFree(coords); - - if (num == 0) { - //All detections were NULL?! - psTrace("ppMops.merge", 3, "All %ld detections were NULL\n", detections->n); - return NULL; - } - psTrace("ppMops.merge", 2, "%ld sources in merged detections list\n", merged->num); - - merged->seeing /= (float) num; - - return merged; + psFree(coords); + } + psFree(tree); + psFree(raMerged); + psFree(decMerged); + psFree(sourceMerged); + psFree(indexMerged); + + // Remove duplicates + for (int i = 0; i < detections->n; i++) { + ppMopsDetections *det = detections->data[i]; // Detections of interest + if (!det) { + continue; + } else if (det->num == 0) { + continue; + } + psTrace("ppMops.merge", 3, "Purging %d duplicates from input %d\n", dupNum->data.U32[i], i); + +#define VECTOR_PURGE_CASE(TYPE) \ + case PS_TYPE_##TYPE: { \ + long j = 0; \ + for (long i = 0; i < vector->n; i++) { \ + if (!dupes->data.U8[i]) { \ + if (i == j) { \ + j++; \ + continue; \ + } \ + vector->data.TYPE[j++] = vector->data.TYPE[i]; \ + } \ + } \ + vector->n = j; \ + } \ + break + + psVector *dupes = duplicates->data[i]; // Duplicates + psArray *table = psListToArray(det->table->list); // Table of data + long newLength = -1; + for (int t = 0; t < table->n; t++) { + psMetadataItem *item = table->data[t]; // Table item + psAssert(item->type == PS_DATA_VECTOR, "Table column is not a vector: %x", item->type); + psVector *vector = item->data.V; // Vector to purge + switch (vector->type.type) { + VECTOR_PURGE_CASE(U8); + VECTOR_PURGE_CASE(U16); + VECTOR_PURGE_CASE(U32); + VECTOR_PURGE_CASE(U64); + VECTOR_PURGE_CASE(S8); + VECTOR_PURGE_CASE(S16); + VECTOR_PURGE_CASE(S32); + VECTOR_PURGE_CASE(S64); + VECTOR_PURGE_CASE(F32); + VECTOR_PURGE_CASE(F64); + default: + psAbort("Unrecognised vector type: %x", vector->type.type); + } + if (newLength == -1) { + newLength = vector->n; + } else if (vector->n != newLength) { + psAbort("Unexpected new length found : %ld expected: %ld", + vector->n, newLength); + } + + } + // XXX IS this safe? Perhaps save in numGood? + det->num = newLength; + psFree(table); + } + psFree(dupNum); + psFree(duplicates); + + return true; } Index: /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsRead.c =================================================================== --- /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsRead.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsRead.c (revision 32630) @@ -20,10 +20,12 @@ psArray *detections = psArrayAlloc(num); // Array of detections, to return for (int i = 0; i < num; i++) { - psFits *fits = psFitsOpen(inNames->data[i], "r"); // FITS file - + const char *name = inNames->data[i]; + + psFits *fits = psFitsOpen(name, "r"); // FITS file if (!fits) { psError(PS_ERR_IO, false, "Unable to open input %d", i); return false; } + psMetadata *header = psFitsReadHeader(NULL, fits); // Primary header if (!header) { @@ -74,4 +76,8 @@ } ppMopsDetections *det = ppMopsDetectionsAlloc(size); + detections->data[i] = det; + det->component = psStringNCopy(name, strrchr(name, '.') - name); // Strip off extension + det->num = size; + det->diffSkyfileId = diffSkyfileId; psTrace("ppMops.read", 3, "Reading %ld rows from %s\n", size, (const char*)inNames->data[i]); @@ -90,135 +96,84 @@ psMetadataLookupF32(NULL, header, "FWHM_MIN")); - int naxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1"); // Number of columns - int naxis2 = psMetadataLookupS32(NULL, header, "IMNAXIS2"); // Number of rows + det->naxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1"); // Number of columns + det->naxis2 = psMetadataLookupS32(NULL, header, "IMNAXIS2"); // Number of rows psFree(header); - psArray *table = psFitsReadTable(fits); // Table of interest + psMetadata *table = psFitsReadTableAllColumns(fits); // Table of interest if (!table) { psError(PS_ERR_IO, false, "Unable to read table %d", i); - return false; - } + return NULL; + } + det->table = table; psFitsClose(fits); + if (args->version == 0) { + if (skyChipPsfVersion < 2) { + // XXX: TODO: Do we need to add dummy vectors for the missing columns? + } + } + + psVector *ra = psMetadataLookupVector(NULL, table, "RA_PSF"); + psVector *dec = psMetadataLookupVector(NULL, table, "DEC_PSF"); + + det->raErr = psVectorAlloc(size, PS_TYPE_F64); + det->decErr = psVectorAlloc(size, PS_TYPE_F64); + det->mask = psVectorAlloc(size, PS_TYPE_U8); + + // convert ra and dec to radians for use in the purge duplicates function + det->ra = (psVector*)psBinaryOp(NULL, ra, "*", psScalarAlloc(DEG_TO_RAD(1.0), PS_TYPE_F64)); + det->dec = (psVector*)psBinaryOp(NULL, dec, "*", psScalarAlloc(DEG_TO_RAD(1.0), PS_TYPE_F64)); + det->x = psMemIncrRefCounter(psMetadataLookupVector(NULL, table, "X_PSF")); + det->y = psMemIncrRefCounter(psMetadataLookupVector(NULL, table, "Y_PSF")); + if (!det->ra || !det->dec || !det->x || !det->y) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find all of RA, Dec, X and Y columns"); + return NULL; + } + + // Add our new vectors to the table so that duplicates and masked items may be purged + psMetadataAddVector(table, PS_LIST_HEAD, "DEC_ERR", 0, NULL, det->decErr); + psMetadataAddVector(table, PS_LIST_HEAD, "RA_ERR", 0, NULL, det->raErr); + + psTrace("ppMops.read", 2, "Read %ld rows from %s\n", det->num, det->component); + + psVector *mag = psMetadataLookupVector(NULL, table, "PSF_INST_MAG"); + psVector *magErr = psMetadataLookupVector(NULL, table, "PSF_INST_MAG_SIG"); + psVector *xErrV = psMetadataLookupVector(NULL, table, "X_PSF_SIG"); + psVector *yErrV = psMetadataLookupVector(NULL, table, "Y_PSF_SIG"); + psVector *scaleV = psMetadataLookupVector(NULL, table, "PLTSCALE"); + psVector *angleV = psMetadataLookupVector(NULL, table, "POSANGLE"); + psVector *flagsV = psMetadataLookupVector(NULL, table, "FLAGS"); double plateScale = 0.0; // Plate scale long numGood = 0; // Number of good rows - for (long j = 0; j < size; j++) { - psMetadata *row = table->data[j]; // Row of interest - - psU32 flags = psMetadataLookupU32(NULL, row, "FLAGS"); + for (long row = 0; row < size; row++) { + + psU32 flags = flagsV->data.U32[row]; // psFitsTableGetU32(NULL, table, row, "FLAGS"); if (flags & SOURCE_MASK) { - psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of flags: %ud", j, i, flags); + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of flags: %ud", row, i, flags); + det->mask->data.U8[row] = 0xFF; continue; } - det->x->data.F32[numGood] = psMetadataLookupF32(NULL, row, "X_PSF"); - det->y->data.F32[numGood] = psMetadataLookupF32(NULL, row, "Y_PSF"); - det->ra->data.F64[numGood] = DEG_TO_RAD(psMetadataLookupF64(NULL, row, "RA_PSF")); - det->dec->data.F64[numGood] = DEG_TO_RAD(psMetadataLookupF64(NULL, row, "DEC_PSF")); - det->mag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_MAG"); - det->magErr->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_MAG_SIG"); - det->chi2->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_CHISQ"); - det->dof->data.S32[numGood] = psMetadataLookupS32(NULL, row, "PSF_NDOF"); - det->cr->data.F32[numGood] = psMetadataLookupF32(NULL, row, "CR_NSIGMA"); - det->extended->data.F32[numGood] = psMetadataLookupF32(NULL, row, "EXT_NSIGMA"); - det->psfMajor->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_MAJOR"); - det->psfMinor->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_MINOR"); - det->psfTheta->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_THETA"); - det->quality->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_QF"); - det->numPix->data.S32[numGood] = psMetadataLookupS32(NULL, row, "PSF_NPIX"); - det->xxMoment->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_XX"); - det->xyMoment->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_XY"); - det->yyMoment->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_YY"); - det->flags->data.U32[numGood] = psMetadataLookupU32(NULL, row, "FLAGS"); - det->diffSkyfileId->data.S64[numGood] = diffSkyfileId; - det->naxis1->data.S32[numGood] = naxis1; - det->naxis2->data.S32[numGood] = naxis2; - det->nPos->data.S32[numGood] = psMetadataLookupS32(NULL, row, "DIFF_NPOS"); - det->fPos->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_FRATIO"); - det->nRatioBad->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_NRATIO_BAD"); - det->nRatioMask->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_NRATIO_MASK"); - det->nRatioAll->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_NRATIO_ALL"); - - //Additions of 2010-10-25 - if (args->version == 2) { - //Values are set only if the version is 2 - if (skyChipPsfVersion == 2) { - det->psfInstFlux->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_FLUX"); - det->psfInstFluxSig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_FLUX_SIG"); - det->apMag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_MAG"); - det->apMagRaw->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_MAG_RAW"); - det->apMagRadius->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_MAG_RADIUS"); - det->apFlux->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_FLUX"); - det->apFluxSig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_FLUX_SIG"); - det->peakFluxAsMag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PEAK_FLUX_AS_MAG"); - det->calPsfMag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "CAL_PSF_MAG"); - det->calPsfMagSig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "CAL_PSF_MAG_SIG"); - det->sky->data.F32[numGood] = psMetadataLookupF32(NULL, row, "SKY"); - det->skySig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "SKY_SIGMA"); - det->qualityPerfect->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_QF_PERFECT"); - det->momentsR1->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_R1"); - det->momentsRH->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_RH"); - det->kronFlux->data.F32[numGood] = psMetadataLookupF32(NULL, row, "KRON_FLUX"); - det->kronFluxErr->data.F32[numGood] = psMetadataLookupF32(NULL, row, "KRON_FLUX_ERR"); - det->kronFluxInner->data.F32[numGood] = psMetadataLookupF32(NULL, row, "KRON_FLUX_INNER"); - det->kronFluxOuter->data.F32[numGood] = psMetadataLookupF32(NULL, row, "KRON_FLUX_OUTER"); - det->diffRP->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_R_P"); - det->diffSnP->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_SN_P"); - det->diffRM->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_R_M"); - det->diffSnM->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_SN_M"); - det->flags2->data.U32[numGood] = psMetadataLookupU32(NULL, row, "FLAGS2"); - det->ippIdet->data.U32[numGood] = psMetadataLookupU32(NULL, row, "IPP_IDET"); - det->nFrames->data.U16[numGood] = psMetadataLookupU16(NULL, row, "N_FRAMES"); - det->padding->data.S16[numGood] = psMetadataLookupS16(NULL, row, "PADDING"); - } else { - det->psfInstFlux->data.F32[numGood] = NAN; - det->psfInstFluxSig->data.F32[numGood] = NAN; - det->apMag->data.F32[numGood] = NAN; - det->apMagRaw->data.F32[numGood] = NAN; - det->apMagRadius->data.F32[numGood] = NAN; - det->apFlux->data.F32[numGood] = NAN; - det->apFluxSig->data.F32[numGood] = NAN; - det->peakFluxAsMag->data.F32[numGood] = NAN; - det->calPsfMag->data.F32[numGood] = NAN; - det->calPsfMagSig->data.F32[numGood] = NAN; - det->sky->data.F32[numGood] = NAN; - det->skySig->data.F32[numGood] = NAN; - det->qualityPerfect->data.F32[numGood] = NAN; - det->momentsR1->data.F32[numGood] = NAN; - det->momentsRH->data.F32[numGood] = NAN; - det->kronFlux->data.F32[numGood] = NAN; - det->kronFluxErr->data.F32[numGood] = NAN; - det->kronFluxInner->data.F32[numGood] = NAN; - det->kronFluxOuter->data.F32[numGood] = NAN; - det->diffRP->data.F32[numGood] = NAN; - det->diffSnP->data.F32[numGood] = NAN; - det->diffRM->data.F32[numGood] = NAN; - det->diffSnM->data.F32[numGood] = NAN; - det->flags2->data.U32[numGood] = 0; - det->ippIdet->data.U32[numGood] = 0; - det->nFrames->data.U16[numGood] = 0; - det->padding->data.S16[numGood] = 0; - } - } - // Calculate error in RA, Dec - double xErr = psMetadataLookupF64(NULL, row, "X_PSF_SIG"); //SC: Warning! Promotion of F32 - double yErr = psMetadataLookupF64(NULL, row, "Y_PSF_SIG"); //SC: Warning! Promotion of F32 - double scale = psMetadataLookupF64(NULL, row, "PLTSCALE"); //SC: Warning! Promotion of F32 - double angle = psMetadataLookupF64(NULL, row, "POSANGLE"); //SC: Warning! Promotion of F32 - - if (!isfinite(det->x->data.F32[numGood]) || !isfinite(det->y->data.F32[numGood]) || - !isfinite(det->ra->data.F64[numGood]) || !isfinite(det->dec->data.F64[numGood]) || - !isfinite(det->mag->data.F32[numGood]) || !isfinite(det->magErr->data.F32[numGood]) || + + double xErr = xErrV->data.F32[row]; + double yErr = yErrV->data.F32[row]; + double scale = scaleV->data.F32[row]; + double angle = angleV->data.F32[row]; + + if (!isfinite(det->x->data.F32[row]) || !isfinite(det->y->data.F32[row]) || + !isfinite(det->ra->data.F64[row]) || !isfinite(det->dec->data.F64[row]) || + !isfinite(mag->data.F32[row]) || !isfinite(magErr->data.F32[row]) || !isfinite(xErr) || !isfinite(yErr) || !isfinite(scale) || !isfinite(angle)) { psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of non-finite values: " "%f %f %lf %lf %f %f %f %f %f %f", - j, i, - det->x->data.F32[numGood], det->y->data.F32[numGood], - det->ra->data.F64[numGood], det->dec->data.F64[numGood], - det->mag->data.F32[numGood], det->magErr->data.F32[numGood], + row, i, + det->x->data.F32[row], det->y->data.F32[row], + det->ra->data.F64[row], det->dec->data.F64[row], + mag->data.F32[row], magErr->data.F32[row], xErr, yErr, scale, angle); + det->mask->data.U8[row] = 0xFF; continue; } @@ -229,8 +184,8 @@ double xErr2 = PS_SQR(xErr), yErr2 = PS_SQR(yErr); double errScale = scale / 3600.0; - det->raErr->data.F64[numGood] = errScale * sqrt(cosAngle2 * xErr2 + sinAngle2 * yErr2); - det->decErr->data.F64[numGood] = errScale * sqrt(sinAngle2 * xErr2 + cosAngle2 * yErr2); - - det->mask->data.U8[numGood] = 0; + det->raErr->data.F64[row] = errScale * sqrt(cosAngle2 * xErr2 + sinAngle2 * yErr2); + det->decErr->data.F64[row] = errScale * sqrt(sinAngle2 * xErr2 + cosAngle2 * yErr2); + + det->mask->data.U8[row] = 0; plateScale += scale; numGood++; @@ -238,72 +193,12 @@ det->seeing *= ((float) plateScale) / ((float) numGood); - det->x->n = numGood; - det->y->n = numGood; - det->ra->n = numGood; - det->dec->n = numGood; - det->raErr->n = numGood; - det->decErr->n = numGood; - det->mag->n = numGood; - det->magErr->n = numGood; - det->chi2->n = numGood; - det->dof->n = numGood; - det->cr->n = numGood; - det->extended->n = numGood; - det->psfMajor->n = numGood; - det->psfMinor->n = numGood; - det->psfTheta->n = numGood; - det->quality->n = numGood; - det->numPix->n = numGood; - det->xxMoment->n = numGood; - det->xyMoment->n = numGood; - det->yyMoment->n = numGood; - det->flags->n = numGood; - det->diffSkyfileId->n = numGood; - det->naxis1->n = numGood; - det->naxis2->n = numGood; - det->mask->n = numGood; - det->nPos->n = numGood; - det->fPos->n = numGood; - det->nRatioBad->n = numGood; - det->nRatioMask->n = numGood; - det->nRatioAll->n = numGood; - det->psfInstFlux->n = numGood; - det->psfInstFluxSig->n = numGood; - det->apMag->n = numGood; - det->apMagRaw->n = numGood; - det->apMagRadius->n = numGood; - det->apFlux->n = numGood; - det->apFluxSig->n = numGood; - det->peakFluxAsMag->n = numGood; - det->calPsfMag->n = numGood; - det->calPsfMagSig->n = numGood; - det->sky->n = numGood; - det->skySig->n = numGood; - det->qualityPerfect->n = numGood; - det->momentsR1->n = numGood; - det->momentsRH->n = numGood; - det->kronFlux->n = numGood; - det->kronFluxErr->n = numGood; - det->kronFluxInner->n = numGood; - det->kronFluxOuter->n = numGood; - det->diffRP->n = numGood; - det->diffSnP->n = numGood; - det->diffRM->n = numGood; - det->diffSnM->n = numGood; - det->flags2->n = numGood; - det->ippIdet->n = numGood; - det->nFrames->n = numGood; - det->padding->n = numGood; - - det->num = numGood; + // Are we using numGood for anything outside of this function? + det->numGood = numGood; if (isfinite(args->zp) && numGood > 0) { - psBinaryOp(det->mag, det->mag, "+", psScalarAlloc(args->zp, PS_TYPE_F32)); - } - - psTrace("ppMops.read", 2, "Read %ld good rows from %s\n", numGood, (const char*)inNames->data[i]); - - psFree(table); - detections->data[i] = det; + psBinaryOp(mag, mag, "+", psScalarAlloc(args->zp, PS_TYPE_F32)); + } + + psTrace("ppMops.read", 2, "Read %ld good rows from %s\n", numGood, (const char*)name); } Index: /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsWrite.c =================================================================== --- /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsWrite.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ppTranslate/src/ppMopsWrite.c (revision 32630) @@ -9,7 +9,9 @@ #include "ppTranslateVersion.h" -bool ppMopsWrite(const ppMopsDetections *det, const ppMopsArguments *args) +static bool addOutputColumn(psMetadata *table, const psArray *detections, long total, char *outColName, char *inColName, bool convertTo32); +static bool addSkyfileIDColumn(psMetadata *table, const psArray *detections, long total, char *colName); + +bool ppMopsWrite(const psArray *detections, const ppMopsArguments *args) { - psTrace("ppMops.write", 1, "Writing %ld rows to %s", det->num, args->output); psFits *fits = psFitsOpen(args->output, "w"); // FITS file @@ -36,24 +38,47 @@ psMetadataAddBool(header, PS_LIST_TAIL, "DIFF_POS", 0, "Positive subtraction?", args->positive); - psMetadataAddF64(header, PS_LIST_TAIL, "MJD-OBS", 0, "MJD of exposure midpoint", det->mjd); - psMetadataAddStr(header, PS_LIST_TAIL, "RA", 0, "Right Ascension of boresight", det->raBoresight); - psMetadataAddStr(header, PS_LIST_TAIL, "DEC", 0, "Declination of boresight", det->decBoresight); - psMetadataAddF64(header, PS_LIST_TAIL, "TEL_ALT", 0, "Telescope altitude", det->alt); - psMetadataAddF64(header, PS_LIST_TAIL, "TEL_AZ", 0, "Telescope azimuth", det->az); - psMetadataAddF64(header, PS_LIST_TAIL, "EXPTIME", 0, "Exposure time (sec)", det->exptime); - psMetadataAddF64(header, PS_LIST_TAIL, "ROTANGLE", 0, "Rotator position angle", det->posangle); - psMetadataAddStr(header, PS_LIST_TAIL, "FILTER", 0, "Filter name", det->filter); - psMetadataAddF32(header, PS_LIST_TAIL, "AIRMASS", 0, "Airmass of exposure", det->airmass); + // Get these header words from the first input non null input + ppMopsDetections *det = NULL; + for (int d = 0; d < detections->n && det == NULL; d++) { + det = detections->data[d]; + } + if (det != NULL) { + psMetadataAddF64(header, PS_LIST_TAIL, "MJD-OBS", 0, "MJD of exposure midpoint", det->mjd); + psMetadataAddStr(header, PS_LIST_TAIL, "RA", 0, "Right Ascension of boresight", det->raBoresight); + psMetadataAddStr(header, PS_LIST_TAIL, "DEC", 0, "Declination of boresight", det->decBoresight); + psMetadataAddF64(header, PS_LIST_TAIL, "TEL_ALT", 0, "Telescope altitude", det->alt); + psMetadataAddF64(header, PS_LIST_TAIL, "TEL_AZ", 0, "Telescope azimuth", det->az); + psMetadataAddF64(header, PS_LIST_TAIL, "EXPTIME", 0, "Exposure time (sec)", det->exptime); + psMetadataAddF64(header, PS_LIST_TAIL, "ROTANGLE", 0, "Rotator position angle", det->posangle); + psMetadataAddStr(header, PS_LIST_TAIL, "FILTER", 0, "Filter name", det->filter); + psMetadataAddF32(header, PS_LIST_TAIL, "AIRMASS", 0, "Airmass of exposure", det->airmass); + psMetadataAddF32(header, PS_LIST_TAIL, "SEEING", 0, "Mean seeing", det->seeing); + } else { + psWarning("no inputs with surviving detections. output header will be incomplete"); + } psMetadataAddStr(header, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", OBSERVATORY_CODE); - psMetadataAddF32(header, PS_LIST_TAIL, "SEEING", 0, "Mean seeing", det->seeing); psMetadataAddF32(header, PS_LIST_TAIL, "MAGZP", 0, "Magnitude zero point", args->zp); psMetadataAddF32(header, PS_LIST_TAIL, "MAGZPERR", 0, "Error in magnitude zero point", args->zpErr); psMetadataAddF32(header, PS_LIST_TAIL, "ASTRORMS", 0, "RMS of astrometric fit", args->rmsAstrom); - //New field in header that tells about the CMF version + + //field in header that tells about the CMF version char cmfVersion[8]; sprintf(cmfVersion, PS1_DV_FORMAT, args->version); psMetadataAddStr(header, PS_LIST_TAIL, "CMFVERSION", 0, "CMF version", cmfVersion); - if (det->num == 0) { + // Find the total number of detections + + long total = 0; + for (long i=0; in; i++) { + ppMopsDetections *det = detections->data[i]; + if (!det) { + continue; + } + total += det->num; + } + + psTrace("ppMops.write", 1, "Writing %ld rows to %s", total, args->output); + + if (total == 0) { // Write dummy table psMetadata *row = psMetadataAlloc(); // Output row @@ -85,5 +110,5 @@ psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_ALL", 0, "Ratio of positive pixels to all", NAN); - if (args->version == 2) { + if (args->version == 2) { // Write data of version 2 (see ICD) psMetadataAdd (row, PS_LIST_TAIL, "IPP_IDET", PS_DATA_U32, "IPP detection identifier index", @@ -141,5 +166,5 @@ psMetadataAdd (row, PS_LIST_TAIL, "PADDING", PS_DATA_S16, "padding", 0); - } + } if (!psFitsWriteTableEmpty(fits, header, row, OUT_EXTNAME)) { @@ -151,114 +176,70 @@ psFree(row); } else { - psArray *table = psArrayAlloc(det->num); // Table to write - for (long i = 0; i < det->num; i++) { - psMetadata *row = psMetadataAlloc(); // Output row - psMetadataAddF64(row, PS_LIST_TAIL, "RA", 0, "Right ascension (degrees)", - RAD_TO_DEG(det->ra->data.F64[i])); - psMetadataAddF64(row, PS_LIST_TAIL, "RA_ERR", 0, "Right ascension error (degrees)", - det->raErr->data.F64[i]); - psMetadataAddF64(row, PS_LIST_TAIL, "DEC", 0, "Declination (degrees)", - RAD_TO_DEG(det->dec->data.F64[i])); - psMetadataAddF64(row, PS_LIST_TAIL, "DEC_ERR", 0, "Declination error (degrees)", - det->decErr->data.F64[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "MAG", 0, "Magnitude", det->mag->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "MAG_ERR", 0, "Magnitude error", det->magErr->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "PSF_CHI2", 0, "chi^2 of PSF fit", det->chi2->data.F32[i]); - psMetadataAddS32(row, PS_LIST_TAIL, "PSF_DOF", 0, "Degrees of freedom of PSF fit", - det->dof->data.S32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "CR_SIGNIFICANCE", 0, "Significance of CR", - det->cr->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "EXT_SIGNIFICANCE", 0, "Significance of extendedness", - det->extended->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "PSF_MAJOR", 0, "PSF major axis (pixels)", det->psfMajor->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "PSF_MINOR", 0, "PSF minor axis (pixels)", det->psfMinor->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "PSF_THETA", 0, "PSF position angle (deg on chip)", - det->psfTheta->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "PSF_QUALITY", 0, "PSF quality factor", - det->quality->data.F32[i]); - psMetadataAddS32(row, PS_LIST_TAIL, "PSF_NPIX", 0, "Number of pixels in PSF", - det->numPix->data.S32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_XX", 0, "xx moment", det->xxMoment->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_XY", 0, "xy moment", det->xyMoment->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_YY", 0, "yy moment", det->yyMoment->data.F32[i]); - psMetadataAddS32(row, PS_LIST_TAIL, "N_POS", 0, "Number of positive pixels", - det->nPos->data.S32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "F_POS", 0, "Fraction of positive pixels", - det->fPos->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_BAD", 0, "Ratio of positive pixels to negative", - det->nRatioBad->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_MASK", 0, "Ratio of positive pixels to masked", - det->nRatioMask->data.F32[i]); - psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_ALL", 0, "Ratio of positive pixels to all", - det->nRatioAll->data.F32[i]); - psMetadataAddU32(row, PS_LIST_TAIL, "FLAGS", 0, "Detection bit flags", det->flags->data.U32[i]); - psMetadataAddS64(row, PS_LIST_TAIL, "DIFF_SKYFILE_ID", 0, "Identifier for diff skyfile", - det->diffSkyfileId->data.S64[i]); - - if (args->version == 2) { - // Write data of version 2 (see ICD) - psMetadataAdd (row, PS_LIST_TAIL, "IPP_IDET", PS_DATA_U32, "IPP detection identifier index", - det->ippIdet->data.U32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX", PS_DATA_F32, "PSF fit instrumental magnitude", - det->psfInstFlux->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX_SIG",PS_DATA_F32, "Sigma of PSF instrumental magnitude", - det->psfInstFluxSig->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG", PS_DATA_F32, "magnitude in standard aperture", - det->apMag->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RAW", PS_DATA_F32, "magnitude in real aperture", - det->apMagRaw->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS", PS_DATA_F32, "radius used for aperture mags", - det->apMagRadius->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "AP_FLUX", PS_DATA_F32, "instrumental flux in standard aperture", - det->apFlux->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "AP_FLUX_SIG", PS_DATA_F32, "aperture flux error", - det->apFluxSig->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude", - det->peakFluxAsMag->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG", PS_DATA_F32, "PSF Magnitude using supplied calibration", - det->calPsfMag->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG_SIG", PS_DATA_F32, "measured scatter of zero point calibration", - det->calPsfMagSig->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "SKY", PS_DATA_F32, "Sky level", - det->sky->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "SKY_SIGMA", PS_DATA_F32, "Sigma of sky level", - det->skySig->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "PSF_QF_PERFECT", PS_DATA_F32, "PSF coverage/quality factor (poor)", - det->qualityPerfect->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_R1", PS_DATA_F32, "first radial moment", - det->momentsR1->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_RH", PS_DATA_F32, "half radial moment", - det->momentsRH->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX", PS_DATA_F32, "Kron Flux (in 2.5 R1)", - det->kronFlux->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_ERR", PS_DATA_F32, "Kron Flux Error", - det->kronFluxErr->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_INNER", PS_DATA_F32, "Kron Flux (in 1.0 R1)", - det->kronFluxInner->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_OUTER", PS_DATA_F32, "Kron Flux (in 4.0 R1)", - det->kronFluxOuter->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "DIFF_R_P", PS_DATA_F32, "distance to positive match source", - det->diffRP->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "DIFF_SN_P", PS_DATA_F32, "signal-to-noise of pos match src", - det->diffSnP->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "DIFF_R_M", PS_DATA_F32, "distance to negative match source", - det->diffRM->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "DIFF_SN_M", PS_DATA_F32, "signal-to-noise of neg match src", - det->diffSnM->data.F32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "FLAGS2", PS_DATA_U32, "psphot analysis flags (group 2)", - det->flags2->data.U32[i]); - psMetadataAdd (row, PS_LIST_TAIL, "N_FRAMES", PS_DATA_U16, "Number of frames overlapping source center", - det->nFrames->data.U16[i]); - psMetadataAdd (row, PS_LIST_TAIL, "PADDING", PS_DATA_S16, "padding", - det->padding->data.S16[i]); - } - - //Update with the table with the current row - table->data[i] = row; - } - if (!psFitsWriteTable(fits, header, table, OUT_EXTNAME)) { - psErrorStackPrint(stderr, "Unable to write table."); - psFree(header); - psFree(table); + +#define addColumn(_outName, _inName, _convertTo32) \ + if (!addOutputColumn(table, detections, total, _outName, _inName, _convertTo32)) { \ + psError(PS_ERR_UNKNOWN, false, "Failed to add column %s", _outName); \ + return false; \ + } + + // Allocate the output table + psMetadata *table = psMetadataAlloc(); + addColumn("RA", "RA_PSF", 0); + addColumn("RA_ERR", NULL, 0); // calculated from various parameters including X_PSF_SIG Y_PSF_SIG and POSANG + addColumn("DEC", "DEC_PSF", 0); + addColumn("DEC_ERR", NULL, 0); // calculated from various parameters including X_PSF_SIG Y_PSF_SIG and POSANG + addColumn("MAG", "PSF_INST_MAG", 0); + addColumn("MAG_ERR", "PSF_INST_MAG_SIG", 0); + addColumn("PSF_CHI2", "PSF_CHISQ", 0); + addColumn("PSF_DOF", "PSF_NDOF", 1); + addColumn("CR_SIGNIFICANCE", "CR_NSIGMA", 0); + addColumn("EXT_SIGNIFICANCE", "EXT_NSIGMA", 0); + addColumn("PSF_MAJOR", NULL, 0); + addColumn("PSF_MINOR", NULL, 0); + addColumn("PSF_THETA", NULL, 0); + addColumn("PSF_QUALITY", "PSF_QF", 0); + addColumn("PSF_NPIX", NULL, 1); + addColumn("MOMENTS_XX", NULL, 0); + addColumn("MOMENTS_XY", NULL, 0); + addColumn("MOMENTS_YY", NULL, 0); + addColumn("N_POS", "DIFF_NPOS", 1); + addColumn("F_POS", "DIFF_FRATIO", 0); + addColumn("RATIO_BAD", "DIFF_NRATIO_BAD", 0); + addColumn("RATIO_MASK", "DIFF_NRATIO_MASK", 0); + addColumn("RATIO_ALL", "DIFF_NRATIO_ALL", 0); + addColumn("FLAGS", "FLAGS", 1); + addSkyfileIDColumn(table, detections, total, "DIFF_SKYFILE_ID"); + if (args->version == 2) { + addColumn("IPP_IDET", NULL, 1); + addColumn("PSF_INST_FLUX", NULL, 0); + addColumn("PSF_INST_FLUX_SIG", NULL, 0); + addColumn("AP_MAG", NULL, 0); + addColumn("AP_MAG_RAW", NULL, 0); + addColumn("AP_MAG_RADIUS", NULL, 0); + addColumn("AP_FLUX", NULL, 0); + addColumn("AP_FLUX_SIG", NULL, 0); + addColumn("PEAK_FLUX_AS_MAG", NULL, 0); + addColumn("CAL_PSF_MAG", NULL, 0); + addColumn("CAL_PSF_MAG_SIG", NULL, 0); + addColumn("SKY", NULL, 0); + addColumn("SKY_SIGMA", NULL, 0); + addColumn("PSF_QF_PERFECT", NULL, 0); + addColumn("MOMENTS_R1", NULL, 0); + addColumn("MOMENTS_RH", NULL, 0); + addColumn("KRON_FLUX", NULL, 0); + addColumn("KRON_FLUX_ERR", NULL, 0); + addColumn("KRON_FLUX_INNER", NULL, 0); + addColumn("KRON_FLUX_OUTER", NULL, 0); + addColumn("DIFF_R_P", NULL, 0); + addColumn("DIFF_SN_P", NULL, 0); + addColumn("DIFF_R_M", NULL, 0); + addColumn("DIFF_SN_M", NULL, 0); + addColumn("FLAGS2", NULL, 1); + addColumn("IPP_IDET", NULL, 0); + addColumn("N_FRAMES", NULL, 0); + addColumn("PADDING", NULL, 0); + } + if (!psFitsWriteTableAllColumns(fits, header, table, OUT_EXTNAME)) { + psError(psErrorCodeLast(), false, "Unable to write table"); return false; } @@ -273,2 +254,133 @@ return true; } + +static bool addOutputColumn(psMetadata *table, const psArray *detections, long outputSize, char *outColumnName, char *inColumnName, bool convertTo32) +{ + if (inColumnName == NULL) { + inColumnName = outColumnName; + } + + psVector *out = NULL; + if (convertTo32) { + // psFitsReadTableAllColumns reads columns of cfitsio type LONG and ULONG into a 64 bit integers + // We want to write 32 bits to the output. + int next = 0; + for (long i=0; in; i++) { + ppMopsDetections *det = detections->data[i]; + if (!det || det->num == 0) { + // no detections survived for this input + continue; + } + psVector *in = psMetadataLookupVector(NULL, det->table, inColumnName); + if (!in) { + psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s", inColumnName); + return false; + } + if (in->type.type != PS_TYPE_S64 && in->type.type != PS_TYPE_U64) { + psError(PS_ERR_PROGRAMMING, true, "input column to convert is not S64 or U64: %s %d", + inColumnName, in->type.type); + return false; + } + if (out == NULL) { + // First time through set up the output vector and the copy parameters + if (in->type.type == PS_TYPE_S64) { + out = psVectorAlloc(outputSize, PS_TYPE_S32); + } else { + out = psVectorAlloc(outputSize, PS_TYPE_U32); + } + } + for (long d=0; d < det->num; d++) { + if (in->type.type == PS_TYPE_S64) { + out->data.S32[next++] = in->data.S64[d]; + } else { + out->data.U32[next++] = in->data.U64[d]; + } + } + } + } else { + void *next = NULL; + int elementSize = 0; // size of elements in vector... We are making assumptions here about the organization of primitives in memory so we can use memcopy + for (long i=0; in; i++) { + ppMopsDetections *det = detections->data[i]; + if (!det || det->num == 0) { + // no detections survived for this input + continue; + } + psVector *in = psMetadataLookupVector(NULL, det->table, inColumnName); + if (!in) { + psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s", inColumnName); + return false; + } + if (out == NULL) { + // First time through set up the output vector and the copy parameters + out = psVectorAlloc(outputSize, in->type.type); + next = (void *) out->data.U8; + switch (in->type.type) { + case PS_TYPE_S8: + elementSize = sizeof(psS8); + break; + case PS_TYPE_U8: + elementSize = sizeof(psU8); + break; + case PS_TYPE_S16: + elementSize = sizeof(psS16); + break; + case PS_TYPE_U16: + elementSize = sizeof(psU16); + break; + case PS_TYPE_S32: + elementSize = sizeof(psS32); + break; + case PS_TYPE_U32: + elementSize = sizeof(psU32); + break; + case PS_TYPE_S64: + elementSize = sizeof(psS64); + break; + case PS_TYPE_U64: + elementSize = sizeof(psU64); + break; + case PS_TYPE_F32: + elementSize = sizeof(psF32); + break; + case PS_TYPE_F64: + elementSize = sizeof(psF64); + break; + default: + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Unknown vector type %d", in->type.type); + return false; + + } + } + // We are doing nasty things here so we can use memcpy. + // It would be safer to do a proper loop over the elements. + long toCopy = det->num * elementSize; + memcpy(next, in->data.U8, toCopy); + next += toCopy; + } + } + + // Finally add the new column to the output table + psMetadataAddVector(table, PS_LIST_TAIL, outColumnName, 0, NULL, out); + psFree(out); // drop reference + + return true; +} +static bool addSkyfileIDColumn(psMetadata *table, const psArray *detections, long total, char *colName) +{ + psVector *out = psVectorAlloc(total, PS_TYPE_S64); + long next = 0; + for (long i = 0; in; i++) { + ppMopsDetections *det = detections->data[i]; + if (!det) { + continue; + } + psS64 diffSkyfileId = det->diffSkyfileId; + for (long j = 0; j < det->num; j++) { + out->data.S64[next++] = diffSkyfileId; + } + } + psMetadataAddVector(table, PS_LIST_TAIL, colName, 0, NULL, out); + psFree(out); + return true; +} Index: /branches/eam_branches/ipp-20110906/ppViz/src/ppCoord/ppCoordLoop.c =================================================================== --- /branches/eam_branches/ipp-20110906/ppViz/src/ppCoord/ppCoordLoop.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/ppViz/src/ppCoord/ppCoordLoop.c (revision 32630) @@ -31,6 +31,9 @@ psProject(&tp, &sky, astromFPA->toSky); + psTrace("ppCoord",2,"sky2pix tr1: %f %f\n",tp.x,tp.y); psPlaneTransformApply(&fp, astromFPA->fromTPA, &tp); + psTrace("ppCoord",2,"sky2pix tr2: %f %f\n",fp.x,fp.y); psPlaneTransformApply(&pix, astromChip->fromFPA, &fp); + psTrace("ppCoord",2,"sky2pix tr3: %f %f\n",pix.x,pix.y); *x = pix.x; @@ -261,10 +264,12 @@ psPlaneTransformApply(fp, chip->toFPA, pix); + psTrace("ppCoord",2,"pix2sky tr1: %f %f\n",fp->x,fp->y); psPlaneTransformApply(tp, astromFile->fpa->toTPA, fp); + psTrace("ppCoord",2,"pix2sky tr2: %f %f\n",tp->x,tp->y); psDeproject(sky, tp, astromFile->fpa->toSky); - + psTrace("ppCoord",2,"pix2sky tr3: %f %f\n",sky->r,sky->d); if (!data->radians) { - sky->r *= M_PI / 180.0; - sky->d *= M_PI / 180.0; + sky->r *= 180.0 / M_PI; + sky->d *= 180.0 / M_PI; } Index: /branches/eam_branches/ipp-20110906/psLib/src/fits/psFitsImage.c =================================================================== --- /branches/eam_branches/ipp-20110906/psLib/src/fits/psFitsImage.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/psLib/src/fits/psFitsImage.c (revision 32630) @@ -772,5 +772,5 @@ (options->scaling == PS_FITS_SCALE_LOG_STDEV_BOTH)))) { fits_write_key_dbl(fits->fd, "BOFFSET", boffset, 12, - "Scaling: ORIGINAL = 10**(TRUE) + BOFFSET)", &status); + "Scaling: UNCOMP = 10**(TRUE) + BOFFSET", &status); } if (options&&(((options->scaling == PS_FITS_SCALE_ASINH_RANGE)|| @@ -779,10 +779,8 @@ (options->scaling == PS_FITS_SCALE_ASINH_STDEV_NEGATIVE)|| (options->scaling == PS_FITS_SCALE_ASINH_STDEV_BOTH)))) { + fits_write_key_dbl(fits->fd, "BSOFTEN", bsoften, 12, + "Scaling: LINEAR = 2 * BSOFTEN * sinh(TRUE/a)",&status); fits_write_key_dbl(fits->fd, "BOFFSET", boffset, 12, - "Scaling: NORM = DISK + BOFFSET", &status); - fits_write_key_dbl(fits->fd, "BSOFTEN", bsoften, 12, - "Scaling: TRUE = 2 * BSOFTEN * sinh(NORM/a)",&status); - - + "Scaling: UNCOMP = BOFFSET + LINEAR",&status); } Index: /branches/eam_branches/ipp-20110906/psLib/src/fits/psFitsTableNew.c =================================================================== --- /branches/eam_branches/ipp-20110906/psLib/src/fits/psFitsTableNew.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/psLib/src/fits/psFitsTableNew.c (revision 32630) @@ -117,5 +117,6 @@ psFitsTable* psFitsTableAlloc(int numCols, int numRows) { - if (numCols <= 0) { + if (numCols < 0) { + // empty table psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Can't allocate a psFitsTable with %d columns.", numCols); @@ -127,6 +128,10 @@ table->numRows = numRows; table->numCols = numCols; - table->columns = psAlloc(sizeof(psFitsTableColumn) * numCols); - memset(table->columns, sizeof(psFitsTableColumn) * numCols, 0); + if (numCols > 0) { + table->columns = psAlloc(sizeof(psFitsTableColumn) * numCols); + memset(table->columns, sizeof(psFitsTableColumn) * numCols, 0); + } else { + table->columns = NULL; + } psMemSetDeallocator(table, (psFreeFunc) freeTable); @@ -141,5 +146,6 @@ if (!readTableCheck(fits)) { if (emptyTableCheck(fits)) { - return NULL; + psFitsTable *table = psFitsTableAlloc(0, 0); + return table; } psError(PS_ERR_BAD_FITS, true, "Extension is not a table"); Index: /branches/eam_branches/ipp-20110906/psLib/src/sys/psThread.c =================================================================== --- /branches/eam_branches/ipp-20110906/psLib/src/sys/psThread.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/psLib/src/sys/psThread.c (revision 32630) @@ -36,6 +36,6 @@ static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; // Mutex for locking threads -static psList *pending = NULL; // queue of pending jobs -static psList *done = NULL; // queue of done jobs +static volatile psList *pending = NULL; // queue of pending jobs +static volatile psList *done = NULL; // queue of done jobs static pthread_t *threads = NULL; // array of the POSIX thread handles static psArray *pool = NULL; // array of defined threads @@ -116,5 +116,5 @@ done = psListAlloc(NULL); } - psListAdd(done, PS_LIST_TAIL, job); + psListAdd((psList *)done, PS_LIST_TAIL, job); psFree(job); return task->function(job); @@ -125,5 +125,5 @@ pending = psListAlloc(NULL); } - psListAdd(pending, PS_LIST_TAIL, job); + psListAdd((psList *)pending, PS_LIST_TAIL, job); psFree(job); psThreadUnlock(); @@ -139,5 +139,5 @@ } - psThreadJob *job = psListGetAndRemove(pending, PS_LIST_HEAD); + psThreadJob *job = psListGetAndRemove((psList *)pending, PS_LIST_HEAD); return job; } @@ -150,5 +150,5 @@ } - psThreadJob *job = psListGetAndRemove(done, PS_LIST_HEAD); + psThreadJob *job = psListGetAndRemove((psList *) done, PS_LIST_HEAD); return job; } @@ -253,5 +253,5 @@ done = psListAlloc(NULL); } - psListAdd(done, PS_LIST_TAIL, job); + psListAdd((psList *)done, PS_LIST_TAIL, job); psFree(job); Index: /branches/eam_branches/ipp-20110906/psModules/src/imcombine/pmPSFEnvelope.c =================================================================== --- /branches/eam_branches/ipp-20110906/psModules/src/imcombine/pmPSFEnvelope.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/psModules/src/imcombine/pmPSFEnvelope.c (revision 32630) @@ -380,5 +380,5 @@ // measure the source moments: tophat windowing, no pixel S/N cutoff - if (!pmSourceMoments(source, maxRadius, 0.0, 0.0, 0.0, maskVal)) { + if (!pmSourceMoments(source, maxRadius, 0.25*maxRadius, 0.0, 0.0, maskVal)) { // Can't do anything about it; limp along as best we can psErrorClear(); Index: /branches/eam_branches/ipp-20110906/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /branches/eam_branches/ipp-20110906/pstamp/scripts/pstamp_checkdependent.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/pstamp/scripts/pstamp_checkdependent.pl (revision 32630) @@ -240,8 +240,10 @@ my $state = $chip->{state}; my $data_state = $chip->{data_state}; + my $fault = $chip->{fault}; if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or - ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) { - - print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state cannot update\n"; + ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop') or + ($fault eq $PSTAMP_GONE)) { + + print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state fault: $fault cannot update\n"; my $error_code; if (($state eq 'error_cleaned') or ($data_state ='error_cleaned')) { Index: /branches/eam_branches/ipp-20110906/pstamp/src/ppstampOptions.c =================================================================== --- /branches/eam_branches/ipp-20110906/pstamp/src/ppstampOptions.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/pstamp/src/ppstampOptions.c (revision 32630) @@ -28,4 +28,6 @@ options->roip.dDEC = 0; options->chipName = NULL; + options->cellName = NULL; + options->stage = NULL; options->censorMasked = true; Index: /branches/eam_branches/ipp-20110906/psvideophot/src/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20110906/psvideophot/src/Makefile.am (revision 32629) +++ /branches/eam_branches/ipp-20110906/psvideophot/src/Makefile.am (revision 32630) @@ -1,3 +1,4 @@ -bin_PROGRAMS = psvideophot +bin_PROGRAMS = psvideophot \ + dumpvideo noinst_HEADERS = \ @@ -40,4 +41,9 @@ psvideophotVersion.c +dumpvideo_CFLAGS = $(PSVIDEOPHOT_CFLAGS) $(PPSTATS_CFLAGS) $(PSASTRO_CFLAGS) $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) +dumpvideo_LDFLAGS = $(PSVIDEOPHOT_LIBS) $(PSASTRO_LIBS) $(PPSTATS_LIBS) $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) +dumpvideo_SOURCES = \ + dumpvideo.c + CLEANFILES = *~ Index: /branches/eam_branches/ipp-20110906/psvideophot/src/dumpvideo.c =================================================================== --- /branches/eam_branches/ipp-20110906/psvideophot/src/dumpvideo.c (revision 32630) +++ /branches/eam_branches/ipp-20110906/psvideophot/src/dumpvideo.c (revision 32630) @@ -0,0 +1,75 @@ +#include +#include +#include +#include + +int main (int argc, char **argv) { + + if (argc < 4) { + fprintf (stderr, "USAGE: dumpvideo (input) (extname) (output) [-includetable]\n"); + exit (2); + } + + pmConfig *config = pmConfigRead(&argc, argv, NULL); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't find site configuration!\n"); + exit(EXIT_FAILURE); + } + + + + char *input = argv[1]; // Input file name + char *extname = argv[2]; // Extension of interest + char *output = argv[3]; // Output file name + + bool includetable = false; + if (argc > 4) { + includetable = true; + } + + psString inputResolved = pmConfigConvertFilename(input, config, false, false); + psFits *infile = psFitsOpen (inputResolved, "r"); // Input file + if (!infile) { + fprintf (stderr, "Unable to open input file %s", input); + exit (PS_EXIT_DATA_ERROR); + } + + psString outputResolved = pmConfigConvertFilename(output, config, true, true); + psFits *outfile = psFitsOpen (outputResolved, "w"); // Output file + if (!outfile) { + fprintf (stderr, "Unable to open output file %s", output); + exit(PS_EXIT_DATA_ERROR); + } + + if (!psFitsMoveExtName (infile, extname)) { + fprintf (stderr, "Unable to find extname %s\n", extname); + exit(PS_EXIT_DATA_ERROR); + } + + psMetadata *header = psFitsReadHeader (NULL, infile); + + psRegion region = {0, 0, 0, 0}; + psArray *cube = psFitsReadImageCube (infile, region); + + psFitsWriteImageCube (outfile, header, cube, extname); + + if (includetable) { + psString video_extname = NULL; + psStringAppend(&video_extname, "video_table_%s", extname); + if (!psFitsMoveExtName (infile, video_extname)) { + fprintf (stderr, "Unable to find extname %s\n", video_extname); + exit(PS_EXIT_DATA_ERROR); + } + psArray *table = psFitsReadTable(infile); + if (!psFitsWriteTable(outfile, NULL, table, video_extname)) { + fprintf (stderr, "Unable to write video table video_extname %s\n", video_extname); + exit(PS_EXIT_DATA_ERROR); + } + } + + psFitsClose(infile); + psFitsClose(outfile); + + return PS_EXIT_SUCCESS; +} + Index: /branches/eam_branches/ipp-20110906/psvideophot/src/psvideoLoop.c =================================================================== --- /branches/eam_branches/ipp-20110906/psvideophot/src/psvideoLoop.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/psvideophot/src/psvideoLoop.c (revision 32630) @@ -44,4 +44,7 @@ } + psString class_id = psMetadataLookupStr(&status, config->arguments, "CLASS_ID"); + psString cell_id = psMetadataLookupStr(&status, config->arguments, "CELL_ID"); + pmConfigCamerasCull(config, NULL); @@ -56,13 +59,20 @@ } + psArray *output = NULL; + pmChip *chip; // Chip from FPA while ((chip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) { psLogMsg ("psvideophotLoop", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + psString chipName = psMetadataLookupStr(&status, chip->concepts, "CHIP.NAME"); + if (strcmp(chipName, class_id)) { + continue; + } if (!chip->process || !chip->file_exists) { - continue; + ESCAPE("target chip has no data"); } if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { ESCAPE("load failure for Chip"); } + (void) class_id; double threshold = THRESHOLD; @@ -72,10 +82,16 @@ while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) { psLogMsg ("psvideophotLoop", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + psString cellName = psMetadataLookupStr(&status, cell->concepts, "CELL.NAME"); + if (strcmp(cellName, cell_id)) { + continue; + } if (!cell->process || !cell->file_exists) { - continue; + ESCAPE("target cell has no data"); } if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { ESCAPE("load failure for Cell"); } + + psLogMsg("psvideophotLoop", 1, "Found target cell %s %d. %ld readouts\n", cellName, view->cell, cell->readouts->n); // Put version information into the header @@ -87,9 +103,7 @@ if (cell->readouts->n <2) { - continue; + psLogMsg("psvideophotLoop", 1, "cell %s has only %ld readouts\n", cellName, cell->readouts->n); + ESCAPE("invalid video Cell"); } - - psString cellName = psMetadataLookupStr(&status, cell->concepts, "CELL.NAME"); - psLogMsg("psvideophotLoop", 1, "Found video cell %s %d\n", cellName, view->cell); // get the position of this cell on the chip @@ -114,6 +128,8 @@ // go find the video extension data - psString video_extension_name = NULL; - psStringAppend(&video_extension_name, "video_table_%s", cellName); + psString video_extension_name = psMetadataLookupStr(&status, config->arguments, "VIDEOEXTNAME"); + if (!video_extension_name) { + psStringAppend(&video_extension_name, "video_table_%s", cellName); + } psArray *video_table = readVideoTable(input, video_extension_name); if (!video_table) { @@ -121,5 +137,8 @@ ESCAPE("invalid image file"); } - psMetadataAddArray(cell->analysis, PS_LIST_TAIL, "VIDEO_DATA", 0, "", video_table); + psFree(video_extension_name); + // psMetadataAddArray(cell->analysis, PS_LIST_TAIL, "VIDEO_DATA", 0, "", video_table); + + output = (psArray *) psMemIncrRefCounter(video_table); if (cell->readouts->n != video_table->n) { @@ -180,4 +199,5 @@ double cnpix2 = psMetadataLookupS32(&status, row, "cnpix2"); + // avoid errors from unused variables (void) frame_start; (void) frame_complete; (void) rowpre; (void) cnpix1; (void) cnpix2; (void) fwhm_x; (void) fwhm_y; (void) fwhm; @@ -191,8 +211,9 @@ double star_readout_y = (centroid_y - cnpix2) / yBin; +#ifdef notdef printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.6f %6.3f %6.3f %6.3f\n", frame, flux, snr, sky, star_readout_x, star_readout_y, frame_complete - frame_start, fwhm, fwhm_x, fwhm_y); - +#endif psStatsInit(stats); @@ -206,19 +227,32 @@ double stddev = stats->sampleStdev; double median = stats->sampleMedian; + double vpFlux = 0; + double vpSum = 0; + double xy[2] = {0, 0}; if (stddev > threshold) { - double sum = 0; - // XXX: use a point data type - double xy[2]; - Centering(readout->image, EDGE, &sum, xy, median); - double ourFlux; - ourFlux = SubSum(readout->image, xy, median); - - printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.2f\n", - frame, ourFlux, ourFlux/stddev, median, xy[0], xy[1], stddev); + Centering(readout->image, EDGE, &vpSum, xy, median); + + vpFlux = SubSum(readout->image, xy, median); + +// printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.2f\n", +// frame, vpFlux, vpFlux/median, median, xy[0], xy[1], stddev); } else { - printf("%4d %9.2f %9.2f %8.2f\n", - frame, 0.0, stddev, median); - } +// printf("%4d %9.2f %9.2f %8.2f\n", +// frame, 0.0, stddev, median); + } + if (frame % 60 == 0) { + printf("frame flux sky snr X Y dt fwhm fwhm_x fwhm_y vp_flux vp_median vp_sum vp_stddev vp_x vp_y\n"); + } + printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.6f %6.3f %6.3f %6.3f %9.2f %9.2f %9.2f %9.2f %.1f %.1f\n", + frame, flux, sky, snr, star_readout_x, star_readout_y, frame_complete - frame_start, + fwhm, fwhm_x, fwhm_y, + vpFlux, median, vpSum, stddev, xy[0], xy[1]); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_flux", 0, "vp flux", vpFlux); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_median", 0, "vp meidan", median); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_stddev", 0, "vp meidan", stddev); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_sum", 0, "vp sum of source", vpSum); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_x", 0, "centroid x from vp", xy[0]); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_y", 0, "centroid y from vp", xy[1]); } psLogMsg ("psvideophotLoop", 5, "Done processing video cell"); @@ -247,11 +281,22 @@ psFree(view); - // Dump configuration - psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG"); - if (dump_file) { - if (!pmConfigDump(config, dump_file)) { - ESCAPE("Unable to dump configuration."); - } - } + psString outputName = psMetadataLookupStr(&status, config->arguments, "OUTPUT"); + psString realName = pmConfigConvertFilename(outputName, config, true, true); + if (!realName) { + psError(PS_ERR_UNKNOWN, false, "failed to convert filename for %s", outputName); + ESCAPE("cannot save output"); + } + psFits *outFits = psFitsOpen(realName, "w"); + if (!outFits) { + psError(PS_ERR_UNKNOWN, false, "failed to open output filename for %s", outputName); + ESCAPE(" "); + } + + psMetadata *outputHeader = lastHDU ? lastHDU->header : NULL; + if (!psFitsWriteTable(outFits, outputHeader, output, "videophotemetry")) { + psError(PS_ERR_UNKNOWN, false, "failed to write videophotometry table"); + ESCAPE("cannot write output"); + } + psFitsClose(outFits); return true; Index: /branches/eam_branches/ipp-20110906/psvideophot/src/psvideophotArguments.c =================================================================== --- /branches/eam_branches/ipp-20110906/psvideophot/src/psvideophotArguments.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/psvideophot/src/psvideophotArguments.c (revision 32630) @@ -6,24 +6,10 @@ static void usage (void) { - fprintf(stderr, "USAGE: psvideophot [-file INPUT.fits] [-list INPUT.txt] OUTPUT\n\n"); + fprintf(stderr, "USAGE: psvideophot [-file INPUT.fits] [-list INPUT.txt] OUTPUT -class_id CLASS_ID -cell_id CELL_ID\n\n"); fprintf(stderr, "Optional arguments:\n"); fprintf(stderr, "\t-stats STATS.mdc: Output statistics into STATS.mdc\n"); - fprintf(stderr, "\t-isfringe: The input image contains fringe data.\n"); - fprintf(stderr, "\t-isdark: The input image contains dark data.\n"); - fprintf(stderr, "\t-usemask MASKVAL: Use this mask value (override recipe).\n"); - fprintf(stderr, "\t-chip CHIPNUM: Only process this chip number.\n"); - fprintf(stderr, "\t-norm VALUE: Divide through by this value when done.\n"); - fprintf(stderr, "\t-normlist file.mdc: normalizations by class_id.\n"); - fprintf(stderr, "\t-video: process video cells\n"); + fprintf(stderr, "\t-video_extname VIDEOEXTNAME: EXTNAME for video table\n"); fprintf(stderr, "\n"); fprintf(stderr, "Input options (single file / file list):\n"); - fprintf(stderr, "\t-noisemap/-noisemaplist: Noise Map image.\n"); - fprintf(stderr, "\t-bias/-biaslist: Bias image.\n"); - fprintf(stderr, "\t-dark/-darklist: Dark image.\n"); - fprintf(stderr, "\t-shutter/-shutterlist: Shutter image.\n"); - fprintf(stderr, "\t-flat/-flatlist: Flat image.\n"); - fprintf(stderr, "\t-mask/-masklist: Mask image.\n"); - fprintf(stderr, "\t-fringe/-fringelist: Fringe image and data.\n"); - fprintf(stderr, "\t-linearity/-linearlist: linearity correction file.\n"); fprintf(stderr, "\n"); exit (2); @@ -63,4 +49,29 @@ // psMetadata *options = psMetadataAlloc (); // psMetadataAddPtr (config->arguments, PS_LIST_TAIL, "PSVIDEOPHOT.OPTIONS", PS_DATA_METADATA, "", options); + + if ((argnum = psArgumentGet(argc, argv, "-class_id"))) { + psArgumentRemove(argnum, &argc, argv); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "CLASS_ID", PS_META_REPLACE, + "Chip to process", argv[argnum]); + psArgumentRemove(argnum, &argc, argv); + } else { + fprintf(stderr, "-class_id is required\n"); + usage(); + } + if ((argnum = psArgumentGet(argc, argv, "-cell_id"))) { + psArgumentRemove(argnum, &argc, argv); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "CELL_ID", PS_META_REPLACE, + "Cell to process", argv[argnum]); + psArgumentRemove(argnum, &argc, argv); + } else { + fprintf(stderr, "-cell_id is required\n"); + usage(); + } + if ((argnum = psArgumentGet(argc, argv, "-video_extname"))) { + psArgumentRemove(argnum, &argc, argv); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "VIDEOEXTNAME", PS_META_REPLACE, + "EXTNAME of video table", argv[argnum]); + psArgumentRemove(argnum, &argc, argv); + } if ((argnum = psArgumentGet(argc, argv, "-stats"))) { @@ -117,4 +128,5 @@ // the input file is a required argument; if not found, we will exit pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "ASTROM", "-astrom", "-astromlist"); // if these command-line options are supplied, load the file name lists into config->arguments @@ -168,5 +180,5 @@ // Add the input and output images (which remain on the command-line) to the arguments list - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[1]); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output table", argv[1]); return config; Index: /branches/eam_branches/ipp-20110906/psvideophot/src/psvideophotParseCamera.c =================================================================== --- /branches/eam_branches/ipp-20110906/psvideophot/src/psvideophotParseCamera.c (revision 32629) +++ /branches/eam_branches/ipp-20110906/psvideophot/src/psvideophotParseCamera.c (revision 32630) @@ -7,6 +7,4 @@ psvideophotOptions *psvideophotParseCamera(pmConfig *config) { - bool status = false; - if (!psvideophotDefineFile(config, NULL, "PSVIDEOPHOT.INPUT", "INPUT", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_NONE)) { psError(PS_ERR_IO, false, "Can't find an input image source"); @@ -16,155 +14,13 @@ psAssert(input, "We just put it there!"); +#ifdef notyet // add recipe options supplied on command line + bool status = false; psMetadata *recipe = psMetadataLookupPtr(&status, config->recipes, RECIPE_NAME); +#endif // parse the options from the metadata format to the psvideophotOptions structure psvideophotOptions *options = psvideophotOptionsParse(config); - // the following are defined from the argument list, if given, - // otherwise they revert to the config information - // not all input or output images are used in a given recipe - - int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); - if (nThreads > 0) { - int nScanRows = psMetadataLookupS32(&status, recipe, "SCAN.ROWS"); - pmDetrendSetThreadTasks(nScanRows); - } - - // the following files are output targets - pmFPAfile *outImage = pmFPAfileDefineOutput(config, input->fpa, "PSVIDEOPHOT.OUTPUT"); - if (!outImage) { - psError(PS_ERR_IO, false, _("Unable to generate output file from PSVIDEOPHOT.OUTPUT")); - psFree(options); - return NULL; - } - if (outImage->type != PM_FPA_FILE_OBJ) { - psError(PS_ERR_IO, true, "PSVIDEOPHOT.OUTPUT is not of type OBJ"); - psFree(options); - return NULL; - } - - // XXX should these be bound explicitly to the outImage->fpa rather than the input->fpa? - pmFPAfile *chipImage = pmFPAfileDefineChipMosaic(config, input->fpa, "PSVIDEOPHOT.CHIP"); - if (!chipImage) { - psError(PS_ERR_IO, false, _("Unable to generate new file from PSVIDEOPHOT.CHIP")); - psFree(options); - return NULL; - } - if (chipImage->type != PM_FPA_FILE_OBJ) { - psError(PS_ERR_IO, true, "PSVIDEOPHOT.CHIP is not of type OBJ"); - psFree(options); - return NULL; - } - - - // chipImage -> psphotInput (pmFPAfileDefineFromFile) : fpa is constructed - // psphotInput -> psphotOutput (pmFPAfileDefineOutputFromFile) : fpa is equated - // psphotOutput -> psastroInput (pmFPAfileDefineInput) : fpa is ref-copied - // psastroInput -> psastroModel (pmFPAfileDefineFromArgs : fpa is ref-copied - // psastroInput -> psastroModel (pmFPAfileDefineFromConf : fpa is constructed - // psastroInput -> psastroModel (pmFPAfileDefineFromDetDB : fpa is constructed (pmDetrendSelect uses input concepts ) - - // For photometry, we operate on the chip-mosaicked image - // we create a copy of the mosaicked image for psphot so we can write out a clean image - - - // this file is just used as a carrier; output files (eg, PSPHOT.RESID) are defined by - // psphotDefineFiles - pmFPAfile *psphotInput = pmFPAfileDefineFromFile (config, chipImage, 1, 1, "PSPHOT.INPUT"); - PS_ASSERT (psphotInput, false); - - // specify the number of psphot input images - psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.NUM", PS_META_REPLACE, "number of inputs", 1); - - // define associated psphot input/output files - if (!psphotDefineFiles (config, psphotInput)) { - psError(PSPHOT_ERR_CONFIG, false, - "Trouble defining the additional input/output files for psphot"); - return false; - } - - - // For photometry, we operate on the chip-mosaicked image - - - // save any of these files? - outImage->save = options->BaseFITS; - - - chipImage->save = options->ChipFITS; - - - - // outImage is used as a carrier: input to chipImage -> require the data to remain at the CHIP level - outImage->freeLevel = PS_MIN(outImage->freeLevel, PM_FPA_LEVEL_CHIP); - outImage->dataLevel = outImage->freeLevel; - outImage->fileLevel = PS_MIN(outImage->fileLevel, outImage->dataLevel); - - // outMask and outVariance must be freed at the same level as outImage (all freed by pmFPAFreeData) - - - // the input data is the same as the outImage data : force the free levels to match - input->freeLevel = PS_MIN(outImage->freeLevel, input->freeLevel); - - // define the binned target files (which may just be carriers for some camera configurations) - - // Chip selection: turn on only the chips specified (pass status to suppress missing-key log msg) - char *chipLine = psMetadataLookupStr(&status, config->arguments, "CHIP_SELECTIONS"); - psArray *chips = psStringSplitArray(chipLine, ",", false); - if (chips->n > 0) { - pmFPASelectChip (input->fpa, -1, true); // deselect all chips - for (int i = 0; i < chips->n; i++) { - int chipNum = atoi(chips->data[i]); - if (! pmFPASelectChip(input->fpa, chipNum, false)) { - psError(PS_ERR_IO, false, "Chip number %d doesn't exist in camera.\n", chipNum); - psFree(options); - return false; - } - } - } - psFree (chips); - - if (psMetadataLookupBool(NULL, config->arguments, "INPUT_IS_FRINGE")) { - // It's a fringe file, so change the file type - input->type = PM_FPA_FILE_FRINGE; - outImage->type = PM_FPA_FILE_FRINGE; - } - if (psMetadataLookupBool(NULL, config->arguments, "INPUT_IS_DARK")) { - // It's a dark file, so change the file type - input->type = PM_FPA_FILE_DARK; - outImage->type = PM_FPA_FILE_DARK; - // Turn off compression --- there are just too many nasties that can happen - psFree(outImage->compression); - outImage->compression = NULL; - psFree(outImage->options); - outImage->options = NULL; - } - - // Turn off mask and variance output if we're not doing anything interesting - - if (psTraceGetLevel("psvideophot.config") > 0) { - // Get a look inside all the files. - psMetadataIterator *filesIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD, NULL); - psMetadataItem *item; // Item from iteration - fprintf(stderr, "Files:\n"); - while ((item = psMetadataGetAndIncrement(filesIter))) { - pmFPAfile *file = item->data.V; // File of interest - fprintf(stderr, "%s: %p %p %p (%p) %p\n", file->name, - file->src, file->fpa, - file->camera, file->fpa->camera, file->format); - } - psFree(filesIter); - } - - // Change the input dark type between the old (IMAGE) and new (multi-DARK). - // Hopefully this is a temporary change until we all move over to using the new dark types - bool mdok; // Status of MD lookup - if (options->doDark && psMetadataLookupBool(&mdok, recipe, "OLDDARK")) { - pmFPAfile *dark = psMetadataLookupPtr(NULL, config->files, "PSVIDEOPHOT.DARK"); - assert(dark); - dark->type = PM_FPA_FILE_IMAGE; - } - return (options); } Index: /branches/eam_branches/ipp-20110906/tools/LAP_regiontool.pl =================================================================== --- /branches/eam_branches/ipp-20110906/tools/LAP_regiontool.pl (revision 32630) +++ /branches/eam_branches/ipp-20110906/tools/LAP_regiontool.pl (revision 32630) @@ -0,0 +1,2754 @@ +#!/usr/bin/env perl + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +use Pod::Usage qw( pod2usage ); + + +my $seq_id = 8; +my $label = 'LAP.ThreePi.20110809'; +my $dist_group = 'LAP.ThreePi'; +my $tess_id = 'RINGS.V3'; +my $radius = 5; +my $dbname = 'gpc1'; +my @filters = ('g.00000','r.00000','i.00000','z.00000','y.00000'); +my $degrees = 0; + +my ($ra_min,$ra_max,$dec_min,$dec_max); + +GetOptions( + 'dbname=s' => \$dbname, + 'label|s=s' => \$label, + 'dist_group=s' => \$dist_group, + 'tess_id=s' => \$tess_id, + 'radius=s' => \$radius, + 'seq_id=s' => \$seq_id, + 'ra_min|r=s' => \$ra_min, + 'ra_max|R=s' => \$ra_max, + 'dec_min|d=s' => \$dec_min, + 'dec_max|D=s' => \$dec_max, + 'degrees' => \$degrees, + ) || pod2usage( 2 ); + +die("Unknown option: @ARGV") if @ARGV; + +unless (defined($ra_min)&& + defined($ra_max)&& + defined($dec_min)&& + defined($dec_max)) { + die("Region not defined"); +} + +# Convert to degrees and add overlap boundaries. + +if ($degrees) { + $ra_min = ($ra_min) - 1; + $ra_max = ($ra_max) + 1; +} +else { + $ra_min = (15 * $ra_min) - 1; + $ra_max = (15 * $ra_max) + 1; +} +$dec_min = $dec_min - 1; +$dec_max = $dec_max + 1; + + + +# Create a list of projection cells to be included in this region +my @included = (); +my %iRA = (); +my %iDEC = (); +while() { + chomp; + my ($ra,$dec,$proj) = split /\s+/; + + if (($ra > $ra_min)&&($ra < $ra_max)&& + ($dec > $dec_min)&&($dec < $dec_max)) { + push @included, $proj; + $iRA{$proj} = $ra; + $iDEC{$proj} = $dec; + } +} + + +my $command_base = "laptool -definerun -dbname $dbname -seq_id $seq_id -label $label -dist_group $dist_group -tess_id $tess_id -radius $radius"; + + +while ($#included > -1) { + my @blocks = (); + my %Ndec = (); + my $proj; + + while (scalar(keys %Ndec) < 4) { + $proj = shift(@included); + $Ndec{$iDEC{$proj}} = 1; + push @blocks, $proj; + if ($#included == -1) { + last; + } + } + + if ($#included > -1) { + unshift @included, $proj; + pop @blocks; + } + + foreach my $filter (@filters) { + foreach my $proj (@blocks) { + print "$command_base -filter $filter -projection_cell $proj -ra $iRA{$proj} -decl $iDEC{$proj}\n"; + } + } + my $last_filter = pop(@filters); + unshift(@filters,$last_filter); +# print @blocks ; +} + + + +__DATA__ +0 -90 skycell.0000 +0 -86 skycell.0001 +40 -86 skycell.0002 +80 -86 skycell.0003 +120 -86 skycell.0004 +160 -86 skycell.0005 +200 -86 skycell.0006 +240 -86 skycell.0007 +280 -86 skycell.0008 +320 -86 skycell.0009 +0 -82 skycell.0010 +24 -82 skycell.0011 +48 -82 skycell.0012 +72 -82 skycell.0013 +96 -82 skycell.0014 +120 -82 skycell.0015 +144 -82 skycell.0016 +168 -82 skycell.0017 +192 -82 skycell.0018 +216 -82 skycell.0019 +240 -82 skycell.0020 +264 -82 skycell.0021 +288 -82 skycell.0022 +312 -82 skycell.0023 +336 -82 skycell.0024 +0 -78 skycell.0025 +17.1429 -78 skycell.0026 +34.2857 -78 skycell.0027 +51.4286 -78 skycell.0028 +68.5714 -78 skycell.0029 +85.7143 -78 skycell.0030 +102.857 -78 skycell.0031 +120 -78 skycell.0032 +137.143 -78 skycell.0033 +154.286 -78 skycell.0034 +171.429 -78 skycell.0035 +188.571 -78 skycell.0036 +205.714 -78 skycell.0037 +222.857 -78 skycell.0038 +240 -78 skycell.0039 +257.143 -78 skycell.0040 +274.286 -78 skycell.0041 +291.429 -78 skycell.0042 +308.571 -78 skycell.0043 +325.714 -78 skycell.0044 +342.857 -78 skycell.0045 +0 -74 skycell.0046 +13.3333 -74 skycell.0047 +26.6667 -74 skycell.0048 +40 -74 skycell.0049 +53.3333 -74 skycell.0050 +66.6667 -74 skycell.0051 +80 -74 skycell.0052 +93.3333 -74 skycell.0053 +106.667 -74 skycell.0054 +120 -74 skycell.0055 +133.333 -74 skycell.0056 +146.667 -74 skycell.0057 +160 -74 skycell.0058 +173.333 -74 skycell.0059 +186.667 -74 skycell.0060 +200 -74 skycell.0061 +213.333 -74 skycell.0062 +226.667 -74 skycell.0063 +240 -74 skycell.0064 +253.333 -74 skycell.0065 +266.667 -74 skycell.0066 +280 -74 skycell.0067 +293.333 -74 skycell.0068 +306.667 -74 skycell.0069 +320 -74 skycell.0070 +333.333 -74 skycell.0071 +346.667 -74 skycell.0072 +0 -70 skycell.0073 +10.9091 -70 skycell.0074 +21.8182 -70 skycell.0075 +32.7273 -70 skycell.0076 +43.6364 -70 skycell.0077 +54.5455 -70 skycell.0078 +65.4545 -70 skycell.0079 +76.3636 -70 skycell.0080 +87.2727 -70 skycell.0081 +98.1818 -70 skycell.0082 +109.091 -70 skycell.0083 +120 -70 skycell.0084 +130.909 -70 skycell.0085 +141.818 -70 skycell.0086 +152.727 -70 skycell.0087 +163.636 -70 skycell.0088 +174.545 -70 skycell.0089 +185.455 -70 skycell.0090 +196.364 -70 skycell.0091 +207.273 -70 skycell.0092 +218.182 -70 skycell.0093 +229.091 -70 skycell.0094 +240 -70 skycell.0095 +250.909 -70 skycell.0096 +261.818 -70 skycell.0097 +272.727 -70 skycell.0098 +283.636 -70 skycell.0099 +294.545 -70 skycell.0100 +305.455 -70 skycell.0101 +316.364 -70 skycell.0102 +327.273 -70 skycell.0103 +338.182 -70 skycell.0104 +349.091 -70 skycell.0105 +0 -66 skycell.0106 +9.23077 -66 skycell.0107 +18.4615 -66 skycell.0108 +27.6923 -66 skycell.0109 +36.9231 -66 skycell.0110 +46.1538 -66 skycell.0111 +55.3846 -66 skycell.0112 +64.6154 -66 skycell.0113 +73.8462 -66 skycell.0114 +83.0769 -66 skycell.0115 +92.3077 -66 skycell.0116 +101.538 -66 skycell.0117 +110.769 -66 skycell.0118 +120 -66 skycell.0119 +129.231 -66 skycell.0120 +138.462 -66 skycell.0121 +147.692 -66 skycell.0122 +156.923 -66 skycell.0123 +166.154 -66 skycell.0124 +175.385 -66 skycell.0125 +184.615 -66 skycell.0126 +193.846 -66 skycell.0127 +203.077 -66 skycell.0128 +212.308 -66 skycell.0129 +221.538 -66 skycell.0130 +230.769 -66 skycell.0131 +240 -66 skycell.0132 +249.231 -66 skycell.0133 +258.462 -66 skycell.0134 +267.692 -66 skycell.0135 +276.923 -66 skycell.0136 +286.154 -66 skycell.0137 +295.385 -66 skycell.0138 +304.615 -66 skycell.0139 +313.846 -66 skycell.0140 +323.077 -66 skycell.0141 +332.308 -66 skycell.0142 +341.538 -66 skycell.0143 +350.769 -66 skycell.0144 +0 -62 skycell.0145 +8 -62 skycell.0146 +16 -62 skycell.0147 +24 -62 skycell.0148 +32 -62 skycell.0149 +40 -62 skycell.0150 +48 -62 skycell.0151 +56 -62 skycell.0152 +64 -62 skycell.0153 +72 -62 skycell.0154 +80 -62 skycell.0155 +88 -62 skycell.0156 +96 -62 skycell.0157 +104 -62 skycell.0158 +112 -62 skycell.0159 +120 -62 skycell.0160 +128 -62 skycell.0161 +136 -62 skycell.0162 +144 -62 skycell.0163 +152 -62 skycell.0164 +160 -62 skycell.0165 +168 -62 skycell.0166 +176 -62 skycell.0167 +184 -62 skycell.0168 +192 -62 skycell.0169 +200 -62 skycell.0170 +208 -62 skycell.0171 +216 -62 skycell.0172 +224 -62 skycell.0173 +232 -62 skycell.0174 +240 -62 skycell.0175 +248 -62 skycell.0176 +256 -62 skycell.0177 +264 -62 skycell.0178 +272 -62 skycell.0179 +280 -62 skycell.0180 +288 -62 skycell.0181 +296 -62 skycell.0182 +304 -62 skycell.0183 +312 -62 skycell.0184 +320 -62 skycell.0185 +328 -62 skycell.0186 +336 -62 skycell.0187 +344 -62 skycell.0188 +352 -62 skycell.0189 +0 -58 skycell.0190 +7.2 -58 skycell.0191 +14.4 -58 skycell.0192 +21.6 -58 skycell.0193 +28.8 -58 skycell.0194 +36 -58 skycell.0195 +43.2 -58 skycell.0196 +50.4 -58 skycell.0197 +57.6 -58 skycell.0198 +64.8 -58 skycell.0199 +72 -58 skycell.0200 +79.2 -58 skycell.0201 +86.4 -58 skycell.0202 +93.6 -58 skycell.0203 +100.8 -58 skycell.0204 +108 -58 skycell.0205 +115.2 -58 skycell.0206 +122.4 -58 skycell.0207 +129.6 -58 skycell.0208 +136.8 -58 skycell.0209 +144 -58 skycell.0210 +151.2 -58 skycell.0211 +158.4 -58 skycell.0212 +165.6 -58 skycell.0213 +172.8 -58 skycell.0214 +180 -58 skycell.0215 +187.2 -58 skycell.0216 +194.4 -58 skycell.0217 +201.6 -58 skycell.0218 +208.8 -58 skycell.0219 +216 -58 skycell.0220 +223.2 -58 skycell.0221 +230.4 -58 skycell.0222 +237.6 -58 skycell.0223 +244.8 -58 skycell.0224 +252 -58 skycell.0225 +259.2 -58 skycell.0226 +266.4 -58 skycell.0227 +273.6 -58 skycell.0228 +280.8 -58 skycell.0229 +288 -58 skycell.0230 +295.2 -58 skycell.0231 +302.4 -58 skycell.0232 +309.6 -58 skycell.0233 +316.8 -58 skycell.0234 +324 -58 skycell.0235 +331.2 -58 skycell.0236 +338.4 -58 skycell.0237 +345.6 -58 skycell.0238 +352.8 -58 skycell.0239 +0 -54 skycell.0240 +6.54545 -54 skycell.0241 +13.0909 -54 skycell.0242 +19.6364 -54 skycell.0243 +26.1818 -54 skycell.0244 +32.7273 -54 skycell.0245 +39.2727 -54 skycell.0246 +45.8182 -54 skycell.0247 +52.3636 -54 skycell.0248 +58.9091 -54 skycell.0249 +65.4545 -54 skycell.0250 +72 -54 skycell.0251 +78.5455 -54 skycell.0252 +85.0909 -54 skycell.0253 +91.6364 -54 skycell.0254 +98.1818 -54 skycell.0255 +104.727 -54 skycell.0256 +111.273 -54 skycell.0257 +117.818 -54 skycell.0258 +124.364 -54 skycell.0259 +130.909 -54 skycell.0260 +137.455 -54 skycell.0261 +144 -54 skycell.0262 +150.545 -54 skycell.0263 +157.091 -54 skycell.0264 +163.636 -54 skycell.0265 +170.182 -54 skycell.0266 +176.727 -54 skycell.0267 +183.273 -54 skycell.0268 +189.818 -54 skycell.0269 +196.364 -54 skycell.0270 +202.909 -54 skycell.0271 +209.455 -54 skycell.0272 +216 -54 skycell.0273 +222.545 -54 skycell.0274 +229.091 -54 skycell.0275 +235.636 -54 skycell.0276 +242.182 -54 skycell.0277 +248.727 -54 skycell.0278 +255.273 -54 skycell.0279 +261.818 -54 skycell.0280 +268.364 -54 skycell.0281 +274.909 -54 skycell.0282 +281.455 -54 skycell.0283 +288 -54 skycell.0284 +294.545 -54 skycell.0285 +301.091 -54 skycell.0286 +307.636 -54 skycell.0287 +314.182 -54 skycell.0288 +320.727 -54 skycell.0289 +327.273 -54 skycell.0290 +333.818 -54 skycell.0291 +340.364 -54 skycell.0292 +346.909 -54 skycell.0293 +353.455 -54 skycell.0294 +0 -50 skycell.0295 +6 -50 skycell.0296 +12 -50 skycell.0297 +18 -50 skycell.0298 +24 -50 skycell.0299 +30 -50 skycell.0300 +36 -50 skycell.0301 +42 -50 skycell.0302 +48 -50 skycell.0303 +54 -50 skycell.0304 +60 -50 skycell.0305 +66 -50 skycell.0306 +72 -50 skycell.0307 +78 -50 skycell.0308 +84 -50 skycell.0309 +90 -50 skycell.0310 +96 -50 skycell.0311 +102 -50 skycell.0312 +108 -50 skycell.0313 +114 -50 skycell.0314 +120 -50 skycell.0315 +126 -50 skycell.0316 +132 -50 skycell.0317 +138 -50 skycell.0318 +144 -50 skycell.0319 +150 -50 skycell.0320 +156 -50 skycell.0321 +162 -50 skycell.0322 +168 -50 skycell.0323 +174 -50 skycell.0324 +180 -50 skycell.0325 +186 -50 skycell.0326 +192 -50 skycell.0327 +198 -50 skycell.0328 +204 -50 skycell.0329 +210 -50 skycell.0330 +216 -50 skycell.0331 +222 -50 skycell.0332 +228 -50 skycell.0333 +234 -50 skycell.0334 +240 -50 skycell.0335 +246 -50 skycell.0336 +252 -50 skycell.0337 +258 -50 skycell.0338 +264 -50 skycell.0339 +270 -50 skycell.0340 +276 -50 skycell.0341 +282 -50 skycell.0342 +288 -50 skycell.0343 +294 -50 skycell.0344 +300 -50 skycell.0345 +306 -50 skycell.0346 +312 -50 skycell.0347 +318 -50 skycell.0348 +324 -50 skycell.0349 +330 -50 skycell.0350 +336 -50 skycell.0351 +342 -50 skycell.0352 +348 -50 skycell.0353 +354 -50 skycell.0354 +0 -46 skycell.0355 +5.625 -46 skycell.0356 +11.25 -46 skycell.0357 +16.875 -46 skycell.0358 +22.5 -46 skycell.0359 +28.125 -46 skycell.0360 +33.75 -46 skycell.0361 +39.375 -46 skycell.0362 +45 -46 skycell.0363 +50.625 -46 skycell.0364 +56.25 -46 skycell.0365 +61.875 -46 skycell.0366 +67.5 -46 skycell.0367 +73.125 -46 skycell.0368 +78.75 -46 skycell.0369 +84.375 -46 skycell.0370 +90 -46 skycell.0371 +95.625 -46 skycell.0372 +101.25 -46 skycell.0373 +106.875 -46 skycell.0374 +112.5 -46 skycell.0375 +118.125 -46 skycell.0376 +123.75 -46 skycell.0377 +129.375 -46 skycell.0378 +135 -46 skycell.0379 +140.625 -46 skycell.0380 +146.25 -46 skycell.0381 +151.875 -46 skycell.0382 +157.5 -46 skycell.0383 +163.125 -46 skycell.0384 +168.75 -46 skycell.0385 +174.375 -46 skycell.0386 +180 -46 skycell.0387 +185.625 -46 skycell.0388 +191.25 -46 skycell.0389 +196.875 -46 skycell.0390 +202.5 -46 skycell.0391 +208.125 -46 skycell.0392 +213.75 -46 skycell.0393 +219.375 -46 skycell.0394 +225 -46 skycell.0395 +230.625 -46 skycell.0396 +236.25 -46 skycell.0397 +241.875 -46 skycell.0398 +247.5 -46 skycell.0399 +253.125 -46 skycell.0400 +258.75 -46 skycell.0401 +264.375 -46 skycell.0402 +270 -46 skycell.0403 +275.625 -46 skycell.0404 +281.25 -46 skycell.0405 +286.875 -46 skycell.0406 +292.5 -46 skycell.0407 +298.125 -46 skycell.0408 +303.75 -46 skycell.0409 +309.375 -46 skycell.0410 +315 -46 skycell.0411 +320.625 -46 skycell.0412 +326.25 -46 skycell.0413 +331.875 -46 skycell.0414 +337.5 -46 skycell.0415 +343.125 -46 skycell.0416 +348.75 -46 skycell.0417 +354.375 -46 skycell.0418 +0 -42 skycell.0419 +5.29412 -42 skycell.0420 +10.5882 -42 skycell.0421 +15.8824 -42 skycell.0422 +21.1765 -42 skycell.0423 +26.4706 -42 skycell.0424 +31.7647 -42 skycell.0425 +37.0588 -42 skycell.0426 +42.3529 -42 skycell.0427 +47.6471 -42 skycell.0428 +52.9412 -42 skycell.0429 +58.2353 -42 skycell.0430 +63.5294 -42 skycell.0431 +68.8235 -42 skycell.0432 +74.1176 -42 skycell.0433 +79.4118 -42 skycell.0434 +84.7059 -42 skycell.0435 +90 -42 skycell.0436 +95.2941 -42 skycell.0437 +100.588 -42 skycell.0438 +105.882 -42 skycell.0439 +111.176 -42 skycell.0440 +116.471 -42 skycell.0441 +121.765 -42 skycell.0442 +127.059 -42 skycell.0443 +132.353 -42 skycell.0444 +137.647 -42 skycell.0445 +142.941 -42 skycell.0446 +148.235 -42 skycell.0447 +153.529 -42 skycell.0448 +158.824 -42 skycell.0449 +164.118 -42 skycell.0450 +169.412 -42 skycell.0451 +174.706 -42 skycell.0452 +180 -42 skycell.0453 +185.294 -42 skycell.0454 +190.588 -42 skycell.0455 +195.882 -42 skycell.0456 +201.176 -42 skycell.0457 +206.471 -42 skycell.0458 +211.765 -42 skycell.0459 +217.059 -42 skycell.0460 +222.353 -42 skycell.0461 +227.647 -42 skycell.0462 +232.941 -42 skycell.0463 +238.235 -42 skycell.0464 +243.529 -42 skycell.0465 +248.824 -42 skycell.0466 +254.118 -42 skycell.0467 +259.412 -42 skycell.0468 +264.706 -42 skycell.0469 +270 -42 skycell.0470 +275.294 -42 skycell.0471 +280.588 -42 skycell.0472 +285.882 -42 skycell.0473 +291.176 -42 skycell.0474 +296.471 -42 skycell.0475 +301.765 -42 skycell.0476 +307.059 -42 skycell.0477 +312.353 -42 skycell.0478 +317.647 -42 skycell.0479 +322.941 -42 skycell.0480 +328.235 -42 skycell.0481 +333.529 -42 skycell.0482 +338.824 -42 skycell.0483 +344.118 -42 skycell.0484 +349.412 -42 skycell.0485 +354.706 -42 skycell.0486 +0 -38 skycell.0487 +5 -38 skycell.0488 +10 -38 skycell.0489 +15 -38 skycell.0490 +20 -38 skycell.0491 +25 -38 skycell.0492 +30 -38 skycell.0493 +35 -38 skycell.0494 +40 -38 skycell.0495 +45 -38 skycell.0496 +50 -38 skycell.0497 +55 -38 skycell.0498 +60 -38 skycell.0499 +65 -38 skycell.0500 +70 -38 skycell.0501 +75 -38 skycell.0502 +80 -38 skycell.0503 +85 -38 skycell.0504 +90 -38 skycell.0505 +95 -38 skycell.0506 +100 -38 skycell.0507 +105 -38 skycell.0508 +110 -38 skycell.0509 +115 -38 skycell.0510 +120 -38 skycell.0511 +125 -38 skycell.0512 +130 -38 skycell.0513 +135 -38 skycell.0514 +140 -38 skycell.0515 +145 -38 skycell.0516 +150 -38 skycell.0517 +155 -38 skycell.0518 +160 -38 skycell.0519 +165 -38 skycell.0520 +170 -38 skycell.0521 +175 -38 skycell.0522 +180 -38 skycell.0523 +185 -38 skycell.0524 +190 -38 skycell.0525 +195 -38 skycell.0526 +200 -38 skycell.0527 +205 -38 skycell.0528 +210 -38 skycell.0529 +215 -38 skycell.0530 +220 -38 skycell.0531 +225 -38 skycell.0532 +230 -38 skycell.0533 +235 -38 skycell.0534 +240 -38 skycell.0535 +245 -38 skycell.0536 +250 -38 skycell.0537 +255 -38 skycell.0538 +260 -38 skycell.0539 +265 -38 skycell.0540 +270 -38 skycell.0541 +275 -38 skycell.0542 +280 -38 skycell.0543 +285 -38 skycell.0544 +290 -38 skycell.0545 +295 -38 skycell.0546 +300 -38 skycell.0547 +305 -38 skycell.0548 +310 -38 skycell.0549 +315 -38 skycell.0550 +320 -38 skycell.0551 +325 -38 skycell.0552 +330 -38 skycell.0553 +335 -38 skycell.0554 +340 -38 skycell.0555 +345 -38 skycell.0556 +350 -38 skycell.0557 +355 -38 skycell.0558 +0 -34 skycell.0559 +4.73684 -34 skycell.0560 +9.47368 -34 skycell.0561 +14.2105 -34 skycell.0562 +18.9474 -34 skycell.0563 +23.6842 -34 skycell.0564 +28.4211 -34 skycell.0565 +33.1579 -34 skycell.0566 +37.8947 -34 skycell.0567 +42.6316 -34 skycell.0568 +47.3684 -34 skycell.0569 +52.1053 -34 skycell.0570 +56.8421 -34 skycell.0571 +61.5789 -34 skycell.0572 +66.3158 -34 skycell.0573 +71.0526 -34 skycell.0574 +75.7895 -34 skycell.0575 +80.5263 -34 skycell.0576 +85.2632 -34 skycell.0577 +90 -34 skycell.0578 +94.7368 -34 skycell.0579 +99.4737 -34 skycell.0580 +104.211 -34 skycell.0581 +108.947 -34 skycell.0582 +113.684 -34 skycell.0583 +118.421 -34 skycell.0584 +123.158 -34 skycell.0585 +127.895 -34 skycell.0586 +132.632 -34 skycell.0587 +137.368 -34 skycell.0588 +142.105 -34 skycell.0589 +146.842 -34 skycell.0590 +151.579 -34 skycell.0591 +156.316 -34 skycell.0592 +161.053 -34 skycell.0593 +165.789 -34 skycell.0594 +170.526 -34 skycell.0595 +175.263 -34 skycell.0596 +180 -34 skycell.0597 +184.737 -34 skycell.0598 +189.474 -34 skycell.0599 +194.211 -34 skycell.0600 +198.947 -34 skycell.0601 +203.684 -34 skycell.0602 +208.421 -34 skycell.0603 +213.158 -34 skycell.0604 +217.895 -34 skycell.0605 +222.632 -34 skycell.0606 +227.368 -34 skycell.0607 +232.105 -34 skycell.0608 +236.842 -34 skycell.0609 +241.579 -34 skycell.0610 +246.316 -34 skycell.0611 +251.053 -34 skycell.0612 +255.789 -34 skycell.0613 +260.526 -34 skycell.0614 +265.263 -34 skycell.0615 +270 -34 skycell.0616 +274.737 -34 skycell.0617 +279.474 -34 skycell.0618 +284.211 -34 skycell.0619 +288.947 -34 skycell.0620 +293.684 -34 skycell.0621 +298.421 -34 skycell.0622 +303.158 -34 skycell.0623 +307.895 -34 skycell.0624 +312.632 -34 skycell.0625 +317.368 -34 skycell.0626 +322.105 -34 skycell.0627 +326.842 -34 skycell.0628 +331.579 -34 skycell.0629 +336.316 -34 skycell.0630 +341.053 -34 skycell.0631 +345.789 -34 skycell.0632 +350.526 -34 skycell.0633 +355.263 -34 skycell.0634 +0 -30 skycell.0635 +4.55696 -30 skycell.0636 +9.11392 -30 skycell.0637 +13.6709 -30 skycell.0638 +18.2278 -30 skycell.0639 +22.7848 -30 skycell.0640 +27.3418 -30 skycell.0641 +31.8987 -30 skycell.0642 +36.4557 -30 skycell.0643 +41.0127 -30 skycell.0644 +45.5696 -30 skycell.0645 +50.1266 -30 skycell.0646 +54.6835 -30 skycell.0647 +59.2405 -30 skycell.0648 +63.7975 -30 skycell.0649 +68.3544 -30 skycell.0650 +72.9114 -30 skycell.0651 +77.4684 -30 skycell.0652 +82.0253 -30 skycell.0653 +86.5823 -30 skycell.0654 +91.1392 -30 skycell.0655 +95.6962 -30 skycell.0656 +100.253 -30 skycell.0657 +104.81 -30 skycell.0658 +109.367 -30 skycell.0659 +113.924 -30 skycell.0660 +118.481 -30 skycell.0661 +123.038 -30 skycell.0662 +127.595 -30 skycell.0663 +132.152 -30 skycell.0664 +136.709 -30 skycell.0665 +141.266 -30 skycell.0666 +145.823 -30 skycell.0667 +150.38 -30 skycell.0668 +154.937 -30 skycell.0669 +159.494 -30 skycell.0670 +164.051 -30 skycell.0671 +168.608 -30 skycell.0672 +173.165 -30 skycell.0673 +177.722 -30 skycell.0674 +182.278 -30 skycell.0675 +186.835 -30 skycell.0676 +191.392 -30 skycell.0677 +195.949 -30 skycell.0678 +200.506 -30 skycell.0679 +205.063 -30 skycell.0680 +209.62 -30 skycell.0681 +214.177 -30 skycell.0682 +218.734 -30 skycell.0683 +223.291 -30 skycell.0684 +227.848 -30 skycell.0685 +232.405 -30 skycell.0686 +236.962 -30 skycell.0687 +241.519 -30 skycell.0688 +246.076 -30 skycell.0689 +250.633 -30 skycell.0690 +255.19 -30 skycell.0691 +259.747 -30 skycell.0692 +264.304 -30 skycell.0693 +268.861 -30 skycell.0694 +273.418 -30 skycell.0695 +277.975 -30 skycell.0696 +282.532 -30 skycell.0697 +287.089 -30 skycell.0698 +291.646 -30 skycell.0699 +296.203 -30 skycell.0700 +300.759 -30 skycell.0701 +305.316 -30 skycell.0702 +309.873 -30 skycell.0703 +314.43 -30 skycell.0704 +318.987 -30 skycell.0705 +323.544 -30 skycell.0706 +328.101 -30 skycell.0707 +332.658 -30 skycell.0708 +337.215 -30 skycell.0709 +341.772 -30 skycell.0710 +346.329 -30 skycell.0711 +350.886 -30 skycell.0712 +355.443 -30 skycell.0713 +0 -26 skycell.0714 +4.39024 -26 skycell.0715 +8.78049 -26 skycell.0716 +13.1707 -26 skycell.0717 +17.561 -26 skycell.0718 +21.9512 -26 skycell.0719 +26.3415 -26 skycell.0720 +30.7317 -26 skycell.0721 +35.122 -26 skycell.0722 +39.5122 -26 skycell.0723 +43.9024 -26 skycell.0724 +48.2927 -26 skycell.0725 +52.6829 -26 skycell.0726 +57.0732 -26 skycell.0727 +61.4634 -26 skycell.0728 +65.8537 -26 skycell.0729 +70.2439 -26 skycell.0730 +74.6341 -26 skycell.0731 +79.0244 -26 skycell.0732 +83.4146 -26 skycell.0733 +87.8049 -26 skycell.0734 +92.1951 -26 skycell.0735 +96.5854 -26 skycell.0736 +100.976 -26 skycell.0737 +105.366 -26 skycell.0738 +109.756 -26 skycell.0739 +114.146 -26 skycell.0740 +118.537 -26 skycell.0741 +122.927 -26 skycell.0742 +127.317 -26 skycell.0743 +131.707 -26 skycell.0744 +136.098 -26 skycell.0745 +140.488 -26 skycell.0746 +144.878 -26 skycell.0747 +149.268 -26 skycell.0748 +153.659 -26 skycell.0749 +158.049 -26 skycell.0750 +162.439 -26 skycell.0751 +166.829 -26 skycell.0752 +171.22 -26 skycell.0753 +175.61 -26 skycell.0754 +180 -26 skycell.0755 +184.39 -26 skycell.0756 +188.78 -26 skycell.0757 +193.171 -26 skycell.0758 +197.561 -26 skycell.0759 +201.951 -26 skycell.0760 +206.341 -26 skycell.0761 +210.732 -26 skycell.0762 +215.122 -26 skycell.0763 +219.512 -26 skycell.0764 +223.902 -26 skycell.0765 +228.293 -26 skycell.0766 +232.683 -26 skycell.0767 +237.073 -26 skycell.0768 +241.463 -26 skycell.0769 +245.854 -26 skycell.0770 +250.244 -26 skycell.0771 +254.634 -26 skycell.0772 +259.024 -26 skycell.0773 +263.415 -26 skycell.0774 +267.805 -26 skycell.0775 +272.195 -26 skycell.0776 +276.585 -26 skycell.0777 +280.976 -26 skycell.0778 +285.366 -26 skycell.0779 +289.756 -26 skycell.0780 +294.146 -26 skycell.0781 +298.537 -26 skycell.0782 +302.927 -26 skycell.0783 +307.317 -26 skycell.0784 +311.707 -26 skycell.0785 +316.098 -26 skycell.0786 +320.488 -26 skycell.0787 +324.878 -26 skycell.0788 +329.268 -26 skycell.0789 +333.659 -26 skycell.0790 +338.049 -26 skycell.0791 +342.439 -26 skycell.0792 +346.829 -26 skycell.0793 +351.22 -26 skycell.0794 +355.61 -26 skycell.0795 +0 -22 skycell.0796 +4.28571 -22 skycell.0797 +8.57143 -22 skycell.0798 +12.8571 -22 skycell.0799 +17.1429 -22 skycell.0800 +21.4286 -22 skycell.0801 +25.7143 -22 skycell.0802 +30 -22 skycell.0803 +34.2857 -22 skycell.0804 +38.5714 -22 skycell.0805 +42.8571 -22 skycell.0806 +47.1429 -22 skycell.0807 +51.4286 -22 skycell.0808 +55.7143 -22 skycell.0809 +60 -22 skycell.0810 +64.2857 -22 skycell.0811 +68.5714 -22 skycell.0812 +72.8571 -22 skycell.0813 +77.1429 -22 skycell.0814 +81.4286 -22 skycell.0815 +85.7143 -22 skycell.0816 +90 -22 skycell.0817 +94.2857 -22 skycell.0818 +98.5714 -22 skycell.0819 +102.857 -22 skycell.0820 +107.143 -22 skycell.0821 +111.429 -22 skycell.0822 +115.714 -22 skycell.0823 +120 -22 skycell.0824 +124.286 -22 skycell.0825 +128.571 -22 skycell.0826 +132.857 -22 skycell.0827 +137.143 -22 skycell.0828 +141.429 -22 skycell.0829 +145.714 -22 skycell.0830 +150 -22 skycell.0831 +154.286 -22 skycell.0832 +158.571 -22 skycell.0833 +162.857 -22 skycell.0834 +167.143 -22 skycell.0835 +171.429 -22 skycell.0836 +175.714 -22 skycell.0837 +180 -22 skycell.0838 +184.286 -22 skycell.0839 +188.571 -22 skycell.0840 +192.857 -22 skycell.0841 +197.143 -22 skycell.0842 +201.429 -22 skycell.0843 +205.714 -22 skycell.0844 +210 -22 skycell.0845 +214.286 -22 skycell.0846 +218.571 -22 skycell.0847 +222.857 -22 skycell.0848 +227.143 -22 skycell.0849 +231.429 -22 skycell.0850 +235.714 -22 skycell.0851 +240 -22 skycell.0852 +244.286 -22 skycell.0853 +248.571 -22 skycell.0854 +252.857 -22 skycell.0855 +257.143 -22 skycell.0856 +261.429 -22 skycell.0857 +265.714 -22 skycell.0858 +270 -22 skycell.0859 +274.286 -22 skycell.0860 +278.571 -22 skycell.0861 +282.857 -22 skycell.0862 +287.143 -22 skycell.0863 +291.429 -22 skycell.0864 +295.714 -22 skycell.0865 +300 -22 skycell.0866 +304.286 -22 skycell.0867 +308.571 -22 skycell.0868 +312.857 -22 skycell.0869 +317.143 -22 skycell.0870 +321.429 -22 skycell.0871 +325.714 -22 skycell.0872 +330 -22 skycell.0873 +334.286 -22 skycell.0874 +338.571 -22 skycell.0875 +342.857 -22 skycell.0876 +347.143 -22 skycell.0877 +351.429 -22 skycell.0878 +355.714 -22 skycell.0879 +0 -18 skycell.0880 +4.18605 -18 skycell.0881 +8.37209 -18 skycell.0882 +12.5581 -18 skycell.0883 +16.7442 -18 skycell.0884 +20.9302 -18 skycell.0885 +25.1163 -18 skycell.0886 +29.3023 -18 skycell.0887 +33.4884 -18 skycell.0888 +37.6744 -18 skycell.0889 +41.8605 -18 skycell.0890 +46.0465 -18 skycell.0891 +50.2326 -18 skycell.0892 +54.4186 -18 skycell.0893 +58.6047 -18 skycell.0894 +62.7907 -18 skycell.0895 +66.9767 -18 skycell.0896 +71.1628 -18 skycell.0897 +75.3488 -18 skycell.0898 +79.5349 -18 skycell.0899 +83.7209 -18 skycell.0900 +87.907 -18 skycell.0901 +92.093 -18 skycell.0902 +96.2791 -18 skycell.0903 +100.465 -18 skycell.0904 +104.651 -18 skycell.0905 +108.837 -18 skycell.0906 +113.023 -18 skycell.0907 +117.209 -18 skycell.0908 +121.395 -18 skycell.0909 +125.581 -18 skycell.0910 +129.767 -18 skycell.0911 +133.953 -18 skycell.0912 +138.14 -18 skycell.0913 +142.326 -18 skycell.0914 +146.512 -18 skycell.0915 +150.698 -18 skycell.0916 +154.884 -18 skycell.0917 +159.07 -18 skycell.0918 +163.256 -18 skycell.0919 +167.442 -18 skycell.0920 +171.628 -18 skycell.0921 +175.814 -18 skycell.0922 +180 -18 skycell.0923 +184.186 -18 skycell.0924 +188.372 -18 skycell.0925 +192.558 -18 skycell.0926 +196.744 -18 skycell.0927 +200.93 -18 skycell.0928 +205.116 -18 skycell.0929 +209.302 -18 skycell.0930 +213.488 -18 skycell.0931 +217.674 -18 skycell.0932 +221.86 -18 skycell.0933 +226.047 -18 skycell.0934 +230.233 -18 skycell.0935 +234.419 -18 skycell.0936 +238.605 -18 skycell.0937 +242.791 -18 skycell.0938 +246.977 -18 skycell.0939 +251.163 -18 skycell.0940 +255.349 -18 skycell.0941 +259.535 -18 skycell.0942 +263.721 -18 skycell.0943 +267.907 -18 skycell.0944 +272.093 -18 skycell.0945 +276.279 -18 skycell.0946 +280.465 -18 skycell.0947 +284.651 -18 skycell.0948 +288.837 -18 skycell.0949 +293.023 -18 skycell.0950 +297.209 -18 skycell.0951 +301.395 -18 skycell.0952 +305.581 -18 skycell.0953 +309.767 -18 skycell.0954 +313.953 -18 skycell.0955 +318.14 -18 skycell.0956 +322.326 -18 skycell.0957 +326.512 -18 skycell.0958 +330.698 -18 skycell.0959 +334.884 -18 skycell.0960 +339.07 -18 skycell.0961 +343.256 -18 skycell.0962 +347.442 -18 skycell.0963 +351.628 -18 skycell.0964 +355.814 -18 skycell.0965 +0 -14 skycell.0966 +4.09091 -14 skycell.0967 +8.18182 -14 skycell.0968 +12.2727 -14 skycell.0969 +16.3636 -14 skycell.0970 +20.4545 -14 skycell.0971 +24.5455 -14 skycell.0972 +28.6364 -14 skycell.0973 +32.7273 -14 skycell.0974 +36.8182 -14 skycell.0975 +40.9091 -14 skycell.0976 +45 -14 skycell.0977 +49.0909 -14 skycell.0978 +53.1818 -14 skycell.0979 +57.2727 -14 skycell.0980 +61.3636 -14 skycell.0981 +65.4545 -14 skycell.0982 +69.5455 -14 skycell.0983 +73.6364 -14 skycell.0984 +77.7273 -14 skycell.0985 +81.8182 -14 skycell.0986 +85.9091 -14 skycell.0987 +90 -14 skycell.0988 +94.0909 -14 skycell.0989 +98.1818 -14 skycell.0990 +102.273 -14 skycell.0991 +106.364 -14 skycell.0992 +110.455 -14 skycell.0993 +114.545 -14 skycell.0994 +118.636 -14 skycell.0995 +122.727 -14 skycell.0996 +126.818 -14 skycell.0997 +130.909 -14 skycell.0998 +135 -14 skycell.0999 +139.091 -14 skycell.1000 +143.182 -14 skycell.1001 +147.273 -14 skycell.1002 +151.364 -14 skycell.1003 +155.455 -14 skycell.1004 +159.545 -14 skycell.1005 +163.636 -14 skycell.1006 +167.727 -14 skycell.1007 +171.818 -14 skycell.1008 +175.909 -14 skycell.1009 +180 -14 skycell.1010 +184.091 -14 skycell.1011 +188.182 -14 skycell.1012 +192.273 -14 skycell.1013 +196.364 -14 skycell.1014 +200.455 -14 skycell.1015 +204.545 -14 skycell.1016 +208.636 -14 skycell.1017 +212.727 -14 skycell.1018 +216.818 -14 skycell.1019 +220.909 -14 skycell.1020 +225 -14 skycell.1021 +229.091 -14 skycell.1022 +233.182 -14 skycell.1023 +237.273 -14 skycell.1024 +241.364 -14 skycell.1025 +245.455 -14 skycell.1026 +249.545 -14 skycell.1027 +253.636 -14 skycell.1028 +257.727 -14 skycell.1029 +261.818 -14 skycell.1030 +265.909 -14 skycell.1031 +270 -14 skycell.1032 +274.091 -14 skycell.1033 +278.182 -14 skycell.1034 +282.273 -14 skycell.1035 +286.364 -14 skycell.1036 +290.455 -14 skycell.1037 +294.545 -14 skycell.1038 +298.636 -14 skycell.1039 +302.727 -14 skycell.1040 +306.818 -14 skycell.1041 +310.909 -14 skycell.1042 +315 -14 skycell.1043 +319.091 -14 skycell.1044 +323.182 -14 skycell.1045 +327.273 -14 skycell.1046 +331.364 -14 skycell.1047 +335.455 -14 skycell.1048 +339.545 -14 skycell.1049 +343.636 -14 skycell.1050 +347.727 -14 skycell.1051 +351.818 -14 skycell.1052 +355.909 -14 skycell.1053 +0 -10 skycell.1054 +4.04494 -10 skycell.1055 +8.08989 -10 skycell.1056 +12.1348 -10 skycell.1057 +16.1798 -10 skycell.1058 +20.2247 -10 skycell.1059 +24.2697 -10 skycell.1060 +28.3146 -10 skycell.1061 +32.3596 -10 skycell.1062 +36.4045 -10 skycell.1063 +40.4494 -10 skycell.1064 +44.4944 -10 skycell.1065 +48.5393 -10 skycell.1066 +52.5843 -10 skycell.1067 +56.6292 -10 skycell.1068 +60.6742 -10 skycell.1069 +64.7191 -10 skycell.1070 +68.764 -10 skycell.1071 +72.809 -10 skycell.1072 +76.8539 -10 skycell.1073 +80.8989 -10 skycell.1074 +84.9438 -10 skycell.1075 +88.9888 -10 skycell.1076 +93.0337 -10 skycell.1077 +97.0787 -10 skycell.1078 +101.124 -10 skycell.1079 +105.169 -10 skycell.1080 +109.213 -10 skycell.1081 +113.258 -10 skycell.1082 +117.303 -10 skycell.1083 +121.348 -10 skycell.1084 +125.393 -10 skycell.1085 +129.438 -10 skycell.1086 +133.483 -10 skycell.1087 +137.528 -10 skycell.1088 +141.573 -10 skycell.1089 +145.618 -10 skycell.1090 +149.663 -10 skycell.1091 +153.708 -10 skycell.1092 +157.753 -10 skycell.1093 +161.798 -10 skycell.1094 +165.843 -10 skycell.1095 +169.888 -10 skycell.1096 +173.933 -10 skycell.1097 +177.978 -10 skycell.1098 +182.022 -10 skycell.1099 +186.067 -10 skycell.1100 +190.112 -10 skycell.1101 +194.157 -10 skycell.1102 +198.202 -10 skycell.1103 +202.247 -10 skycell.1104 +206.292 -10 skycell.1105 +210.337 -10 skycell.1106 +214.382 -10 skycell.1107 +218.427 -10 skycell.1108 +222.472 -10 skycell.1109 +226.517 -10 skycell.1110 +230.562 -10 skycell.1111 +234.607 -10 skycell.1112 +238.652 -10 skycell.1113 +242.697 -10 skycell.1114 +246.742 -10 skycell.1115 +250.787 -10 skycell.1116 +254.831 -10 skycell.1117 +258.876 -10 skycell.1118 +262.921 -10 skycell.1119 +266.966 -10 skycell.1120 +271.011 -10 skycell.1121 +275.056 -10 skycell.1122 +279.101 -10 skycell.1123 +283.146 -10 skycell.1124 +287.191 -10 skycell.1125 +291.236 -10 skycell.1126 +295.281 -10 skycell.1127 +299.326 -10 skycell.1128 +303.371 -10 skycell.1129 +307.416 -10 skycell.1130 +311.461 -10 skycell.1131 +315.506 -10 skycell.1132 +319.551 -10 skycell.1133 +323.596 -10 skycell.1134 +327.64 -10 skycell.1135 +331.685 -10 skycell.1136 +335.73 -10 skycell.1137 +339.775 -10 skycell.1138 +343.82 -10 skycell.1139 +347.865 -10 skycell.1140 +351.91 -10 skycell.1141 +355.955 -10 skycell.1142 +0 -6 skycell.1143 +4.04494 -6 skycell.1144 +8.08989 -6 skycell.1145 +12.1348 -6 skycell.1146 +16.1798 -6 skycell.1147 +20.2247 -6 skycell.1148 +24.2697 -6 skycell.1149 +28.3146 -6 skycell.1150 +32.3596 -6 skycell.1151 +36.4045 -6 skycell.1152 +40.4494 -6 skycell.1153 +44.4944 -6 skycell.1154 +48.5393 -6 skycell.1155 +52.5843 -6 skycell.1156 +56.6292 -6 skycell.1157 +60.6742 -6 skycell.1158 +64.7191 -6 skycell.1159 +68.764 -6 skycell.1160 +72.809 -6 skycell.1161 +76.8539 -6 skycell.1162 +80.8989 -6 skycell.1163 +84.9438 -6 skycell.1164 +88.9888 -6 skycell.1165 +93.0337 -6 skycell.1166 +97.0787 -6 skycell.1167 +101.124 -6 skycell.1168 +105.169 -6 skycell.1169 +109.213 -6 skycell.1170 +113.258 -6 skycell.1171 +117.303 -6 skycell.1172 +121.348 -6 skycell.1173 +125.393 -6 skycell.1174 +129.438 -6 skycell.1175 +133.483 -6 skycell.1176 +137.528 -6 skycell.1177 +141.573 -6 skycell.1178 +145.618 -6 skycell.1179 +149.663 -6 skycell.1180 +153.708 -6 skycell.1181 +157.753 -6 skycell.1182 +161.798 -6 skycell.1183 +165.843 -6 skycell.1184 +169.888 -6 skycell.1185 +173.933 -6 skycell.1186 +177.978 -6 skycell.1187 +182.022 -6 skycell.1188 +186.067 -6 skycell.1189 +190.112 -6 skycell.1190 +194.157 -6 skycell.1191 +198.202 -6 skycell.1192 +202.247 -6 skycell.1193 +206.292 -6 skycell.1194 +210.337 -6 skycell.1195 +214.382 -6 skycell.1196 +218.427 -6 skycell.1197 +222.472 -6 skycell.1198 +226.517 -6 skycell.1199 +230.562 -6 skycell.1200 +234.607 -6 skycell.1201 +238.652 -6 skycell.1202 +242.697 -6 skycell.1203 +246.742 -6 skycell.1204 +250.787 -6 skycell.1205 +254.831 -6 skycell.1206 +258.876 -6 skycell.1207 +262.921 -6 skycell.1208 +266.966 -6 skycell.1209 +271.011 -6 skycell.1210 +275.056 -6 skycell.1211 +279.101 -6 skycell.1212 +283.146 -6 skycell.1213 +287.191 -6 skycell.1214 +291.236 -6 skycell.1215 +295.281 -6 skycell.1216 +299.326 -6 skycell.1217 +303.371 -6 skycell.1218 +307.416 -6 skycell.1219 +311.461 -6 skycell.1220 +315.506 -6 skycell.1221 +319.551 -6 skycell.1222 +323.596 -6 skycell.1223 +327.64 -6 skycell.1224 +331.685 -6 skycell.1225 +335.73 -6 skycell.1226 +339.775 -6 skycell.1227 +343.82 -6 skycell.1228 +347.865 -6 skycell.1229 +351.91 -6 skycell.1230 +355.955 -6 skycell.1231 +0 -2 skycell.1232 +4 -2 skycell.1233 +8 -2 skycell.1234 +12 -2 skycell.1235 +16 -2 skycell.1236 +20 -2 skycell.1237 +24 -2 skycell.1238 +28 -2 skycell.1239 +32 -2 skycell.1240 +36 -2 skycell.1241 +40 -2 skycell.1242 +44 -2 skycell.1243 +48 -2 skycell.1244 +52 -2 skycell.1245 +56 -2 skycell.1246 +60 -2 skycell.1247 +64 -2 skycell.1248 +68 -2 skycell.1249 +72 -2 skycell.1250 +76 -2 skycell.1251 +80 -2 skycell.1252 +84 -2 skycell.1253 +88 -2 skycell.1254 +92 -2 skycell.1255 +96 -2 skycell.1256 +100 -2 skycell.1257 +104 -2 skycell.1258 +108 -2 skycell.1259 +112 -2 skycell.1260 +116 -2 skycell.1261 +120 -2 skycell.1262 +124 -2 skycell.1263 +128 -2 skycell.1264 +132 -2 skycell.1265 +136 -2 skycell.1266 +140 -2 skycell.1267 +144 -2 skycell.1268 +148 -2 skycell.1269 +152 -2 skycell.1270 +156 -2 skycell.1271 +160 -2 skycell.1272 +164 -2 skycell.1273 +168 -2 skycell.1274 +172 -2 skycell.1275 +176 -2 skycell.1276 +180 -2 skycell.1277 +184 -2 skycell.1278 +188 -2 skycell.1279 +192 -2 skycell.1280 +196 -2 skycell.1281 +200 -2 skycell.1282 +204 -2 skycell.1283 +208 -2 skycell.1284 +212 -2 skycell.1285 +216 -2 skycell.1286 +220 -2 skycell.1287 +224 -2 skycell.1288 +228 -2 skycell.1289 +232 -2 skycell.1290 +236 -2 skycell.1291 +240 -2 skycell.1292 +244 -2 skycell.1293 +248 -2 skycell.1294 +252 -2 skycell.1295 +256 -2 skycell.1296 +260 -2 skycell.1297 +264 -2 skycell.1298 +268 -2 skycell.1299 +272 -2 skycell.1300 +276 -2 skycell.1301 +280 -2 skycell.1302 +284 -2 skycell.1303 +288 -2 skycell.1304 +292 -2 skycell.1305 +296 -2 skycell.1306 +300 -2 skycell.1307 +304 -2 skycell.1308 +308 -2 skycell.1309 +312 -2 skycell.1310 +316 -2 skycell.1311 +320 -2 skycell.1312 +324 -2 skycell.1313 +328 -2 skycell.1314 +332 -2 skycell.1315 +336 -2 skycell.1316 +340 -2 skycell.1317 +344 -2 skycell.1318 +348 -2 skycell.1319 +352 -2 skycell.1320 +356 -2 skycell.1321 +0 2 skycell.1322 +4 2 skycell.1323 +8 2 skycell.1324 +12 2 skycell.1325 +16 2 skycell.1326 +20 2 skycell.1327 +24 2 skycell.1328 +28 2 skycell.1329 +32 2 skycell.1330 +36 2 skycell.1331 +40 2 skycell.1332 +44 2 skycell.1333 +48 2 skycell.1334 +52 2 skycell.1335 +56 2 skycell.1336 +60 2 skycell.1337 +64 2 skycell.1338 +68 2 skycell.1339 +72 2 skycell.1340 +76 2 skycell.1341 +80 2 skycell.1342 +84 2 skycell.1343 +88 2 skycell.1344 +92 2 skycell.1345 +96 2 skycell.1346 +100 2 skycell.1347 +104 2 skycell.1348 +108 2 skycell.1349 +112 2 skycell.1350 +116 2 skycell.1351 +120 2 skycell.1352 +124 2 skycell.1353 +128 2 skycell.1354 +132 2 skycell.1355 +136 2 skycell.1356 +140 2 skycell.1357 +144 2 skycell.1358 +148 2 skycell.1359 +152 2 skycell.1360 +156 2 skycell.1361 +160 2 skycell.1362 +164 2 skycell.1363 +168 2 skycell.1364 +172 2 skycell.1365 +176 2 skycell.1366 +180 2 skycell.1367 +184 2 skycell.1368 +188 2 skycell.1369 +192 2 skycell.1370 +196 2 skycell.1371 +200 2 skycell.1372 +204 2 skycell.1373 +208 2 skycell.1374 +212 2 skycell.1375 +216 2 skycell.1376 +220 2 skycell.1377 +224 2 skycell.1378 +228 2 skycell.1379 +232 2 skycell.1380 +236 2 skycell.1381 +240 2 skycell.1382 +244 2 skycell.1383 +248 2 skycell.1384 +252 2 skycell.1385 +256 2 skycell.1386 +260 2 skycell.1387 +264 2 skycell.1388 +268 2 skycell.1389 +272 2 skycell.1390 +276 2 skycell.1391 +280 2 skycell.1392 +284 2 skycell.1393 +288 2 skycell.1394 +292 2 skycell.1395 +296 2 skycell.1396 +300 2 skycell.1397 +304 2 skycell.1398 +308 2 skycell.1399 +312 2 skycell.1400 +316 2 skycell.1401 +320 2 skycell.1402 +324 2 skycell.1403 +328 2 skycell.1404 +332 2 skycell.1405 +336 2 skycell.1406 +340 2 skycell.1407 +344 2 skycell.1408 +348 2 skycell.1409 +352 2 skycell.1410 +356 2 skycell.1411 +0 6 skycell.1412 +4.04494 6 skycell.1413 +8.08989 6 skycell.1414 +12.1348 6 skycell.1415 +16.1798 6 skycell.1416 +20.2247 6 skycell.1417 +24.2697 6 skycell.1418 +28.3146 6 skycell.1419 +32.3596 6 skycell.1420 +36.4045 6 skycell.1421 +40.4494 6 skycell.1422 +44.4944 6 skycell.1423 +48.5393 6 skycell.1424 +52.5843 6 skycell.1425 +56.6292 6 skycell.1426 +60.6742 6 skycell.1427 +64.7191 6 skycell.1428 +68.764 6 skycell.1429 +72.809 6 skycell.1430 +76.8539 6 skycell.1431 +80.8989 6 skycell.1432 +84.9438 6 skycell.1433 +88.9888 6 skycell.1434 +93.0337 6 skycell.1435 +97.0787 6 skycell.1436 +101.124 6 skycell.1437 +105.169 6 skycell.1438 +109.213 6 skycell.1439 +113.258 6 skycell.1440 +117.303 6 skycell.1441 +121.348 6 skycell.1442 +125.393 6 skycell.1443 +129.438 6 skycell.1444 +133.483 6 skycell.1445 +137.528 6 skycell.1446 +141.573 6 skycell.1447 +145.618 6 skycell.1448 +149.663 6 skycell.1449 +153.708 6 skycell.1450 +157.753 6 skycell.1451 +161.798 6 skycell.1452 +165.843 6 skycell.1453 +169.888 6 skycell.1454 +173.933 6 skycell.1455 +177.978 6 skycell.1456 +182.022 6 skycell.1457 +186.067 6 skycell.1458 +190.112 6 skycell.1459 +194.157 6 skycell.1460 +198.202 6 skycell.1461 +202.247 6 skycell.1462 +206.292 6 skycell.1463 +210.337 6 skycell.1464 +214.382 6 skycell.1465 +218.427 6 skycell.1466 +222.472 6 skycell.1467 +226.517 6 skycell.1468 +230.562 6 skycell.1469 +234.607 6 skycell.1470 +238.652 6 skycell.1471 +242.697 6 skycell.1472 +246.742 6 skycell.1473 +250.787 6 skycell.1474 +254.831 6 skycell.1475 +258.876 6 skycell.1476 +262.921 6 skycell.1477 +266.966 6 skycell.1478 +271.011 6 skycell.1479 +275.056 6 skycell.1480 +279.101 6 skycell.1481 +283.146 6 skycell.1482 +287.191 6 skycell.1483 +291.236 6 skycell.1484 +295.281 6 skycell.1485 +299.326 6 skycell.1486 +303.371 6 skycell.1487 +307.416 6 skycell.1488 +311.461 6 skycell.1489 +315.506 6 skycell.1490 +319.551 6 skycell.1491 +323.596 6 skycell.1492 +327.64 6 skycell.1493 +331.685 6 skycell.1494 +335.73 6 skycell.1495 +339.775 6 skycell.1496 +343.82 6 skycell.1497 +347.865 6 skycell.1498 +351.91 6 skycell.1499 +355.955 6 skycell.1500 +0 10 skycell.1501 +4.04494 10 skycell.1502 +8.08989 10 skycell.1503 +12.1348 10 skycell.1504 +16.1798 10 skycell.1505 +20.2247 10 skycell.1506 +24.2697 10 skycell.1507 +28.3146 10 skycell.1508 +32.3596 10 skycell.1509 +36.4045 10 skycell.1510 +40.4494 10 skycell.1511 +44.4944 10 skycell.1512 +48.5393 10 skycell.1513 +52.5843 10 skycell.1514 +56.6292 10 skycell.1515 +60.6742 10 skycell.1516 +64.7191 10 skycell.1517 +68.764 10 skycell.1518 +72.809 10 skycell.1519 +76.8539 10 skycell.1520 +80.8989 10 skycell.1521 +84.9438 10 skycell.1522 +88.9888 10 skycell.1523 +93.0337 10 skycell.1524 +97.0787 10 skycell.1525 +101.124 10 skycell.1526 +105.169 10 skycell.1527 +109.213 10 skycell.1528 +113.258 10 skycell.1529 +117.303 10 skycell.1530 +121.348 10 skycell.1531 +125.393 10 skycell.1532 +129.438 10 skycell.1533 +133.483 10 skycell.1534 +137.528 10 skycell.1535 +141.573 10 skycell.1536 +145.618 10 skycell.1537 +149.663 10 skycell.1538 +153.708 10 skycell.1539 +157.753 10 skycell.1540 +161.798 10 skycell.1541 +165.843 10 skycell.1542 +169.888 10 skycell.1543 +173.933 10 skycell.1544 +177.978 10 skycell.1545 +182.022 10 skycell.1546 +186.067 10 skycell.1547 +190.112 10 skycell.1548 +194.157 10 skycell.1549 +198.202 10 skycell.1550 +202.247 10 skycell.1551 +206.292 10 skycell.1552 +210.337 10 skycell.1553 +214.382 10 skycell.1554 +218.427 10 skycell.1555 +222.472 10 skycell.1556 +226.517 10 skycell.1557 +230.562 10 skycell.1558 +234.607 10 skycell.1559 +238.652 10 skycell.1560 +242.697 10 skycell.1561 +246.742 10 skycell.1562 +250.787 10 skycell.1563 +254.831 10 skycell.1564 +258.876 10 skycell.1565 +262.921 10 skycell.1566 +266.966 10 skycell.1567 +271.011 10 skycell.1568 +275.056 10 skycell.1569 +279.101 10 skycell.1570 +283.146 10 skycell.1571 +287.191 10 skycell.1572 +291.236 10 skycell.1573 +295.281 10 skycell.1574 +299.326 10 skycell.1575 +303.371 10 skycell.1576 +307.416 10 skycell.1577 +311.461 10 skycell.1578 +315.506 10 skycell.1579 +319.551 10 skycell.1580 +323.596 10 skycell.1581 +327.64 10 skycell.1582 +331.685 10 skycell.1583 +335.73 10 skycell.1584 +339.775 10 skycell.1585 +343.82 10 skycell.1586 +347.865 10 skycell.1587 +351.91 10 skycell.1588 +355.955 10 skycell.1589 +0 14 skycell.1590 +4.09091 14 skycell.1591 +8.18182 14 skycell.1592 +12.2727 14 skycell.1593 +16.3636 14 skycell.1594 +20.4545 14 skycell.1595 +24.5455 14 skycell.1596 +28.6364 14 skycell.1597 +32.7273 14 skycell.1598 +36.8182 14 skycell.1599 +40.9091 14 skycell.1600 +45 14 skycell.1601 +49.0909 14 skycell.1602 +53.1818 14 skycell.1603 +57.2727 14 skycell.1604 +61.3636 14 skycell.1605 +65.4545 14 skycell.1606 +69.5455 14 skycell.1607 +73.6364 14 skycell.1608 +77.7273 14 skycell.1609 +81.8182 14 skycell.1610 +85.9091 14 skycell.1611 +90 14 skycell.1612 +94.0909 14 skycell.1613 +98.1818 14 skycell.1614 +102.273 14 skycell.1615 +106.364 14 skycell.1616 +110.455 14 skycell.1617 +114.545 14 skycell.1618 +118.636 14 skycell.1619 +122.727 14 skycell.1620 +126.818 14 skycell.1621 +130.909 14 skycell.1622 +135 14 skycell.1623 +139.091 14 skycell.1624 +143.182 14 skycell.1625 +147.273 14 skycell.1626 +151.364 14 skycell.1627 +155.455 14 skycell.1628 +159.545 14 skycell.1629 +163.636 14 skycell.1630 +167.727 14 skycell.1631 +171.818 14 skycell.1632 +175.909 14 skycell.1633 +180 14 skycell.1634 +184.091 14 skycell.1635 +188.182 14 skycell.1636 +192.273 14 skycell.1637 +196.364 14 skycell.1638 +200.455 14 skycell.1639 +204.545 14 skycell.1640 +208.636 14 skycell.1641 +212.727 14 skycell.1642 +216.818 14 skycell.1643 +220.909 14 skycell.1644 +225 14 skycell.1645 +229.091 14 skycell.1646 +233.182 14 skycell.1647 +237.273 14 skycell.1648 +241.364 14 skycell.1649 +245.455 14 skycell.1650 +249.545 14 skycell.1651 +253.636 14 skycell.1652 +257.727 14 skycell.1653 +261.818 14 skycell.1654 +265.909 14 skycell.1655 +270 14 skycell.1656 +274.091 14 skycell.1657 +278.182 14 skycell.1658 +282.273 14 skycell.1659 +286.364 14 skycell.1660 +290.455 14 skycell.1661 +294.545 14 skycell.1662 +298.636 14 skycell.1663 +302.727 14 skycell.1664 +306.818 14 skycell.1665 +310.909 14 skycell.1666 +315 14 skycell.1667 +319.091 14 skycell.1668 +323.182 14 skycell.1669 +327.273 14 skycell.1670 +331.364 14 skycell.1671 +335.455 14 skycell.1672 +339.545 14 skycell.1673 +343.636 14 skycell.1674 +347.727 14 skycell.1675 +351.818 14 skycell.1676 +355.909 14 skycell.1677 +0 18 skycell.1678 +4.18605 18 skycell.1679 +8.37209 18 skycell.1680 +12.5581 18 skycell.1681 +16.7442 18 skycell.1682 +20.9302 18 skycell.1683 +25.1163 18 skycell.1684 +29.3023 18 skycell.1685 +33.4884 18 skycell.1686 +37.6744 18 skycell.1687 +41.8605 18 skycell.1688 +46.0465 18 skycell.1689 +50.2326 18 skycell.1690 +54.4186 18 skycell.1691 +58.6047 18 skycell.1692 +62.7907 18 skycell.1693 +66.9767 18 skycell.1694 +71.1628 18 skycell.1695 +75.3488 18 skycell.1696 +79.5349 18 skycell.1697 +83.7209 18 skycell.1698 +87.907 18 skycell.1699 +92.093 18 skycell.1700 +96.2791 18 skycell.1701 +100.465 18 skycell.1702 +104.651 18 skycell.1703 +108.837 18 skycell.1704 +113.023 18 skycell.1705 +117.209 18 skycell.1706 +121.395 18 skycell.1707 +125.581 18 skycell.1708 +129.767 18 skycell.1709 +133.953 18 skycell.1710 +138.14 18 skycell.1711 +142.326 18 skycell.1712 +146.512 18 skycell.1713 +150.698 18 skycell.1714 +154.884 18 skycell.1715 +159.07 18 skycell.1716 +163.256 18 skycell.1717 +167.442 18 skycell.1718 +171.628 18 skycell.1719 +175.814 18 skycell.1720 +180 18 skycell.1721 +184.186 18 skycell.1722 +188.372 18 skycell.1723 +192.558 18 skycell.1724 +196.744 18 skycell.1725 +200.93 18 skycell.1726 +205.116 18 skycell.1727 +209.302 18 skycell.1728 +213.488 18 skycell.1729 +217.674 18 skycell.1730 +221.86 18 skycell.1731 +226.047 18 skycell.1732 +230.233 18 skycell.1733 +234.419 18 skycell.1734 +238.605 18 skycell.1735 +242.791 18 skycell.1736 +246.977 18 skycell.1737 +251.163 18 skycell.1738 +255.349 18 skycell.1739 +259.535 18 skycell.1740 +263.721 18 skycell.1741 +267.907 18 skycell.1742 +272.093 18 skycell.1743 +276.279 18 skycell.1744 +280.465 18 skycell.1745 +284.651 18 skycell.1746 +288.837 18 skycell.1747 +293.023 18 skycell.1748 +297.209 18 skycell.1749 +301.395 18 skycell.1750 +305.581 18 skycell.1751 +309.767 18 skycell.1752 +313.953 18 skycell.1753 +318.14 18 skycell.1754 +322.326 18 skycell.1755 +326.512 18 skycell.1756 +330.698 18 skycell.1757 +334.884 18 skycell.1758 +339.07 18 skycell.1759 +343.256 18 skycell.1760 +347.442 18 skycell.1761 +351.628 18 skycell.1762 +355.814 18 skycell.1763 +0 22 skycell.1764 +4.28571 22 skycell.1765 +8.57143 22 skycell.1766 +12.8571 22 skycell.1767 +17.1429 22 skycell.1768 +21.4286 22 skycell.1769 +25.7143 22 skycell.1770 +30 22 skycell.1771 +34.2857 22 skycell.1772 +38.5714 22 skycell.1773 +42.8571 22 skycell.1774 +47.1429 22 skycell.1775 +51.4286 22 skycell.1776 +55.7143 22 skycell.1777 +60 22 skycell.1778 +64.2857 22 skycell.1779 +68.5714 22 skycell.1780 +72.8571 22 skycell.1781 +77.1429 22 skycell.1782 +81.4286 22 skycell.1783 +85.7143 22 skycell.1784 +90 22 skycell.1785 +94.2857 22 skycell.1786 +98.5714 22 skycell.1787 +102.857 22 skycell.1788 +107.143 22 skycell.1789 +111.429 22 skycell.1790 +115.714 22 skycell.1791 +120 22 skycell.1792 +124.286 22 skycell.1793 +128.571 22 skycell.1794 +132.857 22 skycell.1795 +137.143 22 skycell.1796 +141.429 22 skycell.1797 +145.714 22 skycell.1798 +150 22 skycell.1799 +154.286 22 skycell.1800 +158.571 22 skycell.1801 +162.857 22 skycell.1802 +167.143 22 skycell.1803 +171.429 22 skycell.1804 +175.714 22 skycell.1805 +180 22 skycell.1806 +184.286 22 skycell.1807 +188.571 22 skycell.1808 +192.857 22 skycell.1809 +197.143 22 skycell.1810 +201.429 22 skycell.1811 +205.714 22 skycell.1812 +210 22 skycell.1813 +214.286 22 skycell.1814 +218.571 22 skycell.1815 +222.857 22 skycell.1816 +227.143 22 skycell.1817 +231.429 22 skycell.1818 +235.714 22 skycell.1819 +240 22 skycell.1820 +244.286 22 skycell.1821 +248.571 22 skycell.1822 +252.857 22 skycell.1823 +257.143 22 skycell.1824 +261.429 22 skycell.1825 +265.714 22 skycell.1826 +270 22 skycell.1827 +274.286 22 skycell.1828 +278.571 22 skycell.1829 +282.857 22 skycell.1830 +287.143 22 skycell.1831 +291.429 22 skycell.1832 +295.714 22 skycell.1833 +300 22 skycell.1834 +304.286 22 skycell.1835 +308.571 22 skycell.1836 +312.857 22 skycell.1837 +317.143 22 skycell.1838 +321.429 22 skycell.1839 +325.714 22 skycell.1840 +330 22 skycell.1841 +334.286 22 skycell.1842 +338.571 22 skycell.1843 +342.857 22 skycell.1844 +347.143 22 skycell.1845 +351.429 22 skycell.1846 +355.714 22 skycell.1847 +0 26 skycell.1848 +4.39024 26 skycell.1849 +8.78049 26 skycell.1850 +13.1707 26 skycell.1851 +17.561 26 skycell.1852 +21.9512 26 skycell.1853 +26.3415 26 skycell.1854 +30.7317 26 skycell.1855 +35.122 26 skycell.1856 +39.5122 26 skycell.1857 +43.9024 26 skycell.1858 +48.2927 26 skycell.1859 +52.6829 26 skycell.1860 +57.0732 26 skycell.1861 +61.4634 26 skycell.1862 +65.8537 26 skycell.1863 +70.2439 26 skycell.1864 +74.6341 26 skycell.1865 +79.0244 26 skycell.1866 +83.4146 26 skycell.1867 +87.8049 26 skycell.1868 +92.1951 26 skycell.1869 +96.5854 26 skycell.1870 +100.976 26 skycell.1871 +105.366 26 skycell.1872 +109.756 26 skycell.1873 +114.146 26 skycell.1874 +118.537 26 skycell.1875 +122.927 26 skycell.1876 +127.317 26 skycell.1877 +131.707 26 skycell.1878 +136.098 26 skycell.1879 +140.488 26 skycell.1880 +144.878 26 skycell.1881 +149.268 26 skycell.1882 +153.659 26 skycell.1883 +158.049 26 skycell.1884 +162.439 26 skycell.1885 +166.829 26 skycell.1886 +171.22 26 skycell.1887 +175.61 26 skycell.1888 +180 26 skycell.1889 +184.39 26 skycell.1890 +188.78 26 skycell.1891 +193.171 26 skycell.1892 +197.561 26 skycell.1893 +201.951 26 skycell.1894 +206.341 26 skycell.1895 +210.732 26 skycell.1896 +215.122 26 skycell.1897 +219.512 26 skycell.1898 +223.902 26 skycell.1899 +228.293 26 skycell.1900 +232.683 26 skycell.1901 +237.073 26 skycell.1902 +241.463 26 skycell.1903 +245.854 26 skycell.1904 +250.244 26 skycell.1905 +254.634 26 skycell.1906 +259.024 26 skycell.1907 +263.415 26 skycell.1908 +267.805 26 skycell.1909 +272.195 26 skycell.1910 +276.585 26 skycell.1911 +280.976 26 skycell.1912 +285.366 26 skycell.1913 +289.756 26 skycell.1914 +294.146 26 skycell.1915 +298.537 26 skycell.1916 +302.927 26 skycell.1917 +307.317 26 skycell.1918 +311.707 26 skycell.1919 +316.098 26 skycell.1920 +320.488 26 skycell.1921 +324.878 26 skycell.1922 +329.268 26 skycell.1923 +333.659 26 skycell.1924 +338.049 26 skycell.1925 +342.439 26 skycell.1926 +346.829 26 skycell.1927 +351.22 26 skycell.1928 +355.61 26 skycell.1929 +0 30 skycell.1930 +4.55696 30 skycell.1931 +9.11392 30 skycell.1932 +13.6709 30 skycell.1933 +18.2278 30 skycell.1934 +22.7848 30 skycell.1935 +27.3418 30 skycell.1936 +31.8987 30 skycell.1937 +36.4557 30 skycell.1938 +41.0127 30 skycell.1939 +45.5696 30 skycell.1940 +50.1266 30 skycell.1941 +54.6835 30 skycell.1942 +59.2405 30 skycell.1943 +63.7975 30 skycell.1944 +68.3544 30 skycell.1945 +72.9114 30 skycell.1946 +77.4684 30 skycell.1947 +82.0253 30 skycell.1948 +86.5823 30 skycell.1949 +91.1392 30 skycell.1950 +95.6962 30 skycell.1951 +100.253 30 skycell.1952 +104.81 30 skycell.1953 +109.367 30 skycell.1954 +113.924 30 skycell.1955 +118.481 30 skycell.1956 +123.038 30 skycell.1957 +127.595 30 skycell.1958 +132.152 30 skycell.1959 +136.709 30 skycell.1960 +141.266 30 skycell.1961 +145.823 30 skycell.1962 +150.38 30 skycell.1963 +154.937 30 skycell.1964 +159.494 30 skycell.1965 +164.051 30 skycell.1966 +168.608 30 skycell.1967 +173.165 30 skycell.1968 +177.722 30 skycell.1969 +182.278 30 skycell.1970 +186.835 30 skycell.1971 +191.392 30 skycell.1972 +195.949 30 skycell.1973 +200.506 30 skycell.1974 +205.063 30 skycell.1975 +209.62 30 skycell.1976 +214.177 30 skycell.1977 +218.734 30 skycell.1978 +223.291 30 skycell.1979 +227.848 30 skycell.1980 +232.405 30 skycell.1981 +236.962 30 skycell.1982 +241.519 30 skycell.1983 +246.076 30 skycell.1984 +250.633 30 skycell.1985 +255.19 30 skycell.1986 +259.747 30 skycell.1987 +264.304 30 skycell.1988 +268.861 30 skycell.1989 +273.418 30 skycell.1990 +277.975 30 skycell.1991 +282.532 30 skycell.1992 +287.089 30 skycell.1993 +291.646 30 skycell.1994 +296.203 30 skycell.1995 +300.759 30 skycell.1996 +305.316 30 skycell.1997 +309.873 30 skycell.1998 +314.43 30 skycell.1999 +318.987 30 skycell.2000 +323.544 30 skycell.2001 +328.101 30 skycell.2002 +332.658 30 skycell.2003 +337.215 30 skycell.2004 +341.772 30 skycell.2005 +346.329 30 skycell.2006 +350.886 30 skycell.2007 +355.443 30 skycell.2008 +0 34 skycell.2009 +4.73684 34 skycell.2010 +9.47368 34 skycell.2011 +14.2105 34 skycell.2012 +18.9474 34 skycell.2013 +23.6842 34 skycell.2014 +28.4211 34 skycell.2015 +33.1579 34 skycell.2016 +37.8947 34 skycell.2017 +42.6316 34 skycell.2018 +47.3684 34 skycell.2019 +52.1053 34 skycell.2020 +56.8421 34 skycell.2021 +61.5789 34 skycell.2022 +66.3158 34 skycell.2023 +71.0526 34 skycell.2024 +75.7895 34 skycell.2025 +80.5263 34 skycell.2026 +85.2632 34 skycell.2027 +90 34 skycell.2028 +94.7368 34 skycell.2029 +99.4737 34 skycell.2030 +104.211 34 skycell.2031 +108.947 34 skycell.2032 +113.684 34 skycell.2033 +118.421 34 skycell.2034 +123.158 34 skycell.2035 +127.895 34 skycell.2036 +132.632 34 skycell.2037 +137.368 34 skycell.2038 +142.105 34 skycell.2039 +146.842 34 skycell.2040 +151.579 34 skycell.2041 +156.316 34 skycell.2042 +161.053 34 skycell.2043 +165.789 34 skycell.2044 +170.526 34 skycell.2045 +175.263 34 skycell.2046 +180 34 skycell.2047 +184.737 34 skycell.2048 +189.474 34 skycell.2049 +194.211 34 skycell.2050 +198.947 34 skycell.2051 +203.684 34 skycell.2052 +208.421 34 skycell.2053 +213.158 34 skycell.2054 +217.895 34 skycell.2055 +222.632 34 skycell.2056 +227.368 34 skycell.2057 +232.105 34 skycell.2058 +236.842 34 skycell.2059 +241.579 34 skycell.2060 +246.316 34 skycell.2061 +251.053 34 skycell.2062 +255.789 34 skycell.2063 +260.526 34 skycell.2064 +265.263 34 skycell.2065 +270 34 skycell.2066 +274.737 34 skycell.2067 +279.474 34 skycell.2068 +284.211 34 skycell.2069 +288.947 34 skycell.2070 +293.684 34 skycell.2071 +298.421 34 skycell.2072 +303.158 34 skycell.2073 +307.895 34 skycell.2074 +312.632 34 skycell.2075 +317.368 34 skycell.2076 +322.105 34 skycell.2077 +326.842 34 skycell.2078 +331.579 34 skycell.2079 +336.316 34 skycell.2080 +341.053 34 skycell.2081 +345.789 34 skycell.2082 +350.526 34 skycell.2083 +355.263 34 skycell.2084 +0 38 skycell.2085 +5 38 skycell.2086 +10 38 skycell.2087 +15 38 skycell.2088 +20 38 skycell.2089 +25 38 skycell.2090 +30 38 skycell.2091 +35 38 skycell.2092 +40 38 skycell.2093 +45 38 skycell.2094 +50 38 skycell.2095 +55 38 skycell.2096 +60 38 skycell.2097 +65 38 skycell.2098 +70 38 skycell.2099 +75 38 skycell.2100 +80 38 skycell.2101 +85 38 skycell.2102 +90 38 skycell.2103 +95 38 skycell.2104 +100 38 skycell.2105 +105 38 skycell.2106 +110 38 skycell.2107 +115 38 skycell.2108 +120 38 skycell.2109 +125 38 skycell.2110 +130 38 skycell.2111 +135 38 skycell.2112 +140 38 skycell.2113 +145 38 skycell.2114 +150 38 skycell.2115 +155 38 skycell.2116 +160 38 skycell.2117 +165 38 skycell.2118 +170 38 skycell.2119 +175 38 skycell.2120 +180 38 skycell.2121 +185 38 skycell.2122 +190 38 skycell.2123 +195 38 skycell.2124 +200 38 skycell.2125 +205 38 skycell.2126 +210 38 skycell.2127 +215 38 skycell.2128 +220 38 skycell.2129 +225 38 skycell.2130 +230 38 skycell.2131 +235 38 skycell.2132 +240 38 skycell.2133 +245 38 skycell.2134 +250 38 skycell.2135 +255 38 skycell.2136 +260 38 skycell.2137 +265 38 skycell.2138 +270 38 skycell.2139 +275 38 skycell.2140 +280 38 skycell.2141 +285 38 skycell.2142 +290 38 skycell.2143 +295 38 skycell.2144 +300 38 skycell.2145 +305 38 skycell.2146 +310 38 skycell.2147 +315 38 skycell.2148 +320 38 skycell.2149 +325 38 skycell.2150 +330 38 skycell.2151 +335 38 skycell.2152 +340 38 skycell.2153 +345 38 skycell.2154 +350 38 skycell.2155 +355 38 skycell.2156 +0 42 skycell.2157 +5.29412 42 skycell.2158 +10.5882 42 skycell.2159 +15.8824 42 skycell.2160 +21.1765 42 skycell.2161 +26.4706 42 skycell.2162 +31.7647 42 skycell.2163 +37.0588 42 skycell.2164 +42.3529 42 skycell.2165 +47.6471 42 skycell.2166 +52.9412 42 skycell.2167 +58.2353 42 skycell.2168 +63.5294 42 skycell.2169 +68.8235 42 skycell.2170 +74.1176 42 skycell.2171 +79.4118 42 skycell.2172 +84.7059 42 skycell.2173 +90 42 skycell.2174 +95.2941 42 skycell.2175 +100.588 42 skycell.2176 +105.882 42 skycell.2177 +111.176 42 skycell.2178 +116.471 42 skycell.2179 +121.765 42 skycell.2180 +127.059 42 skycell.2181 +132.353 42 skycell.2182 +137.647 42 skycell.2183 +142.941 42 skycell.2184 +148.235 42 skycell.2185 +153.529 42 skycell.2186 +158.824 42 skycell.2187 +164.118 42 skycell.2188 +169.412 42 skycell.2189 +174.706 42 skycell.2190 +180 42 skycell.2191 +185.294 42 skycell.2192 +190.588 42 skycell.2193 +195.882 42 skycell.2194 +201.176 42 skycell.2195 +206.471 42 skycell.2196 +211.765 42 skycell.2197 +217.059 42 skycell.2198 +222.353 42 skycell.2199 +227.647 42 skycell.2200 +232.941 42 skycell.2201 +238.235 42 skycell.2202 +243.529 42 skycell.2203 +248.824 42 skycell.2204 +254.118 42 skycell.2205 +259.412 42 skycell.2206 +264.706 42 skycell.2207 +270 42 skycell.2208 +275.294 42 skycell.2209 +280.588 42 skycell.2210 +285.882 42 skycell.2211 +291.176 42 skycell.2212 +296.471 42 skycell.2213 +301.765 42 skycell.2214 +307.059 42 skycell.2215 +312.353 42 skycell.2216 +317.647 42 skycell.2217 +322.941 42 skycell.2218 +328.235 42 skycell.2219 +333.529 42 skycell.2220 +338.824 42 skycell.2221 +344.118 42 skycell.2222 +349.412 42 skycell.2223 +354.706 42 skycell.2224 +0 46 skycell.2225 +5.625 46 skycell.2226 +11.25 46 skycell.2227 +16.875 46 skycell.2228 +22.5 46 skycell.2229 +28.125 46 skycell.2230 +33.75 46 skycell.2231 +39.375 46 skycell.2232 +45 46 skycell.2233 +50.625 46 skycell.2234 +56.25 46 skycell.2235 +61.875 46 skycell.2236 +67.5 46 skycell.2237 +73.125 46 skycell.2238 +78.75 46 skycell.2239 +84.375 46 skycell.2240 +90 46 skycell.2241 +95.625 46 skycell.2242 +101.25 46 skycell.2243 +106.875 46 skycell.2244 +112.5 46 skycell.2245 +118.125 46 skycell.2246 +123.75 46 skycell.2247 +129.375 46 skycell.2248 +135 46 skycell.2249 +140.625 46 skycell.2250 +146.25 46 skycell.2251 +151.875 46 skycell.2252 +157.5 46 skycell.2253 +163.125 46 skycell.2254 +168.75 46 skycell.2255 +174.375 46 skycell.2256 +180 46 skycell.2257 +185.625 46 skycell.2258 +191.25 46 skycell.2259 +196.875 46 skycell.2260 +202.5 46 skycell.2261 +208.125 46 skycell.2262 +213.75 46 skycell.2263 +219.375 46 skycell.2264 +225 46 skycell.2265 +230.625 46 skycell.2266 +236.25 46 skycell.2267 +241.875 46 skycell.2268 +247.5 46 skycell.2269 +253.125 46 skycell.2270 +258.75 46 skycell.2271 +264.375 46 skycell.2272 +270 46 skycell.2273 +275.625 46 skycell.2274 +281.25 46 skycell.2275 +286.875 46 skycell.2276 +292.5 46 skycell.2277 +298.125 46 skycell.2278 +303.75 46 skycell.2279 +309.375 46 skycell.2280 +315 46 skycell.2281 +320.625 46 skycell.2282 +326.25 46 skycell.2283 +331.875 46 skycell.2284 +337.5 46 skycell.2285 +343.125 46 skycell.2286 +348.75 46 skycell.2287 +354.375 46 skycell.2288 +0 50 skycell.2289 +6 50 skycell.2290 +12 50 skycell.2291 +18 50 skycell.2292 +24 50 skycell.2293 +30 50 skycell.2294 +36 50 skycell.2295 +42 50 skycell.2296 +48 50 skycell.2297 +54 50 skycell.2298 +60 50 skycell.2299 +66 50 skycell.2300 +72 50 skycell.2301 +78 50 skycell.2302 +84 50 skycell.2303 +90 50 skycell.2304 +96 50 skycell.2305 +102 50 skycell.2306 +108 50 skycell.2307 +114 50 skycell.2308 +120 50 skycell.2309 +126 50 skycell.2310 +132 50 skycell.2311 +138 50 skycell.2312 +144 50 skycell.2313 +150 50 skycell.2314 +156 50 skycell.2315 +162 50 skycell.2316 +168 50 skycell.2317 +174 50 skycell.2318 +180 50 skycell.2319 +186 50 skycell.2320 +192 50 skycell.2321 +198 50 skycell.2322 +204 50 skycell.2323 +210 50 skycell.2324 +216 50 skycell.2325 +222 50 skycell.2326 +228 50 skycell.2327 +234 50 skycell.2328 +240 50 skycell.2329 +246 50 skycell.2330 +252 50 skycell.2331 +258 50 skycell.2332 +264 50 skycell.2333 +270 50 skycell.2334 +276 50 skycell.2335 +282 50 skycell.2336 +288 50 skycell.2337 +294 50 skycell.2338 +300 50 skycell.2339 +306 50 skycell.2340 +312 50 skycell.2341 +318 50 skycell.2342 +324 50 skycell.2343 +330 50 skycell.2344 +336 50 skycell.2345 +342 50 skycell.2346 +348 50 skycell.2347 +354 50 skycell.2348 +0 54 skycell.2349 +6.54545 54 skycell.2350 +13.0909 54 skycell.2351 +19.6364 54 skycell.2352 +26.1818 54 skycell.2353 +32.7273 54 skycell.2354 +39.2727 54 skycell.2355 +45.8182 54 skycell.2356 +52.3636 54 skycell.2357 +58.9091 54 skycell.2358 +65.4545 54 skycell.2359 +72 54 skycell.2360 +78.5455 54 skycell.2361 +85.0909 54 skycell.2362 +91.6364 54 skycell.2363 +98.1818 54 skycell.2364 +104.727 54 skycell.2365 +111.273 54 skycell.2366 +117.818 54 skycell.2367 +124.364 54 skycell.2368 +130.909 54 skycell.2369 +137.455 54 skycell.2370 +144 54 skycell.2371 +150.545 54 skycell.2372 +157.091 54 skycell.2373 +163.636 54 skycell.2374 +170.182 54 skycell.2375 +176.727 54 skycell.2376 +183.273 54 skycell.2377 +189.818 54 skycell.2378 +196.364 54 skycell.2379 +202.909 54 skycell.2380 +209.455 54 skycell.2381 +216 54 skycell.2382 +222.545 54 skycell.2383 +229.091 54 skycell.2384 +235.636 54 skycell.2385 +242.182 54 skycell.2386 +248.727 54 skycell.2387 +255.273 54 skycell.2388 +261.818 54 skycell.2389 +268.364 54 skycell.2390 +274.909 54 skycell.2391 +281.455 54 skycell.2392 +288 54 skycell.2393 +294.545 54 skycell.2394 +301.091 54 skycell.2395 +307.636 54 skycell.2396 +314.182 54 skycell.2397 +320.727 54 skycell.2398 +327.273 54 skycell.2399 +333.818 54 skycell.2400 +340.364 54 skycell.2401 +346.909 54 skycell.2402 +353.455 54 skycell.2403 +0 58 skycell.2404 +7.2 58 skycell.2405 +14.4 58 skycell.2406 +21.6 58 skycell.2407 +28.8 58 skycell.2408 +36 58 skycell.2409 +43.2 58 skycell.2410 +50.4 58 skycell.2411 +57.6 58 skycell.2412 +64.8 58 skycell.2413 +72 58 skycell.2414 +79.2 58 skycell.2415 +86.4 58 skycell.2416 +93.6 58 skycell.2417 +100.8 58 skycell.2418 +108 58 skycell.2419 +115.2 58 skycell.2420 +122.4 58 skycell.2421 +129.6 58 skycell.2422 +136.8 58 skycell.2423 +144 58 skycell.2424 +151.2 58 skycell.2425 +158.4 58 skycell.2426 +165.6 58 skycell.2427 +172.8 58 skycell.2428 +180 58 skycell.2429 +187.2 58 skycell.2430 +194.4 58 skycell.2431 +201.6 58 skycell.2432 +208.8 58 skycell.2433 +216 58 skycell.2434 +223.2 58 skycell.2435 +230.4 58 skycell.2436 +237.6 58 skycell.2437 +244.8 58 skycell.2438 +252 58 skycell.2439 +259.2 58 skycell.2440 +266.4 58 skycell.2441 +273.6 58 skycell.2442 +280.8 58 skycell.2443 +288 58 skycell.2444 +295.2 58 skycell.2445 +302.4 58 skycell.2446 +309.6 58 skycell.2447 +316.8 58 skycell.2448 +324 58 skycell.2449 +331.2 58 skycell.2450 +338.4 58 skycell.2451 +345.6 58 skycell.2452 +352.8 58 skycell.2453 +0 62 skycell.2454 +8 62 skycell.2455 +16 62 skycell.2456 +24 62 skycell.2457 +32 62 skycell.2458 +40 62 skycell.2459 +48 62 skycell.2460 +56 62 skycell.2461 +64 62 skycell.2462 +72 62 skycell.2463 +80 62 skycell.2464 +88 62 skycell.2465 +96 62 skycell.2466 +104 62 skycell.2467 +112 62 skycell.2468 +120 62 skycell.2469 +128 62 skycell.2470 +136 62 skycell.2471 +144 62 skycell.2472 +152 62 skycell.2473 +160 62 skycell.2474 +168 62 skycell.2475 +176 62 skycell.2476 +184 62 skycell.2477 +192 62 skycell.2478 +200 62 skycell.2479 +208 62 skycell.2480 +216 62 skycell.2481 +224 62 skycell.2482 +232 62 skycell.2483 +240 62 skycell.2484 +248 62 skycell.2485 +256 62 skycell.2486 +264 62 skycell.2487 +272 62 skycell.2488 +280 62 skycell.2489 +288 62 skycell.2490 +296 62 skycell.2491 +304 62 skycell.2492 +312 62 skycell.2493 +320 62 skycell.2494 +328 62 skycell.2495 +336 62 skycell.2496 +344 62 skycell.2497 +352 62 skycell.2498 +0 66 skycell.2499 +9.23077 66 skycell.2500 +18.4615 66 skycell.2501 +27.6923 66 skycell.2502 +36.9231 66 skycell.2503 +46.1538 66 skycell.2504 +55.3846 66 skycell.2505 +64.6154 66 skycell.2506 +73.8462 66 skycell.2507 +83.0769 66 skycell.2508 +92.3077 66 skycell.2509 +101.538 66 skycell.2510 +110.769 66 skycell.2511 +120 66 skycell.2512 +129.231 66 skycell.2513 +138.462 66 skycell.2514 +147.692 66 skycell.2515 +156.923 66 skycell.2516 +166.154 66 skycell.2517 +175.385 66 skycell.2518 +184.615 66 skycell.2519 +193.846 66 skycell.2520 +203.077 66 skycell.2521 +212.308 66 skycell.2522 +221.538 66 skycell.2523 +230.769 66 skycell.2524 +240 66 skycell.2525 +249.231 66 skycell.2526 +258.462 66 skycell.2527 +267.692 66 skycell.2528 +276.923 66 skycell.2529 +286.154 66 skycell.2530 +295.385 66 skycell.2531 +304.615 66 skycell.2532 +313.846 66 skycell.2533 +323.077 66 skycell.2534 +332.308 66 skycell.2535 +341.538 66 skycell.2536 +350.769 66 skycell.2537 +0 70 skycell.2538 +10.9091 70 skycell.2539 +21.8182 70 skycell.2540 +32.7273 70 skycell.2541 +43.6364 70 skycell.2542 +54.5455 70 skycell.2543 +65.4545 70 skycell.2544 +76.3636 70 skycell.2545 +87.2727 70 skycell.2546 +98.1818 70 skycell.2547 +109.091 70 skycell.2548 +120 70 skycell.2549 +130.909 70 skycell.2550 +141.818 70 skycell.2551 +152.727 70 skycell.2552 +163.636 70 skycell.2553 +174.545 70 skycell.2554 +185.455 70 skycell.2555 +196.364 70 skycell.2556 +207.273 70 skycell.2557 +218.182 70 skycell.2558 +229.091 70 skycell.2559 +240 70 skycell.2560 +250.909 70 skycell.2561 +261.818 70 skycell.2562 +272.727 70 skycell.2563 +283.636 70 skycell.2564 +294.545 70 skycell.2565 +305.455 70 skycell.2566 +316.364 70 skycell.2567 +327.273 70 skycell.2568 +338.182 70 skycell.2569 +349.091 70 skycell.2570 +0 74 skycell.2571 +13.3333 74 skycell.2572 +26.6667 74 skycell.2573 +40 74 skycell.2574 +53.3333 74 skycell.2575 +66.6667 74 skycell.2576 +80 74 skycell.2577 +93.3333 74 skycell.2578 +106.667 74 skycell.2579 +120 74 skycell.2580 +133.333 74 skycell.2581 +146.667 74 skycell.2582 +160 74 skycell.2583 +173.333 74 skycell.2584 +186.667 74 skycell.2585 +200 74 skycell.2586 +213.333 74 skycell.2587 +226.667 74 skycell.2588 +240 74 skycell.2589 +253.333 74 skycell.2590 +266.667 74 skycell.2591 +280 74 skycell.2592 +293.333 74 skycell.2593 +306.667 74 skycell.2594 +320 74 skycell.2595 +333.333 74 skycell.2596 +346.667 74 skycell.2597 +0 78 skycell.2598 +17.1429 78 skycell.2599 +34.2857 78 skycell.2600 +51.4286 78 skycell.2601 +68.5714 78 skycell.2602 +85.7143 78 skycell.2603 +102.857 78 skycell.2604 +120 78 skycell.2605 +137.143 78 skycell.2606 +154.286 78 skycell.2607 +171.429 78 skycell.2608 +188.571 78 skycell.2609 +205.714 78 skycell.2610 +222.857 78 skycell.2611 +240 78 skycell.2612 +257.143 78 skycell.2613 +274.286 78 skycell.2614 +291.429 78 skycell.2615 +308.571 78 skycell.2616 +325.714 78 skycell.2617 +342.857 78 skycell.2618 +0 82 skycell.2619 +24 82 skycell.2620 +48 82 skycell.2621 +72 82 skycell.2622 +96 82 skycell.2623 +120 82 skycell.2624 +144 82 skycell.2625 +168 82 skycell.2626 +192 82 skycell.2627 +216 82 skycell.2628 +240 82 skycell.2629 +264 82 skycell.2630 +288 82 skycell.2631 +312 82 skycell.2632 +336 82 skycell.2633 +0 86 skycell.2634 +40 86 skycell.2635 +80 86 skycell.2636 +120 86 skycell.2637 +160 86 skycell.2638 +200 86 skycell.2639 +240 86 skycell.2640 +280 86 skycell.2641 +320 86 skycell.2642 +0 90 skycell.2643 Index: /branches/eam_branches/ipp-20110906/tools/czarcal =================================================================== --- /branches/eam_branches/ipp-20110906/tools/czarcal (revision 32630) +++ /branches/eam_branches/ipp-20110906/tools/czarcal (revision 32630) @@ -0,0 +1,136 @@ +#!/usr/bin/env perl + +# create an iCalendar with IPP czar assignments for a given number of days + +use strict; +use warnings; + +use Time::Local; +use Time::localtime; +use Time::gmtime; + + +# number of days that an individual is assinged to be czar +my $czar_days = 2; +my $first_czar = shift; +my $start_date = shift; +my $num_assignments = shift; + +$num_assignments = 1 unless $num_assignments; + + +# list of all czars +my @staff = qw( +mark +gene +bill +roy +serge +heather +chris +); + +die "usage: $0 \n czar list: @staff\n" + unless $start_date and $first_czar; + +my @list = get_czars($first_czar); +die "failed to find $first_czar in @list\n" if !scalar @list; + +my $begin_vcalendar = +"BEGIN:VCALENDAR +CALSCALE:GREGORIAN +VERSION:2.0 +X-WR-CALNAME:IPP Czar calendar for import +METHOD:PUBLISH +PRODID:-//IfA.hawaii.edu//czarcal 1.0//EN"; + +my $end_vcalendar = +"END:VCALENDAR +"; + + + +my $year = substr $start_date, 0, 4; +my $month = substr $start_date, 4, 2; +my $day = substr $start_date, 6, 2; + +my $tm = localtime(timelocal(0, 0, 0, $day, $month - 1, $year - 1900)); +my $time = timelocal(0, 0, 0, $day, $month - 1, $year - 1900); + +print "$begin_vcalendar\n"; + +my $j = 0; +my $this_czar = shift @list; +for (my $i = 0; $i < $num_assignments; $i++) { + $tm = localtime($time); + if ($tm->wday != 6 and $tm->wday != 0) { + + assign_czar($this_czar, $tm->year+1900, $tm->mon+1, $tm->mday); + + if (++$j == $czar_days) { + if (!scalar @list) { + @list = get_czars(); + } + $this_czar = shift @list; + $j = 0; + } + } else { + # bump counter to account for weekend day + $num_assignments++; + } + $time += 86400; +} + + +print "$end_vcalendar\n"; + +sub get_czars { + my $czar = shift; + + my @czarlist = @staff; + if ($czar) { + for (my $i = 0; $i < scalar @staff; $i++) { + my $c = $staff[$i]; + last if $c eq $czar; + shift @czarlist; + } + } + return @czarlist; +} + +my $num_events = 0; +sub assign_czar { +# print STDERR "assign: @_\n"; + my $czar = shift; + my $year = shift; + my $month = shift; + my $day = shift; + + $num_events++; + + my $start = sprintf "%4d%02d%02d", $year, $month, $day; + my $start_ticks = timelocal(0, 0, 0, $day, $month - 1, $year - 1900); + my $end_ticks = $start_ticks + 86400; + + my $tm = localtime($end_ticks); + my $end = sprintf "%4d%02d%02d", $tm->year+1900, $tm->mon + 1, $tm->mday; + + $tm = gmtime(time()); + + my $now = sprintf "%4d%02d%02dT%02d%02d%02dZ", $tm->year+1900, $tm->mon + 1, $tm->mday, $tm->hour, $tm->min, $tm->sec; + + my $uid = "$now-$num_events". '@czarcal.ipp.ifa.hawaii.edu'; + + print "BEGIN:VEVENT\n"; + print "CREATED:$now\n"; + print "UID:$uid\n"; + print "DTEND;VALUE=DATE:$end\n"; + print "TRANSP:TRANSPARENT\n"; + print "SUMMARY:czar $czar\n"; + print "DTSTART;VALUE=DATE:$start\n"; + print "DTSTAMP:$now\n"; + print "SEQUENCE:0\n"; + print "END:VEVENT\n"; +} + + Index: /branches/eam_branches/ipp-20110906/tools/diskspace/merge_counts_and_sizes.pl =================================================================== --- /branches/eam_branches/ipp-20110906/tools/diskspace/merge_counts_and_sizes.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/tools/diskspace/merge_counts_and_sizes.pl (revision 32630) @@ -5,13 +5,20 @@ use PS::IPP::Config; -getopts('N',\%opt); +getopts('ND:',\%opt); my $ipprc = PS::IPP::Config->new("GPC1"); my $nebcon = $ipprc->nebulous(); -my @lt = gmtime; -my $dt = DateTime->now; -$dt->subtract( days => $lt[6] ); -my $datestring = $dt->ymd; + +my $datestring; +if (exists($opt{D})) { + $datestring = $opt{D}; +} +else { + my @lt = gmtime; + my $dt = DateTime->now; + $dt->subtract( days => $lt[6] ); + $datestring = $dt->ymd; +} my $summary_file = "neb:///ipp_diskspace/${datestring}/summary.dat"; @@ -22,12 +29,15 @@ chomp; unless ($_ =~ /^#/) { - ($stage_product,$size_in_gb) = split /\s+/; + ($stage_product,$size_in_gb,$count) = split /\s+/; ($stage,$product) = split /_/, $stage_product; $size{$stage}{$product} = $size_in_gb; + $diskcount{$stage}{$product} = $count; $total += $size_in_gb; } } close(S); + +$diskcount{"EXTERNAL"}{"UNKNOWN"} = 1; my $count_file = "neb:///ipp_diskspace/${datestring}/run_im_counts.dat"; @@ -85,4 +95,5 @@ if ($N{$stage}{$product} > 0) { $VpN{$stage}{$state} += $map{$stage}{$product}{$state} * $size{$stage}{$product} / $N{$stage}{$product}; + $DVpN{$stage}{$state} += $map{$stage}{$product}{$state} * $size{$stage}{$product} / $diskcount{$stage}{$product}; } # print STDERR ">>$stage $product $state $map{$stage}{$product}{$state} $N{$stage}{$product} $count{$stage}{$state} $size{$stage}{$product} $VpN{$stage}{$state}\n"; @@ -94,10 +105,11 @@ if (exists($opt{N})) { - my $dt = DateTime->now; +# my $dt = DateTime->now; - my @lt = gmtime; - my $dt_data = DateTime->now; - $dt->subtract( days => $lt[6]); - my $datestring = $dt->ymd; +# my @lt = gmtime; +# my $dt_data = DateTime->now; +# $dt->subtract( days => $lt[6]); +# my $datestring = $dt->ymd; + my $ipprc = PS::IPP::Config->new("GPC1"); my $nebcon = $ipprc->nebulous(); @@ -124,4 +136,7 @@ $label = "PERMANENT"; } + elsif ($stage eq 'EXTERNAL') { + $label = 'EXTERNAL'; + } else { if (($state eq 'goto_cleaned')|| @@ -145,9 +160,10 @@ } $labels{$label} += $V; - printf("%-8s %-18s %-10s %14.4f %12.6f %9d\n", - $stage,$state,$label,$V,$VpN{$stage}{$state},$count{$stage}{$state}); + $alt_labels{$label} += $DVpN{$stage}{$state} * $count{$stage}{$state}; + printf("%-8s %-18s %-10s\t%14.4f %12.6f\t%14.6f %12.6f\t%9d\n", + $stage,$state,$label,$V,$VpN{$stage}{$state},$DVpN{$stage}{$state} * $count{$stage}{$state},$DVpN{$stage}{$state},$count{$stage}{$state}); if (exists($opt{N})) { - printf($out_fh "%-8s %-18s %-10s %14.4f %12.6f %9d\n", - $stage,$state,$label,$V,$VpN{$stage}{$state},$count{$stage}{$state}); + printf($out_fh "%-8s %-18s %-10s\t%14.4f %12.6f\t%14.6f %12.6f\t%9d\n", + $stage,$state,$label,$V,$VpN{$stage}{$state},$DVpN{$stage}{$state} * $count{$stage}{$state},$DVpN{$stage}{$state},$count{$stage}{$state}); } } @@ -156,14 +172,15 @@ foreach $l (keys %labels) { $sum += $labels{$l}; - print STDERR "\t$l \t$labels{$l}\n"; + $alt_sum += $alt_labels{$l}; + print STDERR "\t$l \t$labels{$l}\t$alt_labels{$l}\n"; if (exists($opt{N})) { - print $out_fh "\t$l \t$labels{$l}\n"; + print $out_fh "\t$l \t$labels{$l}\t$alt_labels{$l}\n"; } } -print STDERR "\tTOTAL \t$sum\n"; +print STDERR "\tTOTAL \t$sum\t$alt_sum\n"; print STDERR "\tEXP_TOT \t$total\n"; if (exists($opt{N})) { - print $out_fh "\tTOTAL \t$sum\n"; + print $out_fh "\tTOTAL \t$sum\t$alt_sum\n"; print $out_fh "\tEXP_TOT \t$total\n"; close($out_fh); Index: /branches/eam_branches/ipp-20110906/tools/diskspace/nebulous_data.pl =================================================================== --- /branches/eam_branches/ipp-20110906/tools/diskspace/nebulous_data.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/tools/diskspace/nebulous_data.pl (revision 32630) @@ -61,5 +61,18 @@ my $neb_dir = "/data/${neb}/nebulous/"; +my $total_volume = 0; +my $used_volume = 0; +open(DF,"df -B 1 /export/${neb}/ |") || die "Cannot run df\n"; +while() { + chomp; + if ($_ =~ /${neb}/) { + $used_volume = (split /\s+/, $_)[2]; + } +} +close(DF); + open(FIND,"find $neb_dir -type f -printf \"%f %s\n\" |") || die "Cannot run find.\n"; + + while () { chomp; @@ -88,6 +101,8 @@ $count{$stage}{$product} += 1; $volume{$stage}{$product} += $size; + $total_volume += $size; } close(FIND); + @@ -98,5 +113,8 @@ } } - +if ($used_volume > 0) { + my $external_volume = $used_volume - $total_volume; + print $out_fh "EXTERNAL UNKNOWN 1 $external_volume\n"; +} close($out_fh); Index: /branches/eam_branches/ipp-20110906/tools/diskspace/nebulous_wiki.pl =================================================================== --- /branches/eam_branches/ipp-20110906/tools/diskspace/nebulous_wiki.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/tools/diskspace/nebulous_wiki.pl (revision 32630) @@ -7,12 +7,16 @@ $opt{S} = 0; -getopts('S',\%opt); +getopts('SD:',\%opt); -my $dt = DateTime->now; - -my @lt = gmtime; -my $dt_data = DateTime->now; -$dt->subtract( days => $lt[6]); -my $datestring = $dt->ymd; +my $datestring; +if (exists($opt{D})) { + $datestring = $opt{D}; +} +else { + my @lt = gmtime; + my $dt = DateTime->now; + $dt->subtract( days => $lt[6] ); + $datestring = $dt->ymd; +} @files = sort ( split /\n/, `neb-ls -p neb:///ipp_diskspace/${datestring}/%neb_usage.dat`); @@ -32,4 +36,5 @@ $sum{$volume} += $size; $sizes{$what} += $size; + $counts{$what} += $count; } close(F); @@ -65,4 +70,5 @@ print "\n"; } + my $dt = DateTime->now; $date = $dt->ymd; print "${presep}${date}${sep}"; @@ -88,5 +94,6 @@ foreach $w (sort (keys %sizes)) { $val = $sizes{$w} / 1024 / 1024 / 1024; - print $out_fh "$w $val\n"; + $v2 = $counts{$w}; + print $out_fh "$w $val $v2\n"; } close($out_fh); Index: /branches/eam_branches/ipp-20110906/tools/diskspace/run_counter.pl =================================================================== --- /branches/eam_branches/ipp-20110906/tools/diskspace/run_counter.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/tools/diskspace/run_counter.pl (revision 32630) @@ -77,4 +77,5 @@ } } +print $out_fh "EXTERNAL any 1\n"; close($out_fh); Index: /branches/eam_branches/ipp-20110906/tools/diskspace/sizes_from_counts.pl =================================================================== --- /branches/eam_branches/ipp-20110906/tools/diskspace/sizes_from_counts.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/tools/diskspace/sizes_from_counts.pl (revision 32630) @@ -1,7 +1,28 @@ #! /usr/local/bin/perl -w + + +use Getopt::Std; +use DateTime; +use PS::IPP::Config; + +getopts('D:',\%opt); + my $count_list = shift; +my $ipprc = PS::IPP::Config->new("GPC1"); +my $nebcon = $ipprc->nebulous(); + + +if (exists($opt{D})) { + my $datestring = $opt{D}; + my $usage_data = "neb:///ipp_diskspace/${datestring}/usage_report.dat"; + my $real_file = $ipprc->file_resolve($usage_data); + open(DATA,$real_file) || die "Cannot find usage_report.dat\n"; +} while() { chomp; + if ($_ =~ /^\s+/) { + next; + } ($stage,$state,$label,undef,$rate,$count) = split /\s+/; $data{$stage}{$state}{L} = $label; Index: /branches/eam_branches/ipp-20110906/tools/neb-reverse =================================================================== --- /branches/eam_branches/ipp-20110906/tools/neb-reverse (revision 32629) +++ /branches/eam_branches/ipp-20110906/tools/neb-reverse (revision 32630) @@ -19,4 +19,5 @@ # Convert colons to slashes. $neb_key =~ s%:%/%g; + $neb_key =~ s%.fz$%%; print "$neb_key\n"; } Index: /branches/eam_branches/ipp-20110906/tools/refstacktool =================================================================== --- /branches/eam_branches/ipp-20110906/tools/refstacktool (revision 32629) +++ /branches/eam_branches/ipp-20110906/tools/refstacktool (revision 32630) @@ -103,9 +103,21 @@ my $n_exps = 0; my $exp_id_last = 0; + +my %exp_hash; while (my $exp = $stmt->fetchrow_hashref()) { # avoid exposures processed multiple times my $exp_id = $exp->{exp_id}; - next if $exp_id_last == $exp_id; + + if ($exp_hash{$exp_id}) { +# print "skipping duplicate for $exp_id\n"; + next; + } +# This doesn't work since we are ordered by fwhm_major +# if ($exp_id_last == $exp_id) { +# print "skipping duplicate for $exp_id\n"; +# } $exp_id_last = $exp_id; + + $exp_hash{$exp_id} = 1; $n_exps++; Index: /branches/eam_branches/ipp-20110906/tools/regpeek.pl =================================================================== --- /branches/eam_branches/ipp-20110906/tools/regpeek.pl (revision 32629) +++ /branches/eam_branches/ipp-20110906/tools/regpeek.pl (revision 32630) @@ -43,5 +43,9 @@ foreach my $row (@{ $data }) { my ($exp_name,$class_id,$bt_state,$data_state,$uri,$exp_id) = @{ $row }; - print "$exp_name $class_id $bt_state $data_state $uri\n"; + print "$exp_name $class_id $bt_state $data_state $uri\t"; + if (($bt_state > -10)&&($data_state eq 'check_burntool')) { + print "#??? regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state pending_burntool -dbname gpc1"; + } + print "\n"; if ($logs) { $reg_log = $uri;
No imageNo imageNo imageNo image
No imageNo imageNo image
Update UpperRight
Update UpperRight
Problem in Upper part!
Problem in Upper part!
Update UpperRight
Problem in Upper part!
4z
5y
6z