IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 25051


Ignore:
Timestamp:
Aug 11, 2009, 3:54:00 PM (17 years ago)
Author:
watersc1
Message:

Bringing branch up-to-date at revision 25049

Location:
branches/czw_branch/cleanup
Files:
100 edited
50 copied

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/cleanup

  • branches/czw_branch/cleanup/Nebulous-Server/Changes

    r24951 r25051  
    3636      instance and just those that are available
    3737    - add Nebulous::Server->prune_object() API
     38    - don't replicate to the volume being implied by the key being replicated!
    3839     
    39400.16

  • branches/czw_branch/cleanup/Nebulous-Server/lib/Nebulous/Server.pm

    r24951 r25051  
    696696    $log->debug("entered - @_");
    697697
    698     my ($key, $vol_name) = validate_pos(@_,
     698    my ($key, $dest_vol_name) = validate_pos(@_,
    699699        {
    700700            type        => SCALAR,
    … …  
    726726    # instance on it.  If all avilable volume already have an instance on them
    727727    # then we should throw an error
    728 
    729     # vol_name overrides the key implied volume
    730     eval {
    731         $key = parse_neb_key($key, $vol_name);
     728    # volume names implied as part of the key are *IGNORED* as the source and
     729    # *SHOULD NOT* be used as the destination either
     730
     731    eval {
     732        $key = parse_neb_key($key);
    732733    };
    733734    $log->logdie("$@") if $@;
    734     $vol_name = $key->volume;
    735735
    736736    my $db  = $self->db($key);
    737737
    738     if (defined $vol_name
     738    # puke if the source volume is bogus, we may want to actually use this as
     739    # the instance to be copied later
     740    if (defined $key->volume
    739741        and not $self->_is_valid_volume_name($key, $key->volume)) {
    740742        unless ($key->hard_volume) {
    741             $log->warn( "$vol_name is not a known volume name" );
    742             $vol_name = undef;
     743            $log->warn($key->volume . " not a known volume name");
    743744        } else {
    744            $log->logdie("$vol_name is not a valid volume name");
    745         }
     745           $log->logdie("$key is not a valid volume name");
     746        }
     747    }
     748    # puke if the source volume is bogus, we may want to actually use this as
     749    # the instance to be copied later
     750    if (defined $dest_vol_name
     751        and not $self->_is_valid_volume_name($key, $dest_vol_name)) {
     752           $log->logdie($key->volume . " is not a valid volume name");
    746753    }
    747754       
    748755    my ($vol_id, $vol_host, $vol_path, $vol_xattr);
    749     if (defined $vol_name) {
     756    if (defined $dest_vol_name) {
    750757        ($vol_id, $vol_host, $vol_path, $vol_xattr)
    751             = $self->_get_storage_volume($key, $vol_name);
     758            = $self->_get_storage_volume($key, $dest_vol_name);
    752759    } else {
    753760        ($vol_id, $vol_host, $vol_path, $vol_xattr)
    … …  
    810817            $db->rollback;
    811818            # handle soft volumes
    812             if (defined $vol_name and not defined $key->hard_volume) {
     819            if (defined $dest_vol_name and not defined $key->hard_volume) {
    813820                $log->debug("retrying with 'any' volume");
    814821                return $self->replicate_object($key->path, 'any');

  • branches/czw_branch/cleanup/Nebulous-Server/lib/Nebulous/Server/Config.pm

    r24370 r25051  
    1313
    1414use Log::Log4perl qw( :levels );
    15 use Params::Validate qw( validate validate_pos SCALAR ARRAYREF );
     15use Params::Validate qw( validate validate_pos SCALAR ARRAYREF UNDEF );
    1616
    1717our %LEVELS = (
    … …  
    3232    dsn         => { type => SCALAR },
    3333    dbuser      => { type => SCALAR },
    34     dbpasswd    => { type => SCALAR },
     34    dbpasswd    => { type => SCALAR | UNDEF },
    3535};
    3636
    … …  
    4848    dsn                 => { type => SCALAR, optional => 1 },
    4949    dbuser              => { type => SCALAR, optional => 1 },
    50     dbpasswd            => { type => SCALAR, optional => 1 },
     50    dbpasswd            => { type => SCALAR | UNDEF, optional => 1 },
    5151    memcached_servers   => { type => ARRAYREF, optional => 1 },
    5252};

  • branches/czw_branch/cleanup/Nebulous-Server/t/05_server_replicate_object.t

    r24372 r25051  
    88use warnings FATAL => qw( all );
    99
    10 use Test::More tests => 26;
     10use Test::More tests => 25;
    1111
    1212use lib qw( ./t ./lib );

  • branches/czw_branch/cleanup/Nebulous-Server/t/09_server_delete_instance.t

    r24637 r25051  
    5858    my $key = "foo";
    5959    my $uri1 = $neb->create_object($key, 'node01');
    60     my $uri2 = $neb->replicate_object($key, 'node2');
     60    my $uri2 = $neb->replicate_object($key, 'node02');
    6161
    6262    # make one of the instances unavailable

  • branches/czw_branch/cleanup/Nebulous/Changes

    r24951 r25051  
    88    - add Nebulous::Client->prune() method
    99    - add Nebulous::Client->there_can_be_only_one() method
     10    - add neb-cull --one_only flag
    1011
    11120.10

  • branches/czw_branch/cleanup/Nebulous/MANIFEST

    r24951 r25051  
    102102t/70_neb-ls.t
    103103t/71_client_there_can_be_only_one.t
     104t/72_neb-cull.t
    104105t/90_nebclient.t
    105106t/TEST.PL

  • branches/czw_branch/cleanup/Nebulous/bin/neb-cull

    r16183 r25051  
    1616use Pod::Usage qw( pod2usage );
    1717
    18 my ($server, $volume);
     18my ($server, $volume, $one_only);
    1919
    2020$server = $ENV{'NEB_SERVER'} unless $server;
    … …  
    2323    'server|s=s'    => \$server,
    2424    'volume|v=s'    => \$volume,
     25    'one_only|o'    => \$one_only,
    2526) || pod2usage( 2 );
    2627
    … …  
    4041    unless defined $neb;
    4142
     43my @cull_args;   
     44push @cull_args, $key;
    4245if (defined $volume) {
    43     $neb->cull($key, $volume)
    44         or die "failed to replicate Nebulous key: $key";
     46    push @cull_args, $volume;
     47}
     48
     49if ($one_only) {
     50    my $status = $neb->there_can_be_only_one(@cull_args);
     51    if ($status == 0) {
     52        die "not enough instances of Nebulous key: $key";
     53    }
     54    if (not defined $status or $status < 1) {
     55        die "failed to cull Nebulous key: $key - " . $neb->err
     56            if defined $neb->err;
     57        die "failed to cull Nebulous key: $key";
     58    }
    4559} else {
    46     $neb->cull($key)
    47         or die "failed to replicate Nebulous key: $key";
     60    unless (defined $neb->cull(@cull_args)) {
     61        die "failed to cull Nebulous key: $key - " . $neb->err;
     62    }
    4863}
    4964
    … …  
    5873=head1 SYNOPSIS
    5974
    60     neb-cull [--server <URL>] [--volume <volume name>] <key>
     75    neb-cull [--server <URL>] [--volume <volume name>] [--one_only] <key>
    6176
    6277=head1 DESCRIPTION
    … …  
    7792
    7893Symbolic name of the volume to create the new instance on.
     94
     95Optional.
     96
     97=item * --one_only|-o
     98
     99Removes all but one instances of an object.  Any inaccessible instances are
     100removed.  If C<--volume> is specified, and an instance exists on that volume,
     101all instances but the one on that volume are removed.
    79102
    80103Optional.
    … …  
    110133=head1 COPYRIGHT
    111134
    112 Copyright (C) 2007-2008  Joshua Hoblitt.  All rights reserved.
     135Copyright (C) 2007-2009  Joshua Hoblitt.  All rights reserved.
    113136
    114137This program is free software; you can redistribute it and/or modify it under

  • branches/czw_branch/cleanup/Nebulous/lib/Nebulous/Client.pm

    r24951 r25051  
    297297        my $instances = $stats[6];
    298298        if (defined $instances and $instances < 2) {
    299             $log->debug("can not cull - not enough instances");
     299            $self->set_err("can not cull - not enough instances");
    300300            $log->debug("leaving");
    301301
    … …  
    305305        $locations = $self->find_instances($key, $vol_name);
    306306        unless (defined $locations) {
    307             $log->debug( "no instances" );
     307            $self->set_err( "no instances" );
    308308            $log->debug( "leaving" );
    309309
    … …  
    313313        $locations = $self->find_instances($key);
    314314        unless ($locations) {
    315             $log->debug("no instances");
     315            $self->set_err( "no instances" );
    316316            $log->debug("leaving");
    317317
    … …  
    320320
    321321        if (scalar @{ $locations } < 2) {
    322             $log->debug("can not cull - not enough instances");
     322            $self->set_err("can not cull - not enough instances");
    323323            $log->debug("leaving");
    324324
    … …  
    365365        if (not defined $self->prune($key)) {
    366366            # use the prune() method to also determine if $key is valid
    367             $removed = undef;
    368             return;
     367            die "invalid key: $key";
    369368        }
    370369
    … …  
    377376        if (defined $locations) {
    378377            my $instances = $self->find_instances($key);
     378            if (scalar @$instances < 2) {
     379                die "not enough instances";
     380            }
    379381            foreach my $victim (@$instances) {
    380382                next if $victim eq $locations->[0];
    … …  
    387389            # start at one so cull() is called one less time then the # of
    388390            # instances
     391            if ($stats->[6] < 2) {
     392                die "not enough instances";
     393            }
    389394            for (my $i = 1; $i < $stats->[6]; $i++) {
    390395                $self->cull($key);
    … …  
    395400    };
    396401    if ($@) {
     402        if ($@ =~ qr/invalid key/) {
     403            $self->set_err($@);
     404            return;
     405        }
    397406        $log->logdie($@);
    398407    }

  • branches/czw_branch/cleanup/Nebulous/nebclient/configure.ac

    r20208 r25051  
    55AC_PREREQ(2.61)
    66
    7 AC_INIT([nebclient], [0.08], [jhoblitt@cpan.org])
     7AC_INIT([nebclient], [0.17], [jhoblitt@cpan.org])
    88AC_CONFIG_SRCDIR([nebulous.wsdl])
    99

  • branches/czw_branch/cleanup/Nebulous/nebclient/src/nebclient.c

    r24951 r25051  
    5151static char *nullstrncpy(char *dest, const char *src, size_t n);
    5252
     53
    5354nebServer *nebServerAlloc(const char *endpoint)
    5455{
    55     nebServer       *server;
    56 
    57     server          = xmalloc(sizeof(nebServer));
     56    nebServer *server = xmalloc(sizeof(nebServer));
    5857
    5958    if (endpoint) {
    … …  
    7776}
    7877
     78
    7979void nebServerFree(nebServer *server)
    8080{
    … …  
    9292}
    9393
     94
    9495char *nebCreate(nebServer *server, const char *key, const char *volume, char **URI)
    9596{
     97    REQUIRE_SERVER;
     98
     99    if (!key) {
     100        nebSetErr(server, "parameter 'key' may not be NULL");
     101
     102        return NULL;
     103    }
     104
    96105    struct ns1__create_USCOREobjectResponse response;
    97     char            *filename;
    98 
    99     REQUIRE_SERVER;
    100 
    101     if (!key) {
    102         nebSetErr(server, "parameter 'key' may not be NULL");
    103 
    104         return NULL;
    105     }
    106 
    107106    if (soap_call_ns1__create_USCOREobject(server->soap, server->endpoint, NULL,
    108107            (char *)key, (char *)volume, (char **)&response) != SOAP_OK) {
    … …  
    117116    }
    118117
     118    char *filename;
    119119    if (!nebParseURI(server, (char *)response.result, &filename)) {
    120120        nebSetErr(server, "can not parse URI");
     121        nebFree(filename);
     122
    121123        return NULL;
    122124    }
    … …  
    131133}
    132134
     135
    133136int nebOpenCreate(nebServer *server, const char *key, const char *volume, char **URI)
    134137{
    135     char            *filename;
    136     int             file;
    137 
    138     REQUIRE_SERVER;
    139 
    140     if (!key) {
    141         nebSetErr(server, "parameter 'key' may not be NULL");
    142 
    143         return -1;
    144     }
    145 
    146     filename = nebCreate(server, key, volume, URI);
    147 
    148     file = open(filename, O_RDWR|O_TRUNC, 0660);
     138    REQUIRE_SERVER;
     139
     140    if (!key) {
     141        nebSetErr(server, "parameter 'key' may not be NULL");
     142
     143        return -1;
     144    }
     145
     146    char *filename = nebCreate(server, key, volume, URI);
     147    if (!filename) {
     148        return -1;
     149    }
     150
     151    int file = open(filename, O_RDWR|O_TRUNC, 0660);
    149152    if (file == -1) {
    150153        nebSetErr(server, "can not open %s: %s", filename, strerror(errno));
    … …  
    159162}
    160163
     164
    161165bool nebReplicate(nebServer *server, const char *key, const char *volume, char **URI)
    162166{
    163     struct ns1__replicate_USCOREobjectResponse response;
    164     char            *filename;
    165     int             sourceFH, destFH;
    166 
    167167    REQUIRE_SERVER;
    168168
    … …  
    177177    // we don't do this, it's possible that open will find & open the new
    178178    // instance that we're in the process of creating
    179     sourceFH = nebOpen(server, (char *)key, NEB_READ);
     179    int sourceFH = nebOpen(server, (char *)key, NEB_READ);
    180180    if (sourceFH == -1) {
    181181        nebSetErr(server, "can not open key %s", key);
    182         nebFree(filename);
    183182
    184183        return false;
    … …  
    186185
    187186    // ask the server for a new instance attached to our key
     187    struct ns1__replicate_USCOREobjectResponse response;
    188188    if (soap_call_ns1__replicate_USCOREobject(server->soap, server->endpoint,
    189189            NULL, (char *)key, (char *)volume, (char **)&response) != SOAP_OK) {
    … …  
    193193    }
    194194
     195    char *filename = NULL;
    195196    if (!nebParseURI(server, (char *)response.result, &filename)) {
    196197        nebSetErr(server, "can not parse URI");
    … …  
    199200    }
    200201
    201     destFH = open(filename, O_RDWR|O_TRUNC, 0660);
     202    int destFH = open(filename, O_RDWR|O_TRUNC, 0660);
    202203    if (destFH == -1) {
    203204        nebSetErr(server, "can not open %s: %s\n", filename, strerror(errno));
    … …  
    234235}
    235236
     237
    236238bool nebCull(nebServer *server, const char *key)
    237239{
    238     nebObjectInstances *locations;
    239 
    240     REQUIRE_SERVER;
    241 
    242     if (!key) {
    243         nebSetErr(server, "parameter 'key' may not be NULL");
    244 
    245         return false;
    246     }
    247 
    248     locations = nebFindInstances(server, key, NULL);
     240    REQUIRE_SERVER;
     241
     242    if (!key) {
     243        nebSetErr(server, "parameter 'key' may not be NULL");
     244
     245        return false;
     246    }
     247
     248    nebObjectInstances *locations = nebFindInstances(server, key, NULL);
    249249    if (!locations || locations->n <= 1) {
    250250        nebSetErr(server, "can not cull - not enough instances");
     251        if (locations) {
     252            nebObjectInstancesFree(locations);
     253        }
    251254
    252255        return false;
    … …  
    264267}
    265268
     269
    266270bool nebLock(nebServer *server, const char *key, nebRW flag)
    267271{
    268     time_t          endtime;
    269     char            *type;
    270     int             response;
    271 
    272     REQUIRE_SERVER;
    273 
    274     if (!key) {
    275         nebSetErr(server, "parameter 'key' may not be NULL");
    276 
    277         return false;
    278     }
    279 
    280     type = flag == NEB_READ ? "read" : "write";
    281 
    282     endtime = time(NULL) + LOCK_TIMEOUT;
     272    REQUIRE_SERVER;
     273
     274    if (!key) {
     275        nebSetErr(server, "parameter 'key' may not be NULL");
     276
     277        return false;
     278    }
     279
     280    char *type = flag == NEB_READ ? "read" : "write";
     281
     282    time_t endtime = time(NULL) + LOCK_TIMEOUT;
    283283
    284284    while (time(NULL) < endtime) {
     285        int response;
    285286        if (soap_call_ns1__lock_USCOREobject(server->soap, server->endpoint,
    286287            NULL, (char *)key, type, &response) != SOAP_OK) {
    … …  
    302303}
    303304
     305
    304306bool nebUnlock(nebServer *server, const char *key, nebRW flag)
    305307{
    306     char            *type;
    307     int             response;
    308 
    309     REQUIRE_SERVER;
    310 
    311     if (!key) {
    312         nebSetErr(server, "parameter 'key' may not be NULL");
    313 
    314         return false;
    315     }
    316 
    317     type = flag == NEB_READ ? "read" : "write";
    318 
     308    REQUIRE_SERVER;
     309
     310    if (!key) {
     311        nebSetErr(server, "parameter 'key' may not be NULL");
     312
     313        return false;
     314    }
     315
     316    char *type = flag == NEB_READ ? "read" : "write";
     317
     318    int response;
    319319    if (soap_call_ns1__unlock_USCOREobject(server->soap, server->endpoint,
    320320        NULL, (char *)key, type, &response) != SOAP_OK) {
    … …  
    466466nebObjectInstances *nebFindInstances(nebServer *server, const char *key, const char *volume)
    467467{
     468    REQUIRE_SERVER;
     469
     470    if (!key) {
     471        nebSetErr(server, "parameter 'key' may not be NULL");
     472
     473        return NULL;
     474    }
     475
    468476    struct ns1__find_USCOREinstancesResponse response;
    469     int             resultElements = 0;
    470     char            **resultArray;
    471     nebObjectInstances *locations = NULL;
    472 
    473     REQUIRE_SERVER;
    474 
    475     if (!key) {
    476         nebSetErr(server, "parameter 'key' may not be NULL");
    477 
    478         return NULL;
    479     }
    480 
    481477    // FIXME is this leaking memory when response goes out of scope?  the gsoap
    482478    // manual seems to 'suggest' that this is temporary data that gets cleaed
    … …  
    488484    }
    489485
    490     resultElements = response.result->__size;
    491 
     486    int resultElements = response.result->__size;
     487
     488    nebObjectInstances *locations = NULL;
    492489    if (resultElements) {
    493490        locations = xmalloc(sizeof(nebObjectInstances));
    494491        locations->n = resultElements;
    495492       
    496         resultArray    = response.result->__ptr;
     493        char **resultArray = response.result->__ptr;
    497494
    498495        locations->URI = xmalloc(sizeof(char*) * resultElements);
    … …  
    508505}
    509506
     507
    510508void nebObjectInstancesFree(nebObjectInstances *locations)
    511509{
    … …  
    521519}
    522520
     521
    523522char *nebFind(nebServer *server, const char *key)
    524523{
    525     nebObjectInstances *locations;
    526     char            *filename;
    527 
    528524    REQUIRE_SERVER;
    529525
    … …  
    534530    }
    535531   
    536     locations = nebFindInstances(server, key, NULL);
     532    nebObjectInstances *locations = nebFindInstances(server, key, NULL);
    537533    if (!locations) {
    538534        if (!strstr(nebErr(server), "no instances on storage volume")) {
    … …  
    548544    }
    549545
     546    char *filename;
    550547    if (!nebParseURI(server, locations->URI[0], &filename)) {
    551548        nebSetErr(server, "can not parse URI");
    … …  
    560557}
    561558
     559
    562560int nebOpen(nebServer *server, const char *key, nebRW flag)
    563561{
    564     nebObjectInstances *locations;
    565     char            *filename;
    566     int             fh;
    567 
    568     REQUIRE_SERVER;
    569 
    570     if (!key) {
    571         nebSetErr(server, "parameter 'key' may not be NULL");
    572 
    573         return -1;
    574     }
    575 
    576     locations = nebFindInstances(server, key, NULL);
     562    REQUIRE_SERVER;
     563
     564    if (!key) {
     565        nebSetErr(server, "parameter 'key' may not be NULL");
     566
     567        return -1;
     568    }
     569
     570    nebObjectInstances *locations = nebFindInstances(server, key, NULL);
    577571    if (!locations) {
    578572        nebSetErr(server, "no instances found");
    … …  
    581575    }
    582576
     577    char *filename;
    583578    if (!nebParseURI(server, locations->URI[0], &filename)) {
    584579        nebSetErr(server, "can not parse URI");
    … …  
    588583    }
    589584
     585    int fh = 0;
    590586    if (flag == NEB_WRITE) {
    591587        if (locations->n > 1) {
    … …  
    613609}
    614610
     611
    615612bool nebDelete(nebServer *server, const char *key)
    616613{
    617     nebObjectInstances *locations;
    618 
    619     REQUIRE_SERVER;
    620 
    621     if (!key) {
    622         nebSetErr(server, "parameter 'key' may not be NULL");
    623 
    624         return false;
    625     }
    626 
    627     locations = nebFindInstances(server, key, "any");
     614    REQUIRE_SERVER;
     615
     616    if (!key) {
     617        nebSetErr(server, "parameter 'key' may not be NULL");
     618
     619        return false;
     620    }
     621
     622    nebObjectInstances *locations = nebFindInstances(server, key, "any");
    628623    if (!locations) {
    629624        nebSetErr(server, "no instances found");
    … …  
    646641}
    647642
     643
    648644bool nebCopy(nebServer *server, const char *key, const char *newKey)
    649645{
    650     int             sourceFH;
    651     int             destFH;
    652 
    653646    REQUIRE_SERVER;
    654647
    … …  
    665658    }
    666659
    667     sourceFH = nebOpen(server, (char *)key, NEB_READ);
     660    int sourceFH = nebOpen(server, (char *)key, NEB_READ);
    668661    if (sourceFH < 0) {
    669662        // propigate nebOpen() error
    … …  
    672665    }
    673666
    674     destFH = nebOpenCreate(server, newKey, NULL, NULL);
     667    int destFH = nebOpenCreate(server, newKey, NULL, NULL);
    675668    if (destFH < 0) {
    676669        // propigate nebCreate() error
    … …  
    696689}
    697690
     691
    698692bool nebMove(nebServer *server, const char *key, const char *newKey)
    699693{
    … …  
    723717}
    724718
     719
    725720bool nebSwap(nebServer *server, const char *key1, const char *key2)
    726721{
    727     struct ns1__create_USCOREobjectResponse response;
    728 
    729722    REQUIRE_SERVER;
    730723
    … …  
    741734    }
    742735
     736    struct ns1__create_USCOREobjectResponse response;
    743737    if (soap_call_ns1__swap_USCOREobjects(server->soap, server->endpoint,
    744738            NULL, (char *)key1, (char *)key2, (char **)&response) != SOAP_OK) {
    … …  
    749743}
    750744
     745
    751746bool nebDeleteInstance(nebServer *server, const char *key, const char *URI)
    752747{
    753     int             response;
    754     char            *filename;
    755     bool            status;
    756 
    757748    REQUIRE_SERVER;
    758749
    … …  
    763754    }
    764755
     756    char *filename;
    765757    if (!nebParseURI(server, URI, &filename)) {
    766758        nebSetErr(server, "can not parse URI");
    … …  
    769761    }
    770762
     763    int response;
    771764    if (soap_call_ns1__delete_USCOREinstance(server->soap, server->endpoint,
    772765        NULL, (char *)key, (char *)URI, &response) != SOAP_OK) {
    … …  
    776769    }
    777770
     771    bool status;
    778772    if (response > 0) {
    779773        status = nebNukeFile(server, filename);
    … …  
    787781}
    788782
     783
    789784nebObjectStat *nebStat(nebServer *server, const char *key)
    790785{
    791     struct ns1__stat_USCOREobjectResponse response;
    792     nebObjectStat   *stat;
    793     int             resultElements = 0;
    794     char            **resultArray = NULL;
    795 
    796786    REQUIRE_SERVER;
    797787
    … …  
    802792    }
    803793
    804     stat = xmalloc(sizeof(nebObjectStat));
     794    nebObjectStat *stat = xmalloc(sizeof(nebObjectStat));
    805795
    806796    // FIXME is this leaking memory when response goes out of scope?  the gsoap
    807797    // manual seems to 'suggest' that this is temporary data that gets cleaed
    808798    // up on the next soap function call
     799    struct ns1__stat_USCOREobjectResponse response;
    809800    if (soap_call_ns1__stat_USCOREobject(server->soap, server->endpoint,
    810801            NULL, (char *)key, &response) != SOAP_OK) {
    … …  
    813804    }
    814805
    815     resultElements = response.result->__size;
    816     resultArray    = response.result->__ptr;
     806    int resultElements = response.result->__size;
     807    char **resultArray    = response.result->__ptr;
    817808
    818809    if (resultElements != 8) {
    … …  
    821812    }
    822813
    823     nullstrncpy(stat->so_id, resultArray[0], 256);
    824     nullstrncpy(stat->ext_id, resultArray[1], 256);
    825     nullstrncpy(stat->read_lock, resultArray[2], 256);
     814    stat->so_id     = atol(resultArray[0]);
     815    stat->ext_id    = atol(resultArray[1]);
     816    stat->read_lock = atoi(resultArray[2]);
    826817    nullstrncpy(stat->write_lock, resultArray[3], 256);
    827818    nullstrncpy(stat->epoch, resultArray[4], 256);
    … …  
    833824}
    834825
     826
    835827void nebObjectStatFree(nebObjectStat *stat)
    836828{
    … …  
    839831    }
    840832
    841     xfree(stat->so_id);
    842     xfree(stat->ext_id);
    843     xfree(stat->read_lock);
    844     xfree(stat->write_lock);
    845     xfree(stat->epoch);
    846     xfree(stat->mtime);
    847     xfree(stat);
     833    nebFree(stat);
    848834}
    849835
    … …  
    895881    return response;
    896882}
     883
    897884
    898885int nebThereCanBeOnlyOne(nebServer *server, const char *key, const char *volume)
    … …  
    936923        nebObjectInstancesFree(locations);
    937924    } else {
     925        nebObjectInstancesFree(locations);
    938926        // nuke whatever
    939927        nebObjectStat *stat = nebStat(server, key);
    … …  
    941929            return -1;
    942930        }
     931
     932        int available = stat->available;
     933        nebObjectStatFree(stat);
     934
    943935        // start at one so cull() is called one less time then the # of
    944936        // instances
    945         for (long i = 1; i < stat->available; i++) {
     937        for (long i = 1; i < available; i++) {
    946938            if (!nebCull(server, key)) {
    947                 nebObjectStatFree(stat);
    948939                return -1;
    949940            }
    950941            removed++;
    951942        }
    952         nebObjectStatFree(stat);
    953943    }
    954944
    … …  
    962952}
    963953
     954
    964955void nebFree(void *ptr) {
     956    if (!ptr) { return; }
    965957    xfree(ptr);
    966958}
    … …  
    969961static void p_nebSetErr(const char * filename, unsigned long lineno, const char *function, nebServer *server, const char *format, ...)
    970962{
    971     va_list         args;
    972     size_t          err_size;
    973     char            err_location[ERRBUF_SIZE];
    974     size_t          err_location_size;
    975 
    976     err_location_size = snprintf(err_location, ERRBUF_SIZE, "%s:%lu %s() - ",
     963    char err_location[ERRBUF_SIZE];
     964    size_t err_location_size = snprintf(err_location, ERRBUF_SIZE, "%s:%lu %s() - ",
    977965            filename, lineno, function);
    978966
    … …  
    987975
    988976    while (1) {
     977        va_list args;
    989978        va_start(args, format);
    990979        // append error string after error location info
    991         err_size = vsnprintf(
     980        size_t err_size = vsnprintf(
    992981                server->err_buf + err_location_size,
    993982                server->err_buf_size - err_location_size,
    … …  
    10131002}
    10141003
     1004
    10151005static void nebSetServerErr(nebServer *server)
    10161006{
    1017     const char      **code;
    1018     const char      **string;
    1019 
    1020     REQUIRE_SERVER;
    1021 
    1022     code   = soap_faultcode(server->soap);
    1023     string = soap_faultstring(server->soap);
     1007    REQUIRE_SERVER;
     1008
     1009    const char **code   = soap_faultcode(server->soap);
     1010    const char **string = soap_faultstring(server->soap);
    10241011
    10251012    nebSetErr(server, "%s - %s", *code, *string);
    10261013}
     1014
    10271015
    10281016#if NEBCOPYFILE
    10291017static off_t nebCopyFile(nebServer *server, const char *source, const char *dest)
    10301018{
    1031     int             sourceFH;
    1032     int             destFH;
    1033     off_t           bytesCopied;
    1034 
    1035     REQUIRE_SERVER;
    1036 
    1037     sourceFH = open(source, O_RDONLY);
     1019    REQUIRE_SERVER;
     1020
     1021    int sourceFH = open(source, O_RDONLY);
    10381022    if (sourceFH == -1) {
    10391023        nebSetErr(server, "can not open %s: %s", source, strerror(errno));
    … …  
    10421026    }
    10431027
    1044     destFH = open(dest, O_WRONLY|O_CREAT|O_CREAT, 0600);
     1028    int destFH = open(dest, O_WRONLY|O_CREAT|O_CREAT, 0600);
    10451029    if (destFH == -1) {
    10461030        nebSetErr(server, "can not open %s: %s", dest, strerror(errno));
    … …  
    10491033    }
    10501034
    1051     bytesCopied = nebCopyFilehandle(server, sourceFH, destFH);
     1035    off_t bytesCopied = nebCopyFilehandle(server, sourceFH, destFH);
    10521036    if (bytesCopied < 0) {
    10531037        return -1;
    … …  
    10701054#endif
    10711055
     1056
    10721057static off_t nebCopyFilehandle(nebServer *server, int sourceFH, int destFH)
    10731058{
    1074     off_t           bytesTotal;
    1075     off_t           bytesRemaining;
    1076     off_t           writeSize;
    1077     char            writeBuf[64 * 1024];
     1059    REQUIRE_SERVER;
     1060
    10781061    struct stat     sourceStat;
    1079 
    1080     REQUIRE_SERVER;
    1081 
    10821062    if(fstat(sourceFH, &sourceStat)) {
    10831063        nebSetErr(server, "can not stat filehandles: %s", strerror(errno));
    … …  
    10871067
    10881068    // the size of the file to copied
    1089     bytesTotal = sourceStat.st_size;
     1069    off_t bytesTotal = sourceStat.st_size;
    10901070
    10911071    /*
    … …  
    10951075    */
    10961076
    1097     bytesRemaining = bytesTotal;
     1077    off_t bytesRemaining = bytesTotal;
    10981078
    10991079    while (bytesRemaining) {
     1080        off_t writeSize;
     1081        char writeBuf[64 * 1024];
    11001082        if (bytesRemaining <= sizeof(writeBuf)) {
    11011083            writeSize = bytesRemaining;
    … …  
    11221104}
    11231105
     1106
    11241107static size_t nebParseURI(nebServer *server, const char *URI, char **filename)
    11251108{
    1126     regex_t         myregex;
    1127     regmatch_t      mymatch[2];
    1128     char            errbuf[ERRBUF_SIZE];
    1129     char            *pattern = "^file://(.*)$";
    1130     int             status;
    1131     size_t          matchStart, matchEnd;
    1132     size_t          matchLength;
    1133     size_t          filename_size;
    1134 
    11351109    REQUIRE_SERVER;
    11361110
    … …  
    11471121    }
    11481122
    1149     status = regcomp(&myregex, pattern, REG_EXTENDED);
     1123    regex_t myregex;
     1124    regmatch_t mymatch[2];
     1125    char *pattern = "^file://(.*)$";
     1126    char errbuf[ERRBUF_SIZE];
     1127
     1128    int status = regcomp(&myregex, pattern, REG_EXTENDED);
    11501129    if (status != 0) {
    11511130        regerror(status, &myregex, errbuf, ERRBUF_SIZE);
    … …  
    11651144    regfree(&myregex);
    11661145
    1167     matchStart = (size_t)mymatch[1].rm_so;
    1168     matchEnd   = (size_t)mymatch[1].rm_eo;
    1169     matchLength = matchEnd - matchStart;
     1146    size_t matchStart = (size_t)mymatch[1].rm_so;
     1147    size_t matchEnd   = (size_t)mymatch[1].rm_eo;
     1148    size_t matchLength = matchEnd - matchStart;
    11701149
    11711150    if (matchStart == -1) {
    … …  
    11731152    }
    11741153
    1175     filename_size = matchLength + 1;
     1154    size_t filename_size = matchLength + 1;
    11761155    *filename = xmalloc(filename_size);
    11771156
    … …  
    11801159    return strlen(*filename);
    11811160}
     1161
    11821162
    11831163static bool nebNukeFile(nebServer *server, const char *filename)
    … …  
    11991179    return true;
    12001180}
     1181
    12011182
    12021183static char *nullstrncpy(char *dest, const char *src, size_t n) {
    … …  
    12061187    }
    12071188
     1189    if (!dest) {
     1190        dest = xmalloc(n + 1);
     1191    }
     1192
    12081193    return strncpy(dest, src, n);
    12091194}

  • branches/czw_branch/cleanup/Nebulous/nebclient/src/nebclient.h

    r24951 r25051  
    3838
    3939typedef struct {
    40     char            so_id[256];         ///< storage object ID
    41     char            ext_id[256];        ///< storage object key (name)
    42     char            read_lock[256];     ///< read lock value
     40    long            so_id;              ///< storage object ID
     41    long            ext_id;             ///< storage object key (name)
     42    int             read_lock;          ///< read lock value
    4343    char            write_lock[256];    ///< write lock value
    4444    char            epoch[256];         ///< creation time stamp

  • branches/czw_branch/cleanup/Nebulous/nebclient/tests/tests.c

    r24951 r25051  
    9898
    9999    char *copies = nebGetXattr(server, key, "user.copies");
    100     neb_ok(server, strcmp(copies, "2") == 0, "get user.copies xattr");
     100    neb_ok(server, copies != NULL && strcmp(copies, "2") == 0, "get user.copies xattr");
    101101    char **xattrs = NULL;
    102102    int n_xattrs = nebListXattr(server, key, &xattrs);
    103     printf("# %s\n\n", xattrs[0]);
    104     neb_ok(server, n_xattrs == 1, "count of xattrs");
    105     neb_ok(server, strcmp(xattrs[0], "user.copies") == 0, "user.copies xattr exists");
     103    neb_ok(server, xattrs != NULL && n_xattrs == 1, "count of xattrs");
     104    neb_ok(server, xattrs != NULL && strcmp(xattrs[0], "user.copies") == 0, "user.copies xattr exists");
    106105    neb_ok(server, nebRemoveXattr(server, key, "user.copies"), "remove user.copies xattr");
    107106

  • branches/czw_branch/cleanup/Ohana/src/addstar/Makefile

    r21508 r25051  
    5858$(SRC)/greference.$(ARCH).o \
    5959$(SRC)/grefstars.$(ARCH).o \
     60$(SRC)/GetZeroPointExposure.$(ARCH).o \
    6061$(SRC)/LoadStars.$(ARCH).o \
    6162$(SRC)/LoadHeaders.$(ARCH).o \
    … …  
    199200$(SRC)/replace_match.$(ARCH).o \
    200201$(SRC)/update_coords.$(ARCH).o \
     202$(SRC)/SkyRegionUtils.$(ARCH).o \
    201203$(SRC)/StarOps.$(ARCH).o \
    202204$(SRC)/ConfigInit.$(ARCH).o \

  • branches/czw_branch/cleanup/Ohana/src/addstar/doc/notes.txt

    r21508 r25051  
     1
     22009.08.02
     3
     4  ZERO POINTS: addstar currently has a couple of inconsistent ways that it
     5  measures or uses image zero points:
     6
     7  * the standard zero point: data in the database is saved as M_inst +
     8    25.0.  This value is written to the image header and checked when
     9    the image is loaded.  This is completely archaic, but removing
     10    this concept from the DVO system will require some work and some
     11    care.  This value is NOT reflected in the image->Mcal values
     12
     13  * on-the-fly calibration: addstar was used by CFHT skyprobe to
     14    measure the sky transparency.  This calculation was done by
     15    measuring the zero point between the Tycho stars in the database
     16    and the stars in the image.  Star data is saved in
     17    AddToCalibration as they are found (in find_matches, or
     18    equivalent).  At the end of the match processing, the zero point
     19    for the image was calculated with FindCalibration and saved to the
     20    image table but NOT the measure table.  For skyprobe, the stars
     21    are normally not saved to the database -- the Tycho stars are kept
     22    in the measure & average tables, and used as the reference for the
     23    zero point calculation above.
     24
     25  For Pan-STARRS, the zero point analysis is performed by psastro
     26  before we run addstar.  However, there are a few ways we could
     27  choose to apply the zero point calculation:
     28
     29  * use the per-chip zero point reported in the individual chip
     30    headers (easy, but fairly wrong)
     31
     32  * calculate the per-exposure zero point from the chip headers and
     33    supply.
     34
     35  * calculate the per-exposure zero point from the MATCHED_REFS table
     36
     37  We also need to consider the zero point error.  We currently have
     38  only dM (poisson error) + dMcal (calibration error).  We need to
     39  divide dMsys into dMcal (photometry calibration error) and dMap
     40  (aperture correction error).
     41
     42  * zero point options:
     43    * NOMINAL : set Mcal = 0.0
     44    * CHIP_HEADER : each chip has zero point in header
     45    * PHU_HEADER : a single exposure-wide value in the header
     46    * CHIP_AVERAGE : determine zero point for exposure from chip zero points
     47    * MATCHED_REFS : recalculate from table
    148
    2492009.02.11

  • branches/czw_branch/cleanup/Ohana/src/addstar/include/2mass.h

    r17439 r25051  
    3333e_time    get2mass_date (char *ptr, int Nbound, int Nmax);
    3434
    35 int       load2mass_as_rawdata (SkyTable *skytable, char *filename, AddstarClientOptions options);
     35int       load2mass_as_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options);
    3636SkyTable *load2mass_acc (char *path, char *accel);
    3737int       get2mass_3star (Stars *star, char *line, int Nmax);

  • branches/czw_branch/cleanup/Ohana/src/addstar/include/addstar.h

    r24400 r25051  
    7373char   TYCHO_DIR[256];
    7474char   SubpixDatafile[256];
     75char   *USE_NAME;
    7576char   PASSWORD[80];
    7677char   HOSTNAME[80];
    … …  
    118119double  FAKE_THETA;     // boresite angle for fake images
    119120
     121char    ZERO_POINT_OPTION[64];
     122float   ZERO_POINT_OFFSET;
     123
    120124// carries the mosaic into gstars
    121125
    … …  
    187191HeaderSet *MatchHeaders           PROTO((int **extsize, int *nimage, int mode, Header **headers, int Nheaders));
    188192int        LoadData               PROTO((FILE *f, char *file, Image **images, int *nvalid, Stars **stars, int *Nstars, Header **headers, int *extsize, HeaderSet *headerSets, int NheaderSets));
     193int        GetZeroPointExposure   PROTO((Header **headers, HeaderSet *headerSets, int Nimages));
    189194
    190195int        in_image               PROTO((double r, double d, Image *image));

  • branches/czw_branch/cleanup/Ohana/src/addstar/src/ConfigInit.c

    r21508 r25051  
    7777  if (!ScanConfig (config, "EXTNAME-KEYWORD",        "%s",  0, ExtnameKeyword)) {
    7878      strcpy (ExtnameKeyword, "EXTNAME");
     79  }
     80
     81  // if this config variable is not set, do not set any zero point offset
     82  if (!ScanConfig (config, "ZERO_POINT_OPTION", "%s",  0, ZERO_POINT_OPTION)) {
     83      strcpy (ZERO_POINT_OPTION, "NOMINAL");
    7984  }
    8085

  • branches/czw_branch/cleanup/Ohana/src/addstar/src/LoadData.c

    r21508 r25051  
    2323
    2424  // find image rootname
    25   name = filebasename (file);
     25  if (USE_NAME) {
     26      name = filebasename (USE_NAME);
     27  } else {
     28      name = filebasename (file);
     29  }
     30
     31  // if zero points are calculated for the full exposure using more than just the matched chip header,
     32  // we need to perform that analysis here
     33  GetZeroPointExposure (headers, headerSets, Nimages);
    2634
    2735  // now run through the images, interpret the headers and read the stars

  • branches/czw_branch/cleanup/Ohana/src/addstar/src/ReadImageHeader.c

    r21508 r25051  
    88  double tmp, sec, Cerror, ZeroPt, FWHM_X, FWHM_Y;
    99  char *c, photname[64], line[80];
     10  PhotCode *photcodeData = NULL;
    1011
    1112  // zero out the entire image structure
    … …  
    104105      return (FALSE);
    105106    }
    106     photcode = GetPhotcodeCodebyName (photname);
    107     if (photcode == 0) {
     107    photcodeData = GetPhotcodebyName (photname);
     108    if (photcodeData == NULL) {
    108109      fprintf (stderr, "photcode %s not found in photcode table\n", photname);
    109110      return (FALSE);
    110111    }
     112    photcode = photcodeData[0].code;
    111113  }
    112114  if (photcode == 0) {
    … …  
    191193  }
    192194
     195  // XXX this is archaic: we used to set a fixed zero point of 25 to shift data into the range
     196  // 0 - 32 so it would fit in an unsigned int.  This is also needed because some programs like
     197  // sextractor will put in an arbitrary zero point that we need to understand to get back to
     198  // instrumental mags. 
    193199  gfits_scan (header, "ZERO_PT", "%lf", 1, &ZeroPt);
    194200  if (ZeroPt != GetZeroPoint()) {
    … …  
    197203  }
    198204
     205  // if it exists, find the
     206  if (!strcasecmp(ZERO_POINT_OPTION, "CHIP_HEADER")) {
     207      float ZPT_OBS;
     208      if (!photcodeData || !gfits_scan (header, "ZPT_OBS", "%f", 1, &ZPT_OBS)) {
     209          fprintf (stderr, "zero point not supplied in header\n");
     210          ZERO_POINT_OFFSET = 0.0;
     211      } else {
     212          ZERO_POINT_OFFSET = 0.001*photcodeData[0].C - ZPT_OBS;
     213      }   
     214  }
     215
    199216  /* secz is in units milli-airmass */
    200   image[0].Mcal = 0.0;
     217  image[0].Mcal = ZERO_POINT_OFFSET;
    201218  image[0].Xm   = NAN_S_SHORT;
    202219  image[0].flags = 0;

  • branches/czw_branch/cleanup/Ohana/src/addstar/src/SkyRegionUtils.c

    r14401 r25051  
    1111  ALLOCATE (subset, SkyList, 1);
    1212  ALLOCATE (subset[0].regions, SkyRegion *, NSUBSET);
     13  ALLOCATE (subset[0].filename, char *, NSUBSET);
    1314  subset[0].ownElements = FALSE; // free these elements when freeing the list
    1415
    … …  
    2021
    2122    subset[0].regions[Nsubset] = input[0].regions[i];
     23    subset[0].filename[Nsubset] = input[0].filename[i];
    2224    Nsubset ++;
    23     CHECK_REALLOCATE (subset[0].regions, SkyRegion *, NSUBSET, Nsubset, 100);
     25    if (Nsubset >= NSUBSET) {
     26        NSUBSET += 100;
     27        REALLOCATE (subset[0].regions, SkyRegion *, NSUBSET);
     28        REALLOCATE (subset[0].filename, char *, NSUBSET);
     29    }
    2430    subset[0].Nregions = Nsubset;
    2531  }

  • branches/czw_branch/cleanup/Ohana/src/addstar/src/args.c

    r21508 r25051  
    4646  if ((N = get_argument (argc, argv, "-list"))) {
    4747    options.filelist = TRUE;
     48    remove_argument (N, &argc, argv);
     49  }
     50
     51  USE_NAME = NULL;
     52  if ((N = get_argument (argc, argv, "-use-name"))) {
     53    remove_argument (N, &argc, argv);
     54    USE_NAME = strcreate (argv[N]);
    4855    remove_argument (N, &argc, argv);
    4956  }

  • branches/czw_branch/cleanup/Ohana/src/addstar/src/args_load2mass.c

    r18374 r25051  
    3838  }
    3939
     40  /* only add to existing regions */
     41  options.existing_regions = FALSE;
     42  if ((N = get_argument (argc, argv, "-existing-regions"))) {
     43    options.existing_regions = TRUE;
     44    remove_argument (N, &argc, argv);
     45  }
    4046  /* only add to existing objects */
    4147  options.only_match = FALSE;
    … …  
    6268  options.timeref = 0;
    6369  options.mosaic = FALSE;
    64   options.existing_regions = FALSE;
    6570  options.skip_missed = FALSE;
    6671  options.closest = FALSE;

  • branches/czw_branch/cleanup/Ohana/src/addstar/src/load2mass.c

    r7780 r25051  
    77  int i;
    88  SkyTable *sky, *sky2mass;
     9  SkyList *skylist = NULL;
     10  SkyList *overlap = NULL;
    911  AddstarClientOptions options;
    1012
    … …  
    1315  options = args_load2mass (argc, argv, options);
    1416
     17  // load the full sky description table:
    1518  sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE);
    1619  SkyTableSetFilenames (sky, CATDIR, "cpt");
    1720 
     21  // generate the subset matching the user-selected region
     22  skylist = SkyListByPatch (sky, -1, &UserPatch);
     23
     24  // if we only match to existing (already populated) regions, limit the select to those regions:
     25  if (options.existing_regions) {
     26    SkyList *tmp;
     27    tmp = SkyListExistingSubset (skylist, CATDIR);
     28    SkyListFree (skylist);
     29    skylist = tmp;
     30  }
     31
    1832  path = TWO_MASS_DIR_AS;
    1933
    … …  
    2337 
    2438  for (i = 0; i < sky2mass[0].Nregions; i++) {
     39    // check if any of the skylist entries overlap this 2mass catalog:
     40    overlap = SkyListByBounds_List (skylist, -1, sky2mass[0].regions[i].Rmin, sky2mass[0].regions[i].Rmax, sky2mass[0].regions[i].Dmin, sky2mass[0].regions[i].Dmax);
     41    if (overlap[0].Nregions == 0) {
     42      SkyListFree (overlap);
     43      continue;
     44    }
     45   
    2546    fprintf (stderr, "loading %s\n", sky2mass[0].filename[i]);
    26     load2mass_as_rawdata (sky, sky2mass[0].filename[i], options);
     47    load2mass_as_rawdata (overlap, sky2mass[0].filename[i], options);
     48    SkyListFree (overlap);
    2749  }
    2850  exit (0);

  • branches/czw_branch/cleanup/Ohana/src/addstar/src/load2mass_as_rawdata.c

    r21508 r25051  
    1010# define DEBUG 0
    1111
    12 int load2mass_as_rawdata (SkyTable *skytable, char *filename, AddstarClientOptions options) {
     12int load2mass_as_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options) {
    1313 
    1414  int i, j, verbose;
    … …  
    109109      if (tstars[i].D > UserPatch.Dmax) continue;
    110110
     111      // identify the relevant catalog
     112      skylist = SkyRegionByPoint_List (skytable, -1, tstars[i].R, tstars[i].D);
     113      if (skylist[0].Nregions == 0) {
     114          SkyListFree (skylist);
     115          continue;
     116      }
     117      region = skylist[0].regions[0];
     118      if (DEBUG) fprintf (stderr, "writing to %s\n", skylist[0].filename[0]);
     119
    111120      // collect array of (Stars *) stars in a new output catalog
    112121      Nstars = 0;
    … …  
    114123      ALLOCATE (stars, Stars *, NSTARS);
    115124
    116       // identify the relevant catalog
    117       skylist = SkyRegionByPoint (skytable, -1, tstars[i].R, tstars[i].D);
    118       region = skylist[0].regions[0];
    119       if (DEBUG) fprintf (stderr, "writing to %s\n", skylist[0].filename[0]);
    120 
     125      // loop over stars in this 2mass region that are also in this output region
    121126      for (j = i; j < Ntstars; j++) {
    122127        if (tstars[j].flag) continue;
    … …  
    166171      }
    167172
    168       if (DEBUG) fprintf (stderr, "selected %d stars (%10.6f - %10.6f, %10.6f - %10.6f\n", Nstars,
     173      if (DEBUG) fprintf (stderr, "selected %d stars (%10.6f - %10.6f, %10.6f - %10.6f)\n", Nstars,
    169174                          region[0].Rmin, region[0].Rmax, region[0].Dmin, region[0].Dmax);
    170175

  • branches/czw_branch/cleanup/Ohana/src/libdvo/include/dvo.h

    r24951 r25051  
    311311float PhotInst (Measure *measure);
    312312float PhotCat (Measure *measure);
     313float PhotAper (Measure *measure);
    313314float PhotSys (Measure *measure, Average *average, SecFilt *secfilt);
    314315float PhotRel (Measure *measure, Average *average, SecFilt *secfilt);
    … …  
    441442SkyList   *SkyListByBounds         PROTO((SkyTable *table, int depth, double Rmin, double Rmax, double Dmin, double Dmax));
    442443SkyList   *SkyListChildrenByBounds PROTO((SkyTable *table, int No, int depth, double Rmin, double Rmax, double Dmin, double Dmax));
     444
    443445int        SkyListMerge            PROTO((SkyList **outlist, SkyList *newlist));
    444446int        SkyListFree             PROTO((SkyList *list));
    … …  
    447449int        SkyTableSetFilenames    PROTO((SkyTable *sky, char *path, char *ext));
    448450
     451SkyList   *SkyRegionByPoint_List   PROTO((SkyList *inList, int depth, double ra, double dec));
     452SkyList   *SkyListByBounds_List    PROTO((SkyList *table, int depth, double Rmin, double Rmax, double Dmin, double Dmax));
     453SkyList   *SkyListChildrenByBounds_List PROTO((SkyList *table, int depth, double Rmin, double Rmax, double Dmin, double Dmax));
     454
    449455/* dvo-specific sorting functions */
    450456void sortave (Average *ave, int N);

  • branches/czw_branch/cleanup/Ohana/src/libdvo/src/dvo_photcode_ops.c

    r17253 r25051  
    249249  code = &photcodes[0].code[Np];
    250250  Mcat = measure[0].M - ZERO_POINT + code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C;
     251 
     252  return (Mcat);
     253}
     254
     255float PhotAper (Measure *measure) {
     256
     257  int Np;
     258  float Mcat;
     259  PhotCode *code;
     260
     261  Np = photcodes[0].hashcode[measure[0].photcode];
     262  if (Np == -1) return (NAN);
     263
     264  if (photcodes[0].code[Np].type == PHOT_REF) {
     265    Mcat = measure[0].Map;
     266    return (Mcat);
     267  }
     268  code = &photcodes[0].code[Np];
     269  Mcat = measure[0].Map - ZERO_POINT + code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C;
    251270 
    252271  return (Mcat);

  • branches/czw_branch/cleanup/Ohana/src/libdvo/src/skyregion_ops.c

    r24951 r25051  
    300300          REALLOCATE (list[0].filename, char *, NNEW);
    301301      }
     302    }
     303  }
     304
     305  list[0].Nregions = Nnew;
     306  return (list);
     307}
     308
     309/* find region which overlaps c at given depth (-1 : populated ) */
     310SkyList *SkyRegionByPoint_List (SkyList *inList, int depth, double ra, double dec) {
     311 
     312  int i;
     313  SkyRegion **region;
     314  SkyList *list;
     315
     316  ALLOCATE (list, SkyList, 1);
     317  ALLOCATE (list[0].regions,  SkyRegion *, 1);
     318  ALLOCATE (list[0].filename,  char *, 1);
     319  list[0].Nregions = 0;
     320  list[0].ownElements = FALSE; // this list is only holding a view to the elements
     321
     322  region = inList[0].regions;
     323
     324  for (i = 0; i < inList[0].Nregions; i++) {
     325
     326    // once we pass our desired depth, quit
     327    if ((depth > -1) && (depth < region[i][0].depth)) break;
     328
     329    // skip tables that do not overlap
     330    if (ra  < region[i][0].Rmin) continue;
     331    if (ra  > region[i][0].Rmax) continue;
     332    if (dec < region[i][0].Dmin) continue;
     333    if (dec > region[i][0].Dmax) continue;
     334
     335    // skip tables that are not at our depth
     336    if ((depth >  -1) && (depth > region[i][0].depth)) continue;
     337    if ((depth == -1) && (region[i][0].table == FALSE)) continue;
     338   
     339    list[0].regions[0] = region[i];
     340    list[0].filename[0] = inList[0].filename[i];
     341    list[0].Nregions = 1;
     342    break;
     343  }
     344  return (list);
     345}
     346
     347SkyList *SkyListByBounds_List (SkyList *table, int depth, double Rmin, double Rmax, double Dmin, double Dmax) {
     348
     349  int i, j, Ns;
     350  SkyList *list, *extra;
     351
     352  Rmin = ohana_normalize_angle (Rmin);
     353  Rmax = ohana_normalize_angle (Rmax);
     354
     355  /* handle 0,360 boundary requests */
     356  /* this is probably wrong: I will get duplicates for all Dec bands... */
     357  if (Rmin > Rmax) {
     358    list = SkyListChildrenByBounds_List (table, depth, 0.0, Rmax, Dmin, Dmax);
     359
     360    extra = SkyListChildrenByBounds_List (table, depth, Rmin, 360.0, Dmin, Dmax);
     361    REALLOCATE (list[0].regions, SkyRegion *, list[0].Nregions + extra[0].Nregions);
     362    REALLOCATE (list[0].filename, char *, list[0].Nregions + extra[0].Nregions);
     363    Ns = list[0].Nregions;
     364    for (i = 0; i < extra[0].Nregions; i++) {
     365      // search for pre-existing match
     366      for (j = 0; j < list[0].Nregions; j++) {
     367        if (list[0].regions[j] == extra[0].regions[i]) {
     368          goto skip;
     369        }
     370      }
     371      list[0].regions[Ns] = extra[0].regions[i];
     372      list[0].filename[Ns] = extra[0].filename[i];
     373      Ns ++;
     374    skip:
     375      continue;
     376    }
     377    list[0].Nregions = Ns;
     378    SkyListFree (extra);
     379  } else {
     380    list = SkyListChildrenByBounds_List (table, depth, Rmin, Rmax, Dmin, Dmax);
     381  }
     382
     383  return (list);
     384}
     385
     386SkyList *SkyListChildrenByBounds_List (SkyList *table, int depth, double Rmin, double Rmax, double Dmin, double Dmax) {
     387
     388  int i, j, Ns, Ne, Nnew, NNEW;
     389  int append;
     390  SkyList *children;
     391  SkyList *list;
     392  SkyRegion **region;
     393
     394  Nnew = 0;
     395  NNEW = 50;
     396  ALLOCATE (list, SkyList, 1);
     397  ALLOCATE (list[0].regions, SkyRegion *, NNEW);
     398  ALLOCATE (list[0].filename, char *, NNEW);
     399  list[0].ownElements = FALSE; // this list is only holding a view to the elements
     400
     401  region = table[0].regions;
     402
     403  // can we assume the skylist is globally sorted by depth?  i think so...
     404  for (i = 0; i < table[0].Nregions; i++) {
     405   
     406    // once we pass our desired depth, quit
     407    if ((depth > -1) && (depth < region[i][0].depth)) break;
     408
     409    // skip tables that do not overlap
     410    if (Rmax <= region[i][0].Rmin) continue;
     411    if (Rmin >= region[i][0].Rmax) continue;
     412    if (Dmax <= region[i][0].Dmin) continue;
     413    if (Dmin >= region[i][0].Dmax) continue;
     414
     415    // skip tables that are not at our depth
     416    if ((depth >  -1) && (depth > region[i][0].depth)) continue;
     417    if ((depth == -1) && (region[i][0].table == FALSE)) continue;
     418
     419    list[0].regions[Nnew] = region[i];
     420    list[0].filename[Nnew] = table[0].filename[i];
     421    Nnew ++;
     422    if (Nnew >= NNEW) {
     423      NNEW += 50;
     424      REALLOCATE (list[0].regions, SkyRegion *, NNEW);
     425      REALLOCATE (list[0].filename, char *, NNEW);
    302426    }
    303427  }

  • branches/czw_branch/cleanup/Ohana/src/opihi/dvo/dbExtractMeasures.c

    r21508 r25051  
    117117        case MAG_REF:
    118118          value.Flt = PhotRef  (equiv, average, secfilt, measure);
     119          break;
     120        case MAG_APER:
     121          value.Flt = PhotAper  (measure);
    119122          break;
    120123        case MAG_ERR:

  • branches/czw_branch/cleanup/Ohana/src/opihi/dvo/dbFields.c

    r21508 r25051  
    3737  if (!strcasecmp (string, "ave"))       return (MAG_AVE);
    3838  if (!strcasecmp (string, "ref"))       return (MAG_REF);
     39  if (!strcasecmp (string, "ap"))        return (MAG_APER);
     40  if (!strcasecmp (string, "aper"))      return (MAG_APER);
    3941  if (!strcasecmp (string, "err"))       return (MAG_ERR);
    4042  if (!strcasecmp (string, "photflags")) return (MAG_PHOT_FLAGS);

  • branches/czw_branch/cleanup/Ohana/src/opihi/include/dvoshell.h

    r21508 r25051  
    2727      MAG_INST,
    2828      MAG_CAT,
     29      MAG_APER,
    2930      MAG_SYS,
    3031      MAG_REL,

  • branches/czw_branch/cleanup/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

    r24951 r25051  
    232232        }
    233233        my $out = $image;
     234
    234235        # if uri is nil this will get overridded below
     236        # (this is for raw stage)
    235237        $out->{image}  = $image->{uri};
    236 #        $out->{exp_id} = $image->{exp_id};
    237 #        $out->{exp_name} = $image->{exp_name};
    238 #        $out->{state}  = $image->{state}; # state is undef for rawExp, but that's ok
    239 #        if ($set_class_id) {
    240 #            $class_id = $image->{class_id};
    241 #            $out->{class_id} = $class_id;
    242 #        }
     238        if ($set_class_id) {
     239            $class_id = $image->{class_id};
     240            $out->{component} = $class_id;
     241        }
    243242
    244243        # find the mask and weight images
    … …  
    248247            $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name;
    249248        }
    250 #        $out->{camera} = $camera;
    251 
    252249        $out->{astrom} = find_astrometry($ipprc, $image_db, $image, $verbose) if $want_astrom;
    253250
    … …  
    441438}
    442439
     440# cache of results of ppConfigDump
     441my %astromSources;
     442my $last_exp_id = 0;
     443my $lastAstromFile;
     444
    443445# find the astrometry file for a given exposure
    444446# return undef if no completed camRun exists
    … …  
    449451    my $verbose = shift;
    450452
     453    my $exp_id = $image->{exp_id};
     454    if (($exp_id eq $last_exp_id) and $lastAstromFile) {
     455        # running camtool 60 times is really expensive when the answer is the same
     456        return $lastAstromFile;
     457    }
     458    $last_exp_id = 0;
     459    $lastAstromFile = undef;
     460
    451461    my $missing_tools;
    452462    my $camtool = can_run("camtool") or (warn "Can't find camtool" and $missing_tools = 1);
    … …  
    457467    }
    458468
    459     my $command = "$camtool -dbname $image_db -processedexp -exp_id $image->{exp_id}";
     469    my $command = "$camtool -dbname $image_db -processedexp -exp_id $exp_id";
    460470    # run the tool and parse the output
    461471    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    487497    my $camRoot = $camdata->{path_base};
    488498    my $camera = $image->{camera};
    489     my $astromSource;
    490     {
     499    my $astromSource = $astromSources{$camera};
     500    if (!$astromSource) {
    491501       my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -";
    492502        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    499509            die("Unable to parse metadata config doc");
    500510        $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
     511        $astromSources{$camera} = $astromSource;
    501512    }
    502513
    503514    my $astromFile = $ipprc->filename($astromSource, $camRoot);
     515    if ($astromFile) {
     516        $lastAstromFile = $astromFile;
     517        $last_exp_id = $exp_id;
     518    }
    504519
    505520    return $astromFile;

  • branches/czw_branch/cleanup/arclog/arclog.pl

    r24951 r25051  
    1919use Pod::Usage qw( pod2usage );
    2020
    21 my ($email);
     21my ($debug, $email);
    2222
    2323GetOptions(
    24     'email|s=s'    => \$email,
     24    'debug|d'   => \$debug,
     25    'email|s=s' => \$email,
    2526) || pod2usage( 2 );
    2627
    … …  
    176177        = @$rec{qw( host time device event elapse_time errors )};
    177178    # pack record
    178     $query->execute("$host|$time|$device|$event|$elapse_time|$errors")
     179    my $rec = "$host|$time|$device|$event|$elapse_time|$errors";
     180    $query->execute($rec)
    179181        or die "database error: $!";
     182    warn "inserting $rec\n" if $debug;
    180183
    181184    # do not print filtered records

  • branches/czw_branch/cleanup/arclog/arclog_readdb.pl

    r19139 r25051  
    1919pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    2020
    21 my $filter_records = 1;
     21my $filter_records = 0;
    2222
    2323my $rcfile = "$ENV{HOME}/.arclogrc";
    … …  
    3030die "can not open logdb" unless $dbh;
    3131
    32 my $query = $dbh->prepare("SELECT * FROM log")
     32my $query = $dbh->prepare("SELECT record, stamp FROM log")
    3333    or die "database error: $!";
    3434$query->execute or die "database error: $!";
    … …  
    5252    @rec{qw( host time device event elapse_time errors )}
    5353        = split(/\|/, $row->[0]);
     54    $rec{stamp} = $row->[1];
    5455    $rec{time} = $strptime->parse_datetime($rec{time});
    5556    push @records, \%rec;
    … …  
    6061
    6162RECORDS: foreach my $rec (@records) {
    62     my ($host, $time, $device, $event, $elapse_time, $errors)
    63         = @$rec{qw( host time device event elapse_time errors )};
     63    my ($host, $stamp, $time, $device, $event, $elapse_time, $errors)
     64        = @$rec{qw( host stamp time device event elapse_time errors )};
    6465
    6566    # do not print filtered records
    … …  
    7172
    7273    no warnings qw( uninitialized );
    73     print "$host $time $device $event $elapse_time $errors\n";
     74    print "$stamp - $host $time $device $event $elapse_time $errors\n";
    7475    use warnings;
    7576}

  • branches/czw_branch/cleanup/console/scripts/console

    r24615 r25051  
    6363=over 4
    6464
    65 =item * --db|-d <db name>
     65=item * --user|-u
    6666
    67 Name of database (C<namespace>) to write too.
     67The C<ssh> username to use when connecting to the console port.
    6868
    69 Optional if defined in the F<.nebdiskdrc> file or the appropriate environment
    70 variable is set.
    71 
    72 =item * --user|-u <db username>
    73 
    74 Username to authenticate with.
    75 
    76 Optional if defined in the F<.nebdiskdrc> file or the appropriate environment
    77 variable is set.
    78 
    79 =item * --pass|-p <db password>
    80 
    81 Password to authenticate with.
    82 
    83 Optional if defined in the F<.nebdiskdrc> file or the appropriate environment
    84 variable is set.
    85 
    86 =item * --debug
    87 
    88 This flag prevents the program from "daemonizing" so output can be sent to
    89 C<stdout>/C<stderr> and a debugger used on the running process.
    90 
    91 =item * --pidfile <filename>
    92 
    93 Filename to log the running daemon's PID too.
    94 
    95 Optional, if this option is omitted and no value is defined in the
    96 F<.nebdiskrc> then no pidfile is created.
    97 
    98 =item * --stop|-s
    99 
    100 Uses the pidfile to kill an already running daemon.
    101 
    102 =item * --restart|-r
    103 
    104 Uses the pidfile to kill an already running daemon and then starts a new
    105 instance.
    106 
    107 =item * --verbose|-r
     69=item * --verbose|-v
    10870
    10971Turns on informational/debugging messages to be sent to the
    110 C<stderr>/C<stdout>.  This option is probably not very usefully unless combined
    111 with C<--debug>.
    112 
    113 =back
    114 
    115 =head1 ENVIRONMENT
    116 
    117 These environment variables may be used in place of the specified command line
    118 options.  All command line option will override the corresponding environment
    119 value.
    120 
    121 =over 4
    122 
    123 =item * C<NEB_DB>
    124 
    125 Equivalent to --db|-d
    126 
    127 =item * C<NEB_USER>
    128 
    129 Equivalent to --user|-u
    130 
    131 =item * C<NEB_PASS>
    132 
    133 Equivalent to --pass|-p
     72C<stderr>/C<stdout>.
    13473
    13574=back
    … …  
    13776=head1 RCFILE
    13877
    139 This program will attempt to read an C<rcfile> from F<$HOME/.nebdiskrc> and if
    140 found will use it's values to override program defaults.  Please not that the
    141 precedence for values is: command line, rcfile, environment variables.
    142 
    143 The format of the C<rcfile> is in L<YAML> format and uses the following values:
    144 
    145     ---
    146     db: nebulous
    147     dbpass: '@neb@'
    148     dbuser: nebulous
    149     mounts:
    150       - /mnt
    151       - /tmp
    152       - /usr
    153     pidfile: /var/tmp/nebdiskd
    154     poll_interval: 5
    155 
    156 The values C<db>, C<dbpass>, C<dbuser>, and C<pidfile> have the same semantics
    157 as their command line and/or environment variable equivalents.
    158 
    159 =over 4
    160 
    161 =item * C<mounts>
    162 
    163 A list of "paths" to C<stat(2)> before mounted volumes are polled.  The propose
    164 of this option is to attempt to keep "automounted" volumes mounted while this
    165 program is running.
    166 
    167 This value may be omitted or left blank.
    168 
    169 =item * C<poll_interval>
    170 
    171 The number of seconds to wait between checks for mounted volumes.
    172 
    173 This value may be omitted or left blank.  The default value is C<60>s.
    174 
    175 =back
     78This program will attempt to read an C<rcfile> from F<$HOME/.Consolerc>.
    17679
    17780=head1 CREDITS
    … …  
    209112=head1 SEE ALSO
    210113
    211 L<Config::YAML>
     114L<Config::YAML>, L<Console>, L<console-config>, L<console-ipp-defaults>
    212115
    213116=cut

  • branches/czw_branch/cleanup/console/scripts/console-config

    r24611 r25051  
    152152=head1 RCFILE
    153153
    154 This program will attempt to read an C<rcfile> from F<$HOME/.Consolerc>
    155 
    156 The format of the C<rcfile> is in L<YAML> format and uses the following values:
    157 
    158     ---
    159     db: nebulous
    160     dbpass: '@neb@'
    161     dbuser: nebulous
    162     mounts:
    163       - /mnt
    164       - /tmp
    165       - /usr
    166     pidfile: /var/tmp/nebdiskd
    167     poll_interval: 5
    168 
    169 The values C<db>, C<dbpass>, C<dbuser>, and C<pidfile> have the same semantics
    170 as their command line and/or environment variable equivalents.
    171 
    172 =over 4
    173 
    174 =item * C<mounts>
    175 
    176 A list of "paths" to C<stat(2)> before mounted volumes are polled.  The propose
    177 of this option is to attempt to keep "automounted" volumes mounted while this
    178 program is running.
    179 
    180 This value may be omitted or left blank.
    181 
    182 =item * C<poll_interval>
    183 
    184 The number of seconds to wait between checks for mounted volumes.
    185 
    186 This value may be omitted or left blank.  The default value is C<60>s.
    187 
    188 =back
     154This program will attempt to read an C<rcfile> from F<$HOME/.Consolerc>.
    189155
    190156=head1 CREDITS
    … …  
    222188=head1 SEE ALSO
    223189
    224 L<Config::YAML>
     190L<Config::YAML>, L<Console>, L<console>, L<console-ipp-defaults>
    225191
    226192=cut

  • branches/czw_branch/cleanup/console/scripts/console-ipp-defaults

    r24618 r25051  
    139139=over 4
    140140
    141 =item * --verbose|-r
     141=item * --verbose|-v
    142142
    143143Turns on informational/debugging messages to be sent to the
    … …  
    180180=head1 SEE ALSO
    181181
    182 L<Config::YAML>, L<Console>
     182L<Config::YAML>, L<Console>, L<console>, L<console-config>
    183183
    184184=cut

  • branches/czw_branch/cleanup/dbconfig/changes.txt

    r24951 r25051  
    12011201ALTER TABLE distTarget DROP KEY clean;
    12021202ALTER TABLE distTarget ADD CONSTRAINT UNIQUE(label, filter, stage, clean);
     1203
     1204ALTER TABLE pstampJob DROP COLUMN args;
     1205ALTER TABLE pstampJob DROP COLUMN uri;
     1206ALTER TABLE magicDSRun ADD COLUMN inv_magic_id BIGINT AFTER magic_id;
     1207ALTER TABLE magicDSRun ADD FOREIGN KEY (inv_magic_id) REFERENCES magicRun(magic_id);

  • branches/czw_branch/cleanup/dbconfig/magic.md

    r24552 r25051  
    4747    magic_ds_id S64         0       # Primary Key
    4848    magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
     49    inv_magic_id S64        0      # Primary Key fkey(inverse_magic_id) ref magicRun(magic_id)
    4950    state       STR         0       # Key
    5051    stage       STR         64

  • branches/czw_branch/cleanup/dbconfig/pstamp.md

    r19760 r25051  
    3636    jobType     STR         16
    3737    fault       S32         0
    38     uri         STR         255
    3938    exp_id      S64         0
    4039    outputBase  STR         255
    41     args        STR         511
    4240END

  • branches/czw_branch/cleanup/doc/misc/docgen.pl

    r24951 r25051  
    44#Run Doxygen and copy the html files produced to alala web server
    55#Called daily by cron
    6 #Bill Giebink 12/08
    76
    87use strict;
    … …  
    1110use File::Copy;
    1211
    13 my $srcDir = "/data/ipp003.0/ippTests/src/ipp";
     12my $srcDir = "/data/ipp002.0/ippDoxygen/ipp";
     13system `rm -rf /data/ipp002.0/ippDoxygen/ipp`;
     14system `svn co http://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/ /data/ipp002.0/ippDoxygen/ipp`;
    1415my $destDir = "/data/alala.0/ippDocs";
    1516system `rm -rf $destDir/*`;
    1617
    1718finddepth(\&wanted, $srcDir);
     19chdir $srcDir;
     20system `/usr/bin/doxygen`;
     21system ("cp -r $srcDir/docs/html/* $destDir");
    1822
    1923sub wanted {
    … …  
    2731    my $currDir = $File::Find::dir;
    2832    system ("$currDir/autogen.sh");
    29     system ("doxygen");
    30     system ("mkdir $destDir/$subDir");
    31     system ("cp $currDir/docs/html/* $destDir/$subDir");
    3233  }
    3334}

  • branches/czw_branch/cleanup/ippScripts/scripts/magic_destreak.pl

    r24951 r25051  
    3838
    3939# Parse the command-line arguments
    40 my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $inverse, $cam_path_base);
     40my ($magic_ds_id, $camera, $streaks, $inv_streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
    4141my ($outroot, $recoveryroot);
    4242my ($replace, $release);
    … …  
    4747           'camera=s'       => \$camera,     # camera for evaluating file rules
    4848           'streaks=s'      => \$streaks,    # file containing the list of streaks
     49           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
    4950           'stage=s'        => \$stage,      # raw, chip, warp, or diff
    5051           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
    … …  
    5253           'uri=s'          => \$uri,        # uri of the input image
    5354           'path_base=s'    => \$path_base,  # path_base of the input
    54            'inverse'        => \$inverse,    # Inverse subtraction?
    5555           'cam_path_base=s'=> \$cam_path_base,  # path_base from camera stage (for chip and raw)
    5656           'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
    … …  
    9494} elsif ($stage ne "camera") {
    9595    &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    96 }
     96}   
     97$inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL");
    9798
    9899my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    … …  
    205206
    206207    my ($image, $mask, $ch_mask, $weight, $astrom, $sources);
     208
     209    # if we're destreaking a bothways diff need to combine the
     210    # two streaks files
     211    my ($allstreaks_fh, $allstreaks_name);
    207212
    208213    if ($stage eq "raw") {
    … …  
    234239        $sources    = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $path_base);
    235240    } elsif ($stage eq "diff") {
    236         my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images
    237         $image  = $ipprc->filename($name, $path_base);
    238         $mask   = $ipprc->filename("$name.MASK", $path_base);
    239         $weight = $ipprc->filename("$name.VARIANCE", $path_base);
     241        $image  = $ipprc->filename("PPSUB.OUTPUT", $path_base);
     242        $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
     243        $weight = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
    240244        $sources    = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base);
     245
     246        if ($inv_streaks) {
     247            # create a temporary file containing the contents of the
     248            # two streaks files
     249            ($allstreaks_fh, $allstreaks_name) = tempfile ("/tmp/all.streaks.XXXX",
     250                    UNLINK => !$save_temps);
     251
     252            combine_streaks($allstreaks_fh, $streaks, $inv_streaks);
     253
     254            # apply the combined streaks to both the forward and inverse diffs
     255            $streaks = $allstreaks_name;
     256        }
    241257    }
    242258
    … …  
    266282        }
    267283    }
     284    if (($stage eq "diff") and $inv_streaks) {
     285        $image   = $ipprc->filename("PPSUB.INVERSE", $path_base);
     286        $mask    = $ipprc->filename("PPSUB.INVERSE.MASK", $path_base);
     287        $weight  = $ipprc->filename("PPSUB.INVERSE.VARIANCE", $path_base);
     288        $sources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $path_base);
     289
     290        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
     291
     292        $command .= " -recovery $recoveryroot" if defined $recoveryroot;
     293        $command .= " -mask $mask" if defined $mask;
     294        $command .= " -weight $weight" if defined $weight;
     295        $command .= " -sources $sources" if defined $sources;
     296        $command .= " -replace" if $replace;
     297        $command .= " -release" if $release;
     298        $command .= " -dbname $dbname" if defined $dbname;
     299        unless (defined $no_op) {
     300            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     301                run(command => $command, verbose => $verbose);
     302            unless ($success) {
     303                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     304                &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code);
     305            }
     306        } else {
     307            print "skipping command $command\n";
     308        }
     309    }
    268310} else {
    269311    # camera stage. The only work to do is to censor the detections file
    … …  
    376418}
    377419
     420sub combine_streaks
     421{
     422    my $fout = shift;
     423    my $fn1 = shift;
     424    my $fn2 = shift;
     425
     426    my ($n1, @streaks1) = read_streaks_file($fn1);
     427    my_die("failed to read streaks from $fn1", $magic_ds_id, $component,
     428        $PS_EXIT_UNKNOWN_ERROR) if $n1 < 0;
     429
     430    my ($n2, @streaks2) = read_streaks_file($fn2);
     431    my_die("failed to read streaks from $fn2", $magic_ds_id, $component,
     432        $PS_EXIT_UNKNOWN_ERROR) if $n2 < 0;
     433
     434    print $fout $n1 + $n2 . "\n";
     435
     436    foreach my $line (@streaks1, @streaks2) {
     437        print $fout $line;
     438    }
     439
     440    close $fout
     441        or my_die("failed to close combined streaks file", $magic_ds_id,
     442                     $component, $PS_EXIT_UNKNOWN_ERROR);
     443}
     444
     445sub read_streaks_file
     446{
     447    my $filename = shift;
     448    my $fh;
     449    open $fh, "<$filename" or my_die("failed to open $filename",
     450                    $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR);
     451
     452    # first line is the number of streaks
     453    my $line = <$fh>;
     454    chomp $line;
     455    my $nstreaks = $line;
     456
     457    my @streaks;
     458
     459    foreach $line (<$fh>) {
     460        push @streaks, $line;
     461    }
     462
     463    close $fh;
     464
     465    return ($nstreaks, @streaks);
     466}
     467
    378468
    379469sub my_die

  • branches/czw_branch/cleanup/ippScripts/scripts/magic_destreak_revert.pl

    r24951 r25051  
    180180    $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
    181181} elsif ($stage eq "diff") {
    182     my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images
     182    my $name = "PPSUB.OUTPUT";
    183183    $image  = $ipprc->filename($name, $path_base);
    184184    $mask   = $ipprc->filename("$name.MASK", $path_base);
    185185    $weight = $ipprc->filename("$name.VARIANCE", $path_base);
    186     $sources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base);
     186    $sources = $ipprc->filename("$name.SOURCES", $path_base);
    187187    $bimage  = $ipprc->filename($name, $backup_path_base);
    188188    $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
    189189    $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
    190     $bsources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $backup_path_base);
    191 }
    192 
    193 if ($image) {
    194     revert_file($image, $bimage) or
    195         &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    196 }
    197 
    198 if ($mask) {
    199     if (!revert_file($mask, $bmask)) {
    200         if ($stage eq 'chip') {
    201             # don't fail if the mask file (from the camera stage) fails to revert.
    202             # It probably doesn't exist
    203             # XXX: Handle this properly
    204             print STDERR "failure to revert mask file, ignored\n";
    205         } else {
    206             &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    207         }
    208     }
    209 }
    210 
    211 if ($ch_mask) {
    212     if (!revert_file($ch_mask, $bch_mask)) {
    213         &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    214     }
    215 }
    216 
    217 
    218 if ($weight) {
    219     revert_file($weight, $bweight) or
    220         &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    221 }
    222 
    223 if ($sources) {
    224     revert_file($sources, $bsources) or
    225         &my_die("failed to restore sources file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    226 }
    227 
    228 if ($astrom) {
    229     revert_file($astrom, $bastrom) or
    230         &my_die("failed to restore astrometry file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     190    $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
     191}
     192
     193revert_files($image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
     194
     195if ($stage eq "diff") {
     196    my $name = "PPSUB.INVERSE";
     197    $image  = $ipprc->filename($name, $path_base);
     198    $mask   = $ipprc->filename("$name.MASK", $path_base);
     199    $weight = $ipprc->filename("$name.VARIANCE", $path_base);
     200    $sources = $ipprc->filename("$name.SOURCES", $path_base);
     201    $bimage  = $ipprc->filename($name, $backup_path_base);
     202    $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
     203    $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
     204    $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
     205    revert_files($image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef);
    231206}
    232207
    … …  
    254229
    255230### Pau.
     231
     232sub revert_files {
     233    my $image = shift;
     234    my $mask = shift;
     235    my $weight = shift;
     236    my $sources = shift;
     237    my $astrom = shift;
     238    my $bimage = shift;
     239    my $bmask = shift;
     240    my $bweight = shift;
     241    my $bsources = shift;
     242    my $bastrom = shift;
     243
     244    if ($image) {
     245        revert_file($image, $bimage) or
     246            &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     247    }
     248
     249    if ($mask) {
     250        if (!revert_file($mask, $bmask)) {
     251            if ($stage eq 'chip') {
     252                # don't fail if the mask file (from the camera stage) fails to revert.
     253                # It probably doesn't exist
     254                # XXX: Handle this properly
     255                print STDERR "failure to revert mask file, ignored\n";
     256            } else {
     257                &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     258            }
     259        }
     260    }
     261
     262    if ($ch_mask) {
     263        if (!revert_file($ch_mask, $bch_mask)) {
     264            &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     265        }
     266    }
     267
     268
     269    if ($weight) {
     270        revert_file($weight, $bweight) or
     271            &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     272    }
     273
     274    if ($sources) {
     275        revert_file($sources, $bsources) or
     276            &my_die("failed to restore sources file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     277    }
     278
     279    if ($astrom) {
     280        revert_file($astrom, $bastrom) or
     281            &my_die("failed to restore astrometry file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     282    }
     283}
    256284
    257285sub check_keyword

  • branches/czw_branch/cleanup/ippScripts/scripts/publish_file.pl

    r24707 r25051  
    134134    my $file = $ipprc->file_resolve( $files{$comp} ) or
    135135        &my_die("Unable to resolve file $files{$comp}", $pub_id, $PS_EXIT_SYS_ERROR);
    136     my_die("Unable to find file $file", $pub_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists( $file );
     136    my_die("Unable to find file $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $file );
    137137
    138138    my $zp = $zp{$comp};

  • branches/czw_branch/cleanup/ippTasks/dist.pro

    r24951 r25051  
    44check.globals
    55
    6 #$LOGSUBDIR = $LOGDIR/dist
     6$LOGSUBDIR = $LOGDIR/dist
    77mkdir $LOGSUBDIR
    88

  • branches/czw_branch/cleanup/ippTasks/magic.pro

    r24951 r25051  
    104104    end
    105105end
     106macro magic.ds.off
     107    task magic.destreak.load
     108        active false
     109    end
     110    task magic.destreak.run
     111        active false
     112    end
     113end
     114macro magic.ds.on
     115    task magic.destreak.load
     116        active true
     117    end
     118    task magic.destreak.run
     119        active true
     120    end
     121end
     122
     123macro magic.ds.status
     124    echo magicToDS
     125    book listbook magicToDS
     126    echo ""
     127    echo magicDSToRevert
     128    book listbook magicDSToRevert
     129end
     130
    106131
    107132
    … …  
    380405
    381406  periods      -poll $LOADPOLL
    382   periods      -exec $LOADEXEC
    383   periods      -timeout 20
     407  # this query can take a long time
     408  periods      -exec 10
     409  periods      -timeout 120
    384410  npending     1
    385411
    … …  
    388414
    389415  task.exec
    390     $run = magicdstool -todestreak -limit 32
     416    $run = magicdstool -todestreak -limit 120
    391417    if ($DB:n == 0)
    392418      option DEFAULT
    … …  
    449475    book getword magicToDS $pageName camera -var CAMERA
    450476    book getword magicToDS $pageName streaks_uri -var STREAKS
     477    book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS
    451478    book getword magicToDS $pageName stage -var STAGE
    452479    book getword magicToDS $pageName stage_id -var STAGE_ID
    … …  
    454481    book getword magicToDS $pageName uri -var URI
    455482    book getword magicToDS $pageName path_base -var PATH_BASE
    456     book getword magicToDS $pageName inverse -var INVERSE
    457483    book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE
    458484    book getword magicToDS $pageName outroot -var OUTROOT
    … …  
    473499    # TODO: do not add recoveryroot or replace if they are null or zero
    474500
    475     $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE
    476     if ("$INVERSE" == "T")
    477        # Inverse subtraction
    478        $run = $run --inverse
    479     end
     501    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE
    480502
    481503    add_standard_args run

  • branches/czw_branch/cleanup/ippTasks/pstamp.pro

    r24951 r25051  
    1414# example:
    1515# $PSDBSERVER = -dbserver hostname
    16 $PS_DBSERVER = ""
     16if ($?PSDBSERVER == 0)
     17    $PS_DBSERVER = ""
     18end
    1719
    1820macro pstamp.reset
    … …  
    363365        book setword pstampJob $pageName pantaskState RUN
    364366        book getword pstampJob $pageName job_id -var JOB_ID
     367        book getword pstampJob $pageName jobType -var JOB_TYPE
     368        book getword pstampJob $pageName rownum -var ROWNUM
    365369        book getword pstampJob $pageName dbname -var DBNAME
    366370        book getword pstampJob $pageName outputBase -var OUTPUT_BASE
    … …  
    371375
    372376
    373         $run = pstamp_job_run.pl --job_id $JOB_ID --redirect-output --output_base $OUTPUT_BASE
     377        $run = pstamp_job_run.pl --job_id $JOB_ID --job_type $JOB_TYPE --rownum $ROWNUM --output_base $OUTPUT_BASE --redirect-output
    374378        add_standard_args run
    375379

  • branches/czw_branch/cleanup/ippTasks/survey.pro

    r24951 r25051  
    44check.globals
    55
    6 list DIFF_WW
    7   ThreePi_SouthernRegion.090724
    8 end
    9 
    10 $DIFF_WW_CNT = 0
    11 
    12 task diff.check
     6book create SURVEY_DIFF
     7book create SURVEY_MAGIC
     8
     9$SURVEY_DIFF_DB = 0
     10$SURVEY_MAGIC_DB = 0
     11
     12$SURVEY_EXEC = 120
     13$SURVEY_POLL = 10
     14$SURVEY_TIMEOUT = 600
     15
     16
     17# user functions to manipulate diff labels
     18macro survey.add.diff
     19  if ($0 != 3)
     20    echo "USAGE: survey.add.diff (label) (workdir base)"
     21    break
     22  end
     23  book newpage SURVEY_DIFF $1
     24  book setword SURVEY_DIFF $1 WORKDIR $2
     25  book setword SURVEY_DIFF $1 STATE PENDING
     26end
     27
     28macro survey.del.diff
     29  if ($0 != 2)
     30    echo "USAGE: survey.del.diff (label)"
     31    break
     32  end
     33  book delpage SURVEY_DIFF $1
     34end
     35
     36macro survey.show.diff
     37  if ($0 != 1)
     38    echo "USAGE: survey.show.diff"
     39    break
     40  end
     41  book listbook SURVEY_DIFF
     42end
     43
     44# user functions to manipulate magic labels
     45macro survey.add.magic
     46  if ($0 != 3)
     47    echo "USAGE: survey.add.magic (label) (workdir base)"
     48    break
     49  end
     50  book newpage SURVEY_MAGIC $1
     51  book setword SURVEY_MAGIC $1 WORKDIR $2
     52  book setword SURVEY_MAGIC $1 STATE PENDING
     53end
     54
     55macro survey.del.magic
     56  if ($0 != 2)
     57    echo "USAGE: survey.del.magic (label)"
     58    break
     59  end
     60  book delpage SURVEY_MAGIC $1
     61end
     62
     63macro survey.show.magic
     64  if ($0 != 1)
     65    echo "USAGE: survey.show.magic"
     66    break
     67  end
     68  book listbook SURVEY_MAGIC
     69end
     70
     71
     72task survey.diff
    1373  host local
    1474 
    15   periods      -poll 10
    16   periods      -exec 120
    17   periods      -timeout 600
     75  periods      -poll $SURVEY_POLL
     76  periods      -exec $SURVEY_EXEC
     77  periods      -timeout $SURVEY_TIMEOUT
    1878  npending     1
    1979
    20   stdout $LOGDIR/diffcheck.log
    21   stderr $LOGDIR/diffcheck.log
     80  stdout $LOGDIR/survey.diff.log
     81  stderr $LOGDIR/survey.diff.log
    2282
    2383  # generate diff warp-warp runs
    2484  task.exec
    25     if ($DIFF_WW:n == 0) break
    26     $run = difftool -dbname gpc1 -definewarpwarp -distance 0.1 -good_frac 0.1 -timediff 3600
    27     $run = $run -input_label $DIFF_WW:$DIFF_WW_CNT
    28     $run = $run -template_label $DIFF_WW:$DIFF_WW_CNT
    29     $run = $run -label $DIFF_WW:$DIFF_WW_CNT
    30     $run = $run -workdir neb://@HOST@.0/gpc1/$DIFF_WW:$DIFF_WW_CNT
    31     $run = $run -reduction WARPWARP
    32 
    33     $DIFF_WW_CNT ++
    34     if ($DIFF_WW_CNT >= $DIFF_WW:n)
    35       echo "reset DIFF_WW_CNT (was $DIFF_WW_CNT)"
    36       set DIFF_WW_CNT = 0
    37     end
    38 
     85    book npages SURVEY_DIFF -var N
     86    if ($N == 0)
     87#      echo "No labels for processing"
     88      break
     89    endif
     90
     91    book getpage SURVEY_DIFF 0 -var label -key STATE NEW
     92    if ("$label" == "NULL")
     93      # All labels have been done --- reset
     94#      echo "Resetting labels"
     95      for i 0 $N
     96        book getpage SURVEY_DIFF $i -var label
     97        book setword SURVEY_DIFF $label STATE NEW
     98      end
     99      book getpage SURVEY_DIFF 0 -var label -key STATE NEW
     100
     101      # Select different database
     102      $SURVEY_DIFF_DB ++
     103      if ($SURVEY_DIFF_DB >= $DB:n)
     104        $SURVEY_DIFF_DB = 0
     105      end
     106    end
     107
     108    book setword SURVEY_DIFF $label STATE DONE
     109    book getword SURVEY_DIFF $label WORKDIR -var workdir
     110
     111    $run = difftool -definewarpwarp -distance 0.1 -good_frac 0.1 -timediff 3600
     112    $run = $run -input_label $label
     113    $run = $run -template_label $label
     114    $run = $run -label $label
     115    $run = $run -workdir $workdir/$label/
     116
     117    if ($DB:n == 0)
     118      option DEFAULT
     119    else
     120      $run = $run -dbname $DB:$SURVEY_DIFF_DB
     121      option $DB:$SURVEY_DIFF_DB
     122    end
     123   
     124#    echo $run
    39125    command $run
    40126  end
    … …  
    42128  # success
    43129  task.exit    0
    44     echo "checked for new diff runs"
     130#    echo "Success"
    45131  end
    46132
    … …  
    59145  end
    60146end
     147
     148
     149
     150task survey.magic
     151  host local
     152 
     153  periods      -poll $SURVEY_POLL
     154  periods      -exec $SURVEY_EXEC
     155  periods      -timeout $SURVEY_TIMEOUT
     156  npending     1
     157
     158  stdout $LOGDIR/survey.magic.log
     159  stderr $LOGDIR/survey.magic.log
     160
     161  # generate magic warp-warp runs
     162  task.exec
     163    book npages SURVEY_MAGIC -var N
     164    if ($N == 0)
     165#      echo "No labels for processing"
     166      break
     167    endif
     168
     169    book getpage SURVEY_MAGIC 0 -var label -key STATE NEW
     170    if ("$label" == "NULL")
     171      # All labels have been done --- reset
     172#      echo "Resetting labels"
     173      for i 0 $N
     174        book getpage SURVEY_MAGIC $i -var label
     175        book setword SURVEY_MAGIC $label STATE NEW
     176      end
     177      book getpage SURVEY_MAGIC 0 -var label -key STATE NEW
     178
     179      # Select different database
     180      $SURVEY_MAGIC_DB ++
     181      if ($SURVEY_MAGIC_DB >= $DB:n)
     182        set $SURVEY_MAGIC_DB = 0
     183      end
     184    end
     185
     186    book setword SURVEY_MAGIC $label STATE DONE
     187    book getword SURVEY_MAGIC $label WORKDIR -var workdir
     188
     189    $run = magictool -definebyquery -label $label -diff_label $label -workdir $workdir/$label/
     190
     191    if ($DB:n == 0)
     192      option DEFAULT
     193    else
     194      $run = $run -dbname $DB:$SURVEY_MAGIC_DB
     195      option $DB:$SURVEY_MAGIC_DB
     196    end
     197   
     198#    echo $run
     199    command $run
     200  end
     201
     202  # success
     203  task.exit    0
     204#    echo "Success"
     205  end
     206
     207  # locked list
     208  task.exit    default
     209    showcommand failure
     210  end
     211
     212  task.exit    crash
     213    showcommand crash
     214  end
     215
     216  # operation times out?
     217  task.exit    timeout
     218    showcommand timeout
     219  end
     220end
     221

  • branches/czw_branch/cleanup/ippTools/share/Makefile.am

    r24951 r25051  
    151151     magictool_tomask.sql \
    152152     magictool_toprocess_inputs.sql \
     153     magictool_toprocess_runs.sql \
    153154     magictool_toprocess_tree.sql \
    154155     magictool_toskyfilemask.sql \

  • branches/czw_branch/cleanup/ippTools/share/chiptool_processedimfile.sql

    r24112 r25051  
    88    chipProcessedImfile.bg_mean_stdev,
    99    chipProcessedImfile.path_base,
     10    chipProcessedImfile.magicked,
    1011    chipRun.state,
    1112    chipRun.workdir,

  • branches/czw_branch/cleanup/ippTools/share/difftool_definewarpstack_part1.sql

    r24572 r25051  
    1515    SELECT DISTINCT
    1616        diffRun.*,
     17        warp_id,
    1718        exp_id
    1819    FROM diffRun
    … …  
    2425    JOIN chipRun USING(chip_id)
    2526    WHERE warp1 IS NOT NULL
    26 ) AS diffExp USING(exp_id)
     27) AS diffExp USING(exp_id, warp_id)
    2728WHERE
    2829    warpSkyfile.fault = 0

  • branches/czw_branch/cleanup/ippTools/share/magicdstool_definebyquery_camera.sql

    r24951 r25051  
    66    camRun.cam_id,
    77    magicRun.label,
    8     magicRun.workdir
     8    magicRun.workdir,
     9    CAST(NULL AS SIGNED) AS inv_magic_id,
     10    CAST(NULL AS SIGNED) AS inv_exp_id
    911FROM magicRun
    1012    JOIN magicMask USING(magic_id)

  • branches/czw_branch/cleanup/ippTools/share/magicdstool_definebyquery_chip.sql

    r24951 r25051  
    66    camRun.cam_id,
    77    magicRun.label,
    8     magicRun.workdir
     8    magicRun.workdir,
     9    CAST(NULL AS SIGNED) AS inv_magic_id,
     10    CAST(NULL AS SIGNED) AS inv_exp_id
    911FROM magicRun
    1012    JOIN magicMask USING(magic_id)

  • branches/czw_branch/cleanup/ippTools/share/magicdstool_definebyquery_diff.sql

    r24951 r25051  
     1SELECT *
     2FROM (
    13SELECT
    24    magicRun.magic_id,
    … …  
    68    0 AS cam_id,
    79    magicRun.label,
    8     magicRun.workdir
     10    magicRun.workdir,
     11    CAST(NULL AS SIGNED) AS inv_magic_id,
     12    CAST(NULL AS SIGNED) AS inv_exp_id
    913FROM magicRun
    1014    JOIN magicMask USING(magic_id)
    … …  
    1519    AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s
    1620        OR magicDSRun.magic_ds_id IS NULL)
     21    AND diffRun.bothways = 0
    1722    AND diffRun.magicked  = 0
    1823    AND diffRun.state = 'full'
     24    -- WHERE hook %s
     25UNION
     26SELECT
     27    forwardRun.magic_id,
     28    forwardRun.exp_id,
     29    'diff' AS stage,
     30    forwardRun.diff_id AS stage_id,
     31    0 AS cam_id,
     32    forwardRun.label,
     33    forwardRun.workdir,
     34    inverseRun.magic_id AS inv_magic_id,
     35    inverseRun.exp_id AS inv_exp_id
     36FROM (
     37    SELECT
     38        magicRun.magic_id,
     39        exp_id,
     40        diff_id,
     41        magicRun.label,
     42        magicRun.workdir
     43    FROM magicRun
     44        JOIN magicMask USING(magic_id)
     45        JOIN diffRun USING(diff_id)
     46        LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id
     47                             AND magicDSRun.stage  = 'diff'
     48    WHERE magicRun.state = 'full'
     49        AND magicRun.inverse = 0
     50        AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s
     51            OR magicDSRun.magic_ds_id IS NULL)
     52        AND diffRun.state = 'full'
     53        AND diffRun.bothways
     54        AND diffRun.magicked  = 0
     55        -- WHERE hook %s
     56) AS forwardRun
     57JOIN (
     58    SELECT
     59        magic_id,
     60        exp_id,
     61        diff_id
     62    FROM magicRun
     63        JOIN magicMask USING(magic_id)
     64    WHERE magicRun.state = 'full'
     65        AND magicRun.inverse = 1
     66) AS inverseRun USING(diff_id)
    1967
     68) AS Foo

  • branches/czw_branch/cleanup/ippTools/share/magicdstool_definebyquery_raw.sql

    r24951 r25051  
    66    0 AS cam_id,
    77    magicRun.label,
    8     magicRun.workdir
     8    magicRun.workdir,
     9    CAST(NULL AS SIGNED) AS inv_magic_id,
     10    CAST(NULL AS SIGNED) AS inv_exp_id
    911FROM magicRun
    1012    JOIN magicMask USING(magic_id)

  • branches/czw_branch/cleanup/ippTools/share/magicdstool_definebyquery_warp.sql

    r24951 r25051  
    66    0 AS cam_id,
    77    magicRun.label,
    8     magicRun.workdir
     8    magicRun.workdir,
     9    CAST(NULL AS SIGNED) AS inv_magic_id,
     10    CAST(NULL AS SIGNED) AS inv_exp_id
    911FROM magicRun
    1012    JOIN magicMask USING(magic_id)

  • branches/czw_branch/cleanup/ippTools/share/magicdstool_todestreak.sql

    r24951 r25051  
    88    camera,
    99    magicMask.uri as streaks_uri,
     10    NULL AS inv_streaks_uri,
    1011    stage,
    1112    stage_id,
    … …  
    4041    camera,
    4142    magicMask.uri as streaks_uri,
     43    NULL AS inv_streaks_uri,
    4244    stage,
    4345    stage_id,
    … …  
    7779    camera,
    7880    magicMask.uri as streaks_uri,
     81    NULL AS inv_streaks_uri,
    7982    stage,
    8083    stage_id,
    … …  
    115118    camera,
    116119    magicMask.uri as streaks_uri,
     120    NULL AS inv_streaks_uri,
    117121    stage,
    118122    stage_id,
    … …  
    143147    AND magicDSFile.component IS NULL
    144148UNION
    145 -- diffSkyfiles
     149-- regular diffSkyfiles
    146150SELECT DISTINCT
    147151    magicDSRun.magic_ds_id,
    … …  
    151155    rawExp.camera,
    152156    magicMask.uri as streaks_uri,
     157    NULL AS inv_streaks_uri,
    153158    stage,
    154159    magicRun.diff_id as stage_id,
    … …  
    167172JOIN magicDSRun USING(magic_id)
    168173JOIN magicInputSkyfile USING(magic_id)
     174JOIN diffRun USING(diff_id)
    169175JOIN diffSkyfile
    170176    ON  magicRun.diff_id = diffSkyfile.diff_id
    … …  
    176182    magicDSRun.state = 'new'
    177183    AND magicDSRun.stage = 'diff'
     184    AND diffRun.bothways = 0
    178185    AND diffSkyfile.fault = 0
    179186    AND diffSkyfile.quality = 0
    180187    AND magicDSFile.component IS NULL
     188-- bothways diffSkyfiles
     189UNION
     190SELECT DISTINCT
     191    magicDSRun.magic_ds_id,
     192    magicRun.magic_id,
     193    magicRun.exp_id,
     194    magicDSRun.label,
     195    rawExp.camera,
     196    magicMask.uri as streaks_uri,
     197    (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri,
     198    stage,
     199    magicRun.diff_id as stage_id,
     200    diffSkyfile.skycell_id as component,
     201    NULL AS uri,
     202    diffSkyfile.path_base,
     203    magicRun.inverse,
     204    NULL as cam_path_base,
     205    outroot,
     206    recoveryroot,
     207    re_place,
     208    remove
     209FROM rawExp
     210JOIN magicRun USING (exp_id)
     211JOIN magicMask USING (magic_id)
     212JOIN magicDSRun USING(magic_id)
     213JOIN magicInputSkyfile USING(magic_id)
     214JOIN diffRun USING(diff_id)
     215JOIN diffSkyfile
     216    ON  magicRun.diff_id = diffSkyfile.diff_id
     217    AND magicInputSkyfile.node = diffSkyfile.skycell_id
     218LEFT JOIN magicDSFile
     219    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
     220    AND magicDSFile.component = diffSkyfile.skycell_id
     221WHERE
     222    magicDSRun.state = 'new'
     223    AND magicDSRun.stage = 'diff'
     224    AND diffRun.bothways
     225    AND diffSkyfile.fault = 0
     226    AND diffSkyfile.quality = 0
     227    AND magicDSFile.component IS NULL
    181228) as Foo

  • branches/czw_branch/cleanup/ippTools/share/pstamptool_addjob_otherjob.sql

    r19219 r25051  
    11INSERT INTO pstampJob
    2  (req_id, rownum, state, jobType, uri, exp_id, outputBase, fault)
    3  VALUES( %lld, '%s', '%s', '%s', '%s', %lld, '%s', %s)
     2 (req_id, rownum, state, jobType, exp_id, outputBase, fault)
     3 VALUES( %lld, '%s', '%s', '%s', %lld, '%s', %s)

  • branches/czw_branch/cleanup/ippTools/share/pstamptool_addjob_stampjob.sql

    r19219 r25051  
    11INSERT INTO pstampJob
    2  (req_id, rownum, state, jobType, uri, exp_id, outputBase, fault, args)
    3  VALUES( %lld, '%s', '%s', '%s', '%s', %lld, '%s', %s, '%s')
     2 (req_id, rownum, state, jobType, exp_id, outputBase, fault)
     3 VALUES( %lld, '%s', '%s', '%s', %lld, '%s', %s)

  • branches/czw_branch/cleanup/ippTools/share/pxadmin_create_tables.sql

    r24951 r25051  
    10711071        magic_ds_id BIGINT AUTO_INCREMENT,
    10721072        magic_id BIGINT,
     1073        inv_magic_id BIGINT,
    10731074        state VARCHAR(64),
    10741075        stage VARCHAR(64),
    … …  
    10861087        KEY(label),
    10871088        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id)
     1089        FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id)
    10881090) ENGINE=innodb DEFAULT CHARSET=latin1;
    10891091
    … …  
    12031205        jobType VARCHAR(16),
    12041206        fault SMALLINT,
    1205         uri VARCHAR(255),
    12061207        exp_id BIGINT,
    12071208        outputBase VARCHAR(255),
    1208         args VARCHAR(511),
    12091209        PRIMARY KEY(job_id, req_id),
    12101210        KEY(job_id),

  • branches/czw_branch/cleanup/ippTools/share/stacktool_sumskyfile.sql

    r24124 r25051  
    22    stackSumSkyfile.*,
    33    stackRun.state,
     4    stackRun.tess_id,
    45    stackRun.skycell_id,
     6    stackRun.filter,
    57    stackRun.workdir,
     8    stackRun.label,
    69    (SELECT rawExp.camera FROM
    710        stackInputSkyfile

  • branches/czw_branch/cleanup/ippTools/src/difftool.c

    r24951 r25051  
    582582    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
    583583    PXOPT_COPY_S64(config->args, where,  "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
    584     PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "==");
     584    PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "==");
    585585    PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "==");
    586586    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");

  • branches/czw_branch/cleanup/ippTools/src/disttool.c

    r24951 r25051  
    384384        psStringAppend(&outroot, "%s/%s/%s", workdir, run_tag, stage);
    385385
    386         if (set_label == NULL) {
    387             set_label = target_label;
     386        psString new_label;
     387        if (set_label != NULL) {
     388            new_label = set_label;
     389        } else {
     390            new_label = target_label;
    388391        }
    389392        distRunRow *row = distRunRowAlloc(
    … …  
    392395                stage,
    393396                stage_id,
    394                 set_label,
     397                new_label,
    395398                outroot,
    396399                clean,

  • branches/czw_branch/cleanup/ippTools/src/magicdstool.c

    r24951 r25051  
    160160        return false;
    161161    }
    162 
    163     if (psListLength(where->list)) {
    164         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    165         psStringAppend(&query, "\n AND %s", whereClause);
    166         psFree(whereClause);
    167     }
    168     psFree(where);
    169 
    170     // treat limit == 0 as "no limit"
    171     if (limit) {
    172         psString limitString = psDBGenerateLimitSQL(limit);
    173         psStringAppend(&query, " %s", limitString);
    174         psFree(limitString);
    175     }
    176 
    177     if (!p_psDBRunQueryF(config->dbh, query, rerun ? "\n1 " : "\n0 ")) {
    178         psError(PS_ERR_UNKNOWN, false, "database error");
    179         psFree(query);
    180         return false;
     162    char *rerun_flag =  rerun ? "\n1 " : "\n0" ;
     163
     164    if (stageNum != IPP_STAGE_DIFF) {
     165        if (psListLength(where->list)) {
     166            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     167            psStringAppend(&query, "\nAND %s", whereClause);
     168            psFree(whereClause);
     169        }
     170        psFree(where);
     171
     172        // treat limit == 0 as "no limit"
     173        if (limit) {
     174            psString limitString = psDBGenerateLimitSQL(limit);
     175            psStringAppend(&query, " %s", limitString);
     176            psFree(limitString);
     177        }
     178        if (!p_psDBRunQueryF(config->dbh, query, rerun_flag)) {
     179            psError(PS_ERR_UNKNOWN, false, "database error");
     180            psFree(query);
     181            return false;
     182        }
     183    } else {
     184        // diff stage query has two types bothways and !bothways
     185        // so we need to send the rerun flag and the where data twice
     186        psString whereString = psStringCopy("");
     187        if (psListLength(where->list)) {
     188            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     189            psStringAppend(&whereString, "\nAND %s", whereClause);
     190            psFree(whereClause);
     191        }
     192        psFree(where);
     193
     194        // treat limit == 0 as "no limit"
     195        if (limit) {
     196            psString limitString = psDBGenerateLimitSQL(limit);
     197            psStringAppend(&query, " %s", limitString);
     198            psFree(limitString);
     199        }
     200
     201        if (!p_psDBRunQueryF(config->dbh, query, rerun_flag, whereString, rerun_flag, whereString)) {
     202            psError(PS_ERR_UNKNOWN, false, "database error");
     203            psFree(whereString);
     204            psFree(query);
     205            return false;
     206        }
     207        psFree(whereString);
    181208    }
    182209    psFree(query);
    … …  
    215242        psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id");
    216243        psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
     244        psS64 inv_magic_id = psMetadataLookupS64(NULL, row, "inv_magic_id");
    217245        psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id");
    218246        psString magicRunLabel = psMetadataLookupStr(NULL, row, "label");
    … …  
    232260                0, // magic_ds_id
    233261                magic_id,
     262                inv_magic_id,
    234263                "new",
    235264                stage,
    … …  
    309338            0,          // ID
    310339            magic_id,
     340            0,          // inv_magic_id
    311341            "new",      // state
    312342            stage,

  • branches/czw_branch/cleanup/ippTools/src/magictool.c

    r24951 r25051  
    582582    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    583583    PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
     584    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
    584585
    585586    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0");
    … …  
    803804    // treat limit == 0 as "no limit"
    804805    if (limit) {
    805         psString limitString = psDBGenerateLimitSQL(limit);
     806        // cut limit in half
     807        // hack to prevent pending leaf nodes from blocking branch nodes
     808        psString limitString = psDBGenerateLimitSQL((limit + 1) / 2);
    806809        psStringAppend(&query, " %s", limitString);
    807810        psFree(limitString);
    … …  
    849852
    850853    // look for tree nodes that need to be processed
    851     // XXX: This gets all nodes from all magicRuns that are in 'new'state
    852     // That doens't seem particularly efficient
    853     query = pxDataGet("magictool_toprocess_tree.sql");
     854    query = pxDataGet("magictool_toprocess_runs.sql");
    854855    if (!query) {
    855856        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    … …  
    857858    }
    858859
     860    {
     861        psString limitString = psDBGenerateLimitSQL( 100 );
     862        psStringAppend(&query, " %s", limitString);
     863        psFree(limitString);
     864    }
    859865
    860866    if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString :  "")) {
    … …  
    867873    psFree(query);
    868874
    869     psArray *magicTree = p_psDBFetchResult(config->dbh);
    870     if (!magicTree) {
     875    psArray *magicRuns = p_psDBFetchResult(config->dbh);
     876    if (!magicRuns) {
    871877        psErrorCode err = psErrorCodeLast();
    872878        switch (err) {
    … …  
    881887        return false;
    882888    }
    883     if (!psArrayLength(magicTree)) {
     889    if (!psArrayLength(magicRuns)) {
    884890        psTrace("magictool", PS_LOG_INFO, "no rows found");
    885         psFree(magicTree);
     891        psFree(magicRuns);
    886892        return true;
    887893    }
    888894
    889     // entries are ordered by magic_id
    890     long index = 0;
    891     while (index <  psArrayLength(magicTree)) {
    892         bool status;
     895    query = pxDataGet("magictool_toprocess_tree.sql");
     896    if (!query) {
     897        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
     898        return false;
     899    }
     900
     901    for (psS64 index = 0; index < psArrayLength(magicRuns); index++) {
    893902        if (limit && (psArrayLength(output) >= limit)) {
    894903            break;
    895904        }
    896         psS64 current_magic_id = psMetadataLookupS64(&status, magicTree->data[index], "magic_id");
     905        bool status;
     906        psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id");
    897907        if (!status) {
    898908            psAbort("failed to lookup value for magic_id column");
    899909        }
    900 
    901         // find the end of this block
    902         long first = index;
    903         long last = index;
    904         for (long i = index + 1; i < psArrayLength(magicTree); i++) {
    905             psS64 magic_id = psMetadataLookupS64(&status, magicTree->data[i], "magic_id");
    906             if (!status) {
    907                 psAbort("failed to lookup value for magic_id column");
     910       
     911        whereString = NULL;
     912        psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
     913        if (!p_psDBRunQueryF(config->dbh, query, whereString )) {
     914            psError(PS_ERR_UNKNOWN, false, "database error");
     915            psFree(whereString);
     916            psFree(query);
     917            return false;
     918        }
     919        psFree(whereString);
     920        psArray *magicTree = p_psDBFetchResult(config->dbh);
     921        if (!magicTree) {
     922            psErrorCode err = psErrorCodeLast();
     923            switch (err) {
     924                case PS_ERR_DB_CLIENT:
     925                    psError(PXTOOLS_ERR_SYS, false, "database error");
     926                case PS_ERR_DB_SERVER:
     927                    psError(PXTOOLS_ERR_PROG, false, "database error");
     928                default:
     929                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
    908930            }
    909             if (magic_id != current_magic_id) {
    910                 break;
    911             }
    912             last = i;
    913         }
    914 
    915         index = last + 1;
    916 
    917         psHash *forest = psHashAlloc(last - first + 1);
     931
     932            return false;
     933        }
     934        psS64 length = psArrayLength(magicTree);
     935        if (!length) {
     936            psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id);
     937            psFree(magicTree);
     938            continue;
     939        }
     940
     941        psHash *forest = psHashAlloc(length);
    918942
    919943        // convert the array of metadata into a pxTree structure
    920         for (long i = first; i <= last; i++) {
     944        for (long i = 0; i < length; i++) {
    921945            bool status;
    922946            psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
    … …  
    943967        pxTreeCrawl(root, findReadyNodes, output);
    944968        psFree(root);
    945 
    946     }
    947     psFree(magicTree);
     969        psFree(magicTree);
     970    }
    948971
    949972    if (psArrayLength(output)) {

  • branches/czw_branch/cleanup/ippTools/src/magictoolConfig.c

    r24951 r25051  
    108108    psMetadataAddS64(reverttreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0);
    109109    psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
     110    psMetadataAddStr(reverttreeArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
    110111
    111112    // -inputs

  • branches/czw_branch/cleanup/ippTools/src/pstamptool.c

    r24951 r25051  
    500500    PS_ASSERT_PTR_NON_NULL(config, false);
    501501
    502     PXOPT_LOOKUP_S64(uri,         config->args, "-uri",        true, false);
    503502    PXOPT_LOOKUP_S64(req_id,      config->args, "-req_id",     true, false);
    504503    PXOPT_LOOKUP_STR(rownum,      config->args, "-rownum",     true, false);
    505504    PXOPT_LOOKUP_STR(job_type,    config->args, "-job_type",   false, false);
    506505    PXOPT_LOOKUP_STR(outputBase,  config->args, "-outputBase", true,  false);
    507     PXOPT_LOOKUP_STR(argString,   config->args, "-args",       false, false);
    508506    PXOPT_LOOKUP_STR(stateString, config->args, "-state",      false, false);
    509507    PXOPT_LOOKUP_STR(fault,       config->args, "-fault",      false, false);
    … …  
    519517        return false;
    520518    }
    521 
    522     if (stampJob && !argString) {
    523         psError(PS_ERR_UNKNOWN, true, "-args is required for stamp job");
    524         return false;
    525     }
    526 
    527519    if (stampJob) {
    528520        query = pxDataGet("pstamptool_addjob_stampjob.sql");
    … …  
    535527    }
    536528
    537     if (!p_psDBRunQueryF(config->dbh, query, req_id, rownum, stateString, job_type, uri, exp_id, outputBase, fault, argString)) {
     529    if (!p_psDBRunQueryF(config->dbh, query, req_id, rownum, stateString, job_type, exp_id, outputBase, fault)) {
    538530        psError(PS_ERR_UNKNOWN, false, "database error");
    539531        return false;

  • branches/czw_branch/cleanup/ippTools/src/pstamptoolConfig.c

    r20151 r25051  
    100100    // -addjob
    101101    psMetadata *addjobArgs = psMetadataAlloc();
    102     psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-uri", 0,            "define job file uri", NULL);
    103102    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0,            "define job req_id", 0);
    104103    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0,            "define job rownum", NULL);
    105104    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0,            "define job job_type", "stamp");
    106105    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0,            "define job outputBase", NULL);
    107     psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-args", 0,            "define job args", NULL);
    108106    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0,            "new state", "run");
    109107    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0,           "exposure id", 0);
    … …  
    128126    psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0,            "req_id for which to change state", 0);
    129127    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0,            "new state", NULL);
    130     // psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-result", 0,            "new result", NULL);
    131128    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0,            "new result", NULL);
    132129

  • branches/czw_branch/cleanup/ippTools/src/stacktool.c

    r24951 r25051  
    857857    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
    858858    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
     859    PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "==");
     860    PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "==");
    859861
    860862    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);

  • branches/czw_branch/cleanup/ippTools/src/stacktoolConfig.c

    r24951 r25051  
    159159    psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,            "search by stack ID", 0);
    160160    psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warp ID", 0);
     161    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", 0);
     162    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,         "search by skycell ID", 0);
    161163    psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0,            "search by exposure ID", 0);
    162164    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0,          "search by exposure name", NULL);

  • branches/czw_branch/cleanup/ippTools/src/warptool.c

    r24951 r25051  
    11921192    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
    11931193    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
     1194    PXOPT_COPY_S64(config->args, where, "-warp_skyfile_id", "warpImfile.warp_skyfile_id", "==");
     1195    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
    11941196    PXOPT_COPY_STR(config->args, where, "-tess_id",    "warpSkyfile.tess_id", "==");
    11951197    PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");

  • branches/czw_branch/cleanup/ippTools/src/warptoolConfig.c

    r24951 r25051  
    208208    // -warped
    209209    psMetadata *warpedArgs = psMetadataAlloc();
    210     psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warptool ID", 0);
    211     psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id",  0,            "define skycell ID", NULL);
    212     psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id",  0,            "define tessellation ID", NULL);
    213     psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0,            "define exposure tag", 0);
    214     psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0,          "define exposure tag", 0);
    215     psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0,            "define phase 3 version of exposure tag", 0);
     210    psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_id", 0,           "search by warptool ID", 0);
     211    psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id",  0,       "search by skycell ID", NULL);
     212    psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_skyfile_id", 0,   "search by skyfile ID", 0);
     213    psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id",  0,          "search by tessellation ID", NULL);
     214    psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0,            "search by exposure tag", 0);
     215    psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0,          "search by exposure tag", 0);
     216    psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0,           "search by phase 3 version of exposure tag", 0);
    216217    psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    217     psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     218    psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
    218219
    219220    // -revertwarped

  • branches/czw_branch/cleanup/ippconfig/vysos5/psastro.config

    r21502 r25051  
    1212PSASTRO.GRID.NSTAR.MAX S32     800 # max stars accepted for fitting
    1313
    14 PSASTRO.MAX.NRAW      S32       500   # max stars accepted for fitting (0 for all)
     14PSASTRO.MAX.NRAW      S32      1000   # max stars accepted for fitting (0 for all)
    1515PSASTRO.MAX.NREF      S32      2000   # max stars accepted for fitting (0 for all)
    1616
    … …  
    2626# PSASTRO.GRID.MAX.ANGLE F32 360.0
    2727# PSASTRO.GRID.DEL.ANGLE F32  1.0
    28 PSASTRO.GRID.MIN.ANGLE F32 -10.0 # start angle (degrees)
     28PSASTRO.GRID.MIN.ANGLE F32 +6.0 # start angle (degrees)
    2929PSASTRO.GRID.MAX.ANGLE F32 +10.0
    30 PSASTRO.GRID.DEL.ANGLE F32  2.5
     30PSASTRO.GRID.DEL.ANGLE F32  1.0
    3131
    3232#
    33 PSASTRO.GRID.MIN.SCALE F32  0.90
    34 PSASTRO.GRID.MAX.SCALE F32  1.10
    35 PSASTRO.GRID.DEL.SCALE F32  0.025
     33PSASTRO.GRID.MIN.SCALE F32  0.99
     34PSASTRO.GRID.MAX.SCALE F32  1.01
     35PSASTRO.GRID.DEL.SCALE F32  0.01
    3636
    3737# single-chip radius match in pixels
    … …  
    7171DVO.GETSTAR.PHOTCODE        STR      r
    7272
     73PHOTCODE.DATA MULTI
     74PHOTCODE.DATA METADATA
     75  FILTER   STR r
     76  ZEROPT   F32 19.0
     77  PHOTCODE STR r_SYNTH
     78  GHOST_MAX_MAG                   F32 -20.0
     79END
     80PHOTCODE.DATA METADATA
     81  FILTER   STR g
     82  ZEROPT   F32 15.0
     83  PHOTCODE STR g_SYNTH
     84  GHOST_MAX_MAG                   F32 -20.0
     85END
     86
    7387PSASTRO.PLOT.REF.MAG.MIN  F32 +5.0
    7488PSASTRO.PLOT.REF.MAG.MAX  F32 +12.0
    7589
     90# PSASTRO.MATCH.LUMFUNC  BOOL     TRUE
     91
    7692# DVO.GETSTAR.MAG.MAX         F32      20.0
    7793# XXX need to be able to limit the density!

  • branches/czw_branch/cleanup/magic/remove/src/Line.c

    r24382 r25051  
    444444                if (y >= 0 && y < numRows)
    445445                {
    446                     pixel = psAlloc (sizeof(PixelPos));
    447                     pixel->x = (unsigned int) x;
    448                     pixel->y = (unsigned int) y;
    449                     psArrayAdd (pixels, 1024, pixel);
    450                     psFree (pixel);
     446                    psImageSet(pixels, x, y, 1);
    451447                }
    452448            }
    … …  
    494490                if (x >=0 && x < numCols)
    495491                {
    496                     pixel = psAlloc (sizeof(PixelPos));
    497                     pixel->x = (unsigned int) x;
    498                     pixel->y = (unsigned int) y;
    499                     psArrayAdd (pixels, 1024, pixel);
    500                     psFree (pixel);
     492                    psImageSet(pixels, x, y, 1);
    501493                }
    502494            }

  • branches/czw_branch/cleanup/magic/remove/src/streaksextern.c

    r24382 r25051  
    3434    int i;
    3535    Line line;
    36     StreakPixels *pixels = psArrayAllocEmpty (1024);
     36    StreakPixels *pixels = psImageAlloc(numCols, numRows, PS_TYPE_U8);
     37    psImageInit(pixels, 0);
    3738    int streaksOnComponent = 0;
    3839

  • branches/czw_branch/cleanup/magic/remove/src/streaksextern.h

    r20308 r25051  
    44/** psLib includes */
    55
    6 #include "psArray.h"
     6#include "psImage.h"
    77
    88/** streakremove includes */
    … …  
    3636/** For now, use psArray to define a list of PixelPos pointers */
    3737
    38 typedef psArray StreakPixels;
     38typedef psImage StreakPixels;
    3939
    4040/** Create a list of pixel positions from a Streaks pointer list

  • branches/czw_branch/cleanup/magic/remove/src/streaksremove.c

    r24951 r25051  
    1313static bool readAndCopyToOutput(streakFiles *sf, bool exciseAll);
    1414static void exciseNonWarpedPixels(streakFiles *sfiles, double newMaskValue);
    15 static bool warpedPixel(streakFiles *sfiles, PixelPos *cellCoord);
     15static bool warpedPixel(streakFiles *sfiles, int x, int y);
    1616static void excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, double newMaskValue);
    1717static void writeImages(streakFiles *sf, bool exciseImageCube);
    … …  
    146146                }
    147147
    148                 totalStreakPixels +=  psArrayLength(pixels);
    149148
    150149                psTimerStart("REMOVE_STREAKS");
    151150
    152                 // for each pixel covered by the streak
    153                 for (int i = 0; i < psArrayLength (pixels); ++i) {
    154                     PixelPos *pixelPos = psArrayGet (pixels, i);
    155 
    156                     // if this pixel was not part of the warp, skip because it has already been
    157                     // excised (unless we weren't asked to check)
    158                     if (!checkNonWarpedPixels || warpedPixel(sfiles, pixelPos)) {
    159 
    160                         excisePixel(sfiles, pixelPos->x, pixelPos->y, true, maskStreak);
    161 
    162                     } else {
    163                         // This pixel was not included in any warp and has thus already excised
    164                         // by exciseNonWarpedPixels
     151                for (int y=0 ; y < sfiles->inImage->numRows; y++) {
     152                    for (int x = 0; x < sfiles->inImage->numCols; x++) {
     153                        if (psImageGet(pixels, x, y)) {
     154                            ++totalStreakPixels;
     155                            if (!checkNonWarpedPixels || warpedPixel(sfiles, x, y)) {
     156
     157                                excisePixel(sfiles, x, y, true, maskStreak);
     158
     159                            } else {
     160                                // This pixel was not included in any warp and has thus already excised
     161                                // by exciseNonWarpedPixels
     162                            }
     163                        }
    165164                    }
    166165                }
    … …  
    178177                exciseImageCube = true;
    179178            }
    180             psArrayElementsFree (pixels);
    181179            psFree(pixels);
    182180        }
    … …  
    756754
    757755static bool
    758 warpedPixel(streakFiles *sfiles, PixelPos *cellCoord)
     756warpedPixel(streakFiles *sfiles, int x, int y)
    759757{
    760758    PixelPos chipCoord;
    … …  
    767765    // we clip so that the streak calculation code doesn't have to
    768766    // clipping here insures that we don't touch the overscan regions
    769     if ((cellCoord->x < 0) || (cellCoord->x >= sfiles->inImage->numCols) ||
    770         (cellCoord->y < 0) || (cellCoord->y >= sfiles->inImage->numRows)) {
     767    if ((x < 0) || (x >= sfiles->inImage->numCols) ||
     768        (y < 0) || (y >= sfiles->inImage->numRows)) {
    771769        return false;
    772770    }
    773771
    774     cellToChipInt(&chipCoord.x, &chipCoord.y, sfiles->astrom, cellCoord->x, cellCoord->y);
     772    cellToChipInt(&chipCoord.x, &chipCoord.y, sfiles->astrom, x, y);
    775773
    776774    if (chipCoord.x < 0 || chipCoord.x >= sfiles->warpedPixels->numCols) {
    … …  
    839837    }
    840838
    841     printf("Censored %d sources\n", numCensored);
    842 
    843839    // get rid of unused elements (don't know if this is necessary)
    844840    psArrayRealloc(outTable, j);
    845841
    846842    addDestreakKeyword(in->header);
    847     if (! psFitsWriteTable(out->fits, in->header, outTable, extname)) {
    848         psError(PS_ERR_IO, false, "failed to write table to %s", out->resolved_name);
    849         streaksExit("", PS_EXIT_DATA_ERROR);
     843    if (psArrayLength(outTable) > 0) {
     844        printf("Censored %d sources\n", numCensored);
     845        if (! psFitsWriteTable(out->fits, in->header, outTable, extname)) {
     846            psError(PS_ERR_IO, false, "failed to write table to %s", out->resolved_name);
     847            streaksExit("", PS_EXIT_DATA_ERROR);
     848        }
     849    } else {
     850        printf("Censored ALL %d sources\n", numCensored);
     851        if (! psFitsWriteTableEmpty(out->fits, in->header, inTable->data[0], extname)) {
     852            psError(PS_ERR_IO, false, "failed to write empty table to %s", out->resolved_name);
     853            streaksExit("", PS_EXIT_DATA_ERROR);
     854        }
    850855    }
    851856

  • branches/czw_branch/cleanup/ppMops/src/ppMops.c

    r24706 r25051  
    139139        psMetadataAddF64(outHeader, PS_LIST_TAIL, "TEL_ALT", 0, "Telescope altitude", alt);
    140140        psMetadataAddF64(outHeader, PS_LIST_TAIL, "TEL_AZ", 0, "Telescope azimuth", az);
    141         psMetadataAddF64(outHeader, PS_LIST_TAIL, "DIFFIMID", 0, "Difference image identifier", imageid);
    142         psMetadataAddF64(outHeader, PS_LIST_TAIL, "FPA_ID", 0, "Exposure identifier", fpaid);
     141        psMetadataAddS32(outHeader, PS_LIST_TAIL, "DIFFIMID", 0, "Difference image identifier", imageid);
     142        psMetadataAddS32(outHeader, PS_LIST_TAIL, "FPA_ID", 0, "Exposure identifier", fpaid);
    143143        psMetadataAddStr(outHeader, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", OBSERVATORY_CODE);
    144144        psMetadataAddF32(outHeader, PS_LIST_TAIL, "STARPSF", 0, "Stellar PSF (arcsec)", psf);

  • branches/czw_branch/cleanup/ppStack/src/ppStackMatch.c

    r24951 r25051  
    222222            }
    223223            psFitsClose(fits);
    224 
    225             // Add in variance factor
    226             pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, conv->analysis,
    227                                                                 PM_SUBTRACTION_ANALYSIS_KERNEL); // Kernels
    228             float vf = pmSubtractionVarianceFactor(kernels, 0.0, 0.0, false); // Variance factor
    229             psMetadataItem *vfItem = psMetadataLookup(readout->parent->concepts, "CELL.VARFACTOR");
    230             if (!isfinite(vf)) {
    231                 vf = 1.0;
    232             }
    233             if (isfinite(vfItem->data.F32)) {
    234                 vfItem->data.F32 *= vf;
    235             } else {
    236                 vfItem->data.F32 = vf;
    237             }
    238224
    239225            if (!readImage(&readout->image, options->imageNames->data[index], config) ||
    … …  
    413399            }
    414400
    415             // Set the variance factor
    416             psMetadataItem *vfItem = psMetadataLookup(readout->parent->concepts, "CELL.VARFACTOR");
    417             float vf = psMetadataLookupF32(NULL, conv->analysis, PM_SUBTRACTION_ANALYSIS_VARFACTOR_1);
    418             if (!isfinite(vf)) {
    419                 vf = 1.0;
    420             }
    421             if (isfinite(vfItem->data.F32)) {
    422                 vfItem->data.F32 *= vf;
    423             } else {
    424                 vfItem->data.F32 = vf;
    425             }
    426 
    427401            // Replace original images with convolved
    428402            psFree(readout->image);
    … …  
    512486        psBinaryOp(readout->variance, readout->variance, "*", psScalarAlloc(PS_SQR(norm), PS_TYPE_F32));
    513487    }
    514    
     488
    515489    // Ensure the background value is zero
    516490    psStats *bg = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics for background
    … …  
    521495    } else {
    522496      if (!psMetadataLookupBool(NULL, config->arguments, "PPSTACK.SKIP.BG.SUB")) {
    523         psLogMsg("ppStack", PS_LOG_INFO, "Correcting convolved image background by %lf (+/- %lf)",
    524                 psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), psStatsGetValue(bg, PS_STAT_ROBUST_STDEV));
    525         (void)psBinaryOp(readout->image, readout->image, "-",
    526                         psScalarAlloc(psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), PS_TYPE_F32));
     497        psLogMsg("ppStack", PS_LOG_INFO, "Correcting convolved image background by %lf (+/- %lf)",
     498                psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), psStatsGetValue(bg, PS_STAT_ROBUST_STDEV));
     499        (void)psBinaryOp(readout->image, readout->image, "-",
     500                        psScalarAlloc(psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), PS_TYPE_F32));
    527501      }
    528502    }
    529    
     503
    530504
    531505    // Measure the variance level for the weighting

  • branches/czw_branch/cleanup/psLib/src/fits/psFits.c

    r24285 r25051  
    150150            char errorBuf[MAX_STRING_LENGTH], *errorMsg;
    151151#if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && ! _GNU_SOURCE)
    152             errorMsg = strerror_r(thisErrno, errorBuf, MAX_STRING_LENGTH);
    153 #else
    154152            strerror_r(thisErrno, errorBuf, MAX_STRING_LENGTH);
    155153            errorMsg = errorBuf;
     154#else
     155            errorMsg = strerror_r(thisErrno, errorBuf, MAX_STRING_LENGTH);
    156156#endif
    157157            psError(PS_ERR_IO, true, "Directory (%s) for requested file is not accessible: %s",
    … …  
    209209                char errorBuf[64], *errorMsg;
    210210# if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && ! _GNU_SOURCE)
    211                 errorMsg = strerror_r (errno, errorBuf, 64);
    212 # else
    213211                strerror_r (errno, errorBuf, 64);
    214212                errorMsg = errorBuf;
     213# else
     214                errorMsg = strerror_r (errno, errorBuf, 64);
    215215# endif
    216                 psError (PS_ERR_IO, true, "Failed to delete a previously-existing file: %s", errorMsg);
    217                 fprintf (stderr, "errno: %d, %s, %s : %lx, %lx\n", thisErrno, errorMsg, errorBuf, (long int) errorMsg, (long int) errorBuf);
     216                psError(PS_ERR_IO, true, "Failed to delete a previously-existing file (%s), error %d: %s",
     217                        name, thisErrno, errorMsg);
     218                //fprintf(stderr, "errno: %d, %s, %s : %lx, %lx\n", thisErrno, errorMsg, errorBuf, (long int) errorMsg, (long int) errorBuf);
    218219                return NULL;
    219220            }

  • branches/czw_branch/cleanup/psLib/src/fits/psFitsHeader.c

    r23376 r25051  
    147147    return keyword;
    148148}
    149 
    150149
    151150bool psFitsCheckCompressedImagePHU(const psFits *fits, psMetadata *header)
    … …  
    497496static bool fitsWriteHeader(psFits *fits, // The FITS file handle
    498497                            const psMetadata *output, // Metadata that is to be output into the FITS file
    499                             bool keyStarts // Write out the key starts?
     498                            bool keyStarts, // Write out the key starts?
     499                            bool phuImage   // Are we writing a PHU image?
    500500                           )
    501501{
    502502    int status = 0;                     // Status of cfitsio calls
     503    int extnum = psFitsGetExtNum(fits); // Number of extension
    503504    bool simple = true;                 // If SIMPLE is T, then the file should conform to the FITS standard
    504505    psFitsCompressionType compress = psFitsCompressionGetType(fits); // Compression type
    505     if (psFitsGetExtNum(fits) == 0) {
     506    if (extnum == 0) {
     507
    506508        // We allow the user to write SIMPLE, but it must be boolean
    507509        psMetadataItem *simpleItem = psMetadataLookup(output, "SIMPLE"); // SIMPLE in the header
    … …  
    526528    bool compressing = ((!fits->options || fits->options->conventions.compression) &&
    527529                        compress != PS_FITS_COMPRESS_NONE) ? true : false; // Are we compressing?
    528 
    529530    if (compressing) {
    530531        psMetadataItem *simpleItem = psMetadataLookup(output, "SIMPLE"); // SIMPLE in the header
    … …  
    538539            }
    539540        }
    540         //        int value = simple;             // Temporary holder for boolean
    541         //        fits_update_key(fits->fd, TLOGICAL, "ZSIMPLE", &value,
    542         //                        "Uncompressed file's conformance to FITS standard", &status);
     541        if (simple && phuImage && extnum == 1) {
     542            // ZSIMPLE is required for decompression with funpack, etc.  For funpack to work, ZSIMPLE needs to
     543            // go early in the FITS header (otherwise we get "Extension doesn't start with SIMPLE or XTENSION
     544            // keyword.").  We put it after ZIMAGE by reading ZIMAGE (which sets the insertion pointer) and
     545            // then inserting ZSIMPLE.
     546            char comment[FLEN_CARD];    // Comment for ZIMAGE; unused
     547            int value;                  // Value for ZIMAGE; unused
     548            fits_read_key(fits->fd, TLOGICAL, "ZIMAGE", &value, comment, &status);
     549            fits_insert_key_log(fits->fd, "ZSIMPLE", simple, "Uncompressed file's conforms to FITS", &status);
     550        }
    543551    }
    544552
    … …  
    699707    PS_ASSERT_METADATA_NON_NULL(output, false);
    700708
    701     return fitsWriteHeader(fits, output, true);
     709    return fitsWriteHeader(fits, output, true, false);
     710}
     711
     712bool psFitsWriteHeaderImage(psFits *fits,
     713                            const psMetadata *output,
     714                            bool phuImage
     715                            )
     716{
     717    PS_ASSERT_FITS_NON_NULL(fits, false);
     718    PS_ASSERT_FITS_WRITABLE(fits, false);
     719    PS_ASSERT_METADATA_NON_NULL(output, false);
     720
     721    return fitsWriteHeader(fits, output, true, phuImage);
    702722}
    703723
    … …  
    733753    }
    734754
    735     if (output && !fitsWriteHeader(fits, output, false)) {
     755    if (output && !fitsWriteHeader(fits, output, false, false)) {
    736756        psError(PS_ERR_IO, false, "Unable to write FITS header.\n");
    737757        return false;

  • branches/czw_branch/cleanup/psLib/src/fits/psFitsHeader.h

    r22729 r25051  
    6666);
    6767
     68/** Write a header for an image
     69 *  Principal difference with psFitsWriteHeader is this allows writing ZSIMPLE for uncompressing a single image
     70 */
     71bool psFitsWriteHeaderImage(
     72    psFits *fits,                       ///< FITS file
     73    const psMetadata *output,           ///< Header to output
     74    bool phuImage                       ///< Is this image supposed to be in the PHU?
     75    );
     76
     77
    6878/** Writes a "blank" --- a header only, with no image or table.
    6979 *

  • branches/czw_branch/cleanup/psLib/src/fits/psFitsImage.c

    r24951 r25051  
    616616    }
    617617
     618    bool createPHU = false;             // Are we creating a PHU?
     619
    618620    // Create the image HDU
    619621    int hdus = psFitsGetSize(fits);     // Number of HDUs in file
    … …  
    621623        // We're creating the first image
    622624        fits_create_img(fits->fd, bitPix, naxis, naxes, &status);
     625        createPHU = true;
    623626    } else {
    624627        if (!after) {
    … …  
    627630                // Set status to signal fits_insert_img to insert a new primary HDU
    628631                status = PREPEND_PRIMARY;
     632                createPHU = true;
    629633            } else {
    630634                // Move back one to perform an insert after the previous HDU
    … …  
    641645
    642646    // write the header, if any.
    643     if (header && !psFitsWriteHeader(fits, header)) {
     647    if (header && !psFitsWriteHeaderImage(fits, header, createPHU)) {
    644648        psError(PS_ERR_IO, false, "Unable to write FITS header.\n");
    645649        success = false;

  • branches/czw_branch/cleanup/psLib/src/math/psStats.c

    r24951 r25051  
    129129
    130130# define COUNT_WARNING(LIMIT, INTERVAL, ...) { \
    131         static int nCalls = 1; \
    132         if (nCalls < LIMIT) { \
    133             psWarning(__VA_ARGS__); \
    134         } \
    135         if (!(nCalls % INTERVAL)) { \
    136             psWarning(__VA_ARGS__); \
     131        static int nCalls = 1; \
     132        if (nCalls < LIMIT) { \
     133            psWarning(__VA_ARGS__); \
     134        } \
     135        if (!(nCalls % INTERVAL)) { \
     136            psWarning(__VA_ARGS__); \
    137137            psWarning("(warning raised %d times)", nCalls); \
    138         } \
    139         nCalls ++; \
     138        } \
     139        nCalls ++; \
    140140}
    141  
     141
    142142
    143143/*****************************************************************************/
    … …  
    217217    for (long i = 0; i < numData; i++) {
    218218        // Check if the data is with the specified range
    219         if (!isfinite(data[i]))
    220             continue;
     219        if (!isfinite(data[i]))
     220            continue;
    221221        if (useRange && (data[i] < stats->min))
    222222            continue;
    … …  
    242242
    243243    if (!isnan(mean)) {
    244         stats->results |= PS_STAT_SAMPLE_MEAN;
     244        stats->results |= PS_STAT_SAMPLE_MEAN;
    245245    }
    246246    return true;
    … …  
    278278    for (long i = 0; i < num; i++) {
    279279        // Check if the data is with the specified range
    280         if (!isfinite(vector[i]))
    281             continue;
     280        if (!isfinite(vector[i]))
     281            continue;
    282282        if (useRange && (vector[i] < stats->min))
    283283            continue;
    … …  
    329329    // into the temporary vectors.
    330330    for (long i = 0; i < inVector->n; i++) {
    331         if (!isfinite(input[i]))
    332             continue;
     331        if (!isfinite(input[i]))
     332            continue;
    333333        if (useRange && (input[i] < stats->min))
    334334            continue;
    … …  
    353353    // Sort the temporary vector.
    354354    if (!psVectorSort(outVector, outVector)) { // Sort in-place (since it's a copy, it's OK)
    355         // an error in psVectorSort is a serious error
     355        // an error in psVectorSort is a serious error
    356356        psError(PS_ERR_UNEXPECTED_NULL, false, _("Failed to sort input data."));
    357357        stats->sampleUQ = NAN;
    … …  
    407407    // If the mean is NAN, then generate a warning and set the stdev to NAN.
    408408    if (isnan(stats->sampleMean)) {
    409         COUNT_WARNING(10, 100, "WARNING: vectorSampleStdev(): sample mean is NAN. Setting stats->sampleStdev = NAN.");
     409        COUNT_WARNING(10, 100, "WARNING: vectorSampleStdev(): sample mean is NAN. Setting stats->sampleStdev = NAN.");
    410410        stats->sampleStdev = NAN;
    411411        return true;
    … …  
    425425    for (long i = 0; i < myVector->n; i++) {
    426426        // Check if the data is with the specified range
    427         if (!isfinite(data[i]))
    428             continue;
     427        if (!isfinite(data[i]))
     428            continue;
    429429        if (useRange && (data[i] < stats->min)) {
    430430            continue;
    … …  
    502502    for (long i = 0; i < myVector->n; i++) {
    503503        // Check if the data is with the specified range
    504         if (!isfinite(data[i]))
    505             continue;
     504        if (!isfinite(data[i]))
     505            continue;
    506506        if (useRange && (data[i] < stats->min)) {
    507507            continue;
    … …  
    771771            stats->robustLQ = min;
    772772            stats->robustN50 = numValid;
    773             // XXX this is sort of an invalid / out-of-bounds result: to set or not to set these bits:
     773            // XXX this is sort of an invalid / out-of-bounds result: to set or not to set these bits:
    774774            stats->results |= PS_STAT_ROBUST_MEDIAN;
    775775            stats->results |= PS_STAT_ROBUST_STDEV;
    … …  
    801801        // XXXXX we need to consider this step if errors -> variance
    802802        if (!psVectorHistogram(histogram, myVector, errors, mask, maskVal)) {
    803             // if psVectorHistogram returns false, we have a programming error
     803            // if psVectorHistogram returns false, we have a programming error
    804804            psError(PS_ERR_UNKNOWN, false, "Unable to generate histogram for robust statistics.\n");
    805             psFree(histogram);
    806             psFree(cumulative);
    807             psFree(statsMinMax);
    808             psFree(mask);
    809 
    810             return false;
     805            psFree(histogram);
     806            psFree(cumulative);
     807            psFree(statsMinMax);
     808            psFree(mask);
     809
     810            return false;
    811811        }
    812812        if (psTraceGetLevel("psLib.math") >= 8) {
    … …  
    838838
    839839        // ADD step 3: Interpolate to the exact 50% position in bin units
    840         stats->robustMedian = fitQuadraticSearchForYThenReturnBin(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0);
    841         // float robustBin = fitQuadraticSearchForYThenReturnXusingValues(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0);
    842         // fprintf (stderr, "robustBin : %f vs %f\n", robustBin, stats->robustMedian);
    843 
    844         // convert bin to bin value: this is the robust histogram median.
     840        stats->robustMedian = fitQuadraticSearchForYThenReturnBin(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0);
     841        // float robustBin = fitQuadraticSearchForYThenReturnXusingValues(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0);
     842        // fprintf (stderr, "robustBin : %f vs %f\n", robustBin, stats->robustMedian);
     843
     844        // convert bin to bin value: this is the robust histogram median.
    845845        if (isnan(stats->robustMedian)) {
    846846            COUNT_WARNING(10, 100, "Failed to fit a quadratic and calculate the 50-percent position.\n");
    … …  
    10541054        if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) {
    10551055            psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n");
    1056             return false;
    1057         }
     1056            return false;
     1057        }
    10581058    }
    10591059
    … …  
    11071107        psHistogram *histogram = psHistogramAlloc(min, max, numBins); // A new histogram (without outliers)
    11081108        if (!psVectorHistogram(histogram, myVector, errors, mask, maskVal)) {
    1109             // if psVectorHistogram returns false, we have a programming error
     1109            // if psVectorHistogram returns false, we have a programming error
    11101110            psError(PS_ERR_UNKNOWN, false, "Unable to generate histogram for fitted statistics.\n");
    11111111            psFree(histogram);
    … …  
    11721172            PS_VECTOR_PRINT_F32(y);
    11731173        }
    1174        
    1175         // psMinimizeLMChi2 can return false for bad data as well as for serious failures
     1174
     1175        // psMinimizeLMChi2 can return false for bad data as well as for serious failures
    11761176        if (!psMinimizeLMChi2(minimizer, NULL, params, NULL, x, y, NULL, minimizeLMChi2Gauss1D)) {
    11771177            psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n");
    … …  
    12351235        if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) {
    12361236            psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n");
    1237             return false;
    1238         }
     1237            return false;
     1238        }
    12391239    }
    12401240
    … …  
    13551355        // psVectorInit (fitMask, 0);
    13561356
    1357         // XXX not sure if these should result in errors or not...
     1357        // XXX not sure if these should result in errors or not...
    13581358        if (!psVectorFitPolynomial1D (poly, NULL, 0, y, NULL, x)) {
    13591359            psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n");
    … …  
    14321432        if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) {
    14331433            psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n");
    1434             return false;
    1435         }
     1434            return false;
     1435        }
    14361436    }
    14371437
    … …  
    17291729        if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) {
    17301730            psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n");
    1731             return false;
    1732         }
     1731            return false;
     1732        }
    17331733    }
    17341734
    … …  
    17661766            COUNT_WARNING(10, 100, "Failed to calculate the min/max of the input vector.\n");
    17671767            psFree(statsMinMax);
    1768             goto escape;
     1768            goto escape;
    17691769        }
    17701770
    … …  
    17741774            stats->fittedMean = min;
    17751775            stats->fittedStdev = 0.0;
    1776             stats->results |= PS_STAT_FITTED_MEAN_V4;
    1777             stats->results |= PS_STAT_FITTED_STDEV_V4;
     1776            stats->results |= PS_STAT_FITTED_MEAN_V4;
     1777            stats->results |= PS_STAT_FITTED_STDEV_V4;
    17781778            return true;
    17791779        }
    … …  
    17931793            psFree(histogram);
    17941794            psFree(statsMinMax);
    1795             goto escape;
     1795            goto escape;
    17961796        }
    17971797        if (psTraceGetLevel("psLib.math") >= 8) {
    … …  
    18251825            COUNT_WARNING(10, 100, "Failed to calculate the min/max of the input vector.\n");
    18261826            psFree(statsMinMax);
    1827             goto escape;
     1827            goto escape;
    18281828        }
    18291829
    … …  
    18861886
    18871887            // fit 2nd order polynomial to ln(y) = -(x-xo)^2/2sigma^2
    1888             // XXX this fit may fail with an error for an ill-conditioned matrix (bad data)
    1889             // we probably should be able to handle the data errors gracefully
     1888            // XXX this fit may fail with an error for an ill-conditioned matrix (bad data)
     1889            // we probably should be able to handle the data errors gracefully
    18901890            psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2);
    18911891            bool status = psVectorFitPolynomial1D (poly, NULL, 0, y, NULL, x);
    … …  
    18981898                psFree(histogram);
    18991899                psFree(statsMinMax);
    1900                 goto escape;
     1900                goto escape;
    19011901            }
    19021902
    … …  
    19171917
    19181918                COUNT_WARNING(10, 100, "fit did not converge\n");
    1919                 goto escape;
     1919                goto escape;
    19201920            }
    19211921
    … …  
    19871987                psFree(histogram);
    19881988                psFree(statsMinMax);
    1989                 goto escape;
     1989                goto escape;
    19901990            }
    19911991
    … …  
    28982898*****************************************************************************/
    28992899static psF32 fitQuadraticSearchForYThenReturnBin(const psVector *xVec,
    2900                                                 psVector *yVec,
    2901                                                 psS32 binNum,
    2902                                                 psF32 yVal
     2900                                                psVector *yVec,
     2901                                                psS32 binNum,
     2902                                                psF32 yVal
    29032903    )
    29042904{
    … …  
    29822982        }
    29832983
    2984         // I believe that mathematically the fitted bin position must be between binNum - 1 and binNum + 1
    2985         assert (binValue >= binNum - 1);
    2986         assert (binValue <= binNum + 1);
    2987 
    2988         int fitBin = binValue;
    2989         float dX = xVec->data.F32[fitBin+1] - xVec->data.F32[fitBin];
    2990         float dY = binValue - fitBin;
    2991         tmpFloat = xVec->data.F32[fitBin] + dY * dX;
     2984        // I believe that mathematically the fitted bin position must be between binNum - 1 and binNum + 1
     2985        assert (binValue >= binNum - 1);
     2986        assert (binValue <= binNum + 1);
     2987
     2988        int fitBin = binValue;
     2989        float dX = xVec->data.F32[fitBin+1] - xVec->data.F32[fitBin];
     2990        float dY = binValue - fitBin;
     2991        tmpFloat = xVec->data.F32[fitBin] + dY * dX;
    29922992    } else {
    29932993        // These are special cases where the bin is at the beginning or end of the vector.
    29942994        if (binNum == 0) {
    29952995            // We have two points only at the beginning of the vectors x and y.
    2996             // X = (dX/dY)(Y - Yo) + Xo
    2997             float dX = xVec->data.F32[1] - xVec->data.F32[0];
    2998             float dY = yVec->data.F32[1] - yVec->data.F32[0];       
    2999             tmpFloat = (yVal - yVec->data.F32[0]) * (dX / dY) + xVec->data.F32[0];
     2996            // X = (dX/dY)(Y - Yo) + Xo
     2997            float dX = xVec->data.F32[1] - xVec->data.F32[0];
     2998            float dY = yVec->data.F32[1] - yVec->data.F32[0];
     2999            if (dY == 0.0) {
     3000                tmpFloat = xVec->data.F32[0];
     3001            } else {
     3002                tmpFloat = (yVal - yVec->data.F32[0]) * (dX / dY) + xVec->data.F32[0];
     3003            }
    30003004        } else if (binNum == (xVec->n - 1)) {
    30013005            // We have two points only at the end of the vectors x and y.
    3002             // X = (dX/dY)(Y - Yo) + Xo
    3003             float dX = xVec->data.F32[binNum] - xVec->data.F32[binNum-1];
    3004             float dY = yVec->data.F32[binNum] - yVec->data.F32[binNum-1];           
    3005             tmpFloat = (yVal - yVec->data.F32[binNum-1]) * (dX / dY) + xVec->data.F32[binNum-1];
    3006         }
     3006            // X = (dX/dY)(Y - Yo) + Xo
     3007            float dX = xVec->data.F32[binNum] - xVec->data.F32[binNum-1];
     3008            float dY = yVec->data.F32[binNum] - yVec->data.F32[binNum-1];
     3009            if (dY == 0.0) {
     3010                tmpFloat = xVec->data.F32[binNum-1];
     3011            } else {
     3012                tmpFloat = (yVal - yVec->data.F32[binNum-1]) * (dX / dY) + xVec->data.F32[binNum-1];
     3013            }
     3014        }
    30073015    }
    30083016

  • branches/czw_branch/cleanup/psModules/src/concepts/pmConcepts.c

    r24951 r25051  
    377377        conceptRegisterS32("CELL.XWINDOW", "Start of cell window (pixels)",p_pmConceptParse_Positions,p_pmConceptFormat_Positions, NULL, true, PM_FPA_LEVEL_CELL);
    378378        conceptRegisterS32("CELL.YWINDOW", "Start of cell window (pixels)",p_pmConceptParse_Positions,p_pmConceptFormat_Positions, NULL, true, PM_FPA_LEVEL_CELL);
    379         conceptRegisterF32("CELL.VARFACTOR", "Variance factor for conversion from large to small scales", NULL, NULL, NULL, true, PM_FPA_LEVEL_CELL);
    380379
    381380        // CELL.TRIMSEC
    … …  
    485484        concept[length - 1] = '\0';
    486485
    487         // special variants:
    488         if (!strcmp(concept, "FPA.DATE")) {
    489           psTime *fpaTime = psMetadataLookupPtr(NULL, fpa->concepts, "FPA.TIME");
    490           if (!fpaTime) {
    491             psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Missing concept FPA.TIME needed for FPA.DATE");
    492             psFree(string);
    493             return NULL;
    494           }
    495           psString dateTimeString = psTimeToISO(fpaTime); // String representation
    496           psList *dateTime = psStringSplit(dateTimeString, "T", true);
    497           psFree(dateTimeString);
    498           psString dateString = psMemIncrRefCounter(psListGet(dateTime, PS_LIST_HEAD)); // The date string
    499           psFree (dateTime);
    500 
    501           if (!psStringSubstitute(&string, dateString, "{FPA.DATE}")) {
    502               psError(PS_ERR_UNKNOWN, false, "Unable to replace concept %s", concept);
    503               psFree(string);
    504               psFree(dateString);
    505               return NULL;
    506           }
    507           psFree (dateString);
    508           continue;
    509         }
     486        // special variants:
     487        if (!strcmp(concept, "FPA.DATE")) {
     488          psTime *fpaTime = psMetadataLookupPtr(NULL, fpa->concepts, "FPA.TIME");
     489          if (!fpaTime) {
     490            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Missing concept FPA.TIME needed for FPA.DATE");
     491            psFree(string);
     492            return NULL;
     493          }
     494          psString dateTimeString = psTimeToISO(fpaTime); // String representation
     495          psList *dateTime = psStringSplit(dateTimeString, "T", true);
     496          psFree(dateTimeString);
     497          psString dateString = psMemIncrRefCounter(psListGet(dateTime, PS_LIST_HEAD)); // The date string
     498          psFree (dateTime);
     499
     500          if (!psStringSubstitute(&string, dateString, "{FPA.DATE}")) {
     501              psError(PS_ERR_UNKNOWN, false, "Unable to replace concept %s", concept);
     502              psFree(string);
     503              psFree(dateString);
     504              return NULL;
     505          }
     506          psFree (dateString);
     507          continue;
     508        }
    510509
    511510        psTrace("psModules.concepts", 7, "Interpolating concept %s", concept);

  • branches/czw_branch/cleanup/psModules/src/imcombine/pmSubtractionAnalysis.c

    r23780 r25051  
    129129
    130130
    131     // Set the variance factors
    132     float vf1 = 1.0, vf2 = 1.0;         // Variance factors for each image
    133     switch (kernels->mode) {
    134       case PM_SUBTRACTION_MODE_1:
    135         vf1 = pmSubtractionVarianceFactor(kernels, 0.5, 0.5, false);
    136         break;
    137       case PM_SUBTRACTION_MODE_2:
    138         vf2 = pmSubtractionVarianceFactor(kernels, 0.5, 0.5, false);
    139         break;
    140       case PM_SUBTRACTION_MODE_DUAL:
    141         vf1 = pmSubtractionVarianceFactor(kernels, 0.5, 0.5, false);
    142         vf2 = pmSubtractionVarianceFactor(kernels, 0.5, 0.5, true);
    143         break;
    144       default:
    145         psAbort("Invalid subtraction mode: %x", kernels->mode);
    146     }
    147 
    148     // Weight by the area
    149     if (region) {
    150         float norm = (region->x1 - region->x0 + 1) * (region->y1 - region->y0 + 1) / (numCols * numRows);
    151         vf1 *= norm;
    152         vf2 *= norm;
    153     }
    154 
    155     // Update the variance factor
    156 #define UPDATE_VARFACTOR(VF, ANALYSIS) { \
    157     psMetadataItem *vfItem = psMetadataLookup(analysis, ANALYSIS); \
    158     if (vfItem) { \
    159         psAssert(vfItem->type == PS_TYPE_F32, "Should be the type we said."); \
    160         vfItem->data.F32 += VF; \
    161     } else { \
    162         psMetadataAddF32(analysis, PS_LIST_TAIL, ANALYSIS, 0, "Variance factor weighted by the area", VF); \
    163     } \
    164 }
    165 
    166     UPDATE_VARFACTOR(vf1, PM_SUBTRACTION_ANALYSIS_VARFACTOR_1);
    167     UPDATE_VARFACTOR(vf2, PM_SUBTRACTION_ANALYSIS_VARFACTOR_2);
    168 
    169131    // Kernel shape
    170132    {

  • branches/czw_branch/cleanup/psconfig/pschecklibs

    r24068 r25051  
    100100# search for, and drop, existing libdir entry in libpath?
    101101unshift @libpath, $libdir;
     102
     103# XXX a temporary hack for libreadline (this failed... needed to modify libreadline/support/shobj-conf
     104# $shlib_libs = $ENV{'SHLIB_LIBS'};
     105# unless ($shlib_libs =~ m|-lncurses|) {
     106#     $shlib_libs .= " -lncurses";
     107# }
     108# $ENV{'SHLIB_LIBS'} = $shlib_libs;
    102109
    103110# add the path defined by PATH

  • branches/czw_branch/cleanup/psconfig/tagsets/ipp-2.9.libs

    r24399 r25051  
    2929lib libpthread           NONE           NONE   NONE                     NONE             N NONE NONE NONE
    3030lib libncurses           curses,termcap NONE   ncurses-5.6.tar.gz       ncurses-5.6      N NONE NONE NONE
    31 lib libreadline          NONE           NONE   readline-5.2-p12.tar.gz  readline-5.2-p12 Y NONE NONE NONE
     31lib libreadline          NONE           NONE   readline-5.2-p13.tar.gz  readline-5.2-p13 Y NONE NONE NONE
    3232lib libz                 NONE           NONE   zlib-1.2.3.tar.gz        zlib-1.2.3       N --shared NONE NONE
    3333lib libpng               NONE           NONE   libpng-1.2.15.tar.gz     libpng-1.2.15    N NONE NONE NONE

  • branches/czw_branch/cleanup/psphot/doc/notes.20090523.txt

    r24583 r25051  
    11
    2 20090523 : extended source analysis -- further development
     220090725 : extended source radial profiles
    33
    4  issues to address:
     4 I need to fix the Petrosian and similar code.  There are several issues:
    55
    6  * quality of moments used for initial psf selection
    7    - what is the behavior of moments with Gaussian windowing as function of S/N & size?
    8    - can we choose a sigma value that will give reliable moments for an expected range of seeing?
     6 1) correct implementation of the actual Petrosian radius / Petrosian flux measurement:
    97
    10  * moments for galaxy model guesses
    11    - trade-off between Gaussian windowing, tophat windowing, and S/N windowing in footprint area
     8    R_p : Given F(R) : F(0) / F(R_p) = 0.9 ?? (something like this...)
    129
    13  initial dev work:
     10    Actually implemented by calculating the average surface brightness in annullii:
    1411
    15  * add Gaussian windowing, S/N windowing, (tophat windowing) and options to select
     12    R_p : Given F(R) -> define f(R) = \sum_{R-dR}^{R+dR}F(R) / 4\pi R dR
     13    R_p : f(dR) / F(R_p) = P_ratio
    1614
    17  * check visualization of moments
     15 2) need to construct a clean, normalized radial profile to make this measurement
     16
     17    * generate radial profiles for each of N annular wedges
     18    * find R_50 : f(0) / f(R_50) = 0.5
     19    $ fit the ellipse of R_50 to Rmaj sin(T) + Rmin cos(T)
     20
     21
     22    cos(a+b) = cos(a) cos(b) - sin(a) sin(b)
     23    sin(a+b) = cos(a) sin(b) + cos(b) sin(a)
     24
     25    cos(theta+To) = cos(theta) cos(To) - sin(theta) sin(To)
     26    sin(theta+To) = cos(theta) sin(To) + sin(theta) cos(To)
     27
     28    R_x = R_maj cos(theta + theta_o) = R_maj cos(theta) cos(theta_o) - Rmaj sin(theta) sin(theta_o)
     29    R_y = R_min sin(theta + theta_o) = R_min cos(theta) sin(theta_o) + Rmin sin(theta) cos(theta_o)
     30
     31    R_X = R_x cos(phi) - R_y sin(phi)
     32    R_Y = R_x sin(phi) + R_y cos(phi)
     33
     34    R_X = R_maj cos(phi) cos(theta + To) - R_min sin(phi) sin(theta + To)
     35    R_Y = R_maj sin(phi) cos(theta + To) + R_min cos(phi) sin(theta + To)
     36
     37    R_X = (R_maj cos(phi) cos (To) - R_min sin(phi) sin(To)) cos(theta) - (Rmaj cos(phi) sin(To) + Rmin sin(phi) cos(To)) sin(theta)
     38    R_Y = (R_maj sin(phi) cos (To) + R_min cos(phi) sin(To)) cos(theta) - (Rmaj cos(phi) sin(To) + Rmin sin(phi) cos(To)) sin(theta)
     39   
     40
     41    R_X = Ax cos(theta) + Bx sin(theta)
     42    R_Y = Ay cos(theta) + By sin(theta)
     43   
     44    Ax = \sum (R_X * cos(theta))
     45    Bx = \sum (R_X * sin(theta))
     46    Ay = \sum (R_Y * cos(theta))
     47    By = \sum (R_Y * sin(theta))
     48
     49    R_maj = sqrt(Ax^2 + Ay^2)
     50    R_min = sqrt(Bx^2 + By^2)
     51
     52    phi = atan2(Bx + By, Ax + Ay)
     53
     54    *** get rest of math from my notebook...
     55
     56
    1857
    195820090525 : some clarity of issues:
    … …  
    4079   
    4180 * QGAUSS with see
     81
     8220090523 : extended source analysis -- further development
     83
     84 issues to address:
     85
     86 * quality of moments used for initial psf selection
     87   - what is the behavior of moments with Gaussian windowing as function of S/N & size?
     88   - can we choose a sigma value that will give reliable moments for an expected range of seeing?
     89
     90 * moments for galaxy model guesses
     91   - trade-off between Gaussian windowing, tophat windowing, and S/N windowing in footprint area
     92
     93 initial dev work:
     94
     95 * add Gaussian windowing, S/N windowing, (tophat windowing) and options to select
     96
     97 * check visualization of moments
     98

  • branches/czw_branch/cleanup/pstamp/scripts/pstamp_finish.pl

    r24951 r25051  
    173173    }
    174174
     175    my $exp_info;
     176    my $last_exp_id = -1;
    175177    foreach my $job (@jobs) {
    176178        my $job_id = $job->{job_id};
    … …  
    181183
    182184        my ($row, $req_info, $project) = get_request_info($rows, $rownum);
    183         my $proj_hash = resolve_project($ipprc, $project, $dbname);
     185
     186        my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
    184187        my $image_db = $proj_hash->{dbname};
    185 
    186         # get the metadata for the exposure (if any i.e. stack)
    187         # returns an appropriate string if !$exp_id
    188         my $exp_info = get_exposure_info($image_db, $exp_id);
     188        if (!$image_db) {
     189            carp("failed to find imagedb for project: $project");
     190            stop_request($req_id, $PS_EXIT_CONFIG_ERROR);
     191        }
     192
     193        if ($exp_id ne $last_exp_id) {
     194            # get the metadata for the exposure (if any i.e. stack)
     195            # returns an appropriate string if !$exp_id
     196            $exp_info = get_exposure_info($image_db, $exp_id);
     197            $last_exp_id = $exp_id;
     198        }
    189199
    190200        if (($job_type eq "stamp") || ($job_type eq "get_image")) {
    … …  
    292302    # Create a results file module and provide a list of the names (we have the data in the columns)
    293303    my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|";
    294     $rowinfo   .= "$row->{ID}|$row->{CLASS_ID}|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|";
     304    $rowinfo   .= "$row->{ID}|$row->{TESS_ID}|$row->{COMPONENT}|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|";
    295305    $rowinfo   .= "$row->{REQFILT}|$row->{COORD_MASK}|$row->{CENTER_X}|$row->{CENTER_Y}|";
    296306    $rowinfo   .= "$row->{WIDTH}|$row->{HEIGHT}";

  • branches/czw_branch/cleanup/pstamp/scripts/pstamp_job_run.pl

    r24951 r25051  
    1414use Digest::MD5::File qw( file_md5_hex );
    1515use PS::IPP::PStamp::RequestFile qw( :standard );
    16 
    17 my $verbose;
    18 my $dbname;
    19 my $dbserver;
    20 my $job_id;
    21 my $redirect_output;
    22 my $output_base;
    23 
    24 my $host = hostname();
    25 
     16use IPC::Cmd 0.36 qw( can_run run );
     17
     18use PS::IPP::Metadata::Config;
     19#use PS::IPP::Metadata::Stats;
     20#use PS::IPP::Metadata::List qw( parse_md_list );
     21
     22use PS::IPP::Config qw( :standard );
     23
     24my ($job_id, $redirect_output, $outputBase, $rownum, $jobType);
     25my ($verbose, $dbname, $dbserver, $no_update);
    2626
    2727GetOptions(
    2828    'job_id=s'          =>  \$job_id,
    29     'output_base=s'     =>  \$output_base,
    30     'redirect-output'   => \$redirect_output,
     29    'job_type=s'        =>  \$jobType,
     30    'rownum=s'          =>  \$rownum,
     31    'output_base=s'     =>  \$outputBase,
     32    'redirect-output'   =>  \$redirect_output,
    3133    'dbname=s'          =>  \$dbname,
    3234    'dbserver=s'        =>  \$dbserver,
    3335    'verbose'           =>  \$verbose,
     36    'no-update'         =>  \$no_update,
    3437);
    3538
    3639
     40my $host = hostname();
    3741if ($verbose) {
    3842    print "\n\n";
    … …  
    4145
    4246die "job_id is required" if !$job_id;
    43 die "output_base is required" if !$output_base;
    44 
    45 use IPC::Cmd 0.36 qw( can_run run );
    46 
    47 use PS::IPP::Metadata::Config;
    48 use PS::IPP::Metadata::Stats;
    49 use PS::IPP::Metadata::List qw( parse_md_list );
    50 
    51 use PS::IPP::Config qw($PS_EXIT_SUCCESS
    52                        $PS_EXIT_UNKNOWN_ERROR
    53                        $PS_EXIT_SYS_ERROR
    54                        $PS_EXIT_CONFIG_ERROR
    55                        $PS_EXIT_PROG_ERROR
    56                        $PS_EXIT_DATA_ERROR
    57                        $PS_EXIT_TIMEOUT_ERROR
    58                        metadataLookupStr
    59                        metadataLookupBool
    60                        caturi
    61                        );
     47
     48my_die( "job_type is required", $job_id, $PS_EXIT_PROG_ERROR) if !$jobType;
     49my_die("rownum is required", $job_id, $PS_EXIT_PROG_ERROR) if !$rownum;
     50my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
     51
     52
    6253my $ipprc = PS::IPP::Config->new(); # IPP Configuration
    63 
    6454if ($redirect_output) {
    65     my $logDest = "$output_base.log";
    66     $ipprc->redirect_output($logDest);
    67 }
     55    my $logDest = "$outputBase.log";
     56    $ipprc->redirect_output($logDest)
     57        or my_die ("unable to redirect output to $logDest", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     58}
     59
    6860
    6961if (!$dbserver) {
    … …  
    7870
    7971if ($missing_tools) {
    80     warn("Can't find required tools.");
    81     exit ($PS_EXIT_CONFIG_ERROR);
     72    my_die("Can't find required tools", $job_id, $PS_EXIT_CONFIG_ERROR);
    8273}
    8374
    8475my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    8576
    86 my $psjob;
    87 #Look up the uri for the given job
    88 {
    89     my $command = "$pstamptool -pendingjob -job_id $job_id";
     77my $jobStatus;
     78if ($jobType eq "stamp") {
     79    my $argslist = "$outputBase.args";
     80    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     81    my $argString = <ARGSLIST>;
     82    close ARGSLIST;
     83    chomp $argString;
     84
     85    # XXX: should we do any other sanity checking?
     86    my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
     87
     88    my $command = "$ppstamp $outputBase $argString";
    9089    $command .= " -dbname $dbname" if $dbname;
    9190    $command .= " -dbserver $dbserver" if $dbserver;
    9291    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    9392        run(command => $command, verbose => $verbose);
    94     unless ($success) {
    95         die("Unable to perform pstamptool -pendingjob: $error_code");
    96     }
    97 
    98     if (@$stdout_buf == 0) {
    99         print STDERR "pending pstamp job id $job_id not found\n";
    100         exit 0;
    101     }
    102     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    103         die("Unable to parse metdata config doc");
    104 
    105     my $jobs = parse_md_list($metadata);
    106     my $num = @$jobs;
    107     die "unexpected number of jobs $num found for job: $job_id" if $num > 1;
    108 
    109     $psjob = $jobs->[0];
    110 }
    111 
    112 if (!$psjob) {
    113     print STDERR "postage stamp job $job_id not found\n";
    114     exit 1;
    115 }
    116 
    117 my $rownum = $psjob->{rownum};
    118 my $uri = $psjob->{uri};
    119 my $outputBase = $psjob->{outputBase};
    120 my $argString = $psjob->{args};
    121 my $jobType = $psjob->{jobType};
    122 
    123 my $jobStatus;
    124 if ($jobType eq "stamp") {
    125     my $command = "$ppstamp -file $uri $outputBase $argString";
    126     $command .= " -dbname $dbname" if $dbname;
    127     $command .= " -dbserver $dbserver" if $dbserver;
    128     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    129         run(command => $command, verbose => $verbose);
    13093
    13194    if ($success) {
    132         # XXX shouldn't need to do this, review schema`
    13395        my $dir = dirname($outputBase);
    13496
    13597        my $reglist = "$dir/reglist$job_id";
    13698
    137         open F, ">$reglist" or die "can't open $reglist for output";
    138 
    139         # figure out what output images we should expect
    140 
    141         # Note: we are assuming the contents of the filerules here.
    142         my @extensions = ( "fits", "mk.fits", "wt.fits");
    143 
    144         # we always require an image
     99        open F, ">$reglist" or my_die( "can't open $reglist for output", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     100
     101        # Figure out what output images were produced
     102
     103        # Note: we are assuming the contents of the PSTAMP filerules here.
     104        my %extensions = ( $PSTAMP_SELECT_IMAGE  => "fits",
     105                           $PSTAMP_SELECT_MASK   => "mk.fits",
     106                           $PSTAMP_SELECT_WEIGHT => "wt.fits");
     107
     108        # we always create a stamp of the image
    145109        my $output_mask = $PSTAMP_SELECT_IMAGE;
    146110
    147         # we search the argString for -mask and -weight.
     111        # we search the argString for -mask and -variance.
    148112        # searching the arg string allows us to avoid adding a column in pstampJob
    149         # XXX yeah but does this make us susceptible to user input fouling us up?
    150         # For example, what if somebody sets the output base to file-mask-weight
    151         # TODO: clean this up
    152         $output_mask |= $PSTAMP_SELECT_MASK   if ($argString =~ /-mask/);
    153         $output_mask |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-weight/);
    154 
    155         # XXX we're getting a bit intimate with the bit field definitions here.
    156         # do better use a hash
    157 
    158         my $m = 1;
    159         foreach my $extension (@extensions) {
    160             my $do_this_one = $m & $output_mask;
    161             $m = $m << 1;
     113        $output_mask   |= $PSTAMP_SELECT_MASK   if ($argString =~ /-mask/);
     114        $output_mask   |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-variance/);
     115
     116        foreach my $key (keys (%extensions)) {
     117            my $do_this_one = $key & $output_mask;
     118
    162119            next if (! $do_this_one);
     120
     121            my $extension = $extensions{$key};
    163122
    164123            my $basename = basename($outputBase);
    … …  
    175134    } else {
    176135        $jobStatus = $error_code >> 8;
    177         print STDERR "ppstamp failed with error code: $jobStatus\n";
     136        my_die( "ppstamp failed with error code: $jobStatus", $job_id, $jobStatus);
    178137    }
    179138} elsif ($jobType eq "get_image") {
     139    my_die( "get_image jobs not working right now", $job_id, $PS_EXIT_CONFIG_ERROR);
     140
     141    my $uri = "";
    180142    my $command = "$pstamp_get_image_job --job_id $job_id --uri $uri --out_dir $outputBase --rownum $rownum";
    181143    $command .= " --dbname $dbname" if $dbname;
    … …  
    188150    } else {
    189151        $jobStatus = $error_code >> 8;
    190         print STDERR "ppstamp failed with error code: $jobStatus\n";
     152        my_die( "pstamp_get_image_job failed with error code: $jobStatus", $job_id, $jobStatus);
    191153    }
    192154} elsif ($jobType eq "detect_query") {
    193     die("multiple detect_query jobs not supported yet");
     155    my_die("detect_query jobs not supported yet", $job_id,$PS_EXIT_CONFIG_ERROR);
    194156} else {
    195     die("unknown jobType $jobType found");
    196 }
    197 
    198 # stop the job and set the result value
     157    my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR);
     158}
     159
     160# mark the job stopped in the database
    199161{
    200     my $command = "$pstamptool -updatejob -job_id $job_id -state stop -fault $jobStatus";
     162    my $command = "$pstamptool -updatejob -job_id $job_id -state stop";
    201163    $command .= " -dbname $dbname" if $dbname;
    202164    $command .= " -dbserver $dbserver" if $dbserver;
    203     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    204         run(command => $command, verbose => $verbose);
    205     unless ($success) {
    206         die("Unable to perform $command: $error_code");
    207     }
    208 }
    209 
    210 exit $jobStatus;
     165    if (!$no_update) {
     166        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     167            run(command => $command, verbose => $verbose);
     168        unless ($success) {
     169            die("Unable to perform $command: $error_code");
     170        }
     171    } else {
     172        print STDERR "skipping command: $command\n"
     173    }
     174}
     175
     176exit 0;
    211177
    212178# create a string to be passed as input to dsreg when registering this file in a fileset
    … …  
    219185    if (-e $path) {
    220186        my @finfo = stat($path);
    221         die "failed to stat $path" unless (@finfo);    # XXX clean up
     187        my_die("failed to stat $path", $job_id, $PS_EXIT_UNKNOWN_ERROR) unless (@finfo);
    222188        my $bytes = $finfo[7];
    223189        my $md5sum = file_md5_hex($path);
    … …  
    225191        return "$filename|$bytes|$md5sum|$filetype|";
    226192    } else {
    227         die "$filename not found at $path";
    228     }
    229 }
     193        my_die("$filename not found at $path", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     194    }
     195}
     196
     197sub my_die
     198{
     199    my $msg = shift;            # Warning message on die
     200    my $job_id = shift;         # job identifier
     201    my $exit_code = shift;      # Exit code to add
     202
     203    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     204
     205    carp($msg);
     206    if (defined $job_id and not $no_update) {
     207        my $command = "$pstamptool -updatejob";
     208        $command .= " -job_id $job_id";
     209        $command .= " -fault $exit_code";
     210        $command .= " -dbname $dbname" if defined $dbname;
     211        $command .= " -dbserver $dbserver" if defined $dbserver;
     212        system($command);
     213    }
     214    exit $exit_code;
     215}
     216

  • branches/czw_branch/cleanup/pstamp/scripts/pstamp_parser_run.pl

    r24951 r25051  
    1313use Getopt::Long qw( GetOptions );
    1414use File::Basename qw( basename dirname);
     15use POSIX qw( strftime );
    1516
    1617my $req_id;
    … …  
    7475}
    7576
    76 # workdir is where we download request files to and place any error output from the parser
     77# workdir is where all of the files generated for this request are placed
    7778# NOTE: this location needs to be kept in sync with the web interface ( request.php )
    78 my $workdir = "$pstamp_workdir/$req_id";
    79 
     79my $datestr = strftime "%Y%m%d", gmtime;
     80my $datedir = "$pstamp_workdir/$datestr";
     81if (! -e $datedir ) {
     82    mkdir $datedir or die "failed to create working directory $datedir for request id $req_id";
     83}
     84
     85my $workdir = "$datedir/$req_id";
    8086if (! -e $workdir ) {
    8187    mkdir $workdir or die "failed to create working directory $workdir for request id $req_id";

  • branches/czw_branch/cleanup/pstamp/scripts/pstamp_results_file.pl

    r21196 r25051  
    8888        { name => 'IMG_TYPE',   type => '16A', writetype => TSTRING },       
    8989        { name => 'ID',         type => '16A', writetype => TSTRING },           
    90         { name => 'CLASS_ID',   type => '16A', writetype => TSTRING },   
     90        { name => 'TESS_ID',    type => '64A', writetype => TSTRING },   
     91        { name => 'COMPONENT',  type => '64A', writetype => TSTRING },   
    9192
    9293        # output parameters

  • branches/czw_branch/cleanup/pstamp/scripts/pstampparse.pl

    r24951 r25051  
    133133    # XXX: TODO: sanity check all parameters
    134134
    135     # XXX: TODO: We shouldn't just die in this loop.
     135    # XXX: TODO: We shouldn't really just die in this loop.
    136136    # If we encounter an error for a particular row
    137137    # add a job with the proper fault code. If there is a db or config error we should probably just
    … …  
    146146    $stage = $row->{IMG_TYPE};
    147147    my $id       = $row->{ID};
    148     my $class_id = $row->{CLASS_ID};
    149148    my $component = $row->{COMPONENT};
     149
    150150    my $filter   = $row->{REQFILT};
    151151    my $mjd_min = $row->{MJD_MIN};
    152152    my $mjd_max = $row->{MJD_MAX};
     153
    153154    my $option_mask= $row->{OPTION_MASK};
    154155
    155     die "region of interest is required to make postage stamps"
     156    die "valid region of interest is required to make postage stamps"
    156157        if (($job_type eq "stamp") and ! validROI($row));
    157158
    158159    my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
    159     $class_id = "" if (defined($class_id) and ($class_id eq "null" or $class_id eq "all"));
    160160    $component = "" if (defined($component) and ($component eq "null" or $component eq "all"));
    161161   
    … …  
    163163
    164164
    165     # note: resolve_project remembers the last project returned so avoids running
    166     # pstamptool every time
     165    # note: resolve_project avoids running pstamptool every time by remembering the
     166    # last project resolved
    167167    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
    168168    die "project $project not found\n" unless $proj_hash;
    … …  
    176176    if (@rowList) {
    177177        my $firstRow = $rowList[0];
     178        # XXX: the collecting might work with !$skycenter but I need to think about it
    178179        if ($skycenter and same_images_of_interest($row, $firstRow)) {
    179180            push @rowList, $row;
    … …  
    181182            next;
    182183        } else {
    183             # Process the jobs for this set of rows
    184             $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList);
     184            # queue the jobs for this set of rows
     185            $num_jobs += queueJobs($mode, \@rowList, $imageList);
    185186            @rowList = ();
    186187        }
    … …  
    195196        # request specification. An array reference is returned
    196197        my ($x, $y);
    197         $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $class_id, $skycenter,
     198        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $component, $skycenter,
    198199                $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
    199200    }
    … …  
    203204        next;
    204205    }
     206    $row->{need_magic} = $need_magic;
    205207    push @rowList, $row;
    206208}
    207209
    208210if (@rowList) {
    209     $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList);
     211    $num_jobs += queueJobs($mode, \@rowList, $imageList);
    210212}
    211213
    … …  
    223225
    224226
    225 sub makeJobsForRow
     227sub queueJobsForRow
    226228{
    227229    my $row = shift;
     230    my $stage = shift;
    228231    my $imageList = shift;
    229232    my $have_skycells = shift;
    … …  
    234237    foreach my $image (@$imageList) {
    235238        my $component;
    236         my $class_id;   # get rid of this use componet
    237239        if ($have_skycells) {
    238240            $component = $image->{skycell_id};
    239241        } else {
    240242            $component = $image->{class_id};
    241             $class_id = $component;
    242243        }
    243244
    … …  
    249250        my $roi_string;
    250251
    251         # XXX we're depending on other code to insure valid values for roi components
    252         # this is checked in and ppstamp but I should check here so that we don't get that far,
    253         # but not today
     252        # note values were insured to be numbers in validROI()
    254253        my $x = $row->{CENTER_X};
    255254        my $y = $row->{CENTER_Y};
    … …  
    270269        }
    271270
    272         my $uri = $image->{image};
     271        my $imagefile = $image->{image};
    273272        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
    274             print STDERR "skippping non-magicked image $uri\n" if $verbose;
     273            # XXX: should we add a faulted job so the client can know what happened?
     274            print STDERR "skippping non-magicked image $imagefile\n" if $verbose;
    275275            next;
    276276        }
    … …  
    278278           
    279279        my $args = $roi_string ? $roi_string : "";
    280         $args .= " -class_id $class_id" if $class_id;
     280        if ($stage eq "raw" or $stage eq "chip") {
     281            $args .= " -class_id $component" if $component;
     282        }
    281283
    282284        $job_num++;
    283 
    284         my $output_base = "$out_dir/${rownum}_${job_num}";
    285285
    286286        # add astrometry file for raw and chip images if one is available
    … …  
    289289        }
    290290
     291        $args .= " -file $imagefile";
     292
    291293        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
    292294            $args .= " -mask $image->{mask}";
    293295        }
    294296        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
    295             $args .= " -weight $image->{weight}";
    296         }
     297            $args .= " -variance $image->{weight}";
     298        }
     299
     300        my $output_base = "$out_dir/${rownum}_${job_num}";
     301        my $argslist = "${output_base}.args";
     302
     303        # copy the argument list to a file
     304        open ARGSLIST, ">$argslist" or die "failed to open $argslist";
     305        print ARGSLIST "$args\n";
     306        close ARGSLIST or die "failed to close $argslist";
    297307
    298308        # XXX: TODO: here is where we need to check whether or not the source inputs still exist
    … …  
    302312
    303313        $num_jobs++;
    304         my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}"
    305             . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum"
    306             . " -state $newState";
     314        my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
     315                        . " -outputBase $output_base -rownum $rownum -state $newState";
    307316        $command .= " -exp_id $exp_id" if $exp_id;
    308317        $command .= " -dbname $dbname" if $dbname;
    … …  
    313322            # this is sort of like the mode -noupdate that some other tools support
    314323            print "$command\n";
    315         } else {
     324        } elsif (!$no_update) {
    316325            # mode eq "queue_job"
    317326            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    324333                die "failed to queue job for request $req_id";
    325334            }
     335        } else {
     336            print "skipping command: $command\n";
    326337        }
    327338    }
    … …  
    329340}
    330341
    331 sub makeJobs
     342sub queueJobs
    332343{
    333344    my $mode = shift;
    334     my $stage = shift;
    335     my $need_magic = shift;
    336345    my $rowList = shift;
    337346    my $imageList = shift;
    338347
    339348    my $firstRow = $rowList[0];
     349    my $stage    = $firstRow->{IMG_TYPE};
    340350    my $job_type = $firstRow->{JOB_TYPE};
     351    my $need_magic = $firstRow->{need_magic};
    341352
    342353    my $num_jobs = 0;
    … …  
    381392                    my $astrom = $thisRun->[0]->{astrom};
    382393                    die "no astrometry file found" if !$astrom;
    383                     $command .= " -astrom $astrom";
     394                    my $astrom_resolved = $ipprc->file_resolve($astrom);
     395                    $command .= " -astrom $astrom_resolved";
    384396                }
    385397                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    404416            }
    405417           
    406 
    407                 # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images
    408                 # use -list instead of -file
    409418            foreach my $row (@$rowList) {
    410                 $num_jobs += makeJobsForRow($row, $thisRun, $have_skycells, $need_magic);
     419                $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic);
    411420            }
    412421        }
    … …  
    451460
    452461        my $image = shift @$imageList;
     462        $image->{stage} = $stage;
    453463        push @runList, $image;
    454464        $last_run_id = $run_id;
    … …  
    461471    my $r2 = shift;
    462472
    463     return 0 if ($r1->{project} ne $r2->{project});
    464     return 0 if ($r1->{job_type} ne $r2->{job_type});
    465     return 0 if ($r1->{req_type} ne $r2->{req_type});
    466     return 0 if ($r1->{img_type} ne $r2->{img_type});
    467     return 0 if ($r1->{id} ne $r2->{id});
    468 #    XXX: turn this on when we change the request file format to change class_id into component
    469 #    return false if ($r1->{component} ne $r2->{component});
     473    return 0 if ($r1->{PROJECT}  ne $r2->{PROJECT});
     474    return 0 if ($r1->{JOB_TYPE} ne $r2->{JOB_TYPE});
     475    return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE});
     476    return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
     477    return 0 if ($r1->{ID} ne $r2->{ID});
     478
     479    if (defined($r1->{COMPONENT})) {
     480        return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT});
     481    } elsif (defined($r2->{COMPONENT})) {
     482        return 0;
     483    }
    470484
    471485    return 1;
    472486}
    473487
     488sub validNumber
     489{
     490    my $val = shift;
     491
     492    return $val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/;
     493}
     494
    474495sub validROI
    475496{
    476497    my $row = shift;
    477     return 0 if !defined $row->{CENTER_X};
    478     return 0 if !defined $row->{CENTER_Y};
    479     return 0 if !defined $row->{WIDTH};
    480     return 0 if !defined $row->{HEIGHT};
     498    return 0 if !validNumber($row->{CENTER_X});
     499    return 0 if !validNumber($row->{CENTER_Y});
     500    return 0 if !validNumber($row->{WIDTH});
     501    return 0 if !validNumber($row->{HEIGHT});
    481502
    482503    return 1;

  • branches/czw_branch/cleanup/pstamp/src/ppstampArguments.c

    r18980 r25051  
    2222    fprintf(stderr, "Optional arguments:\n");
    2323    fprintf(stderr, "   [-class_id class_id]     selects class_id (only used with -pixcenter)\n");
    24     fprintf(stderr, "   [-astrom astrom.cmp] : provide an alternative astrometry calibration\n");
    25     fprintf(stderr, "   [-mask   mk_image] :   mask image\n");
    26     fprintf(stderr, "   [-weight wt_image] :   weight image\n");
     24    fprintf(stderr, "   [-astrom astrom.cmp] :   provide an alternative astrometry calibration\n");
     25    fprintf(stderr, "   [-mask   mk_image] :     mask image\n");
     26    fprintf(stderr, "   [-variance var_image] :  variance image\n");
    2727    fprintf(stderr, "\n");
    2828
    … …  
    8181    pmConfigFileSetsMD(config->arguments, &argc, argv, "ASTROM", "-astrom", "-astromlist");
    8282    pmConfigFileSetsMD(config->arguments, &argc, argv, "MASK",   "-mask", "-masklist");
    83     pmConfigFileSetsMD(config->arguments, &argc, argv, "WEIGHT", "-weight", "-weightlist");
     83    pmConfigFileSetsMD(config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist");
    8484
    8585    // the input file is a required argument; if not found, we will exit

  • branches/czw_branch/cleanup/pstamp/src/ppstampParseCamera.c

    r20148 r25051  
    1818    }
    1919    if (doWeight) {
    20         pmFPAfile *outWeight = pmFPAfileDefineSkycell(config, output->fpa, "PPSTAMP.OUTPUT.WEIGHT");
     20        pmFPAfile *outWeight = pmFPAfileDefineSkycell(config, output->fpa, "PPSTAMP.OUTPUT.VARIANCE");
    2121        outWeight->save = true;
    2222    }
    … …  
    5454    }
    5555    bool doWeight = false;
    56     pmFPAfile *weight = pmFPAfileBindFromArgs (&status, input, config, "PPSTAMP.INPUT.WEIGHT", "WEIGHT");
     56    pmFPAfile *weight = pmFPAfileBindFromArgs (&status, input, config, "PPSTAMP.INPUT.VARIANCE", "VARIANCE");
    5757    if (!status) {
    58         psError(PS_ERR_IO, false, "Failed to build FPA from PPSTAMP.INPUT.WEIGHT");
     58        psError(PS_ERR_IO, false, "Failed to build FPA from PPSTAMP.INPUT.VARIANCE");
    5959        return false;
    6060    }

  • branches/czw_branch/cleanup/pstamp/test/pstamp_req_create

    r20257 r25051  
    2626
    2727pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    28 pod2usage( -msg => "Required options: --input --output",
    29            -exitval => 3) unless defined $input and defined $output;
     28pod2usage( -msg => "Required options: --input --output | --req_name",
     29           -exitval => 3) unless defined $input and (defined $output or defined $req_name);
    3030
    3131# The header kewords
    … …  
    5555        { name => 'IMG_TYPE',   type => '16A', writetype => TSTRING },
    5656        { name => 'ID',         type => '16A', writetype => TSTRING },           
    57         { name => 'CLASS_ID',   type => '16A', writetype => TSTRING },
     57        { name => 'TESS_ID',    type => '64A', writetype => TSTRING },
     58        { name => 'COMPONENT',  type => '64A', writetype => TSTRING },
    5859
    5960
    … …  
    9596if ($req_name) {
    9697    $header->[0]->{value} = $req_name;
    97 }
     98} else {
     99    $req_name = $header->[0]->{value};
     100}
     101
     102die "no request name defined" unless defined $req_name;
     103
     104$output = $req_name . ".fits" if !$output;
    98105
    99106my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header);

  • branches/czw_branch/cleanup/pswarp/src/pswarpLoop.c

    r24491 r25051  
    315315
    316316
    317     // Set variance factor
    318     {
    319         bool mdok;                      // Status of MD lookup
    320         float varFactor = psMetadataLookupF32(&mdok, output->analysis, PSWARP_ANALYSIS_VARFACTOR);
    321         long goodPix = psMetadataLookupS64(&mdok, output->analysis, PSWARP_ANALYSIS_GOODPIX);
    322         varFactor /= goodPix;
    323 
    324         psMetadataItem *vfItem = psMetadataLookup(outCell->concepts, "CELL.VARFACTOR"); ///< Item to update
    325         psAssert(vfItem && vfItem->type == PS_TYPE_F32, "Concept should be as specified.");
    326         if (!isfinite(vfItem->data.F32)) {
    327             vfItem->data.F32 = varFactor;
    328         } else {
    329             vfItem->data.F32 *= varFactor;
    330         }
    331     }
    332 
    333317    // Set covariance matrix for output
    334318    {
    … …  
    406390        pmFPAview *view = pmFPAviewAlloc(0); ///< View into skycell
    407391        view->chip = view->cell = view->readout = 0;
    408 
    409         // Need to adjust the weight --- the main operation in psphotReadoutFindPSF is not determining the
    410         // signficance of sources, but looking at the significance of individual pixels.
    411         // We can adjust the weight directly since this is a deep copy
    412         pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa);
    413         float vf = psMetadataLookupF32(NULL, photRO->parent->concepts, "CELL.VARFACTOR"); // Variance factor
    414         psBinaryOp(photRO->variance, photRO->variance, "*", psScalarAlloc(vf, PS_TYPE_F32));
    415392
    416393        // grab the sources of interest from the storage location (pmFPAfile PSPHOT.INPUT.CMF)

  • branches/czw_branch/cleanup/pswarp/src/pswarpTransformReadout.c

    r24494 r25051  
    178178        psTrace("pswarp.transform", 1, "No overlap\n");
    179179    }
    180 
    181     // Store the variance factor and number of good pixels
    182     if (goodPixels > 0) {
    183         // Variance factor: large factor --> small scale
    184         float varFactor = psImageInterpolateVarianceFactor(input->image->numCols / 2.0 + input->image->col0,
    185                                                            input->image->numRows / 2.0 + input->image->row0,
    186                                                            interp->mode);
    187         psMetadataItem *vfItem = psMetadataLookup(output->analysis, PSWARP_ANALYSIS_VARFACTOR);
    188         if (vfItem) {
    189             psMetadataItem *goodpixItem = psMetadataLookup(output->analysis, PSWARP_ANALYSIS_GOODPIX);
    190             psAssert(goodpixItem, "It should be where we left it!");
    191             psAssert(vfItem->type == PS_TYPE_F32 && goodpixItem->type == PS_TYPE_S64,
    192                      "Should be the type we said.");
    193 
    194             vfItem->data.F32 += varFactor * goodPixels;
    195             goodpixItem->data.S64 += goodPixels;
    196         } else {
    197             psMetadataAddF32(output->analysis, PS_LIST_TAIL, PSWARP_ANALYSIS_VARFACTOR, 0,
    198                              "Variance factor weighted by the good pixels", varFactor * goodPixels);
    199             psMetadataAddS64(output->analysis, PS_LIST_TAIL, PSWARP_ANALYSIS_GOODPIX, 0,
    200                              "Number of good pixels", goodPixels);
    201         }
    202     }
    203180    psFree(interp);
    204181
Note: See TracChangeset for help on using the changeset viewer.