IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29515


Ignore:
Timestamp:
Oct 21, 2010, 2:45:13 PM (16 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20100823
Files:
2 deleted
131 edited
30 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20100823/DataStore/lib/DataStore/Utils.pm

    r28273 r29515  
    4242$BYTE_FIELD = qr/^\d+$/;
    4343$MD5_FIELD = qr/^[0-9a-f]{32}$/;
    44 %KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits );
     44%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits IPP-MOPS );
    4545%KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS IPP-DIST);
    4646%KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc dqresults);

  • branches/eam_branches/ipp-20100823/DataStore/scripts/dsgetfileset

    r27905 r29515  
    1717use File::Basename qw( basename );
    1818
    19 my ($uri, $outdir, $timeout, $skip_checks, $unpack, $no_proxy, $verbose);
     19my ($uri, $outdir, $timeout, $skip_checks, $unpack, $first_file, $no_proxy, $verbose);
    2020
    2121GetOptions(
    … …  
    2525    'skip-checks'       => \$skip_checks,
    2626    'unpack'            => \$unpack,
     27    'first-file=s'      => \$first_file,
    2728    'no-proxy'          => \$no_proxy,
    2829    'verbose|v'         => \$verbose,
    … …  
    7576    }
    7677
     78    if ($first_file) {
     79        next if $fs->fileid ne $first_file;
     80        $first_file = undef;
     81    }
    7782    my $uri = $fs->uri;
    7883    my $base = basename($uri);
    … …  
    153158=item * --unpack
    154159
    155 Uncompress fits files if compressed
     160Uncompress fits files if compressed.
     161
     162=item * --first-file <fileid>
     163
     164Skip files in fileset list prior to the given fileid.
    156165
    157166Optional.

  • branches/eam_branches/ipp-20100823/DataStoreServer/scripts/dsreg

    r28800 r29515  
    4242
    4343my $dbname;         # Database name
     44my $dbserver;       # Database host
    4445my $dsroot;         # root directory of the data store
    4546
    … …  
    8283        'ps7=s'         =>      \$ps7,
    8384        'dbname=s'      =>      \$dbname,
     85        'dbserver=s'    =>      \$dbserver,
     86        'ds_dbname=s'   =>      \$dbname,       # some scripts use these names
     87        'ds_dbserver=s' =>      \$dbserver,
    8488        'dsroot=s'      =>      \$dsroot,
    8589        'verbose'       =>      \$verbose,
    8690) or pod2usage(2);
    8791
     92pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
     93
     94# collect all command line argument errors and only fail once below.
    8895my $err = "";
    8996
    … …  
    121128    $dsroot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
    122129    if (!$dsroot) {
    123         die("Data Store root directory not set");
    124     }
    125 }
    126 
     130        &my_die("Data Store root directory not set", $PS_EXIT_CONFIG_ERROR);
     131    }
     132}
    127133
    128134if (!stat("$dsroot/index.txt")) {
    … …  
    131137
    132138# bail if any errors above
    133 show_usage($err)
    134     if ($err);
    135 
    136 show_usage("Invalid product '$product'.")
     139show_usage($err) if ($err);
     140
     141show_usage("Invalid data store product '$product'.")
    137142    unless defined stat("$dsroot/$product/index.txt");
    138143
    139144$dbname      = metadataLookupStr($siteConfig, 'DS_DBNAME') unless defined $dbname;
    140 my $dbserver = metadataLookupStr($siteConfig, 'DS_DBSERVER');
     145$dbserver    = metadataLookupStr($siteConfig, 'DS_DBSERVER') unless defined $dbserver;
    141146my $dbuser   = metadataLookupStr($siteConfig, 'DS_DBUSER');
    142147my $dbpass   = metadataLookupStr($siteConfig, 'DS_DBPASSWORD');
    … …  
    148153
    149154my $dbh = DBI->connect_cached($dsn, $dbuser, $dbpass, \%conn_attrs)
    150         or die "Cannot connect to DB server\n";
     155        or &my_die("Cannot connect to DB server\n", $PS_EXIT_SYS_ERROR);
    151156
    152157print STDERR "dbh: $dbh\n" if $verbose;
    … …  
    161166
    162167    my $stmt = $dbh->prepare("SELECT prod_id, last_fs FROM dsProduct WHERE prod_name = ?");
    163     $stmt->execute($product);
     168    my $execute_status = $stmt->execute($product);
     169    if (!$execute_status) {
     170        print STDERR "failed to execute product lookup\n";
     171        exit 2;
     172    }
    164173
    165174    my $prod_row = $stmt->fetchrow_hashref();
    … …  
    228237        my $rc;
    229238        if (($rc = system "rm -r $fileset_dir")) {
    230             die("failed to remove $fileset_dir error code: $rc");
     239            &my_die("failed to remove $fileset_dir error code: $rc", $PS_EXIT_UNKNOWN_ERROR);
    231240        }
    232241    } else  {
    … …  
    252261
    253262    if (!$prod_id) {
    254         die("product $product not found");
     263        &my_die("product $product not found", $PS_EXIT_UNKNOWN_ERROR);
    255264    }
    256265    my $count = $dbh->do("SELECT fileset_id FROM dsFileset WHERE fileset_name = ?" .
    … …  
    268277    if (! -e $fileset_dir) {
    269278        if (!mkdir $fileset_dir) {
    270             die("failed trying to create fileset directory $fileset_dir");
     279            &my_die("failed trying to create fileset directory $fileset_dir", $PS_EXIT_SYS_ERROR);
    271280        }
    272281        if (! -d $fileset_dir ) {
    273             die("cannot access just created fileset directory $fileset_dir");
     282            &my_die("cannot access just created fileset directory $fileset_dir", $PS_EXIT_SYS_ERROR);
    274283        }
    275284    }
    … …  
    281290            $in = *STDIN;
    282291        } else {
    283             open $in, "<$filelist" or die  "cannot open filelist file $filelist\n";
     292            open $in, "<$filelist" or &my_die("cannot open filelist file $filelist\n", $PS_EXIT_UNKNOWN_ERROR);
    284293        }
    285294
    … …  
    296305            my @fields = split /\|/;
    297306            if (@fields < 4) {
    298                 die "malformed input line at $filelist line $lineno: $_\n";
     307                &my_die("malformed input line at $filelist line $lineno: $_\n", $PS_EXIT_PROG_ERROR);
    299308            }
    300309            $filename = shift @fields;
    … …  
    320329        }
    321330
    322         die("empty filelist\n") if (@files == 0);
     331        &my_die("empty filelist\n", $PS_EXIT_PROG_ERROR) if (@files == 0);
    323332
    324333        # Prepare the fileset directory
    … …  
    350359                    } else {
    351360                        if (!copy($src, $dest)) {
    352                             die("copy($src, $dest) failed");
     361                            &my_die("copy($src, $dest) failed", $PS_EXIT_UNKNOWN_ERROR);
    353362                        }
    354363                    }
    … …  
    359368
    360369                if (!$no_cleanup && system "rm -r $fileset_dir") {
    361                     die("failed to remove $fileset_dir");
     370                    &my_die("failed to remove $fileset_dir", $PS_EXIT_UNKNOWN_ERROR);
    362371                }
    363372                exit $PS_EXIT_UNKNOWN_ERROR;
    … …  
    370379            my $path = "$fileset_dir/$filename";
    371380            if (! -e $path ) {
    372                 die "file $path not found";
     381                &my_die("file $path not found", , $PS_EXIT_UNKNOWN_ERROR);
    373382            }
    374383            # get the size of the file
    375384            my @finfo = stat($path);
    376385            unless (@finfo) {
    377                 die ("can't stat $path");
     386                &my_die ("can't stat $path", $PS_EXIT_UNKNOWN_ERROR);
    378387            }
    379388            # if size was supplied make sure that it matches the actual
    … …  
    382391                my $current_size = $finfo[7];
    383392                if ($file->{bytes} != $current_size) {
    384                     die "size on disk: $current_size does not match supplied"
    385                     . " size: $file->{bytes} for $path";
     393                    &my_die("size on disk: $current_size does not match supplied"
     394                    . " size: $file->{bytes} for $path", $PS_EXIT_PROG_ERROR);
    386395                }
    387396            } else {
    … …  
    391400                # Get MD5 sum
    392401                $file->{md5sum} = file_md5_hex($path);
    393                 die("failed to compute valid md5sum for $path") if !$file->{md5sum};
     402                &my_die("failed to compute valid md5sum for $path", $PS_EXIT_UNKNOWN_ERROR) if !$file->{md5sum};
    394403            }
    395404        }
    … …  
    401410        $dbh->disconnect();
    402411        $dbh = DBI->connect_cached($dsn, $dbuser, $dbpass, \%conn_attrs)
    403             or die "Cannot connect to server\n";
     412            or &my_die("Cannot connect to server\n", $PS_EXIT_SYS_ERROR);
    404413        print STDERR "ping failed new dbh: $dbh\n" if $verbose;
    405414    }
    … …  
    415424                            $ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7));
    416425
    417         die("failed to insert $fileset") if (!defined($count) or ($count == 0E0));
     426        &my_die("failed to insert $fileset", $PS_EXIT_UNKNOWN_ERROR) if (!defined($count) or ($count == 0E0));
    418427
    419428        # I don't use last_insert_id() because I've seen some problems with it and
    … …  
    423432        my $nfs = @fsids;
    424433        if (!$nfs) {
    425             die("fileset '$fileset' missing even though we just added it");
     434            &my_die("fileset '$fileset' missing even though we just added it", $PS_EXIT_UNKNOWN_ERROR);
    426435        } elsif ($nfs > 1) {
    427             die("Fileset '$fileset' already exists under $product.");
     436            &my_die("Fileset '$fileset' already exists under $product.", $PS_EXIT_PROG_ERROR);
    428437        }
    429438        my $fileset_id = $fsids[0];
    … …  
    443452
    444453            if ($count == 0E0) {
    445                 die("failed to insert $filename into $fileset");
     454                &my_die("failed to insert $filename into $fileset", $PS_EXIT_UNKNOWN_ERROR);
    446455            }
    447456        }
    … …  
    450459                                . " WHERE prod_id = $prod_id");
    451460        if ($count == 0E0) {
    452             die("failed to update dsProduct $prod_id");
     461            &my_die("failed to update dsProduct $prod_id", $PS_EXIT_UNKNOWN_ERROR);
    453462        }
    454463
    … …  
    457466        # create the cgi script index.txt by making a copy of its parent's
    458467        if (!copy("$dsroot/$product/index.txt", $index_script_name)) {
    459             die("failed to copy index script to file set");
     468            &my_die("failed to copy index script to file set", $PS_EXIT_UNKNOWN_ERROR);
    460469        }
    461470    };
    … …  
    531540    }
    532541}
     542
     543sub my_die {
     544    my $msg = shift;
     545    my $fault = shift;
     546    carp $msg;
     547    exit $fault;
     548}

  • branches/eam_branches/ipp-20100823/Nebulous/bin/neb-replicate

    r24463 r29515  
    6464
    6565if ($set_copies) {
    66     $neb->setxattr($key, "user.copies", $copies, "replace")
     66    $neb->setxattr($key, "user.copies", $set_copies, "replace")
    6767        or die $neb->err;
    6868}

  • branches/eam_branches/ipp-20100823/Ohana/src/addstar/include/addstar.h

    r29412 r29515  
    121121char    ZERO_POINT_OPTION[64];
    122122float   ZERO_POINT_OFFSET;
     123float   ZERO_POINT_ERROR;
     124float   ZPT_OBS_PHU;
     125float   ZPT_ERR_PHU;
    123126
    124127// carries the mosaic into gstars

  • branches/eam_branches/ipp-20100823/Ohana/src/addstar/src/GetZeroPointExposure.c

    r27435 r29515  
    1717
    1818// PHU_HEADER: in this case, the zero point measured for the entire exposure and reported in
    19 // the PHU header is used to set the zero point offset for each chip.  Note that in this case,
    20 // ZERO_POINT_OFFSET in this funciton is actually set to the zero point, and adjusted to an
    21 // offset in ReadImageHeader based on the per-chip zero points in the photcode database
     19// the PHU header is used to set the zero point offset for each chip. 
     20// Here set ZPT_OBS_PHU and ZPT_ERR_PHU to the observed values from the header.
     21// ZERO_POINT_OFFSET is calculated in ReadImageHeader based on the per-chip zero points in the
     22// photcode database.
    2223
    2324int GetZeroPointExposure (Header **headers, HeaderSet *headerSets, off_t Nimages) {
    … …  
    2627    if (!strcasecmp(ZERO_POINT_OPTION, "NOMINAL")) {
    2728        ZERO_POINT_OFFSET = 0.0;
     29        ZERO_POINT_ERROR = NAN;
    2830        return (TRUE);
    2931    }
    … …  
    3234    if (!strcasecmp(ZERO_POINT_OPTION, "CHIP_HEADER")) {
    3335        ZERO_POINT_OFFSET = 0.0;
     36        ZERO_POINT_ERROR = NAN;
    3437        return (TRUE);
    3538    }
    … …  
    3942
    4043        int i, Nzpt, Nmid, Nhead;
    41         float *zpt, ZPT_OBS;
     44        float *zpt, ZPT_OBS, ZPT_ERR;
    4245        PhotCode *photcode;
    4346        char photname[80];
    … …  
    5356                fprintf (stderr, "zero point not supplied in header\n");
    5457                continue;
     58            }
     59           
     60            if (!gfits_scan (headers[Nhead], "ZPT_ERR", "%f", 1, &ZPT_ERR)) {
     61//              XXX should we emit this message? We currently aren't using this value so no
     62//              fprintf (stderr, "zero point not supplied in header\n");
     63//              continue;
     64                ZPT_ERR = NAN;
    5565            }
    5666           
    … …  
    7888            fprintf (stderr, "WARNING: zero point is not measured, no valid entries in headers\n");
    7989            ZERO_POINT_OFFSET = 0.0;
     90            ZERO_POINT_ERROR = NAN;
    8091            free (zpt);
    8192            return (FALSE);
    … …  
    8697        ZERO_POINT_OFFSET = (Nzpt % 2) ? zpt[Nmid] : 0.5*(zpt[Nmid] + zpt[Nmid-1]);
    8798        free (zpt);
     99        // XXX: TODO calculate someting for ZERO_POINT_ERROR
     100        ZERO_POINT_ERROR = NAN;
    88101        return (TRUE);
    89102    }
    … …  
    92105    if (!strcasecmp(ZERO_POINT_OPTION, "PHU_HEADER")) {
    93106        int i, Nhead;
    94         float ZPT_OBS;
    95107
    96108        ZERO_POINT_OFFSET = 0.0;
     109        ZERO_POINT_ERROR = NAN;
    97110        for (i = 0; i < Nimages; i++) {
    98111            if (strcmp(headerSets[i].exthead, "PHU")) continue;
    99112            Nhead = headerSets[i].extnum_head;
    100113           
     114            float ZPT_OBS, ZPT_ERR;
    101115            if (!gfits_scan (headers[Nhead], "ZPT_OBS", "%f", 1, &ZPT_OBS)) {
    102116                fprintf (stderr, "WARNING: zero point is not measured, no valid entries in headers\n");
    103117                return (FALSE);
    104118            }
    105             ZERO_POINT_OFFSET = ZPT_OBS;
     119            if (!gfits_scan (headers[Nhead], "ZPT_ERR", "%f", 1, &ZPT_ERR)) {
     120                fprintf (stderr, "WARNING: zero point error is not measured\n");
     121//              XXX: Do we want to require ZPT_ERR? for now just set it to zero and proceed.
     122//              return (FALSE);
     123                ZPT_ERR = NAN;
     124            }
     125            ZPT_OBS_PHU = ZPT_OBS;
     126            ZPT_ERR_PHU = ZPT_ERR;
    106127            return (TRUE);
    107128        }

  • branches/eam_branches/ipp-20100823/Ohana/src/addstar/src/ReadImageHeader.c

    r28214 r29515  
    220220          fprintf (stderr, "zero point not supplied in header\n");
    221221          ZERO_POINT_OFFSET = 0.0;
     222          ZERO_POINT_ERROR = NAN;
    222223      } else {
    223224          ZERO_POINT_OFFSET = 0.001*photcodeData[0].C - ZPT_OBS;
     225          float ZPT_ERR;
     226          if (!gfits_scan (header, "ZPT_ERR", "%f", 1, &ZPT_ERR)) {
     227              // XXX: do we want to print this message?
     228              fprintf (stderr, "zero point error not supplied in header\n");
     229              ZPT_ERR = NAN;
     230          }
     231          ZERO_POINT_ERROR = ZPT_ERR;
    224232      }
    225233  }
    … …  
    230238          fprintf (stderr, "photcode data not supplied for this chip\n");
    231239          ZERO_POINT_OFFSET = 0.0;
     240          ZERO_POINT_ERROR = NAN;
    232241      } else {
    233           ZERO_POINT_OFFSET = 0.001*photcodeData[0].C - ZERO_POINT_OFFSET;
     242          ZERO_POINT_OFFSET = 0.001*photcodeData[0].C - ZPT_OBS_PHU;
     243          ZERO_POINT_ERROR =  ZPT_ERR_PHU;
    234244      }
    235245  }
    … …  
    237247  /* secz is in units milli-airmass */
    238248  image[0].Mcal = ZERO_POINT_OFFSET;
     249  image[0].dMcal = ZERO_POINT_ERROR;
    239250  image[0].Xm   = NAN_S_SHORT;
    240251  image[0].flags = 0;

  • branches/eam_branches/ipp-20100823/Ohana/src/dvomerge/src/merge_catalogs_old.c

    r28855 r29515  
    317317    objID ++;
    318318
     319    for (j = 0; j < NsecfiltOut; j++) {
     320        int outputIndex = (Nave * NsecfiltOut) + j;
     321        output[0].secfilt[outputIndex].M     = NAN;
     322        output[0].secfilt[outputIndex].dM    = NAN;
     323        output[0].secfilt[outputIndex].Xm    = NAN_S_SHORT;
     324        output[0].secfilt[outputIndex].M_20  = NAN_S_SHORT;
     325        output[0].secfilt[outputIndex].M_80  = NAN_S_SHORT;
     326        output[0].secfilt[outputIndex].Ncode = 0;
     327        output[0].secfilt[outputIndex].Nused = 0;
     328    }
     329
    319330    for (j = 0; j < NsecfiltIn; j++) {
    320331      int outputIndex;
    … …  
    333344        output[0].secfilt[outputIndex].Ncode = input[0].secfilt[N*NsecfiltIn+j].Ncode;
    334345        output[0].secfilt[outputIndex].Nused = input[0].secfilt[N*NsecfiltIn+j].Nused;
    335       } else {
    336         output[0].secfilt[outputIndex].M     = NAN;
    337         output[0].secfilt[outputIndex].dM    = NAN;
    338         output[0].secfilt[outputIndex].Xm    = NAN_S_SHORT;
    339         output[0].secfilt[outputIndex].M_20  = NAN_S_SHORT;
    340         output[0].secfilt[outputIndex].M_80  = NAN_S_SHORT;
    341         output[0].secfilt[outputIndex].Ncode = 0;
    342         output[0].secfilt[outputIndex].Nused = 0;
    343346      }
    344347    }

  • branches/eam_branches/ipp-20100823/Ohana/src/libdvo/src/dvo_photcode_ops.c

    r28855 r29515  
    539539// Create a map between the secfilt values from one photcode table to another
    540540int *GetSecFiltMap(PhotCodeData *output, PhotCodeData *input) {
    541   int i;
     541  int i, j;
    542542  int NsecfiltIn  = input[0].Nsecfilt;
     543  int NsecfiltOut = output[0].Nsecfilt;
    543544  int *map;
    544545  ALLOCATE(map, int, NsecfiltIn);
    545546
    546   // loop over entries
     547  // loop over entries in the input table
    547548  for (i = 0; i < NsecfiltIn; i++) {
    548549    int code  = input[0].codeNsec[i];
    549     int entry = output[0].hashcode[code];
     550    int entry = -1;
     551    // find the matching entry in the output table
     552    for (j = 0; j < NsecfiltOut; j++) {
     553      int outcode = output[0].codeNsec[j];
     554      if (code == outcode) {
     555         entry = j;
     556         break;
     557      }
     558    }
    550559    if (entry == -1) {
    551       // entry is missing
     560      // entry is missing fail (no printfs in this file)
    552561      free(map);
    553562      return(FALSE);

  • branches/eam_branches/ipp-20100823/Ohana/src/libdvo/src/skyregion_ops.c

    r27435 r29515  
    195195    Dmin = MIN (Dmin, d);
    196196    Dmax = MAX (Dmax, d);
     197  }
     198
     199  if( Rmax > (Rmin+180.)) {
     200   
     201      double temp = Rmax;
     202      Rmax = Rmin;
     203      Rmin = temp;
    197204  }
    198205

  • branches/eam_branches/ipp-20100823/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

    r29124 r29515  
    100100        my $x = $row->{CENTER_X};
    101101        my $y = $row->{CENTER_Y};
    102         my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
     102        my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
    103103        return $results;
    104104    }
    … …  
    164164
    165165    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component,
    166         $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
     166        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
    167167
    168168    return $results;
    … …  
    187187    my $data_group = shift;
    188188    my $option_mask = shift;
     189    my $mjd_min = shift;
     190    my $mjd_max = shift;
    189191    my $verbose  = shift;
    190192
    … …  
    287289            $choose_components = 1;
    288290        }
     291        $command .= " -template" if $inverse;
    289292        $id_opt = $use_imfile_id ? "-diff_skyfile_id" : "-diff_id";
    290293        $image_name  = "PPSUB.OUTPUT";
    … …  
    347350        $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
    348351        $command .= " -dateobs_end $dateobs_end" if $dateobs_end;
     352    } else {
     353        $command .= " -mjd_obs_begin $mjd_min" if $mjd_min;
     354        $command .= " -mjd_obs_end $mjd_max" if $mjd_max;
    349355    }
    350356
    … …  
    361367        # The image selectors are such that multiple runs my have be returned for the same exposure.
    362368        # Return only the latest one.
    363         $images = filterRuns($stage, $need_magic, $images, $verbose);
     369        $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose);
    364370    }
    365371    if ($choose_components) {
    … …  
    379385
    380386        next if $image->{fault};
    381         next if ($stage ne "raw") and $image->{quality};
     387        if (($stage ne "raw") and $image->{quality}) {
     388            print STDERR "Selected $stage image has bad quality. Skipping.\n";
     389            next;
     390        }
    382391
    383392        if ($base_name) {
    … …  
    479488    my $listrun = 0;
    480489    if ($byid) {
    481         if ($skycell_id) {
     490        if ($skycell_id and ($skycell_id ne 'all')) {
    482491            $command .= " -diffskyfile -diff_id $id -skycell_id $skycell_id";
    483492        } else {
    … …  
    634643    my $data_group = shift;
    635644    my $option_mask = shift;
     645    my $mjd_min = shift;
     646    my $mjd_max = shift;
    636647    my $verbose    = shift;
    637648
    … …  
    649660                my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $chip->{id},
    650661                    $tess_id, $chip->{component}, $need_magic,
    651                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
     662                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
    652663
    653664                next if !$these_results;
    … …  
    669680                my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef,
    670681                    $skycell->{tess_id}, $skycell->{component}, $need_magic,
    671                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
     682                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
    672683
    673684                next if !$these_results;
    … …  
    740751        my $astrom_resolved = $ipprc->file_resolve($astrom);
    741752        next if !$astrom_resolved;
     753        if (! -e $astrom_resolved) {
     754            print "$astrom_resolved not found skipping\n";
     755            next;
     756        }
    742757
    743758        my $start_dvo = gettimeofday();
    … …  
    10011016
    10021017    my $ticks = ($mjd - 40587.0) * 86400;
     1018
     1019    # CONVERT from TAI to UTC
     1020    # XXX: do this correctly
     1021    if ($mjd >= 54832) {
     1022        $ticks += -34;
     1023    } else {
     1024        $ticks += -33;
     1025    }
    10031026
    10041027    my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks);
    … …  
    12021225                my $images = runToolAndParse($command, $verbose);
    12031226                if (!defined $images) {
    1204                     print "No components containing coordinates found for ${stage}_id $stage_id\n";
    1205                     foreach my $row (@$rowList) {
    1206                         # XXX: This doesn't seem like the correct error code?
    1207                         $row->{error_code} = $PSTAMP_NO_OVERLAP;
    1208                     }
     1227                    print "No  $c found for $stage ${stage}_id $stage_id\n";
    12091228                    next;
    12101229                }
    … …  
    12351254    my $need_magic = shift;
    12361255    my $inputs     = shift;
     1256    my $inverse     = shift;
    12371257    my $verbose    = shift;
    12381258
    … …  
    12541274    my $printed = 0;
    12551275    foreach my $input (@$inputs) {
    1256         my $exp_id = $input->{exp_id};
     1276        my $exp_id;
     1277        if ($stage ne "diff") {
     1278            $exp_id = $input->{exp_id};
     1279        } elsif (!$inverse) {
     1280            $exp_id = $input->{exp_id_1};
     1281            $input->{exp_id} = $exp_id;
     1282        } else {
     1283            $exp_id = $input->{exp_id_2};
     1284            $input->{exp_id} = $exp_id;
     1285        }
    12571286        my $run_id = $input->{$id_name};
    12581287        my $magicked = $input->{magicked};  # this will be either stageRun.magicked or stage%file.magicked
    … …  
    12601289
    12611290        # can't process run in these states
     1291        # XXX: also goto_purged or goto_scrubbed or drop
    12621292        if (($state eq 'new') or ($state eq 'purged') or ($state eq 'scrubbed')) {
    12631293            print "skipping ${stage}Run $run_id for exp_id $exp_id in state $state\n";

  • branches/eam_branches/ipp-20100823/dbconfig/background.md

    r28486 r29515  
    2121    class_id            STR     64
    2222    path_base           STR     255
     23    data_state          STR     64
    2324    magicked            S64     0
    2425    dtime_script        F32     0.0
    … …  
    5758    skycell_id          STR     64
    5859    path_base           STR     255
     60    data_state          STR     64
    5961    magicked            S64     0
    6062    dtime_script        F32     0.0

  • branches/eam_branches/ipp-20100823/dbconfig/changes.txt

    r28873 r29515  
    19451945ALTER TABLE diffSummary DROP PRIMARY KEY, ADD PRIMARY KEY (diff_id, projection_cell);
    19461946ALTER TABLE stackSummary DROP PRIMARY KEY, ADD PRIMARY KEY (sass_id, projection_cell);
     1947
     1948ALTER TABLE stackSumSkyfile ADD COLUMN mjd_obs double AFTER good_frac;
     1949
     1950-- Version 1.1.65
     1951-- changes to support cleanup of the background preserved images
     1952ALTER TABLE chipBackgroundImfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
     1953ALTER TABLE chipBackgroundImfile ADD KEY(data_state);
     1954ALTER TABLE warpBackgroundSkyfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
     1955ALTER TABLE warpBackgroundSkyfile  ADD KEY(data_state);
     1956
     1957ALTER TABLE magicMask ADD COLUMN path_base VARCHAR(255) AFTER uri;
     1958
     1959-- diff_id was removed from magicInputSkyfile ages ago, but
     1960-- pxadmin_create_tables.sql still had it this command didn't work due to
     1961-- a mysql error.
     1962-- ALTER TABLE magicInputSkyfile DROP COLUMN diff_id;
     1963UPDATE dbversion set schema_version = '1.1.65',  updated= CURRENT_TIMESTAMP();
     1964
     1965-- Version 1.1.66
     1966

  • branches/eam_branches/ipp-20100823/dbconfig/config.md

    r26519 r29515  
    22    pkg_name        STR     ippdb
    33    pkg_namespace   STR     ippdb
    4     pkg_version     STR     1.1.63
     4    pkg_version     STR     1.1.65
    55END

  • branches/eam_branches/ipp-20100823/dbconfig/magic.md

    r27450 r29515  
    4242    magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
    4343    uri         STR         255
     44    path_base   STR         255
    4445    streaks     S32         0
    4546    fault       S16         0       # Key

  • branches/eam_branches/ipp-20100823/dbconfig/pstamp.md

    r28294 r29515  
    4646    options     S64         64
    4747    dep_id      S64         0
     48    fault_count S32         0
    4849END
    4950
    … …  
    5960    outdir      STR         255
    6061    fault       S16         0
     62    fault_count S32         0
    6163END
    6264

  • branches/eam_branches/ipp-20100823/dbconfig/stack.md

    r28375 r29515  
    4949    hostname           STR    64
    5050    good_frac          F32    0.0     # Key
     51    mjd_obs            F64    0.0
    5152    fault              S16    0       # Key
    5253    software_ver    STR         16

  • branches/eam_branches/ipp-20100823/ippMonitor/Makefile.in

    r29124 r29515  
    1616$(DESTBIN)/czartool_getServerStatus.pl \
    1717$(DESTBIN)/czartool_revert.pl \
     18$(DESTBIN)/czartool_serverstoprun.pl \
    1819$(DESTBIN)/build_histogram.dvo \
    1920$(DESTBIN)/helpers.dvo \
    … …  
    205206$(DESTWWW)/scatterCpiBgReMaImage.php \
    206207$(DESTWWW)/scatterCpiBgReMpImage.php \
    207 $(DESTWWW)/scatterCpiBgReSaImage.php
     208$(DESTWWW)/scatterCpiBgReSaImage.php \
     209$(DESTWWW)/exposureStatus.php \
     210$(DESTWWW)/mopsStatus.php \
     211$(DESTWWW)/gpc1MysqlProcessList.php
    208212
    209213PICTURES = \

  • branches/eam_branches/ipp-20100823/ippMonitor/def/autocode.php

    r26456 r29515  
    2525$restricted = 0;
    2626
    27 // define restrictiosn to the queries
     27// define restrictions to the queries
    2828// ** TABLE RESTRICTIONS **
    2929
    … …  
    3333  }
    3434}
     35
     36// Add GROUP BY statements if any
     37// ** GROUP RESTRICTIONS **
    3538
    3639// get the result table count

  • branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_getplot.php

    r28861 r29515  
    1515$label = $_GET[label];
    1616$stage = $_GET[stage];
     17$plottype = $_GET[plottype];
    1718
    1819if ($type=="t")
    19 $filePath = "/tmp/czarplot_".$label."_".$stage."_".$type.".png";
     20$filePath = "/tmp/czarplot_".$plottype."_".$label."_".$stage."_".$type.".png";
    2021else if ($type=="h")
    21 $filePath = "/tmp/czarplot_".$label."_all_stages_".$type.".png";
     22$filePath = "/tmp/czarplot_".$plottype."_".$label."_all_stages_".$type.".png";
    2223else if ($type=="s")
    2324$filePath = "/tmp/czarplot_hosts_space.png";
    … …  
    2930}
    3031
     32
    3133exit();
    3234

  • branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_labels.php

    r29124 r29515  
    2525
    2626menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']);
    27 
    28 $pass = $ID['pass'];
    29 $proj = $ID['proj'];
    30 $menu = $ID['menu'];
    31 
    32 $selectedStage = $_GET[stage];
    33 $selectedLabel = $_GET[label];
    34 $selectedServer = $_GET[server];
    35 $selectedRevertStage = $_GET[revertstage];
    36 $selectedRevertMode = $_GET[revertmode];
    37 
    38 if ($selectedLabel == "") { $selectedLabel = "all_stdscience_labels"; }
    39 if ($selectedStage == "") { $selectedStage = "all_stages"; }
    40 
    41 $nsStatus = getNightlyScienceStatus($czardb);
    42 echo "<p  align=\"center\"> Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses). <br>Documentation can be found <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\">here</a> and cluster load monitored <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&c=IPP%2520Production\">here</a><br>Current nightly science status: $nsStatus</p>";
    43 
    44 
    45 // deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
    46 if ($selectedRevertStage != "" && $selectedRevertMode != "") {
    47 
    48     exec("czartool_revert.pl -t $selectedRevertStage -o $selectedRevertMode", $response, $status);
    49     $currentRevertMode;
    50     if ($response[0] == "off") $currentRevertMode = 0;
    51     else if ($response[0] == "on") $currentRevertMode = 1;
    52 
    53     setRevertStatus($czardb, $selectedRevertStage, $currentRevertMode);
    54 }
    55 $debug = 0;
    56 
    57 $stdsLabels = getLabels($czardb, "stdscience");
    58 $distLabels = getLabels($czardb, "distribution");
    59 $pubLabels = getLabels($czardb, "publishing");
    60 $updateLabels = getLabels($czardb, "update");
    61 
    62 if ($debug) {
    63     echo "prog returned $status, and output:<br>";
    64     for ($i = 0; $i < count($distLabels); $i++) {
    65         echo "$distLabels[$i]<br>";
    66     }
    67 }
    6827
    6928$states=array("full","new","drop","wait");
    … …  
    8241        "summitcopy");
    8342
     43$pass = $ID['pass'];
     44$proj = $ID['proj'];
     45$menu = $ID['menu'];
     46
     47$selectedStage = $_GET[stage];
     48$selectedLabel = $_GET[label];
     49$selectedServer = $_GET[server];
     50$selectedRevertStage = $_GET[revertstage];
     51$selectedRevertMode = $_GET[revertmode];
     52$plotType = $_GET[plottype];
     53$serverCmd = $_GET[servercmd];
     54$allServerCmd = $_GET[allservercmd];
     55
     56if ($selectedLabel == "") { $selectedLabel = "all_stdscience_labels"; }
     57if ($selectedStage == "") { $selectedStage = "all_stages"; }
     58if ($plotType == "") { $plotType = "linear"; }
     59
     60$nsStatus = getNightlyScienceStatus($czardb);
     61$plotTypeLink = ($plotType == "linear") ? "log" : "linear";
     62$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
     63echo "<p  align=\"center\"> Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses). <br>Documentation can be found <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\">here</a> and cluster load monitored <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&c=IPP%2520Production\">here</a><br>Current nightly science status: $nsStatus<br>Use <a href=\"$link\"> $plotTypeLink</a> plots </p>";
     64
     65
     66// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
     67if ($selectedRevertStage && $selectedRevertMode) turnRevertsOnOff($czardb, $selectedRevertStage, $selectedRevertMode);
     68
     69// tell selected server to stop or run
     70if ($selectedServer && $serverCmd) serverStopRun($czardb, $selectedServer,  $serverCmd);
     71
     72// tell all servers to stop or run
     73if ($allServerCmd) {
     74
     75    foreach ($servers as &$server) serverStopRun($czardb, $server,  $allServerCmd);
     76}
     77
     78
     79$debug = 0;
     80
     81$stdsLabels = getLabels($czardb, "stdscience");
     82$distLabels = getLabels($czardb, "distribution");
     83$pubLabels = getLabels($czardb, "publishing");
     84$updateLabels = getLabels($czardb, "update");
     85
     86if ($debug) {
     87    echo "prog returned $status, and output:<br>";
     88    for ($i = 0; $i < count($distLabels); $i++) {
     89        echo "$distLabels[$i]<br>";
     90    }
     91}
     92
     93
    8494// set up the form
    8595echo "<form action=\"czartool_labels.php\" method=\"POST\">\n";
    … …  
    91101
    92102echo "<table>\n";
    93   echo "<tr>\n";
    94     echo "<td>\n";
    95       echo "<img src=\"czartool_getplot.php?type=t&label=$selectedLabel&stage=$selectedStage\"><br>";
    96     echo "</td>\n";
    97 
    98     echo "<td> \n";
    99       createLabelsTable($pass, $proj, $czardb, "stdscience", $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage);
    100     echo "</td>\n";
    101   echo "</tr>\n";
    102 
    103   echo "<tr>\n";
    104     echo "<td>\n";
    105       echo "<img src=\"czartool_getplot.php?type=h&label=$selectedLabel&stage=$selectedStage\"><br>";
    106     echo "</td>\n";
    107 
    108     echo "<td> \n";
    109       createLabelsTable($pass, $proj, $czardb, "update", $updateLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage);
    110     echo "</td>\n";
    111   echo "</tr>\n";
    112 
    113   echo "<tr>\n";
    114     echo "<table>\n";
    115       echo "<tr valign=top>\n";
    116         echo "<td> \n";
    117           echo "<img src=\"czartool_getplot.php?type=s\"><br>";
    118         echo "</td>\n";
    119         echo "<td>\n";
    120           createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage);
    121         echo "</td>\n";
    122 
    123         echo "<td> \n";
    124           $today = date("Y-m-d");
    125           showSummitData($gpc1db, $today);
    126         echo "</td>\n";
    127       echo "</tr>\n";
    128     echo "</table>\n";
    129 
    130     echo "<table>\n";
    131       echo "<tr valign=top>\n";
    132         echo "<td> \n";
    133         echo "</td>\n";
    134         echo "<td>\n";
    135       if ($selectedServer) showServerStatus($selectedServer);
    136         echo "</td>\n";
    137       echo "</tr>\n";
    138     echo "</table>\n";
    139   echo "</tr>\n";
     103echo "<tr>\n";
     104echo "<td>\n";
     105echo "<img src=\"czartool_getplot.php?type=t&label=$selectedLabel&stage=$selectedStage&plottype=$plotType\"><br>";
     106echo "</td>\n";
     107
     108echo "<td> \n";
     109createLabelsTable($pass, $proj, $czardb, "stdscience", $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $plotType);
     110echo "</td>\n";
     111echo "</tr>\n";
     112
     113echo "<tr>\n";
     114echo "<td>\n";
     115echo "<img src=\"czartool_getplot.php?type=h&label=$selectedLabel&stage=$selectedStage&plottype=linear\"><br>";
     116echo "</td>\n";
     117
     118echo "<td> \n";
     119createLabelsTable($pass, $proj, $czardb, "update", $updateLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $plotType);
     120echo "</td>\n";
     121echo "</tr>\n";
     122
     123echo "<tr>\n";
     124echo "<table>\n";
     125echo "<tr valign=top>\n";
     126echo "<td> \n";
     127echo "<img src=\"czartool_getplot.php?type=s\"><br>";
     128echo "</td>\n";
     129echo "<td>\n";
     130createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage, $plotType);
     131echo "</td>\n";
     132
     133echo "<td> \n";
     134$today = date("Y-m-d");
     135showSummitData($gpc1db, $today);
     136echo "</td>\n";
     137echo "</tr>\n";
     138echo "</table>\n";
     139
     140echo "<table>\n";
     141echo "<tr valign=top>\n";
     142echo "<td> \n";
     143echo "</td>\n";
     144echo "<td>\n";
     145if ($selectedServer && !$serverCmd) showServerStatus($selectedServer);
     146echo "</td>\n";
     147echo "</tr>\n";
     148echo "</table>\n";
     149echo "</tr>\n";
    140150echo "</table>\n";
    141151
    … …  
    225235#
    226236###########################################################################
    227 function createLabelsTable($pass, $proj, $db, $server, $labels, $distLabels, $pubLabels, $stages, $states, $selectedState, $selectedLabel, $selectedStage) {
     237function createLabelsTable(
     238        $pass,
     239        $proj,
     240        $db,
     241        $server,
     242        $labels,
     243        $distLabels,
     244        $pubLabels,
     245        $stages,
     246        $states,
     247        $selectedState,
     248        $selectedLabel, $selectedStage, $plotType) {
    228249
    229250    // set up table columns
    … …  
    241262        if ($stage == "burntool") continue;
    242263        $reverting = getRevertStatus($db, $stage);
    243         $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&revertmode=";
     264        $link =
     265            "czartool_labels.php?pass=".$pass
     266            ."&proj=".$proj
     267            ."&label=".$selectedLabel
     268            ."&stage=".$selectedStage
     269            ."&plottype=".$plotType
     270            ."&revertstage=".$stage
     271            ."&revertmode=";
     272
    244273        if(!$reverting) {$label =  "Start";$link = $link . "on";}
    245274        if($reverting) {$label = "Stop";$link = $link . "off";}
    … …  
    254283    write_header_cell($class, "Label (in order of priority)");
    255284    foreach ($stages as &$stage) {
    256        
     285
    257286        if ($stage == $selectedStage) $link = "";
    258         else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $stage;
     287        else $link =
     288            "czartool_labels.php?pass=".$pass
     289                ."&proj=".$proj
     290                ."&label=".$selectedLabel
     291                ."&stage=".$stage
     292                ."&plottype=".$plotType;
     293
    259294        write_table_cell($class, '%s', $link, $stage);
    260295    }
    261296
    262297    if ($selectedStage=="all_stages") $link = "";
    263     else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=all_stages";
     298    else $link =
     299        "czartool_labels.php?pass=".$pass
     300            ."&proj=".$proj
     301            ."&label=".$selectedLabel
     302            ."&stage=all_stages"
     303            ."&plottype=".$plotType;
     304
    264305    write_table_cell($class, '%s', $link, "All stages");
    265306
    … …  
    267308    echo "<tr><td></td>\n";
    268309
    269     $defaultlink = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj;
     310    $defaultlink =
     311        "czartool_labels.php?pass=".$pass
     312        ."&proj=".$proj
     313        ."&label=".$selectedLabel
     314        ."&stage=".$selectedStage
     315        ."&plottype=".$plotType;
    270316
    271317    // write rows
    … …  
    285331        // create link to label summary page for each label
    286332        if ($thisLabel == $selectedLabel) $link = "";
    287         else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $thisLabel . "&stage=" . $selectedStage;
     333        else $link =
     334            "czartool_labels.php?pass=".$pass
     335                ."&proj=".$proj
     336                ."&label=".$thisLabel
     337                ."&stage=".$selectedStage
     338                ."&plottype=".$plotType;
    288339
    289340        echo "<tr><td></td>\n";
    … …  
    342393
    343394    if ($selectedLabel == "all_".$server."_labels") $link = "";
    344     else  $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=all_".$server."_labels&stage=".$selectedStage;
     395    else  $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=all_".$server."_labels&stage=".$selectedStage."&plottype=".$plotType;
    345396
    346397    echo "<tr><td></td>\n";
    … …  
    385436###########################################################################
    386437#
     438#  Turns reverts on or off for a given task
     439#
     440###########################################################################
     441function turnRevertsOnOff($db, $stage, $mode) {
     442
     443    exec("czartool_revert.pl -t $stage -o $mode", $response, $status);
     444    $currentRevertMode;
     445    if ($response[0] == "off") $currentRevertMode = 0;
     446    else if ($response[0] == "on") $currentRevertMode = 1;
     447
     448    setRevertStatus($db, $stage, $currentRevertMode);
     449}
     450
     451###########################################################################
     452#
    387453# Returns whether this stage is reverting or not
    388454#
    … …  
    420486    $faults = $row[1];
    421487
    422     $str = "$pending";
     488    if ($pending == 0)
     489        $str = "";
     490    else
     491        $str = "$pending";
    423492
    424493    if ($state == "new") {
    … …  
    437506#
    438507###########################################################################
    439 function createServersTable($pass, $proj, $db, $servers, $selectedLabel, $selectedStage) {
     508function createServersTable($pass, $proj, $db, $servers, $selectedLabel, $selectedStage, $plotType) {
    440509
    441510    // set up table columns
    … …  
    445514    write_header_cell($class, "Server");
    446515    write_header_cell($class, "Alive?");
    447     write_header_cell($class, "Scheduler running?");
     516    $link = "czartool_labels.php?pass=".$pass
     517        ."&proj=".$proj
     518        ."&label=".$selectedLabel
     519        ."&stage=".$selectedStage
     520        ."&plottype=".$plotType
     521        ."&allservercmd=stop";
     522
     523    write_table_cell($class, '%s', $link, "Stop all");
     524    $link = "czartool_labels.php?pass=".$pass
     525        ."&proj=".$proj
     526        ."&label=".$selectedLabel
     527        ."&stage=".$selectedStage
     528        ."&plottype=".$plotType
     529        ."&allservercmd=run";
     530
     531    write_table_cell($class, '%s', $link, "Run all");
    448532    echo "</tr>\n";
    449533
    450534    foreach ($servers as &$server) {
    451535
    452         $link = "czartool_labels.php?pass=".$pass."&proj=".$proj."&server=".$server."&label=".$selectedLabel."&stage=".$selectedStage;
    453         //      $link = "";
    454 
    455536        getServerStatus($db, $server, $alive, $running);
    456537
    457538        echo "<tr><td></td>\n";
     539        $link = "czartool_labels.php?pass=".$pass
     540            ."&proj=".$proj
     541            ."&server=".$server
     542            ."&label=".$selectedLabel
     543            ."&stage=".$selectedStage
     544            ."&plottype=".$plotType;
     545
    458546        write_table_cell($class, '%s', $link, $server);
    459547        write_table_cell($class, '%s', "", $alive ? "yes" : "NO");
    460         write_table_cell($class, '%s', "", $running ? "yes" : "NO");
     548
     549        if ($alive) {
     550
     551            $link = "czartool_labels.php?pass=".$pass
     552                ."&proj=".$proj
     553                ."&label=".$selectedLabel
     554                ."&stage=".$selectedStage
     555                ."&plottype=".$plotType
     556                ."&server=".$server
     557                ."&servercmd=";
     558
     559            if ($running)  {
     560
     561                $link = $link . "stop";
     562                write_table_cell($class, '%s', $link, "stop");
     563                write_table_cell($class, '%s', "", "");
     564            }
     565            else {
     566
     567                $link = $link . "run";
     568                write_table_cell($class, '%s', "", "");
     569                write_table_cell($class, '%s', $link, "run");
     570            }
     571        }
     572        else {
     573
     574            write_table_cell($class, '%s', "", "");
     575            write_table_cell($class, '%s', "", "");
     576        }
    461577        echo "</tr>\n";
    462578    }
    … …  
    464580    echo "</table>\n";
    465581}
     582
     583###########################################################################
     584#
     585#  Commands a server to stop or run
     586#
     587###########################################################################
     588function serverStopRun($db, $server, $cmd) {
     589
     590    exec("czartool_serverstoprun.pl -s $server -c $cmd", $response, $status);
     591    $alive = 0;
     592    $running = 0;
     593    if ($response[0] == "running") {$alive = 1; $running = 1;}
     594    else if ($response[0] == "stopped") {$alive = 1; $running = 0;}
     595    else if ($response[0] == "dead") {$alive = 0; $running = 0;}
     596    setServerStatus($db, $server, $alive, $running);
     597}
     598
     599###########################################################################
     600#
     601# Sets server status
     602#
     603###########################################################################
     604function setServerStatus($db, $server, $alive, $running) {
     605
     606    $sql = "INSERT INTO servers (server, alive, running) VALUES ('$server', $alive, $running)";
     607
     608    if ($debug) {echo "$sql<br>";}
     609
     610    $qry = $db->query($sql);
     611    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
     612}
     613
    466614
    467615###########################################################################

  • branches/eam_branches/ipp-20100823/ippMonitor/raw/ipp.imfiles.dat

    r29124 r29515  
    3939menulink  | menuselect   | link    | Clean /tmp directory         | cleanTmpDirectory.php
    4040menutop   | menutop      | link    | disk usage                   | diskUsage.php
     41
     42menutop   | menutop      | plain   | &nbsp;                       |
     43menulink  | menuselect   | link    | Exposures Status             | exposureStatus.php
     44menulink  | menuselect   | link    | MOPS Exposures Status        | mopsStatus.php
     45menulink  | menuselect   | link    | GPC1 MySql ProcessList            | gpc1MysqlProcessList.php

  • branches/eam_branches/ipp-20100823/ippMonitor/scripts/generate

    r28969 r29515  
    227227        }
    228228
     229        # fill in GROUP BY
     230        if ($line =~ m|// \*\* GROUP RESTRICTIONS \*\*|) {
     231            &write_group_by;
     232        }
     233
    229234        # fill in table restricts
    230235        if ($line =~ m|// \*\* BUTTON RESTRICTIONS \*\*|) {
    … …  
    331336        print FILE "\$WHERE = check_restrict ('$value', \$WHERE, 'max', 1.0);\n";
    332337    }
     338    # print FILE "\$WHERE = check_ordering ('$GROUPS', \$WHERE);\n";
     339}
     340
     341# Generate GROUP BY statement if any
     342sub write_group_by {
    333343    print FILE "\$WHERE = check_ordering ('$GROUPS', \$WHERE);\n";
    334344}

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl

    r29124 r29515  
    124124    defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or
    125125    defined $test_mode or defined $clean_old or defined $check_mode or
    126     defined $confirm_stacks;
     126    defined $confirm_stacks or defined $burntool_stats;
    127127
    128128# Configurable parameters from our config file.
    129129my @target_list = ();
    130130my @filter_list = ();
     131my %distribution_list = ();
    131132my %tessID_list = ();
    132133my %obsmode_list = ();
    … …  
    135136my %cleanmods_list = ();
    136137my %stackable_list = ();
     138my %extra_processing = ();
    137139my %reduction_class = ();
    138140my %macro_formats = ();
    … …  
    206208                $this_target = ${ $tentry }{value};
    207209                push @target_list, $this_target;
     210                $distribution_list{$this_target} = $this_target;
     211            }
     212            elsif (${ $tentry }{name} eq 'DISTRIBUTION') {
     213                $distribution_list{$this_target} = ${ $tentry }{value};
    208214            }
    209215            elsif (${ $tentry }{name} eq 'TESS') {
    … …  
    222228                $stackable_list{$this_target} = ${ $tentry }{value};
    223229            }
     230            elsif (${ $tentry }{name} eq 'EXTRA_PROCESSING') {
     231                $extra_processing{$this_target} = ${ $tentry }{value};
     232            }
    224233            elsif (${ $tentry }{name} eq 'REDUCTION') {
    225234                $reduction_class{$this_target} = ${ $tentry }{value};
    … …  
    726735        my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target);
    727736        if (defined($burntool_stats)) {
    728             print "$Nexposures $Nimfiles $Nburntooled $Nalready\n";
     737            print "BTSTATS: $date $target $Nexposures $Nimfiles $Nburntooled $Nalready\n";
    729738        }
    730739
    731740        if ($Nexposures == 0) {
    732741            print STDERR "execute_chips: Target $target on $date had no exposures.\n";
    733 #           next;
     742            next;
    734743        }
    735744        if ($Nalready != 0) {
    736745            print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n";
    737 #            next;
     746            next;
    738747        }
    739748        if ($Nimfiles != $Nburntooled) {
    … …  
    11831192        &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
    11841193    }
     1194   
     1195   
    11851196    return(0);
    11861197}
    … …  
    11901201    my $pretend = shift;
    11911202    foreach my $target (@target_list) {
    1192         if ($stackable_list{$target} == 1) {
    1193             foreach my $filter (@filter_list) {
     1203        foreach my $filter (@filter_list) {
     1204            if (exists($extra_processing{$target})) {
     1205                do_extra_processing($date,$target,$filter,$pretend);
     1206            }
     1207            if ($stackable_list{$target} == 1) {
    11941208                my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
    11951209                if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
    … …  
    12311245                }
    12321246            }
     1247            else {
     1248                # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
     1249            }
    12331250        }
    1234         else {
    1235             # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
    1236         }
    1237     }
    1238 
     1251    }
    12391252}
    12401253
    … …  
    12911304
    12921305#
     1306# Extra processing
     1307################################################################################
     1308
     1309sub do_extra_processing {
     1310    my $date = shift;
     1311    my $target = shift;
     1312    my $filter = shift;
     1313    my $pretend = shift;
     1314    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
     1315
     1316    if ($target eq 'OSS') {
     1317        my $db = init_gpc_db();
     1318
     1319        my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
     1320        $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
     1321        print STDERR "$obj_sth\n";
     1322        my $object_ref = $db->selectall_arrayref( $obj_sth );
     1323
     1324        foreach my $object_row (@{ $object_ref }) {
     1325            my $this_object = shift @{ $object_row };
     1326            my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
     1327            $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1328            $input_sth .= " ORDER BY dateobs ";
     1329
     1330            my $warps = $db->selectall_arrayref( $input_sth );
     1331           
     1332            if (($#{ $warps } + 1) % 2 != 0) {
     1333                print STDERR "Number of input warps to make OSS diffs is not even! $#{ $warps }\n";
     1334                die;
     1335            }
     1336           
     1337            while ($#{ $warps } > -1) {
     1338                my $input_warp = shift @{ $warps };
     1339                my $template_warp = shift @{ $warps };
     1340                my $input_exp_id = ${ $input_warp }[0];
     1341                my $template_exp_id = ${ $template_warp }[0];
     1342               
     1343                my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
     1344                $cmd .= "-input_label $label  -template_label $label ";
     1345                $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
     1346                $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
     1347                $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
     1348#               $cmd .= " -pretend ";
     1349                if (defined($pretend)) {
     1350                    $cmd .= ' -pretend ';
     1351                }
     1352                if ($debug == 1) {
     1353                    $cmd .= ' -pretend ';
     1354                }
     1355                print STDERR "EXTRA_PROCESSING: $cmd\n";
     1356                if (($debug == 0)&&(!defined($pretend))) {
     1357                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1358                        run ( command => $cmd, verbose => $verbose );
     1359                    unless ($success) {
     1360                        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1361                        &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1362                    }
     1363                }
     1364            }
     1365        }
     1366    }
     1367}
     1368
     1369           
     1370
     1371
     1372#
    12931373# Auto-Clean
    12941374################################################################################
    … …  
    13801460    my $object   = $object_list{$target};
    13811461    my $comment = $comment_list{$target};
    1382     my $dist_group = $target;
     1462    my $dist_group = $distribution_list{$target};
    13831463    my $data_group = "${target}.${trunc_date}";
    13841464    my $tess_id = $tessID_list{$target};
    … …  
    14271507        my $N = $metadata_out{N_MACROS};
    14281508        $metadata_out{"ns${N}Macro"} = $macro_formats{$proc_mode};
    1429         print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_formats{$proc_mode}\n";
     1509#       print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_formats{$proc_mode}\n";
    14301510        if (defined($date)&&(defined($target))) {
    14311511

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_apply_burntool.pl

    r27299 r29515  
    5555my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
    5656my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1);
     57my $nebreplicate = can_run('neb-replicate') or (warn "Can't find neb-replicate" and $missing_tools = 1);
    5758#my $fpack    = can_run('fpack')    or (warn "Can't find fpack" and $missing_tools = 1);
    5859if ($missing_tools) {
    … …  
    214215
    215216        $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
     217        $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
    216218        $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN);
    217219        if ($status) {

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_cleanup.pl

    r27959 r29515  
    5353}
    5454
     55# set this to 1 to enable checking for files on dead nodes
     56# it is off for now because the implementation is a hack
     57# See comments below.
     58my $check_for_gone = 0;
     59
    5560my $error_state;
    56 if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned";  }
    57 if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; }
    58 if ($mode eq "goto_purged")   { $error_state = "error_purged";   }
     61my $done_state;
     62if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned"; $done_state = "cleaned"; }
     63if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; $done_state = "scrubbed";}
     64if ($mode eq "goto_purged")   { $error_state = "error_purged";   $done_state = "purged";}
    5965
    6066
    … …  
    9298    }
    9399
    94     # if there are no chipProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
    95     # XXX Why? This could just mean there's nothing to cleanup, or that we're trying to rerun an errored run.
     100    # if there are no chipProcessedImfiles (@$stdout_buf == 0) then assume that we're done
     101    # it could be that there are no chipProcessedImfiles at all if say if a run was changed from drop to goto_cleaned
     102    # or of a run was set to update and then back to goto_cleaned before any images were processed
    96103    if (@$stdout_buf == 0)  {
    97         my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state";
     104        my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $done_state";
    98105        $command .= " -dbname $dbname" if defined $dbname;
    99106
    … …  
    126133
    127134                unless ($ipprc->file_exists($config_file)) {
    128                     print STDERR "skipping cleanup for chipRun $stage_id $class_id "
    129                         . " because config file ($config_file) is missing\n";
    130                     $status = 0;
     135                    if (file_gone($config_file)) {
     136                        print STDERR "forcing cleanup for chipRun $stage_id $class_id "
     137                            . " because config file ($config_file) is gone\n";
     138                    } else {
     139                        print STDERR "skipping cleanup for chipRun $stage_id $class_id "
     140                            . " because config file ($config_file) is missing\n";
     141                        $status = 0;
     142                    }
    131143                }
    132144            }
    … …  
    354366
    355367    if (@$stdout_buf == 0) {
    356         # No skycells were found for some reason.
    357         # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
    358         my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state";
     368        # No skycells were found for some reason.
     369        # it could be that there are no warpSkyfiles at all if say if a run was changed from drop to goto_cleaned
     370        # or of a run was cleaned, set to update, and then back to goto_cleaned before any images were successfully
     371        # updated
     372        my $command = "$warptool -updaterun -warp_id $stage_id -set_state $done_state";
    359373        $command .= " -dbname $dbname" if defined $dbname;
    360374
    … …  
    384398
    385399                unless ($ipprc->file_exists($config_file)) {
    386                     print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
    387                         " because config file is missing\n";
    388                     $status = 0;
     400                    if (file_gone($config_file)) {
     401                        print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" .
     402                            " because config file is gone\n";
     403                    } else {
     404                        print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
     405                            " because config file is missing\n";
     406                        $status = 0;
     407                    }
    389408                }
    390409            }
    … …  
    624643
    625644    if (@$stdout_buf == 0) {
    626         # No skycells were found for some reason.
    627         # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
    628         my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
     645        # No skycells were found for some reason.
     646        # it could be that there are no warpSkyfiles at all if say if a run was changed from drop to goto_cleaned
     647        # or of a run was cleaned, set to update, and then back to goto_cleaned before any images were successfully
     648        my $command = "$difftool -updaterun -diff_id $stage_id -set_state $done_state";
    629649        $command .= " -dbname $dbname" if defined $dbname;
    630650
    … …  
    658678
    659679                unless ($ipprc->file_exists($config_file)) {
    660                     print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
    661                         " because config file ($config_file) is missing\n";
    662                     $status = 0;
     680                    if (file_gone($config_file)) {
     681                        print STDERR "forcing cleanup for diffRun $stage_id $skycell_id" .
     682                            " because config file ($config_file) is gone\n";
     683                    } else {
     684                        print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
     685                            " because config file ($config_file) is missing\n";
     686                        $status = 0;
     687                    }
    663688                }
    664689            }
    … …  
    16651690}
    16661691
     1692my $whichnode;
     1693sub file_gone
     1694{
     1695    # if $check_for_gone check whether the only instance of file is on a lost volumen
     1696    # XXX: we don't have a proper interface for this.
     1697    # For now try to use Bill's hack the script 'whichnode'
     1698    return 0 if !$check_for_gone;
     1699
     1700    my $file = shift;
     1701
     1702    if (!$whichnode) {
     1703        $whichnode = can_run('whichnode') or
     1704            &my_die("Can't find whichnode", "chip", $stage_id, $PS_EXIT_CONFIG_ERROR);
     1705    }
     1706
     1707    my $command = "$whichnode $file";
     1708
     1709    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1710        run(command => $command, verbose => $verbose);
     1711    unless ($success) {
     1712        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1713        &my_die("Unable to perform whichnode: $error_code", "chip", $stage_id, $error_code);
     1714    }
     1715
     1716    my @lines = split "\n", (join "", @$stdout_buf);
     1717    my $numGone = 0;
     1718    my $numNotGone = 0;
     1719    foreach my $line (@lines) {
     1720        chomp $line;
     1721
     1722        # output lines are either
     1723        #   "volume available"
     1724        # or
     1725        #   "volume not available"
     1726
     1727        my ($volume, $answer, undef) = split " ", $line;
     1728        # our hack is if the volume has an X in the name it's gone
     1729        if ($volume =~ /X/) {
     1730            print STDERR "$file is on $volume which is gone\n";
     1731            $numGone++;
     1732        } elsif ($answer eq 'not') {
     1733            print STDERR "$file is on $volume which is not available\n";
     1734            $numNotGone++;
     1735        } else {
     1736            print STDERR "unexpected output from whichnode: $line\n";
     1737        }
     1738    }
     1739    # if there are any instances that are not on a gone node return 0
     1740    if ($numNotGone == 0 and $numGone > 0) {
     1741        return 1;
     1742    } else {
     1743        return 0;
     1744    }
     1745}
     1746
    16671747sub addFilename
    16681748{

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_destreak.pl

    r28931 r29515  
    4242# Parse the command-line arguments
    4343my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction);
     44my ($streaks_path_base, $inv_streaks_path_base);
    4445my ($outroot, $recoveryroot, $magicked);
    4546my ($replace, $release);
    … …  
    4950           'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
    5051           'camera=s'       => \$camera,     # camera for evaluating file rules
     52           'streaks_path_base=s'      => \$streaks_path_base,    # path_base for streaks data
     53           'inv_streaks_path_base=s'  => \$inv_streaks_path_base, #path_base for streaks from inverse diff
     54           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
    5155           'streaks=s'      => \$streaks,    # file containing the list of streaks
    5256           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
    … …  
    7882    defined $camera and
    7983    defined $streaks and
     84    defined $streaks_path_base and
    8085    defined $stage and
    8186    defined $stage_id and
    … …  
    108113    &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    109114}
     115$inv_streaks_path_base = undef if defined($inv_streaks_path_base) and ($inv_streaks_path_base eq "NULL");
    110116$inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL");
    111117
    … …  
    240246    my ($allstreaks_fh, $allstreaks_name);
    241247
     248    # Set name of streaks files from their path_base. Note. ne 'NULL' test is for backward compatability
     249    # with runs that don't have path_base set yet
     250    if ($streaks_path_base ne 'NULL') {
     251        $streaks = "$streaks_path_base.streaks";
     252    }
     253    if ($inv_streaks_path_base and ($inv_streaks_path_base ne 'NULL')) {
     254        $inv_streaks = "$inv_streaks_path_base.streaks";
     255    }
     256
     257    my $streaks_resolved = $ipprc->file_resolve($streaks) or &my_die("failed to resolve streaks file $streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
     258    my $inv_streaks_resolved;
     259    if ($inv_streaks) {
     260        $inv_streaks_resolved = $ipprc->file_resolve($inv_streaks) or &my_die("failed to resolve inverse streaks file $inv_streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
     261    }
     262       
     263
    242264    if ($stage eq "raw") {
    243265        $image = $uri;
    … …  
    301323        $sources    = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base);
    302324
    303         if ($inv_streaks) {
     325        if ($inv_streaks_resolved) {
    304326            # create a temporary file containing the contents of the
    305327            # two streaks files
    … …  
    307329                    UNLINK => !$save_temps);
    308330
    309             combine_streaks($allstreaks_fh, $streaks, $inv_streaks);
     331            combine_streaks($allstreaks_fh, $streaks_resolved, $inv_streaks_resolved);
    310332
    311333            # apply the combined streaks to both the forward and inverse diffs
    312             $streaks = $allstreaks_name;
     334            $streaks_resolved = $allstreaks_name;
    313335        }
    314336    }
    315337
    316338    {
    317         my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
     339        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
    318340
    319341        $command .= " -stats $statsFile";
    … …  
    340362        }
    341363    }
    342     if (($stage eq "diff") and $inv_streaks) {
     364    if (($stage eq "diff") and $inv_streaks_resolved) {
    343365        $image   = $ipprc->filename("PPSUB.INVERSE", $path_base);
    344366        $mask    = $ipprc->filename("PPSUB.INVERSE.MASK", $path_base);
    … …  
    349371        my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
    350372
    351         my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
     373        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
    352374        $command .= " -stats $invStatsFile";
    353375

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_process.pl

    r28137 r29515  
    1717use IPC::Cmd 0.36 qw( can_run run );
    1818use File::Temp qw( tempfile );
     19use File::Copy;
    1920use PS::IPP::Metadata::Config;
    2021use PS::IPP::Metadata::List qw( parse_md_list );
    … …  
    4142
    4243# Parse the command-line arguments
    43 my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
     44my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile, $final_outroot);
    4445
    4546GetOptions(
    … …  
    4950           'dbname=s'        => \$dbname,     # Database name
    5051           'baseroot=s'      => \$baseroot,   # Output root name
     52           'final-outroot=s' => \$final_outroot,   # location for final outputs
    5153           'save-temps'      => \$save_temps, # Save temporary files?
    5254           'verbose'         => \$verbose,    # Print stuff?
    … …  
    8688
    8789my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
     90
     91# list of VerifyStreaks input and output files to copy to nebulous
     92my @verify_outputs = qw(
     93clusterPos.txt
     94duplicate.png
     95mask.png
     96original.png
     97original.fits
     98residual.png
     99residual.fits
     100clusters.list
     101);
    88102
    89103### Get a list of inputs
    … …  
    311325}
    312326
    313 
    314327### Input result into database
    315328{
    … …  
    334347
    335348if ($node eq "root") {
     349    # XXXX: Since we just added the result above, all of these my_dies are going to fail
     350    # with a duplicate row error.
     351    # see more comments below
    336352    my $streaks_file = "$outroot.streaks";
    337353    my $resolved = $ipprc->file_resolve($streaks_file);
    … …  
    349365    }
    350366
    351     &run_verifystreaks($baseroot);
    352367
    353368    my $exp_id;                 # Exposure identifier
    … …  
    366381    }
    367382
     383    &run_verifystreaks($baseroot, $exp_id);
     384
    368385    {
    369386        my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file
    … …  
    383400    }
    384401
     402    my $output_streaks = $final_outroot . ".streaks";
     403    if ($output_streaks and ($output_streaks ne $streaks_file)) {
     404        copy_to_nebulous($ipprc, $streaks_file, $output_streaks, 1);
     405        foreach my $f (@verify_outputs) {
     406            my $src = "$baseroot.verify/${exp_id}_$f";
     407            my $dest = "$final_outroot.${f}";
     408            copy_to_nebulous($ipprc, $src, $dest, 1);
     409        }
     410    } else {
     411        $output_streaks = $streaks_file;
     412    }
    385413
    386414    {
    387415        my $command = "$magictool -addmask";
    388416        $command   .= " -magic_id $magic_id";
    389         $command   .= " -uri $streaks_file";
     417        $command   .= " -path_base $final_outroot";
    390418        $command   .= " -streaks $num_streaks";
    391419        $command   .= " -dbname $dbname" if defined $dbname;
    … …  
    397425            unless ($success) {
    398426                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    399                 # This isn't going to work because our result was already entered.
     427                # XXX: This my_die isn't going to work because the result for the root node was already
     428                # added to the database up above.
     429                # There is a magicMask has a fault column. Maybe we should
     430                # use that and add a new revert mode that deletes the magicMask and the magicNodeResult
     431                # for the root node.
    400432                &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code);
    401433            }
    … …  
    406438}
    407439
     440
    408441### Pau.
    409442
    … …  
    411444
    412445    my $baseroot = shift;
     446    my $exp_id = shift;
    413447
    414448    unless ($VerifyStreaks) {
    … …  
    431465    my @files = <$baseroot.*.clusters>;
    432466
    433     unless (open ($FILE, ">$outdir/clusters.list")) {
    434         print "failed to create cluster file $outdir/clusters.list\n";
     467    my $clusters_list = "$outdir/${exp_id}_clusters.list";
     468    unless (open ($FILE, ">$clusters_list")) {
     469        print "failed to create cluster file $clusters_list\n";
    435470        return 1;
    436471    }
    … …  
    441476    close ($FILE);
    442477    if ($status) {
    443         print "failed to create cluster file $outdir/clusters.list\n";
     478        print "failed to create cluster file $clusters_list\n";
    444479        return 1;
    445480    }
    446481
    447     my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap";
     482    my $command = "$VerifyStreaks --out $outdir --clusters $clusters_list $baseroot.root.streakMap";
    448483
    449484    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    513548}
    514549
     550# Copy a file to nebulous and optionally replicate it
     551# We should consider making this an ipprc function. For now try it here so we can print
     552# the right error messages
     553sub copy_to_nebulous {
     554    my $ipprc = shift;
     555    my $src = shift;
     556    my $dest = shift;
     557    my $replicate = shift;
     558
     559    print "copying $src to $dest\n";
     560
     561    $ipprc->file_exists($src) or
     562        &my_die("expected output file does not exist: $src", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     563
     564    # copy the file to it's final destination - which is presumably in nebulous
     565    # we delete it so that all instances are deleted in case it has been replicated
     566    if ($ipprc->file_exists($dest)) {
     567        $ipprc->file_delete($dest) or
     568                &my_die("failed to delete existing file: $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     569    }
     570    my $dest_resolved = $ipprc->file_resolve($dest, 'create');
     571    &my_die("failed to resolve $dest", $PS_EXIT_UNKNOWN_ERROR) if !$dest_resolved;
     572
     573    copy ($src, $dest_resolved) or
     574        &my_die("failed to copy $src to $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     575
     576    if ($replicate and (file_scheme($dest) eq 'neb')) {
     577        my $neb = $ipprc->nebulous();
     578        $neb->setxattr($dest, 'user.copies', 2, 'create') or
     579            &my_die("failed to set user.copies for $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     580
     581        $neb->replicate($dest) or
     582            &my_die("failed to replicate $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     583    }
     584}
     585
    515586sub my_die
    516587{

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_tree.pl

    r27718 r29515  
    8383    unless ($success) {
    8484        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    85         &my_die("Unable to perform magictool -inputfile: $error_code", $magic_id, $error_code);
    86     }
    87 
    88     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     85        &my_die("Unable to perform magictool -inputskyfile: $error_code", $magic_id, $error_code);
     86    }
     87
     88    my $magictool_output = join "", @$stdout_buf;
     89    # if there is no output from magictool that means that there are no
     90    # diffSkyfiles with non-zero quality. Set fault to a special value so
     91    # that these magicRuns can be recognized and dropped.
     92    &my_die("magictool -inputskyfile returned no output. Inputs are probably all bad quality.", $magic_id, 42)
     93        if !$magictool_output;
     94    my $metadata = $mdcParser->parse($magictool_output) or
    8995        &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR);
    9096

  • branches/eam_branches/ipp-20100823/ippTasks/Makefile.am

    r28632 r29515  
    1818        magic.pro \
    1919        destreak.pro \
     20        destreak.cleanup.pro \
    2021        diff.pro \
    2122        stack.pro \

  • branches/eam_branches/ipp-20100823/ippTasks/destreak.pro

    r29124 r29515  
    1010book init magicToDS
    1111book init magicDSToRevert
    12 book init magicDSToCleanup
    1312
    1413### indexes into Database lists
    … …  
    3029$magicDSRevertStage = 0
    3130
     31macro destreak.show.stages
     32    echo $DS_STAGE:n stages enabled for destreak processing
     33    for i 0 $DS_STAGE:n
     34        echo $i $DS_STAGE:$i
     35    end
     36    echo current magicDSStage = $magicDSStage
     37    echo current magicDSRevertStage = $magicDSRevertStage
     38end
     39
    3240### Check status of tasks
    3341macro destreak.status
    34     # since echo and listbook output go to different files these do not come in order
    35     echo magicToDS
    3642    book listbook magicToDS
    37     echo magicDSToRevert
    3843    book listbook magicDSToRevert
    39     echo magicDSToCleanup
    40     book listbook magicDSToCleanup
    4144end
    4245
    … …  
    4548    book init magicToDS
    4649    book init magicDSToRevert
    47     book init magicDSToCleanup
    4850end
    4951
    … …  
    7375end
    7476
    75 macro destreak.cleanup.on
    76     task destreak.cleanup.load
    77         active true
    78     end
    79     task destreak.cleanup.run
    80         active true
    81     end
    82 end
    83 
    8477### Turn tasks off
    8578macro destreak.off
    … …  
    10699    end
    107100end
    108 macro destreak.cleanup.off
    109     task destreak.cleanup.load
    110         active false
    111     end
    112     task destreak.cleanup.run
    113         active false
    114     end
    115 end
    116 
    117101
    118102task           destreak.load
    … …  
    203187    book getword magicToDS $pageName camera -var CAMERA
    204188    book getword magicToDS $pageName streaks_uri -var STREAKS
     189    book getword magicToDS $pageName streaks_path_base -var STREAKS_PATH_BASE
    205190    book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS
     191    book getword magicToDS $pageName inv_streaks_path_base -var INV_STREAKS_PATH_BASE
    206192    book getword magicToDS $pageName stage -var STAGE
    207193    book getword magicToDS $pageName stage_id -var STAGE_ID
    … …  
    229215    # TODO: do not add recoveryroot or replace if they are null or zero
    230216
    231     $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
     217    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
    232218
    233219    add_standard_args run
    … …  
    274260
    275261  periods      -poll $LOADPOLL
    276   #periods      -exec $LOADEXEC
    277262  periods      -exec 30
    278   periods      -timeout 60
     263  periods      -timeout 300
    279264  npending     1
    280265
    … …  
    293278    add_poll_args run
    294279    add_poll_labels run
     280    # since this command can be expensive, reduce the limit
     281    $run = $run -limit 24
    295282    command $run
    296283  end
    … …  
    317304task           destreak.revert.load
    318305  host         local
    319 
    320   periods      -poll $LOADPOLL
    321   periods      -exec $LOADEXEC
    322   periods      -timeout 20
     306  periods      -poll 60.0
     307  periods      -exec 120.
     308  periods      -timeout 120
    323309  npending     1
    324310  active       false
    … …  
    518504end
    519505
    520 task           destreak.cleanup.load
    521   host         local
    522 
    523   periods      -poll $LOADPOLL
    524   periods      -exec $LOADEXEC
    525   periods      -timeout 20
    526   npending     1
    527   active       false
    528 
    529   stdout NULL
    530   stderr $LOGSUBDIR/destreak.cleanup.load.log
    531 
    532   task.exec
    533     $run = magicdstool -tocleanup
    534     if ($DB:n == 0)
    535       option DEFAULT
    536     else
    537 
    538       # save the DB name for the exit tasks
    539       option $DB:$magicDSToCleanup_DB
    540       $run = $run -dbname $DB:$magicDSToCleanup_DB
    541 
    542       # only bump database number after we have gone through all of the stages
    543       $magicDSToCleanup_DB ++
    544       if ($magicDSToCleanup_DB >= $DB:n) set magicDSToCleanup_DB = 0
    545     end
    546     add_poll_args run
    547     add_poll_labels run
    548     command $run
    549   end
    550 
    551   # success
    552   task.exit    0
    553     # convert 'stdout' to book format
    554     ipptool2book stdout magicDSToCleanup -key magic_ds_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    555     if ($VERBOSE > 2)
    556       book listbook magicDSToCleanup
    557     end
    558 
    559     # delete existing entries in the appropriate pantaskStates
    560     process_cleanup magicDSToCleanup
    561   end
    562 
    563   # locked list
    564   task.exit    default
    565     showcommand failure
    566   end
    567 
    568   task.exit    crash
    569     showcommand crash
    570   end
    571 
    572   # operation times out?
    573   task.exit    timeout
    574     showcommand timeout
    575   end
    576 end
    577 
    578 task           destreak.cleanup.run
    579   periods      -poll $RUNPOLL
    580   periods      -exec $RUNEXEC
    581   periods      -timeout 60
    582   active       false
    583 
    584   task.exec
    585     stdout $LOGSUBDIR/destreak.cleanup.run.log
    586     stderr $LOGSUBDIR/destreak.cleanup.run.log
    587 
    588     # if we are unable to run the 'exec', use a long retry time
    589     periods -exec $RUNEXEC
    590 
    591     book npages magicDSToCleanup -var N
    592     if ($N == 0) break
    593     if ($NETWORK == 0) break
    594    
    595     # look for new images (pantaskState == INIT)
    596     book getpage magicDSToCleanup 0 -var pageName -key pantaskState INIT
    597     if ("$pageName" == "NULL") break
    598 
    599     book setword magicDSToCleanup $pageName pantaskState RUN
    600     book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID
    601     book getword magicDSToCleanup $pageName camera -var CAMERA
    602     book getword magicDSToCleanup $pageName stage -var STAGE
    603     book getword magicDSToCleanup $pageName outroot -var OUTROOT
    604     book getword magicDSToCleanup $pageName dbname -var DBNAME
    605 
    606     sprintf logfile "%s/mds.%s.cleanup.log" $OUTROOT $MAGIC_DS_ID
    607 
    608     host anyhost
    609 
    610     $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --stage $STAGE --camera $CAMERA --logfile $logfile
    611 
    612     add_standard_args run
    613 
    614     # save the pageName for future reference below
    615     options $pageName
    616 
    617     # create the command line
    618     if ($VERBOSE > 1)
    619       echo command $run
    620     end
    621     # if we are ready to run, drop the retry timeout low so we fill up the queue
    622     periods -exec 0.05
    623     command $run
    624   end
    625 
    626   # default exit status
    627   task.exit    0
    628     process_exit magicDSToCleanup $options:0 $JOB_STATUS
    629    end
    630 
    631   # locked list
    632   task.exit    default
    633     showcommand failure
    634     process_exit magicDSToCleanup $options:0 $JOB_STATUS
    635   end
    636 
    637   task.exit    crash
    638     showcommand crash
    639     book setword magicDSToCleanup $options:0 pantaskState CRASH
    640   end
    641 
    642   # operation timed out?
    643   task.exit    timeout
    644     showcommand timeout
    645     book setword magicDSToCleanup $options:0 pantaskState TIMEOUT
    646   end
    647 end
    648 

  • branches/eam_branches/ipp-20100823/ippTasks/dist.pro

    r29124 r29515  
    176176  host         local
    177177
    178   periods      -poll $LOADPOLL
     178  periods      -poll 20
    179179  periods      -exec $LOADEXEC
    180   periods      -timeout 30
     180  periods      -timeout 300
    181181  npending     1
    182182
    … …  
    338338
    339339  periods      -poll $LOADPOLL
    340   periods      -exec $LOADEXEC
    341   periods      -timeout 30
     340  periods      -exec 30
     341  periods      -timeout 3000
    342342  npending     1
    343343
    … …  
    479479    end
    480480    add_poll_labels run
    481     echo $run
    482481    command $run
    483482  end

  • branches/eam_branches/ipp-20100823/ippTasks/magic.pro

    r27620 r29515  
    288288    book getword magicToProcess $pageName camera -var CAMERA
    289289    book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE
     290    book getword magicToProcess $pageName raw_workdir -var RAW_TEMPLATE
     291    book getword magicToProcess $pageName exp_name -var EXP_NAME
    290292    book getword magicToProcess $pageName dbname -var DBNAME
    291293
    … …  
    308310    sprintf baseroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
    309311
     312    if ("$NODE" == "root")
     313        # This is the root node, put the streaks file into the rawExps workdir
     314        # which is presumably in nebulous and thus will be replicatable
     315        set.workdir.by.camera $CAMERA FPA $RAW_TEMPLATE $default_host RAW_WORKDIR
     316        sprintf final_outroot "%s/%s.%s/%s.%s.mgc.%s" $RAW_WORKDIR $EXP_NAME $EXP_ID $EXP_NAME $EXP_ID $MAGIC_ID
     317
     318        $EXTRA_ARGS = --final-outroot $final_outroot
     319    else
     320        $EXTRA_ARGS = ""
     321    end
     322
    310323    $logfile = $baseroot.$NODE.log
    311324    if ("$logfile" == "")
    … …  
    319332    mkdir $outpath
    320333
    321     $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile
     334    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile $EXTRA_ARGS
    322335    add_standard_args run
    323336

  • branches/eam_branches/ipp-20100823/ippTasks/nightly_stacks.pro

    r28982 r29515  
    436436
    437437    book getword nsData $options:0 nsNmacros -var ns_Nmacros
    438     for i 0 $ns_Nmacros
    439         sprintf macroName "ns%dMacro" $i
    440         book getword nsData $options:0 $macroName -var macroCmd
    441         $macroCmd
     438    if ("$ns_Nmacros" != "NULL")
     439        for i 0 $ns_Nmacros
     440            sprintf macroName "ns%dMacro" $i
     441            book getword nsData $options:0 $macroName -var macroCmd
     442            $macroCmd
     443        end
    442444    end
    443445
    … …  
    498500
    499501    book getword nsData $options:0 nsNmacros -var ns_Nmacros
    500     for i 0 $ns_Nmacros
    501         sprintf macroName "ns%dMacro" $i
    502         book getword nsData $options:0 $macroName -var macroCmd
    503         $macroCmd
     502    if ("$ns_Nmacros" != "NULL")
     503        for i 0 $ns_Nmacros
     504            sprintf macroName "ns%dMacro" $i
     505            book getword nsData $options:0 $macroName -var macroCmd
     506            $macroCmd
     507        end
    504508    end
    505509
    … …  
    560564
    561565    book getword nsData $options:0 nsNmacros -var ns_Nmacros
    562     for i 0 $ns_Nmacros
    563         sprintf macroName "ns%dMacro" $i
    564         book getword nsData $options:0 $macroName -var macroCmd
    565         $macroCmd
     566    if ("$ns_Nmacros" != "NULL")
     567        for i 0 $ns_Nmacros
     568            sprintf macroName "ns%dMacro" $i
     569            book getword nsData $options:0 $macroName -var macroCmd
     570            $macroCmd
     571        end
    566572    end
    567573
    … …  
    623629
    624630    book getword nsData $options:0 nsNmacros -var ns_Nmacros
    625     for i 0 $ns_Nmacros
    626         sprintf macroName "ns%dMacro" $i
    627         book getword nsData $options:0 $macroName -var macroCmd
    628         $macroCmd
     631    if ("$ns_Nmacros" != "NULL")
     632        for i 0 $ns_Nmacros
     633            sprintf macroName "ns%dMacro" $i
     634            book getword nsData $options:0 $macroName -var macroCmd
     635            $macroCmd
     636        end
    629637    end
    630638
    … …  
    684692
    685693    book getword nsData $options:0 nsNmacros -var ns_Nmacros
    686     for i 0 $ns_Nmacros
    687         sprintf macroName "ns%dMacro" $i
    688         book getword nsData $options:0 $macroName -var macroCmd
    689         $macroCmd
     694    if ("$ns_Nmacros" != "NULL")
     695        for i 0 $ns_Nmacros
     696            sprintf macroName "ns%dMacro" $i
     697            book getword nsData $options:0 $macroName -var macroCmd
     698            $macroCmd
     699        end
    690700    end
    691701
    … …  
    747757
    748758    book getword nsData $options:0 nsNmacros -var ns_Nmacros
    749     for i 0 $ns_Nmacros
    750         sprintf macroName "ns%dMacro" $i
    751         book getword nsData $options:0 $macroName -var macroCmd
    752         $macroCmd
     759    if ("$ns_Nmacros" != "NULL")
     760        for i 0 $ns_Nmacros
     761            sprintf macroName "ns%dMacro" $i
     762            book getword nsData $options:0 $macroName -var macroCmd
     763            $macroCmd
     764        end
    753765    end
    754766
    … …  
    870882
    871883    book getword nsData $options:0 nsNmacros -var ns_Nmacros
    872     for i 0 $ns_Nmacros
    873         sprintf macroName "ns%dMacro" $i
    874         book getword nsData $options:0 $macroName -var macroCmd
    875         $macroCmd
     884    if ("$ns_Nmacros" != "NULL")
     885        for i 0 $ns_Nmacros
     886            sprintf macroName "ns%dMacro" $i
     887            book getword nsData $options:0 $macroName -var macroCmd
     888            $macroCmd
     889        end
    876890    end
    877891

  • branches/eam_branches/ipp-20100823/ippTasks/pstamp.pro

    r28779 r29515  
    1414$pstampRev_DB = 0
    1515$pstampDep_DB = 0
     16$pstampRevDep_DB = 0
    1617$pstampCleanup_DB = 0
     18$pstampStopFaulted_DB = 0
    1719
    1820# set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config
    … …  
    8991        active false
    9092    end
    91     task pstamp.job.revert
    92         active false
    93     end
    9493    task pstamp.dependent.load
    9594        active false
    … …  
    104103        active true
    105104    end
     105    task pstamp.dependent.revert
     106        active true
     107    end
     108    task pstamp.stopfaulted
     109        active true
     110    end
    106111end
    107112macro pstamp.revert.off
    108113    task pstamp.job.revert
     114        active false
     115    end
     116    task pstamp.dependent.revert
     117        active false
     118    end
     119    task pstamp.stopfaulted
    109120        active false
    110121    end
    … …  
    513524        stdout NULL
    514525        stderr $LOGSUBDIR/pstamp.job.revert.log
    515         $run = pstamptool -revertjob -all
     526        $run = pstamptool -revertjob
    516527        if ($DB:n == 0)
    517528            option DEFAULT
    … …  
    798809end
    799810
     811task pstamp.dependent.revert
     812    host        local
     813
     814    periods     -poll $LOADPOLL
     815    periods     -exec 900
     816    periods     -timeout 300
     817    npending    1
     818
     819    task.exec
     820        stdout NULL
     821        stderr $LOGSUBDIR/pstamp.dependent.revert.log
     822        $run = pstamptool -revertdependent
     823        if ($DB:n == 0)
     824            option DEFAULT
     825        else
     826            $run = $run $PS_DBSERVER -dbname $DB:$pstampRevDep_DB
     827            $pstampRevDep_DB ++
     828            if ($pstampRevDep_DB >= $DB:n) set pstampRevDep_DB = 0
     829        end
     830        add_poll_args run
     831        add_poll_labels run
     832        command $run
     833    end
     834
     835    task.exit $EXIT_SUCCESS
     836        # nothing to do
     837    end
     838
     839    task.exit   default
     840        showcommand failure
     841    end
     842
     843    task.exit   crash
     844        showcommand crash
     845    end
     846
     847    task.exit   timeout
     848        showcommand timeout
     849    end
     850end
     851
     852task pstamp.stopfaulted
     853    host        local
     854
     855    periods     -poll $LOADPOLL
     856    periods     -exec 30
     857    periods     -timeout 300
     858    npending    1
     859
     860    task.exec
     861        stderr $LOGSUBDIR/pstamp.stopfaulted.log
     862        stderr $LOGSUBDIR/pstamp.stopfaulted.log
     863        $run = pstampstopfaulted
     864        if ($DB:n == 0)
     865            option DEFAULT
     866        else
     867            $run = $run $PS_DBSERVER -dbname $DB:$pstampStopFaulted_DB
     868            $pstampStopFaulted_DB ++
     869            if ($pstampStopFaulted_DB >= $DB:n) set pstampStopFaulted_DB = 0
     870        end
     871        add_poll_args run
     872        add_poll_labels run
     873        command $run
     874    end
     875
     876    task.exit $EXIT_SUCCESS
     877        # nothing to do
     878    end
     879
     880    task.exit   default
     881        showcommand failure
     882    end
     883
     884    task.exit   crash
     885        showcommand crash
     886    end
     887
     888    task.exit   timeout
     889        showcommand timeout
     890    end
     891end

  • branches/eam_branches/ipp-20100823/ippTasks/publish.pro

    r27074 r29515  
    7575
    7676  task.exec
     77    if ($LABEL:n == 0) break
     78    #otherwise everything will be queued when there is no label.
    7779    $run = pubtool -definerun
    7880    if ($DB:n == 0)

  • branches/eam_branches/ipp-20100823/ippTasks/replicate.pro

    r24847 r29515  
    173173    book getword replicatePending $pageName volume_name -var VOLUME_NAME
    174174    book getword replicatePending $pageName volume_host -var VOLUME_HOST
     175    book getword replicatePending $pageName command     -var COMMAND
    175176
    176177    set.host.for.replicate $VOLUME_HOST
    … …  
    180181
    181182    # these operations do not require a database to be specified
    182     $run = neb-replicate --copies $NEED_COPIES $KEY
     183    $run = $COMMAND $KEY
    183184
    184185    # save the pageName for future reference below

  • branches/eam_branches/ipp-20100823/ippTasks/science.cleanup.pro

    r24595 r29515  
    113113  task.exec
    114114    book npages chipPendingCleanup -var N
    115     if ($N == 0) break
     115    if ($N == 0)
     116        periods -exec $RUNEXEC
     117        break
     118    end
    116119    if ($NETWORK == 0) break
    117120   
    … …  
    143146      echo command $run
    144147    end
     148    periods -exec 0.05
    145149    command $run
    146150  end
    … …  
    569573  task.exec
    570574    book npages warpPendingCleanup -var N
    571     if ($N == 0) break
     575    if ($N == 0)
     576        periods -exec $RUNEXEC
     577        break
     578    end
    572579    if ($NETWORK == 0) break
    573580   
    … …  
    599606      echo command $run
    600607    end
     608    periods -exec 0.05
    601609    command $run
    602610  end
    … …  
    721729  task.exec
    722730    book npages diffCleanup -var N
    723     if ($N == 0) break
     731    if ($N == 0)
     732        periods -exec $RUNEXEC
     733        break
     734    end
    724735    if ($NETWORK == 0) break
    725736   
    … …  
    751762      echo command $run
    752763    end
     764    periods -exec 0.05
    753765    command $run
    754766  end

  • branches/eam_branches/ipp-20100823/ippToPsps/config/stack/tables.xml

    r27361 r29515  
    11<?xml version="1.0" encoding="UTF-8"?>
    22
    3 <tableDescriptions type="ST">
     3<tableDescriptions type="stack">
    44  <table name="StackMeta">
    55    <column name="stackMetaID" type="TLONGLONG" default="0" comment="stack identifier"></column>
    … …  
    3030    <column name="ctype1" type="TSTRING" default=" " comment="name of astrometric projection in RA"></column>
    3131    <column name="ctype2" type="TSTRING" default=" " comment="name of astrometric projection in DEC"></column>
    32     <column name="crval1" type="TFLOAT" default="-999" comment="RA corresponding to reference pixel"></column>
    33     <column name="crval2" type="TFLOAT" default="-999" comment="DEC corresponding to reference pixel"></column>
    34     <column name="crpix1" type="TFLOAT" default="-999" comment="reference pixel value for RA"></column>
    35     <column name="crpix2" type="TFLOAT" default="-999" comment="reference pixel value for DEC"></column>
    36     <column name="pc001001" type="TFLOAT" default="-999" comment="elements of rotation/Dcale matrix"></column>
    37     <column name="pc001002" type="TFLOAT" default="-999" comment="elements of rotation/Dcale matrix"></column>
    38     <column name="pc002001" type="TFLOAT" default="-999" comment="elements of rotation/Dcale matrix"></column>
    39     <column name="pc002002" type="TFLOAT" default="-999" comment="elements of rotation/Dcale matrix"></column>
     32    <column name="crval1" type="TDOUBLE" default="-999" comment="RA corresponding to reference pixel"></column>
     33    <column name="crval2" type="TDOUBLE" default="-999" comment="DEC corresponding to reference pixel"></column>
     34    <column name="crpix1" type="TDOUBLE" default="-999" comment="reference pixel value for RA"></column>
     35    <column name="crpix2" type="TDOUBLE" default="-999" comment="reference pixel value for DEC"></column>
     36    <column name="cdelt1" type="TDOUBLE" default="-999" comment="scale factor for RA"></column>
     37    <column name="cdelt2" type="TDOUBLE" default="-999" comment="scale factor for DEC"></column>
     38    <column name="pc001001" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
     39    <column name="pc001002" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
     40    <column name="pc002001" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
     41    <column name="pc002002" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
    4042    <column name="polyOrder" type="TBYTE" default="255" comment="polynomial order of astrometry fit"></column>
    41     <column name="pca1x3y0" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    42     <column name="pca1x2y1" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    43     <column name="pca1x1y2" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    44     <column name="pca1x0y3" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    45     <column name="pca1x2y0" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    46     <column name="pca1x1y1" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    47     <column name="pca1x0y2" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    48     <column name="pca2x3y0" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    49     <column name="pca2x2y1" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    50     <column name="pca2x1y2" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    51     <column name="pca2x0y3" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    52     <column name="pca2x2y0" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    53     <column name="pca2x1y1" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    54     <column name="pca2x0y2" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     43    <column name="pca1x3y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     44    <column name="pca1x2y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     45    <column name="pca1x1y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     46    <column name="pca1x0y3" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     47    <column name="pca1x2y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     48    <column name="pca1x1y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     49    <column name="pca1x0y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     50    <column name="pca2x3y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     51    <column name="pca2x2y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     52    <column name="pca2x1y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     53    <column name="pca2x0y3" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     54    <column name="pca2x2y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     55    <column name="pca2x1y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     56    <column name="pca2x0y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
    5557    <column name="calibModNum" type="TSHORT" default="0" comment="calibration modification number"></column>
    5658    <column name="dataRelease" type="TBYTE" default="0" comment="Data release"></column>
    … …  
    6971    <column name="projectionCellID" type="TLONG" default="-999" comment="projection cell identifier"></column>
    7072    <column name="stackVer" type="TSHORT" default="-999" comment="version number of this stack"></column>
    71     <column name="obsTime" type="TFLOAT" default="-999" comment=" Time of mid observation (unit = day)"></column>
     73    <column name="obsTime" type="TDOUBLE" default="-999" comment=" Time of mid observation (unit = day)"></column>
    7274    <column name="xPos" type="TFLOAT" default="-999" comment="measured x on CCD from PSF fit"></column>
    7375    <column name="yPos" type="TFLOAT" default="-999" comment="measured y on CCD from PSF fit"></column>
    … …  
    103105    <column name="dataRelease" type="TBYTE" default="0" comment="Data release when this detection was originally taken."></column>
    104106  </table>
    105   <table name="SkinnyObject">
    106     <column name="objID" type="TLONGLONG" default="0" comment="ODM object identifier index"></column>
    107     <column name="ippObjID" type="TLONGLONG" default="0" comment="IPP object number"></column>
    108     <column name="projectionCellID" type="TLONG" default="-999" comment="projection cell identifier at discovery time"></column>
    109     <column name="dataRelease" type="TBYTE" default="0" comment="Data release to propagate to the object"></column>
    110   </table>
    111   <table name="StackOrphan">
    112     <column name="partitionKey" type="TLONGLONG" default="0" comment="partitioning key"></column>
    113     <column name="stackDetectID" type="TLONGLONG" default="0" comment="detection identifier"></column>
     107  <table name="StackApFlx">
     108    <column name="objID" type="TLONGLONG" default="0" comment="ODM object identifier"></column>
     109    <column name="stackDetectID" type="TLONGLONG" default="0" comment="ODM detection identifier"></column>
     110    <column name="ippObjID" type="TLONGLONG" default="0" comment="IPP object identifier"></column>
    114111    <column name="ippDetectID" type="TLONGLONG" default="0" comment="detection ID generated by IPP"></column>
    115112    <column name="filterID" type="TBYTE" default="0" comment="filter identifier"></column>
    116113    <column name="stackTypeID" type="TBYTE" default="0" comment="stack type identifier"></column>
    117     <column name="surveyID" type="TBYTE" default="0" comment="survey identifier"></column>
     114    <column name="surveyID" type="TBYTE" default="0" comment="survey flag identifier"></column>
    118115    <column name="primaryF" type="TBYTE" default="255" comment="identifies best stack detection for Stacks overlapping the same region of the sky."></column>
    119116    <column name="stackMetaID" type="TLONGLONG" default="0" comment="stack identifier"></column>
    120     <column name="skyCellID" type="TLONG" default="-999" comment="skycell identifier"></column>
    121     <column name="stackVer" type="TSHORT" default="-999" comment="version number of this stack"></column>
    122     <column name="xPos" type="TFLOAT" default="-999" comment="measured x on CCD from PSF fit"></column>
    123     <column name="yPos" type="TFLOAT" default="-999" comment="measured y on CCD from PSF fit"></column>
    124     <column name="xPosErr" type="TFLOAT" default="-999" comment="estimated error in x"></column>
    125     <column name="yPosErr" type="TFLOAT" default="-999" comment="estimated error in y"></column>
    126     <column name="instFlux" type="TFLOAT" default="-999" comment="PSF instrumental flux"></column>
    127     <column name="instFluxErr" type="TFLOAT" default="-999" comment="estimated error in instrumental flux"></column>
    128     <column name="peakFlux" type="TFLOAT" default="-999" comment="ratio of peak flux to total flux"></column>
    129     <column name="sky" type="TFLOAT" default="-999" comment="PSF sky level at source (adu)"></column>
    130     <column name="skyErr" type="TFLOAT" default="-999" comment="estimated error in sky"></column>
    131     <column name="sgSep" type="TFLOAT" default="-999" comment="star/galaxy separator"></column>
    132     <column name="psfWidMajor" type="TFLOAT" default="-999" comment="PSF width in major axis"></column>
    133     <column name="psfWidMinor" type="TFLOAT" default="-999" comment="PSF width in minor axis"></column>
    134     <column name="psfTheta" type="TFLOAT" default="-999" comment="PSF orientation angle"></column>
    135     <column name="psfLikelihood" type="TFLOAT" default="-999" comment="PSF likelihood"></column>
    136     <column name="psfCf" type="TFLOAT" default="-999" comment="PSF coverage factor"></column>
    137     <column name="infoFlag" type="TLONG" default="-999" comment="indicator of strange propeties"></column>
    138     <column name="nFrames" type="TLONG" default="-999" comment="number of frames contributing to source"></column>
    139     <column name="wlSigma" type="TFLOAT" default="-999" comment="weak lensing sigma"></column>
    140     <column name="eps1" type="TFLOAT" default="-999" comment="weak lensing vector element eps(1)"></column>
    141     <column name="eps2" type="TFLOAT" default="-999" comment="weak lensing vector element eps(2)"></column>
    142     <column name="Psm11" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sm(1,1)"></column>
    143     <column name="Psm12" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sm(1,2)"></column>
    144     <column name="Psm21" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sm(2,1)"></column>
    145     <column name="Psm22" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sm(2,2)"></column>
    146     <column name="Psh11" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sh(1,1)"></column>
    147     <column name="Psh12" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sh(1,2)"></column>
    148     <column name="Psh21" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sh(2,1)"></column>
    149     <column name="Psh22" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sh(2,2)"></column>
     117    <column name="isophotMag" type="TFLOAT" default="-999" comment="isophotal magnitude"></column>
     118    <column name="isophotMagErr" type="TFLOAT" default="-999" comment="estimated error in isophotal magnitude"></column>
     119    <column name="isophotMajAxis" type="TFLOAT" default="-999" comment="isophotal Major Axis"></column>
     120    <column name="isophotMajAxisErr" type="TFLOAT" default="-999" comment="estimated error in isophotal Major Axis"></column>
     121    <column name="isophotMinAxis" type="TFLOAT" default="-999" comment="isophotal Minor Axis"></column>
     122    <column name="isophotMinAxisErr" type="TFLOAT" default="-999" comment="estimated error in isophotal Minor Axis"></column>
     123    <column name="isophotMajAxisGrad" type="TFLOAT" default="-999" comment="isophotal major axis gradient"></column>
     124    <column name="isophotMinAxisGrad" type="TFLOAT" default="-999" comment="isophotal minor axis gradient"></column>
     125    <column name="isophotPA" type="TFLOAT" default="-999" comment="isophotal position angle"></column>
     126    <column name="isophotPAErr" type="TFLOAT" default="-999" comment="estimated error in isophotal position angle"></column>
     127    <column name="isophotPAGrad" type="TFLOAT" default="-999" comment="isophotal position angle gradient"></column>
     128    <column name="petRadius" type="TFLOAT" default="-999" comment="Petrosian radius"></column>
     129    <column name="petRadiusErr" type="TFLOAT" default="-999" comment="estimated error inPetrosian radius"></column>
     130    <column name="petMag" type="TFLOAT" default="-999" comment="Petrosian magntiude"></column>
     131    <column name="petMagErr" type="TFLOAT" default="-999" comment="estimated error in petMag"></column>
     132    <column name="petR50" type="TFLOAT" default="-999" comment="Petrosian radius at 50% light"></column>
     133    <column name="petR50Err" type="TFLOAT" default="-999" comment="estimated error inPetrosian radius at 50% light"></column>
     134    <column name="petR90" type="TFLOAT" default="-999" comment="Petrosian radius at 90% light"></column>
     135    <column name="petR90Err" type="TFLOAT" default="-999" comment="estimated error in Petrosian radius at 90% light"></column>
     136    <column name="petCf" type="TFLOAT" default="-999" comment="Petrosian fit coverage factor"></column>
     137    <column name="flxR1" type="TFLOAT" default="-999" comment="Flux inside r = 1"></column>
     138    <column name="flxR1Err" type="TFLOAT" default="-999" comment="estimated error is flxR1"></column>
     139    <column name="flxR1Var" type="TFLOAT" default="-999" comment="estimated variance in flxR1"></column>
     140    <column name="flxR2" type="TFLOAT" default="-999" comment="Flux inside r = 2"></column>
     141    <column name="flxR2Err" type="TFLOAT" default="-999" comment="estimated error is flxR2"></column>
     142    <column name="flxR2Var" type="TFLOAT" default="-999" comment="estimated variance in flxR2"></column>
     143    <column name="flxR3" type="TFLOAT" default="-999" comment="Flux inside r = 3"></column>
     144    <column name="flxR3Err" type="TFLOAT" default="-999" comment="estimated error is flxR3"></column>
     145    <column name="flxR3Var" type="TFLOAT" default="-999" comment="estimated variance in flxR3"></column>
     146    <column name="flxR4" type="TFLOAT" default="-999" comment="Flux inside r = 4"></column>
     147    <column name="flxR4Err" type="TFLOAT" default="-999" comment="estimated error is flxR4"></column>
     148    <column name="flxR4Var" type="TFLOAT" default="-999" comment="estimated variance in flxR4"></column>
     149    <column name="flxR5" type="TFLOAT" default="-999" comment="Flux inside r = 5"></column>
     150    <column name="flxR5Err" type="TFLOAT" default="-999" comment="estimated error is flxR5"></column>
     151    <column name="flxR5Var" type="TFLOAT" default="-999" comment="estimated variance in flxR5"></column>
     152    <column name="flxR6" type="TFLOAT" default="-999" comment="Flux inside r = 6"></column>
     153    <column name="flxR6Err" type="TFLOAT" default="-999" comment="estimated error is flxR6"></column>
     154    <column name="flxR6Var" type="TFLOAT" default="-999" comment="estimated variance in flxR6"></column>
     155    <column name="flxR7" type="TFLOAT" default="-999" comment="Flux inside r = 7"></column>
     156    <column name="flxR7Err" type="TFLOAT" default="-999" comment="estimated error is flxR7"></column>
     157    <column name="flxR7Var" type="TFLOAT" default="-999" comment="estimated variance in flxR7"></column>
     158    <column name="flxR8" type="TFLOAT" default="-999" comment="Flux inside r = 8"></column>
     159    <column name="flxR8Err" type="TFLOAT" default="-999" comment="estimated error is flxR8"></column>
     160    <column name="flxR8Var" type="TFLOAT" default="-999" comment="estimated variance in flxR8"></column>
     161    <column name="flxR9" type="TFLOAT" default="-999" comment="Flux inside r = 9"></column>
     162    <column name="flxR9Err" type="TFLOAT" default="-999" comment="estimated error is flxR9"></column>
     163    <column name="flxR9Var" type="TFLOAT" default="-999" comment="estimated variance in flxR9"></column>
     164    <column name="flxR10" type="TFLOAT" default="-999" comment="Flux inside r = 10"></column>
     165    <column name="flxR10Err" type="TFLOAT" default="-999" comment="estimated error is flxR10"></column>
     166    <column name="flxR10Var" type="TFLOAT" default="-999" comment="estimated variance in flxR10"></column>
     167    <column name="c1FlxR1" type="TFLOAT" default="-999" comment="Flux inside r = 1"></column>
     168    <column name="c1FlxR1Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR1"></column>
     169    <column name="c1FlxR1Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR1"></column>
     170    <column name="c1FlxR2" type="TFLOAT" default="-999" comment="Flux inside r = 2"></column>
     171    <column name="c1FlxR2Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR2"></column>
     172    <column name="c1FlxR2Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR2"></column>
     173    <column name="c1FlxR3" type="TFLOAT" default="-999" comment="Flux inside r = 3"></column>
     174    <column name="c1FlxR3Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR3"></column>
     175    <column name="c1FlxR3Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR3"></column>
     176    <column name="c1FlxR4" type="TFLOAT" default="-999" comment="Flux inside r = 4"></column>
     177    <column name="c1FlxR4Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR4"></column>
     178    <column name="c1FlxR4Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR4"></column>
     179    <column name="c1FlxR5" type="TFLOAT" default="-999" comment="Flux inside r = 5"></column>
     180    <column name="c1FlxR5Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR5"></column>
     181    <column name="c1FlxR5Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR5"></column>
     182    <column name="c1FlxR6" type="TFLOAT" default="-999" comment="Flux inside r = 6"></column>
     183    <column name="c1FlxR6Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR6"></column>
     184    <column name="c1FlxR6Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR6"></column>
     185    <column name="c1FlxR7" type="TFLOAT" default="-999" comment="Flux inside r = 7"></column>
     186    <column name="c1FlxR7Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR7"></column>
     187    <column name="c1FlxR7Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR7"></column>
     188    <column name="c1FlxR8" type="TFLOAT" default="-999" comment="Flux inside r = 8"></column>
     189    <column name="c1FlxR8Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR8"></column>
     190    <column name="c1FlxR8Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR8"></column>
     191    <column name="c1FlxR9" type="TFLOAT" default="-999" comment="Flux inside r = 9"></column>
     192    <column name="c1FlxR9Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR9"></column>
     193    <column name="c1FlxR9Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR9"></column>
     194    <column name="c1FlxR10" type="TFLOAT" default="-999" comment="Flux inside r = 10"></column>
     195    <column name="c1FlxR10Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR10"></column>
     196    <column name="c1FlxR10Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR10"></column>
     197    <column name="c2FlxR1" type="TFLOAT" default="-999" comment="Flux inside r = 1"></column>
     198    <column name="c2FlxR1Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR1"></column>
     199    <column name="c2FlxR1Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR1"></column>
     200    <column name="c2FlxR2" type="TFLOAT" default="-999" comment="Flux inside r = 2"></column>
     201    <column name="c2FlxR2Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR2"></column>
     202    <column name="c2FlxR2Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR2"></column>
     203    <column name="c2FlxR3" type="TFLOAT" default="-999" comment="Flux inside r = 3"></column>
     204    <column name="c2FlxR3Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR3"></column>
     205    <column name="c2FlxR3Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR3"></column>
     206    <column name="c2FlxR4" type="TFLOAT" default="-999" comment="Flux inside r = 4"></column>
     207    <column name="c2FlxR4Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR4"></column>
     208    <column name="c2FlxR4Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR4"></column>
     209    <column name="c2FlxR5" type="TFLOAT" default="-999" comment="Flux inside r = 5"></column>
     210    <column name="c2FlxR5Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR5"></column>
     211    <column name="c2FlxR5Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR5"></column>
     212    <column name="c2FlxR6" type="TFLOAT" default="-999" comment="Flux inside r = 6"></column>
     213    <column name="c2FlxR6Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR6"></column>
     214    <column name="c2FlxR6Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR6"></column>
     215    <column name="c2FlxR7" type="TFLOAT" default="-999" comment="Flux inside r = 7"></column>
     216    <column name="c2FlxR7Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR7"></column>
     217    <column name="c2FlxR7Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR7"></column>
     218    <column name="c2FlxR8" type="TFLOAT" default="-999" comment="Flux inside r = 8"></column>
     219    <column name="c2FlxR8Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR8"></column>
     220    <column name="c2FlxR8Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR8"></column>
     221    <column name="c2FlxR9" type="TFLOAT" default="-999" comment="Flux inside r = 9"></column>
     222    <column name="c2FlxR9Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR9"></column>
     223    <column name="c2FlxR9Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR9"></column>
     224    <column name="c2FlxR10" type="TFLOAT" default="-999" comment="Flux inside r = 10"></column>
     225    <column name="c2FlxR10Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR10"></column>
     226    <column name="c2FlxR10Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR10"></column>
     227    <column name="c3FlxR1" type="TFLOAT" default="-999" comment="Flux inside r = 1"></column>
     228    <column name="c3FlxR1Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR1"></column>
     229    <column name="c3FlxR1Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR1"></column>
     230    <column name="c3FlxR2" type="TFLOAT" default="-999" comment="Flux inside r = 2"></column>
     231    <column name="c3FlxR2Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR2"></column>
     232    <column name="c3FlxR2Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR2"></column>
     233    <column name="c3FlxR3" type="TFLOAT" default="-999" comment="Flux inside r = 3"></column>
     234    <column name="c3FlxR3Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR3"></column>
     235    <column name="c3FlxR3Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR3"></column>
     236    <column name="c3FlxR4" type="TFLOAT" default="-999" comment="Flux inside r = 4"></column>
     237    <column name="c3FlxR4Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR4"></column>
     238    <column name="c3FlxR4Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR4"></column>
     239    <column name="c3FlxR5" type="TFLOAT" default="-999" comment="Flux inside r = 5"></column>
     240    <column name="c3FlxR5Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR5"></column>
     241    <column name="c3FlxR5Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR5"></column>
     242    <column name="c3FlxR6" type="TFLOAT" default="-999" comment="Flux inside r = 6"></column>
     243    <column name="c3FlxR6Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR6"></column>
     244    <column name="c3FlxR6Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR6"></column>
     245    <column name="c3FlxR7" type="TFLOAT" default="-999" comment="Flux inside r = 7"></column>
     246    <column name="c3FlxR7Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR7"></column>
     247    <column name="c3FlxR7Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR7"></column>
     248    <column name="c3FlxR8" type="TFLOAT" default="-999" comment="Flux inside r = 8"></column>
     249    <column name="c3FlxR8Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR8"></column>
     250    <column name="c3FlxR8Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR8"></column>
     251    <column name="c3FlxR9" type="TFLOAT" default="-999" comment="Flux inside r = 9"></column>
     252    <column name="c3FlxR9Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR9"></column>
     253    <column name="c3FlxR9Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR9"></column>
     254    <column name="c3FlxR10" type="TFLOAT" default="-999" comment="Flux inside r = 10"></column>
     255    <column name="c3FlxR10Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR10"></column>
     256    <column name="c3FlxR10Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR10"></column>
     257    <column name="logC" type="TFLOAT" default="-999" comment="Abraham concentration index"></column>
     258    <column name="logA" type="TFLOAT" default="-999" comment="Abraham asymmetry index"></column>
    150259    <column name="activeFlag" type="TBYTE" default="-999" comment="indicates whether this detection/orphan is still a detection/orphan"></column>
    151     <column name="assocDate" type="TSTRING" default="28881231" comment="date object association assigned"></column>
    152     <column name="historyModNum" type="TSHORT" default="0" comment="modification number in the O-D association history"></column>
    153     <column name="dataRelease" type="TBYTE" default="0" comment="Data release to reproduce queries through data releases"></column>
    154   </table>
    155   <table name="ObjectCalColor">
     260    <column name="dataRelease" type="TBYTE" default="0" comment="Data release when this detection was taken"></column>
     261  </table>
     262  <table name="StackModelFit">
    156263    <column name="objID" type="TLONGLONG" default="0" comment="ODM object identifier"></column>
    157     <column name="ippObjID" type="TLONGLONG" default="0" comment="ipp object identifier"></column>
     264    <column name="stackDetectID" type="TLONGLONG" default="0" comment="ODM detection identifier"></column>
     265    <column name="ippObjID" type="TLONGLONG" default="0" comment="IPP object identifier"></column>
     266    <column name="ippDetectID" type="TLONGLONG" default="0" comment="detection ID generated by IPP"></column>
    158267    <column name="filterID" type="TBYTE" default="0" comment="filter identifier"></column>
    159     <column name="calColor" type="TFLOAT" default="0" comment=" color adopted for magnitude calculation (unit = mag)"></column>
    160     <column name="calColorErr" type="TFLOAT" default="0" comment=" error in calibrating color (unit = mag)"></column>
    161     <column name="calibModNum" type="TSHORT" default="0" comment="calibration modification number"></column>
    162     <column name="dataRelease" type="TBYTE" default="0" comment="Data release when this color calibration was established"></column>
     268    <column name="stackTypeID" type="TBYTE" default="0" comment="stack type identifier"></column>
     269    <column name="surveyID" type="TBYTE" default="0" comment="survey flag identifier"></column>
     270    <column name="primaryF" type="TBYTE" default="255" comment="identifies best stack detection for Stacks overlapping the same region of the sky."></column>
     271    <column name="stackMetaID" type="TLONGLONG" default="0" comment="stack identifier"></column>
     272    <column name="deVRadius" type="TFLOAT" default="-999" comment="deVaucouleurs radius"></column>
     273    <column name="deVRadiusErr" type="TFLOAT" default="-999" comment="estimated error in deVaucouleurs radius"></column>
     274    <column name="deVMag" type="TFLOAT" default="-999" comment="deVaucouleurs magntiude"></column>
     275    <column name="deVMagErr" type="TFLOAT" default="-999" comment="estimated error in deV_mag"></column>
     276    <column name="deVAb" type="TFLOAT" default="-999" comment="deVaucoulerus axis ratio"></column>
     277    <column name="deVAbErr" type="TFLOAT" default="-999" comment="estimated error in deVaucoulerus axis ratio"></column>
     278    <column name="raDeVOff" type="TFLOAT" default="-999" comment="Offset in RA of deVaucouleurs fit from PSF RA"></column>
     279    <column name="decDeVOff" type="TFLOAT" default="-999" comment="Offset in DEC of deVaucouleurs fit from PSF DEC"></column>
     280    <column name="raDeVOffErr" type="TFLOAT" default="-999" comment="estimated error in ra offset"></column>
     281    <column name="decDeVOffErr" type="TFLOAT" default="-999" comment="estimated error in dec offset"></column>
     282    <column name="deVCf" type="TFLOAT" default="-999" comment="deVaucouleurs fit coverage factor"></column>
     283    <column name="deVLikelihood" type="TFLOAT" default="-999" comment="deVaucouleurs fit likelihood factor"></column>
     284    <column name="deVCovar11" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     285    <column name="deVCovar12" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     286    <column name="deVCovar13" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     287    <column name="deVCovar14" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     288    <column name="deVCovar15" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     289    <column name="deVCovar16" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     290    <column name="deVCovar22" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     291    <column name="deVCovar23" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     292    <column name="deVCovar24" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     293    <column name="deVCovar25" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     294    <column name="deVCovar26" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     295    <column name="deVCovar33" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     296    <column name="deVCovar34" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     297    <column name="deVCovar35" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     298    <column name="deVCovar36" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     299    <column name="deVCovar44" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     300    <column name="deVCovar45" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     301    <column name="deVCovar46" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     302    <column name="deVCovar55" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     303    <column name="deVCovar56" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     304    <column name="deVCovar66" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
     305    <column name="expRadius" type="TFLOAT" default="-999" comment="Exponential fit radius"></column>
     306    <column name="expRadiusErr" type="TFLOAT" default="-999" comment="estimated error in Exponential fit radius"></column>
     307    <column name="expMag" type="TFLOAT" default="-999" comment="Exponential fit magntiude"></column>
     308    <column name="expMagErr" type="TFLOAT" default="-999" comment="estimated error in expMag"></column>
     309    <column name="expAb" type="TFLOAT" default="-999" comment="Exponential fit axis ratio"></column>
     310    <column name="expAbErr" type="TFLOAT" default="-999" comment="estimated error in Exponential fit axis ratio"></column>
     311    <column name="raExpOff" type="TFLOAT" default="-999" comment="Offset in RA of Exponential fit from PSF RA"></column>
     312    <column name="decExpOff" type="TFLOAT" default="-999" comment="Offset in DEC of Exponential fit from PSF DEC"></column>
     313    <column name="raExpOffErr" type="TFLOAT" default="-999" comment="estimated error in raExpOff"></column>
     314    <column name="decExpOffErr" type="TFLOAT" default="-999" comment="estimated error in decExpOff"></column>
     315    <column name="expCf" type="TFLOAT" default="-999" comment="Exponential fit coverage factor"></column>
     316    <column name="expLikelihood" type="TFLOAT" default="-999" comment="Exponential fit likelihood factor"></column>
     317    <column name="expCovar11" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     318    <column name="expCovar12" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     319    <column name="expCovar13" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     320    <column name="expCovar14" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     321    <column name="expCovar15" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     322    <column name="expCovar16" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     323    <column name="expCovar22" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     324    <column name="expCovar23" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     325    <column name="expCovar24" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     326    <column name="expCovar25" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     327    <column name="expCovar26" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     328    <column name="expCovar33" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     329    <column name="expCovar34" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     330    <column name="expCovar35" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     331    <column name="expCovar36" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     332    <column name="expCovar44" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     333    <column name="expCovar45" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     334    <column name="expCovar46" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     335    <column name="expCovar55" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     336    <column name="expCovar56" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     337    <column name="expCovar66" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
     338    <column name="serRadius" type="TFLOAT" default="-999" comment="Sersic radius"></column>
     339    <column name="serRadiusErr" type="TFLOAT" default="-999" comment="estimated error in Sersic radius"></column>
     340    <column name="serMag" type="TFLOAT" default="-999" comment="Sersic magntiude"></column>
     341    <column name="serMagErr" type="TFLOAT" default="-999" comment="estimated error in serMag"></column>
     342    <column name="serAb" type="TFLOAT" default="-999" comment="Sersic axis ratio"></column>
     343    <column name="serAbErr" type="TFLOAT" default="-999" comment="estimated error in Sersic axis ratio"></column>
     344    <column name="serNu" type="TFLOAT" default="-999" comment="Sersic index"></column>
     345    <column name="serNuErr" type="TFLOAT" default="-999" comment="estimated error in Sersic index"></column>
     346    <column name="raSerOff" type="TFLOAT" default="-999" comment="Offset in RA of Sersic fit from PSF RA"></column>
     347    <column name="decSerOff" type="TFLOAT" default="-999" comment="Offset in DEC of Sersic fit from PSF DEC"></column>
     348    <column name="raSerOffErr" type="TFLOAT" default="-999" comment="estimated error in raSerOff"></column>
     349    <column name="decSerOffErr" type="TFLOAT" default="-999" comment="estimated error in decSerOff"></column>
     350    <column name="serCf" type="TFLOAT" default="-999" comment="Sersic fit coverage factor"></column>
     351    <column name="serLikelihood" type="TFLOAT" default="-999" comment="Sersic fit likelihood factor"></column>
     352    <column name="sersicCovar11" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     353    <column name="sersicCovar12" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     354    <column name="sersicCovar13" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     355    <column name="sersicCovar14" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     356    <column name="sersicCovar15" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     357    <column name="sersicCovar16" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     358    <column name="sersicCovar17" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     359    <column name="sersicCovar22" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     360    <column name="sersicCovar23" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     361    <column name="sersicCovar24" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     362    <column name="sersicCovar25" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     363    <column name="sersicCovar26" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     364    <column name="sersicCovar27" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     365    <column name="sersicCovar33" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     366    <column name="sersicCovar34" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     367    <column name="sersicCovar35" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     368    <column name="sersicCovar36" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     369    <column name="sersicCovar37" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     370    <column name="sersicCovar44" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     371    <column name="sersicCovar45" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     372    <column name="sersicCovar46" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     373    <column name="sersicCovar47" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     374    <column name="sersicCovar55" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     375    <column name="sersicCovar56" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     376    <column name="sersicCovar57" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     377    <column name="sersicCovar66" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     378    <column name="sersicCovar67" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     379    <column name="sersicCovar77" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
     380    <column name="activeFlag" type="TBYTE" default="-999" comment="indicates whether this detection/orphan is still a detection/orphan"></column>
     381    <column name="dataRelease" type="TBYTE" default="0" comment="Data release when this detection was taken"></column>
     382  </table>
     383  <table name="StackToImage">
     384    <column name="stackMetaID" type="TLONGLONG" default="0" comment="stack identifier"></column>
     385    <column name="imageID" type="TLONGLONG" default="0" comment="hashed exposure-ccdID identifier"></column>
    163386  </table>
    164387</tableDescriptions>

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl

    r29124 r29515  
    1010use ippToPsps::IppToPspsDb;
    1111use ippToPsps::Datastore;
    12 use ippToPsps::BatchManager;
     12use ippToPsps::Batch;
    1313
    1414my $singleBatch = undef;
    … …  
    1919my $product = undef;
    2020my $filePath = undef;
     21my $removeUnprocessed = undef;
    2122
    2223
    … …  
    2728        'product|p=s' => \$product,
    2829        'location|l=s' => \$filePath,
     30        'remove|r' => \$removeUnprocessed,
    2931        'verbose|v' => \$verbose,
    3032        'save_temps|s' => \$save_temps
    … …  
    3234
    3335if (!defined $product) {
    34     print "* OPTIONAL: a datastore product name        -p <name>\n";
     36    print "* OPTIONAL: a datastore product name            -p <name>\n";
    3537}
    3638if (!defined $singleBatch) {
    37     print "* OPTIONAL: a single batch                  -b <batchNum>        (default = none)\n";
     39    print "* OPTIONAL: a single batch                      -b <batchNum>        (default = none)\n";
    3840}
    3941if (!defined $fromTime) {
    4042    $fromTime = "2010-01-01";
    41     print "* OPTIONAL: from time                       -f <dateTime>        (default = $fromTime)\n";
     43    print "* OPTIONAL: from time                           -f <dateTime>        (default = $fromTime)\n";
    4244}
    4345if (!defined $toTime) {
    4446    $toTime = "2099-12-31";
    45     print "* OPTIONAL: to time                         -t <dateTime>        (default = $toTime)\n";
     47    print "* OPTIONAL: to time                             -t <dateTime>        (default = $toTime)\n";
    4648}
    4749if (!defined $filePath) {
    4850    print "* OPTIONAL: location for files to be deleted    -l <path>\n";
    4951}
     52if (!defined $removeUnprocessed) {
     53    $removeUnprocessed = 0;
     54    print "* OPTIONAL: remove unprocessed files            -r                   (default = $removeUnprocessed)\n";
     55}
    5056if (!defined $verbose) {
    5157    $verbose = 0;
    52     print "* OPTIONAL: run in verbose mode             -v                   (default = $verbose)\n";
     58    print "* OPTIONAL: run in verbose mode                 -v                   (default = $verbose)\n";
    5359}
    5460if (!defined $save_temps) {
    5561    $save_temps = 0;
    56     print "* OPTIONAL: keep temp files                 -t                   (default = $save_temps)\n";
     62    print "* OPTIONAL: keep temp files                     -t                   (default = $save_temps)\n";
    5763}
    5864
    … …  
    6066my $datastore = new ippToPsps::Datastore($product, 0, 0);
    6167my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
    62 my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $filePath, $verbose, $save_temps);
    6368my $odmUrl = "http://web01.psps.ifa.hawaii.edu/a01/OdmWebService/OdmWebService.asmx/GetBatchStatus";
    6469my $ua = LWP::UserAgent->new;
    … …  
    8590
    8691    print "\n";
    87     printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
    88     printf("|      Timestamp       |    Batch     |  Exposure ID | Loaded to ODM? | Merge worthy? | Merged? | Deleted? |\n");
    89     printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
     92    printf("+----------------------+--------------+--------------+----------------+--------------+--------------+---------+----------+\n");
     93    printf("|      Timestamp       |    Batch     |  Exposure ID | Loaded to ODM? | Load failed? |Merge worthy? | Merged? | Deleted? |\n");
     94    printf("+----------------------+--------------+--------------+----------------+--------------+--------------+---------+----------+\n");
    9095
    9196    # loop round batches
    … …  
    95100    my $numRemovedFromDatastore = 0;
    96101    my $numDeleted = 0;
     102    my $newMerged = 0;
    97103    foreach $batch ( @{$batches} ) {
    98         my ($timestamp, $expId, $batchId, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) =  @{$batch};
    99 
    100         if (!$processed) {next;}
     104        my ($timestamp,
     105                $type,
     106                $expId,
     107                $batchId,
     108                $surveyType,
     109                $deleted,
     110                $dvoDb,
     111                $processed,
     112                $onDatastore,
     113                $loadedToOdm,
     114                $loadFailed,
     115                $mergeWorthy,
     116                $merged) = @{$batch};
     117
     118        if (!$onDatastore) {next;}
     119
     120        my $batch = ippToPsps::Batch->existing(
     121                undef,
     122                undef,
     123                $batchId,
     124                $ippToPspsDb,
     125                $filePath,
     126                $verbose,
     127                $save_temps);
     128
     129        if (!$processed) {
     130
     131            if ($removeUnprocessed && defined $filePath) {
     132
     133                if ($batch->deleteDir()) {$numDeleted++;}
     134            }
     135            next;
     136        }
    101137
    102138        $numBatchesToCheck++;
    103139
    104         my $batchName = $batchManager->getBatchName($batchId);
    105 
    106140        # if not merged then update by polling ODM for status
    107         if (!$merged) {
    108 
    109             if (checkODM($batchName, \$loadedToOdm, \$mergeWorthy, \$merged)) {$numChecked++;}
     141        if (!$merged && !$loadFailed) {
     142
     143            if (checkODM($batch->getName(), \$loadedToOdm, \$loadFailed, \$mergeWorthy, \$merged)) {
     144               
     145                $numChecked++;
     146                if ($merged) {$newMerged++;}
     147            }
     148            else {next;}
    110149        }
    111150
    112151        # delete from datastore
    113         if (defined $product && !$deleted && $loadedToOdm && $mergeWorthy) {
    114 
    115             $deleted = $datastore->remove($batchName);
     152        if (defined $product && !$deleted && $loadedToOdm && ($mergeWorthy || $loadFailed)) {
     153
     154
     155            $deleted = $datastore->remove($batch->getName());
    116156            if ($deleted) {
    117157                $ippToPspsDb->setBatchAsDeleted($batchId, $expId);
    … …  
    122162        if (defined $filePath && $merged && $deleted) {
    123163
    124             if($batchManager->deleteBatch($batchId)) {$numDeleted++;}
     164            if($batch->deleteFile($batchId)) {$numDeleted++;}
    125165        }
    126166
    127167        # update database
    128         $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $merged, $deleted);
    129         printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s    | %6s  |  %5s   |\n",
     168        $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $loadFailed, $mergeWorthy, $merged, $deleted);
     169        printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s    |   %6s    | %6s  |  %5s   |\n",
    130170                $timestamp,
    131                 $batchName,
     171                $batch->getName(),
    132172                $expId,
    133173                $loadedToOdm ? "yes" : "no",
     174                $loadFailed ? "yes" : "no",
    134175                $mergeWorthy ? "yes" : "no",
    135176                $merged ? "yes" : "no",
    … …  
    137178    }
    138179
    139     printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
    140 
     180    printf("+----------------------+--------------+--------------+----------------+--------------+--------------+---------+----------+\n");
    141181    printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numBatchesToCheck);
     182    printf( "* %d newly merged batches\n", $newMerged);
    142183    printf( "* Successfully removed %d batch%s from the datastore\n", $numRemovedFromDatastore, ($numRemovedFromDatastore==1) ? "" : "es");
    143184    printf( "* Successfully deleted %d batch%s from local file system\n", $numDeleted, ($numDeleted==1) ? "" : "es");
    … …  
    151192########################################################################################
    152193sub checkODM {
    153     my ($batchName, $loadedToOdm, $mergeWorthy, $merged) = @_;
     194    my ($batchName, $loadedToOdm, $loadFailed, $mergeWorthy, $merged) = @_;
    154195
    155196    my $statusFilter = "*";
    … …  
    171212
    172213    my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_odmXml.XXXX", UNLINK => !$save_temps);
     214    #print $response->content. "\n";
    173215    print $tempFile $response->content;
    174216    close($tempFile);
    175217
    176218    ${$loadedToOdm} = 0;
     219    ${$loadFailed} = 0;
    177220    ${$mergeWorthy} = 0;
    178221    ${$merged} = 0;
    179222
    180     parseXml($tempName, $loadedToOdm, $mergeWorthy, $merged);
     223    parseXml($tempName, $loadedToOdm, $loadFailed, $mergeWorthy, $merged);
     224
     225    unlink($tempFile);
    181226
    182227    return 1;
    183228}
    184229
    185 ######################################################################################y
     230########################################################################################
    186231#
    187232# Parses ODM XML
    … …  
    189234########################################################################################
    190235sub parseXml {
    191     my ($xmlFile, $loadedToOdm, $mergeWorthy, $merged) = @_;
     236    my ($xmlFile, $loadedToOdm, $loadFailed, $mergeWorthy, $merged) = @_;
    192237
    193238    my $parser = XML::LibXML->new;
    … …  
    195240    my $xc = XML::LibXML::XPathContext->new($doc);
    196241    $xc->registerNs('ArrayOfOdmBatchState', 'PanSTARRS.Services.OdmWebService');
    197     my $result = $xc->findvalue('//ArrayOfOdmBatchState:Message');
     242    my $result = $xc->findvalue('//ArrayOfOdmBatchState:BatchState');
     243
     244    if ($result =~ m/failed/) {
     245        ${$loadFailed} = 1;
     246        ${$loadedToOdm} = 1;
     247    return;
     248    }
     249
     250    $result = $xc->findvalue('//ArrayOfOdmBatchState:Message');
    198251    ${$loadedToOdm} = 0;
    199252    ${$mergeWorthy} = 0;

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/getSmfForThisBatch.pl

    r29124 r29515  
    77use ippToPsps::Gpc1Db;
    88use ippToPsps::IppToPspsDb;
    9 use PS::IPP::Config 1.01 qw( :standard );
     9use ippToPsps::DetectionBatch;
    1010
    1111my $camera = undef;
    12 my $singleBatch = undef;
     12my $batch = undef;
    1313my $verbose = undef;
    1414my $save_temps = undef;
    1515
    1616GetOptions(
    17         'batch|b=s' => \$singleBatch,
     17        'batch|b=s' => \$batch,
    1818        'verbose|v' => \$verbose,
    1919        'save_temps|t' => \$save_temps,
    … …  
    2626if (@ARGV) {
    2727    $quit=1;
    28     print "* UNKNKOWN: option                          @ARGV\n";
     28    print "* UNKNOWN: option                          @ARGV\n";
    2929}
    30 if (!defined $singleBatch) {
     30if (!defined $batch) {
    3131    $quit=1;
    3232    print "* REQUIRED: need to provide a batch   -b <batch>\n";
    3333}
    34 if (!defined $verbose) {
     34if (!defined $camera) {
    3535    $camera = "GPC1";
    3636    print "* OPTIONAL: select a camera                 -c                   (default = $camera)\n";
    … …  
    5151my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps);
    5252my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
    53 my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR));
    5453
     54my $detectionBatch = ippToPsps::DetectionBatch->existing(
     55        $camera,
     56        $gpc1Db,
     57        $batch,
     58        $ippToPspsDb,
     59        "./",
     60        $verbose,
     61        $save_temps);
    5562
    56 my $batches;
    57 my $numOfBatches;
    58 
    59 if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getBatch($singleBatch, \$batches);}
    60 if ($numOfBatches < 1) {return 0;}
    61 
    62 
    63 # loop round batches
    64 my $batch;
    65 my $numChecked = 0;
    66 foreach $batch ( @{$batches} ) {
    67     my ($timestamp, $expId, $batchId, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) =  @{$batch};
    68 
    69     print "* found $timestamp, $expId, $batchId, $surveyType, $deleted $dvoDb\n";
    70 
    71     my $nebPath = $gpc1Db->getCameraStageSmfForThisDvoDb($dvoDb, $expId);
    72     if (!$nebPath) { print "* Could not determind neb path for SMF\n"; exit; }
    73 
    74     # get real filename from neb 'key'
    75     my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",$nebPath) or return 0;
    76     my $realFile = $ipprc->file_resolve($fpaObjects) or return 0;
    77 
    78     print "$realFile\n";
    79 
     63my $smf = $detectionBatch->getSmfFile();
     64if (defined $smf) {
     65    print "$smf\n";
    8066}
    81 
    82 
     67else {
     68    print "* Could not find an smf file for batch $batch\n";
     69}

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Datastore.pm

    r28951 r29515  
    2727    bless $self, $class;
    2828    return $self;
     29}
     30
     31########################################################################################
     32#
     33# Returns datastore product
     34#
     35########################################################################################
     36sub getProduct {
     37    my ($self) = @_;
     38    return $self->{_product};
    2939}
    3040
    … …  
    5868    print "* Successfully published $file to datastore\n";
    5969
    60     close($tempFile);
     70    close $tempFile;
     71    if (!$self->{_save_temps}) {unlink $tempFile};
    6172
    6273    return 1;

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Gpc1Db.pm

    r28877 r29515  
    88use ippToPsps::MySQLDb;
    99our @ISA = qw(ippToPsps::MySQLDb);    # inherits from MySQLDb
     10
     11###########################################################################
     12#
     13# Returns path to stack-stage cmf files for this sky_id
     14#
     15###########################################################################
     16sub getStackStageCmfs {
     17    my ($self, $skyId, $nebPath, $numInputs) = @_;
     18
     19    my $query = $self->{_db}->prepare(<<SQL);
     20    SELECT path_base, num_inputs
     21        FROM staticskyResult
     22        WHERE sky_id = $skyId
     23SQL
     24
     25    $query->execute;
     26    my $rows = $query->fetchall_arrayref();
     27
     28    if (@{$rows} < 1 || @{$rows} > 1) {return 0;}
     29
     30    my $row;
     31    foreach $row ( @{$rows} ) {
     32        (${$nebPath}, ${$numInputs}) = @{$row};
     33    }
     34
     35    return 1;
     36}
    1037
    1138###########################################################################
    … …  
    90117###########################################################################
    91118sub getExposureListFromDvoDb {
    92     my ($self, $dvoDb, $exposures) = @_;
     119    my ($self, $dvoDb, $exposures, $lastExposure) = @_;
     120
     121    if (!defined $lastExposure) {$lastExposure = 0;}
    93122
    94123    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    100129        WHERE addRun.dvodb LIKE '$dvoDb'
    101130        AND addRun.state = 'full'
     131        AND rawExp.exp_id > $lastExposure
    102132        ORDER BY exp_id ASC;
    103133SQL
    104134
    105     #AND rawExp.exp_id > 133887
    106     #AND filter = 'r.00000'
    107     #AND rawExp.dateobs <= '2010-03-12'
    108     #AND rawExp.exp_id > $lastExpId
    109     #AND camRun.label LIKE '%$label%'
    110     #AND rawExp.exp_id > 136561
    111     #AND camRun.dist_group LIKE 'sas'
    112135    #AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' Jims Dec range
    113136
    … …  
    115138    ${$exposures} = $query->fetchall_arrayref();
    116139    my $numOfExposures = scalar @{${$exposures}};
    117     if ($numOfExposures > 0) {
    118    
    119         print "* Found $numOfExposures exposures in DVO Db '$dvoDb'\n";
    120         return 1;
    121     }
    122140
    123     print "* No exposures found in DVO Db '$dvoDb'\n";
    124     return 0;
     141    # message
     142    if ($numOfExposures) {print "* Found $numOfExposures exposures";}
     143    else { print "* No exposures found"; }
     144    print " in DVO Db '$dvoDb' (counting from exposure $lastExposure)\n";
     145
     146    return ($numOfExposures) ? 1 : 0;
    125147}
    126148

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPspsDb.pm

    r29124 r29515  
    99our @ISA = qw(ippToPsps::MySQLDb);    # inherits from MySQLDb
    1010
    11 
    1211###########################################################################
    1312#
    … …  
    1918
    2019    my $query = $self->{_db}->prepare(<<SQL);
    21     SELECT created, exp_id, batch_id, survey_id, deleted, dvo_db, processed, on_datastore, loaded_to_ODM, merge_worthy, merged
     20    SELECT created,
     21           batch_type,
     22           exp_id,
     23           batch_id,
     24           survey_id,
     25           deleted,
     26           dvo_db,
     27           processed,
     28           on_datastore,
     29           loaded_to_ODM,
     30           load_failed,
     31           merge_worthy,
     32           merged
    2233        FROM batches
    2334        WHERE created >= '$fromTime'
    24         AND created <= '$toTime';
     35        AND created <= '$toTime'
    2536SQL
    2637
    … …  
    3041    my $count = scalar @{${$batches}};
    3142
    32    printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es");
    3343   return $count;
    3444}
    … …  
    4353
    4454    my $query = $self->{_db}->prepare(<<SQL);
    45     SELECT created, exp_id, batch_id, survey_id, deleted, dvo_db, processed, on_datastore, loaded_to_ODM, merge_worthy, merged
     55    SELECT created,
     56           batch_type,
     57           exp_id,
     58           batch_id,
     59           survey_id,
     60           deleted,
     61           dvo_db,
     62           processed,
     63           on_datastore,
     64           loaded_to_ODM,
     65           load_failed,
     66           merge_worthy,
     67           merged
    4668        FROM batches
    4769        WHERE batch_id = $batch_id
    … …  
    5274    my $count = scalar @{${$batches}};
    5375
    54    printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es");
    5576   return $count;
    5677}
    … …  
    103124#
    104125########################################################################################
    105 sub updateBatch {
    106     my ($self, $batchId, $expId, $processed, $published, $totalDetections, $minObjId, $maxObjId) = @_;
     126sub setAsProcessed {
     127    my ($self, $batchId, $expId, $totalDetections, $minObjId, $maxObjId) = @_;
    107128
    108129    if (!$minObjId) {$minObjId = -1;}
    … …  
    114135    UPDATE batches
    115136        SET
    116           processed = $processed,
    117           on_datastore = $published,
     137          processed = 1,
    118138          total_detections = $totalDetections,
    119139          min_obj_id = $minObjId,
    … …  
    128148#######################################################################################
    129149#
    130 # Checks whether we have successfully processed this exposure
    131 #
    132 ########################################################################################
    133 sub isExposureAlreadyProcessed {
     150# Sets published state of a batch
     151#
     152########################################################################################
     153sub setPublished {
     154    my ($self, $batchId, $expId, $published) = @_;
     155
     156    my $query = $self->{_db}->prepare(<<SQL);
     157    UPDATE batches
     158        SET on_datastore = $published
     159        WHERE batch_id = $batchId
     160        AND exp_id = $expId;
     161SQL
     162
     163    $query->execute; # TODO check response of this
     164}
     165
     166#######################################################################################
     167#
     168# Checks whether we have successfully processed this exposure and loaded it to the datastore
     169#
     170########################################################################################
     171sub isExposureAlreadyPublished {
    134172    my ($self, $expId) = @_;
    135173
    … …  
    138176        FROM batches
    139177        WHERE exp_id = $expId
    140         AND created > '2010-08-12'
    141         AND processed = 1;
     178        AND processed
     179        AND on_datastore;
    142180SQL
    143181
    … …  
    179217    $query->execute;
    180218
    181     print "* New batch '$batchId' for exposure ID = $expId and survey = '$surveyType'\n";
     219    print "* New batch '$batchId' for exposure ID = '$expId' and survey = '$surveyType'\n";
    182220
    183221    return $batchId;
    … …  
    208246#######################################################################################
    209247sub updateODMStatus {
    210     my ($self,$batchId, $expId, $loadedToOdm, $mergeWorthy, $mergeCompleted) = @_;
     248    my ($self,$batchId, $expId, $loadedToOdm, $loadFailed, $mergeWorthy, $mergeCompleted) = @_;
    211249
    212250    my $query = $self->{_db}->prepare(<<SQL);
    213251    UPDATE batches
    214         SET loaded_to_ODM = $loadedToOdm, merge_worthy = $mergeWorthy, merged = $mergeCompleted
     252        SET loaded_to_ODM = $loadedToOdm,
     253            load_failed = $loadFailed,
     254            merge_worthy = $mergeWorthy,
     255            merged = $mergeCompleted
    215256        WHERE batch_id = $batchId
    216257        AND exp_id = $expId;
    … …  
    231272
    232273    my $currentRevision = -1;
    233     my $latestRevision = 6;
    234 
    235     while ($currentRevision != $latestRevision) {
     274
     275    while (1) {
    236276
    237277        $currentRevision = $self->getRevision();
    … …  
    244284        elsif ($currentRevision == 4) {$self->createRevision_5();}
    245285        elsif ($currentRevision == 5) {$self->createRevision_6();}
     286        elsif ($currentRevision == 6) {$self->createRevision_7();}
     287        else {last;}
    246288    }
    247289}
    … …  
    385427#######################################################################################
    386428#
     429# Create revision 7 of the database
     430#
     431#######################################################################################
     432sub createRevision_7 {
     433    my ($self) = @_;
     434
     435    print "* Creating revision 7 of '$self->{_dbName}'\n";
     436
     437    my $query = $self->{_db}->prepare(<<SQL);
     438    ALTER TABLE batches
     439        ADD COLUMN load_failed TINYINT DEFAULT 0
     440SQL
     441        $query->execute;
     442
     443    $self->setRevision(7);
     444}
     445
     446#######################################################################################
     447#
    387448# Sets current revision of ippToPsps database
    388449#

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/MySQLDb.pm

    r28757 r29515  
    6565}                                                                               
    6666
    67 
    6867###########################################################################
    6968#
    70 # Returns 'now' as a timestamp
     69# Adds the provided interval to the provided time
    7170#
    7271###########################################################################
    73 sub getIntervalInPast {
    74     my ($self, $interval) = @_;
     72sub addInterval {
     73    my ($self, $time, $interval) = @_;
    7574
    7675      my $query = $self->{_db}->prepare(<<SQL);
    77           SELECT now() - INTERVAL $interval;
     76          SELECT '$time' + INTERVAL $interval;
    7877SQL
    7978    $query->execute;
    … …  
    8281}
    8382
     83###########################################################################
     84#
     85# Subtracts the provided interval from the provided time
     86#
     87###########################################################################
     88sub subtractInterval {
     89    my ($self, $time, $interval) = @_;
     90
     91      my $query = $self->{_db}->prepare(<<SQL);
     92          SELECT '$time' - INTERVAL $interval;
     93SQL
     94    $query->execute;
     95
     96return scalar $query->fetchrow_array();
     97}
     98
     99###########################################################################
     100#
     101# Returns whether time1 is before time2
     102#
     103###########################################################################
     104sub isBefore {
     105    my ($self, $time1, $time2) = @_;
     106
     107      my $query = $self->{_db}->prepare(<<SQL);
     108          SELECT '$time1' < '$time2';
     109SQL
     110    $query->execute;
     111
     112return scalar $query->fetchrow_array();
     113}
    84114
    85115###########################################################################
    … …  
    97127
    98128return scalar $query->fetchrow_array();
     129}
     130
     131#######################################################################################
     132#
     133# Optimizes a table
     134#
     135#######################################################################################
     136sub optimizeTable {
     137    my ($self, $table) = @_;
     138
     139    my $query = $self->{_db}->prepare(<<SQL);
     140    OPTIMIZE TABLE $table;
     141SQL
     142
     143    my $success = $query->execute;
     144
     145    print "* ";
     146    if (!$success) {print "UN";}
     147    print "successfully optimized '$table' table\n";
     148
     149    return $success;
    99150}
    100151

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/pspsSchema2xml.pl

    r28889 r29515  
    5050
    5151
    52 my $output = new IO::File(">tables.xml");
    53 my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
    54 $writer->xmlDecl('UTF-8');
    55 #    $writer->doctype('manifest', "", "psps-manifest.dtd");
    56 $writer->startTag('tableDescriptions', "type" => "$type");
     52my $tablesOutput = new IO::File(">tables.xml");
     53my $tablesWriter = new XML::Writer(OUTPUT => $tablesOutput, DATA_MODE => 1, DATA_INDENT=>2);
     54$tablesWriter->xmlDecl('UTF-8');
     55$tablesWriter->startTag('tableDescriptions', "type" => "$type");
     56
     57my $mapOutput = new IO::File(">map.xml");
     58my $mapWriter = new XML::Writer(OUTPUT => $mapOutput, DATA_MODE => 1, DATA_INDENT=>2);
     59$mapWriter->xmlDecl('UTF-8');
     60$mapWriter->startTag('tabledata', "type" => "$type");
    5761
    5862if ($type eq "init") {createInit();}
    … …  
    6367
    6468# finish up XML
    65 $writer->endTag();
    66 $writer->end();
     69$tablesWriter->endTag();
     70$tablesWriter->end();
     71
     72# finish up XML
     73$mapWriter->endTag();
     74$mapWriter->end();
    6775
    6876print OUT "\n#endif";
    … …  
    164172    parseTable("StackMeta");
    165173    parseTable("StackDetection");
    166     parseTable("SkinnyObject");
    167     parseTable("StackOrphan");
    168     parseTable("ObjectCalColor");
     174    parseTable("StackApFlx");
     175    parseTable("StackModelFit");
     176    parseTable("StackToImage");
    169177}
    170178
    … …  
    197205
    198206    print OUT "\ntypedef enum {\n";
    199     $writer->startTag('table', "name" => $tableNameOut);
     207    $tablesWriter->startTag('table', "name" => $tableNameOut);
     208    $mapWriter->startTag('table',
     209            "name" => $tableNameOut,
     210            "ippfitsextension" => "");
    200211
    201212    open (SCHEMA, $path);
    … …  
    240251
    241252    if (!$found) {print "Could not find table '$tableName'\n";}
    242     $writer->endTag();
     253    $tablesWriter->endTag();
     254    $mapWriter->endTag();
    243255    print OUT "} ".$tableNameOut.";\n";
    244256
    … …  
    326338    print OUT "  ".uc($tableName)."_".uc($name)." = ".$colNum.",\n";
    327339
    328     $writer->startTag('column',
     340    $tablesWriter->startTag('column',
    329341            "name" => $name,
    330342            "type" => $type,
    331343            "default" => $default,
    332344            "comment" => $comment);
    333 
    334     $writer->endTag();
     345    $tablesWriter->endTag();
     346
     347    $mapWriter->comment(" <map pspsName=\"$name\" ippType=\"$type\" ippName=\"\" comment=\"$comment\"/>");
     348
     349#    $mapWriter->endTag();
    335350
    336351    return $colNum;

  • branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPsps.c

    r28529 r29515  
    2424extern int ippToPsps_batchDifference(IppToPsps *data);
    2525
     26// Gets uncalibrated instrumental flux from magnitude
     27__inline bool ippToPsps_getFlux(const float exposureTime, const float magnitude, float* flux, const float magnitudeErr, float* fluxErr) {
     28
     29    *flux = powf(10.0, -0.4*magnitude) / exposureTime;
     30    if (!isfinite(*flux) || *flux < 0.000001) return false;
     31    if (fluxErr) *fluxErr = fabsf((magnitudeErr * *flux)/1.085736);
     32//      if (fluxErr)    printf("Mag = %03.03f, Flux = %03.03f, Mag err = %03.03f, Flux Err = %03.03f\n", magnitude, *flux, magnitudeErr, *fluxErr);
     33
     34    return true;
     35}
     36
    2637// Destructor
    2738static void ippToPsps_Destructor(IppToPsps* this) {
    … …  
    5566    // save XML document for results
    5667    if (this->resultsXmlDoc) {
    57    
     68
    5869        xmlSaveFormatFileEnc(this->resultsPath, this->resultsXmlDoc, "UTF-8", 1);
    5970
    … …  
    151162
    152163    psMetadata *arguments = psMetadataAlloc();
    153     psMetadataAddStr(arguments, PS_LIST_TAIL, "-expid", 0, "Exposure ID", NULL);
     164    psMetadataAddStr(arguments, PS_LIST_TAIL, "-id", 0, "ID", NULL);
    154165    psMetadataAddStr(arguments, PS_LIST_TAIL, "-expname", 0, "Exposure name", NULL);
    155166    psMetadataAddStr(arguments, PS_LIST_TAIL, "-survey", 0, "Survey type", NULL);
    … …  
    158169    psMetadataAddStr(arguments, PS_LIST_TAIL, "-results", 0, "Path to results output", NULL);
    159170    psMetadataAddStr(arguments, PS_LIST_TAIL, "-config", 0, "Path to config dir", NULL);
    160     psMetadataAddStr(arguments, PS_LIST_TAIL, "-batch", 0, "Product type: [det|stack|diff]", NULL);
     171    psMetadataAddStr(arguments, PS_LIST_TAIL, "-batch", 0, "Product type: [IN|P2|ST|TODO]", NULL);
    161172
    162173    bool ret = true;
    … …  
    168179    else {
    169180
    170         char *tmp = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-expid"));
    171         if (tmp) this->expId = atoi(tmp);
     181        char *tmp = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-id"));
     182        if (tmp) this->id = atoi(tmp);
    172183        //free(tmp); tmp = NULL;
    173184        this->expName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-expname"));
    … …  
    181192        if (!tmp) this->batchType = BATCH_UNDEFINED;
    182193        else if (strcmp(tmp, "test") == 0) this->batchType = BATCH_TEST;
    183         else if (strcmp(tmp, "init") == 0) this->batchType = BATCH_INIT;
    184         else if (strcmp(tmp, "det") == 0) this->batchType = BATCH_DETECTION;
    185         else if (strcmp(tmp, "stack") == 0) this->batchType = BATCH_STACK;
    186         else if (strcmp(tmp, "diff") == 0) this->batchType = BATCH_DIFFERENCE;
     194        else if (strcmp(tmp, "IN") == 0) this->batchType = BATCH_INIT;
     195        else if (strcmp(tmp, "P2") == 0) this->batchType = BATCH_DETECTION;
     196        else if (strcmp(tmp, "ST") == 0) this->batchType = BATCH_STACK;
     197        else if (strcmp(tmp, "TODO") == 0) this->batchType = BATCH_DIFFERENCE;
    187198        else this->batchType = BATCH_UNDEFINED;
    188199
    … …  
    192203                (this->batchType != BATCH_INIT && !this->fitsInPath) ||
    193204                (this->batchType != BATCH_INIT && !this->resultsPath) ||
    194                 (this->batchType != BATCH_INIT && this->expId == -1) ||
     205                (this->batchType != BATCH_INIT && this->id == -1) ||
    195206                !this->fitsOutPath ||
    196207                !this->configsDir ||
    … …  
    211222static int ippToPsps_run(IppToPsps *this) {
    212223
     224    // any files?
     225    if (this->batchType == BATCH_DETECTION ||
     226            this->batchType == BATCH_STACK ||
     227            this->batchType == BATCH_DIFFERENCE ) {
     228
     229        if (this->numOfInputFiles < 1) {
     230
     231            this->exitCode = PS_EXIT_DATA_ERROR;
     232            psError(PS_ERR_UNKNOWN, false, "No input files for this batch");
     233        }
     234
     235    }
     236
    213237    if(this->exitCode == PS_EXIT_SUCCESS) {
    214238
    … …  
    250274    psMemSetDeallocator(this, (psFreeFunc)ippToPsps_Destructor);
    251275
    252     this->expId = -1;
     276    // deal with DVO database
     277    bool haveDvo = false;
     278    for (int i=0;i<*argc; i++) {
     279
     280        if (strcmp(argv[i], "CATDIR") == 0) {haveDvo = true; break;}
     281    }
     282    if (haveDvo) this->dvoConfig = dvoConfigRead(argc, argv);
     283    else this->dvoConfig = NULL;
     284
     285    // remaining args
     286    this->id = -1;
    253287    this->expName = NULL;
    254288    this->surveyType = NULL;
     289    this->surveyID = -1;
    255290    this->fitsInPath = NULL;
    256291    this->resultsPath = NULL;
    … …  
    261296    this->fitsOut = NULL;
    262297    this->configsDir = NULL;
    263     this->dvoConfig = NULL;
    264298    this->config = NULL;
    265     this->dvoConfig = dvoConfigRead(argc, argv);
    266299    this->pmconfig = pmConfigRead(argc, argv, NULL);
    267300    this->exitCode = PS_EXIT_SUCCESS;
    268301    this->run = ippToPsps_run;
     302
     303    // sort out current time
     304    time_t now = time(NULL);;
     305    struct tm *ts = gmtime(&now); // TODO should be UTC
     306    strftime(this->todaysDate, sizeof(this->todaysDate), "%Y-%m-%d", ts);
    269307
    270308    if (!this->pmconfig) {
    … …  
    299337        psStringAppend(&this->configsDir, "/stack");
    300338
    301 
     339    // create filename
    302340    char outputName[100];
    303     sprintf(outputName, "%08d.FITS", this->expId);
     341    sprintf(outputName, "%08d.FITS", this->id);
    304342    psStringAppend(&this->fitsOutPath, "/%s", outputName);
    305343
    … …  
    321359    }
    322360
     361    // get survey ID using config object
     362    if (this->batchType != BATCH_INIT &&
     363            !ippToPspsConfig_getSurveyId(this->config, this->surveyType, &this->surveyID)) {
     364       
     365        this->exitCode = PS_EXIT_CONFIG_ERROR;
     366        return this;
     367    }
     368
    323369    // create XML document for results
    324370    if (this->resultsPath) {

  • branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPsps.h

    r28530 r29515  
    1818#include <libxml/tree.h>
    1919
    20 #define MAXDETECT 30000 //TODO limit ok?
     20#define MAXDETECT 60000 //TODO limit ok?
    2121
    2222typedef struct {
    2323
    24     uint32_t expId;             // the exposure ID to be used
     24    uint32_t id;                 // ID: might be expId for detections, skycell ID etc
    2525    psString expName;           // the exposure name
    2626    psString surveyType;        // the survey type, eg 3PI, MD01, STS, SSS
     27    int8_t surveyID;            // survey ID
    2728    uint8_t batchType;          // PSPS batch type
    2829    psString fitsInPath;        // path to FITS input
    … …  
    3738    dvoConfig* dvoConfig;       // dvo database
    3839    IppToPspsConfig* config;    // config structure
     40    char todaysDate[12];        // today's date
    3941    int exitCode;               // ps exit code
    4042
    … …  
    4547IppToPsps *ippToPsps_Constructor(int *argc, char **argv);
    4648void ippToPsps_VersionPrint(void);
     49bool ippToPsps_getFlux(
     50        const float exposureTime,
     51        const float magnitude,
     52        float* flux,
     53        const float magnitudeErr,
     54        float* fluxErr);
    4755
    4856# endif // IPPTOPSPS_H

  • branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchDetection.c

    r29124 r29515  
    1111#include "ippToPspsConfig.h"
    1212#include "ippToPspsDetEnums.h"
    13 #include "fitsio.h"
    14 
    15 // Gets uncalibrated instrumental flux from magnitude
    16 static __inline bool ippToPsps_getFlux(const float exposureTime, const float magnitude, float* flux, const float magnitudeErr, float* fluxErr) {
    17 
    18     *flux = powf(10.0, -0.4*magnitude) / exposureTime;
    19     if (!isfinite(*flux) || *flux < 0.000001) return false;
    20     if (fluxErr) *fluxErr = fabsf((magnitudeErr * *flux)/1.085736);
    21     //  if (fluxErr)    printf("Mag = %03.03f, Flux = %03.03f, Mag err = %03.03f, Flux Err = %03.03f\n", magnitude, *flux, magnitudeErr, *fluxErr);
    22          
    23     return true;
    24 }
    2513
    2614/**
    … …  
    3523int ippToPsps_batchDetection(IppToPsps *this) {
    3624
    37     if (this->numOfInputFiles < 1) return PS_EXIT_DATA_ERROR;
    38 
    3925    int status = 0;
    4026    fitsfile *fitsIn;         
    41     int nKeys;
    4227
    4328    if (fits_open_file(&fitsIn, this->inputFiles[0], READONLY, &status)) {
    … …  
    4833
    4934    // get primary header and pull stuff out for later
     35    int nKeys;
    5036    fits_get_hdrspace(fitsIn, &nKeys, NULL, &status);
    5137
    … …  
    6551
    6652    // FrameMeta values
    67     fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->expId, &status);
     53    fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->id, &status);
    6854    fits_write_col(this->fitsOut, TSTRING, FRAMEMETA_FRAMENAME, 1, 1, 1, &this->expName, &status);
    6955
    70     int8_t surveyID = -1;
    71     if (!ippToPspsConfig_getSurveyId(this->config, this->surveyType, &surveyID)) {return PS_EXIT_DATA_ERROR;}
    72     fits_write_col(this->fitsOut, TBYTE, FRAMEMETA_SURVEYID, 1, 1, 1, &surveyID, &status);
     56    fits_write_col(this->fitsOut, TBYTE, FRAMEMETA_SURVEYID, 1, 1, 1, &this->surveyID, &status);
    7357
    7458    int8_t filterID = -1;
    … …  
    9781
    9882    // stuff for detections table
    99     time_t now;
    100     struct tm *ts;
    101     char timeStr[12];
    102     now = time(NULL);
    103     ts = gmtime(&now); // TODO should be UTC
    104     strftime(timeStr, sizeof(timeStr), "%Y-%m-%d", ts);
    105 
    10683    uint32_t s,d, invalidDvoRows, smfJumps, unmatched, totalDetections = 0;
    10784
    … …  
    128105
    129106        filterIDs[s] = filterID;
    130         surveyIDs[s] = surveyID;
    131         strcpy(assocDate[s], timeStr);
     107        surveyIDs[s] = this->surveyID;
     108        strcpy(assocDate[s], this->todaysDate);
    132109    }
    133110
    … …  
    192169
    193170            // create unique int from 'frameID' (aka exposure ID) and ccd number
    194             pspsImageId = (this->expId*100) + image->ccdnum;
     171            pspsImageId = (this->id*100) + image->ccdnum;
    195172
    196173            // now get DVO detections
    … …  
    206183            imageFlags = (uint64_t)image->flags;
    207184            fits_write_col(this->fitsOut, TLONGLONG, IMAGEMETA_IMAGEID, 1, 1, 1, &pspsImageId, &status);
    208             fits_write_col(this->fitsOut, TLONG, IMAGEMETA_FRAMEID, 1, 1, 1, &this->expId, &status);
     185            fits_write_col(this->fitsOut, TLONG, IMAGEMETA_FRAMEID, 1, 1, 1, &this->id, &status);
    209186            fits_write_col(this->fitsOut, TSHORT, IMAGEMETA_CCDID, 1, 1, 1, &image->ccdnum, &status);
    210187            fits_write_col(this->fitsOut, TLONG, IMAGEMETA_PHOTOCALID, 1, 1, 1, &image->photcode, &status);

  • branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchStack.c

    r27686 r29515  
    99#include "ippToPsps.h"
    1010#include "ippToPspsConfig.h"
     11#include "ippToPspsStackEnums.h"
     12
    1113
    1214/**
    … …  
    1517int ippToPsps_batchStack(IppToPsps *this) {
    1618
    17 //    char extensionName[20];
    18     uint32_t skycell = 0; // TODO
     19    int status = 0;
     20    fitsfile *fitsIn;
    1921
    20     for (uint16_t i = 0; i<this->numOfInputFiles; i++) {
     22    if (fits_open_file(&fitsIn, this->inputFiles[0], READONLY, &status)) {
    2123
    22         skycell = i; // TODO
     24        fits_report_error(stderr, status);
     25        return PS_EXIT_SYS_ERROR;
     26    }
     27
     28    long removeList[MAXDETECT];
     29    float instMag[MAXDETECT];
     30    float floatnull = -999.0;
     31    int anynull = 0;
     32
     33    // get primary header and pull stuff out for later
     34    int nKeys = 0;
     35    fits_get_hdrspace(fitsIn, &nKeys, NULL, &status);
     36    float exposureTime; status=0; fits_read_key(fitsIn, TFLOAT, "EXPTIME", &exposureTime, NULL, &status);
     37    char filterType[20]; status=0; fits_read_key(fitsIn, TSTRING, "FPA.FILTERID", filterType, NULL, &status);
    2338
    2439
     40    char** assocDate = (char**)calloc(MAXDETECT, sizeof(char**));
     41    for (uint32_t i=0; i<MAXDETECT;i++) assocDate[i] = (char*)calloc(20,sizeof(char));
     42    int8_t filterIDs[MAXDETECT], surveyIDs[MAXDETECT];
     43
     44
     45    // write StackMeta
     46    ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, 1, "StackMeta", true);
     47    fits_write_col(this->fitsOut, TLONG, STACKMETA_SKYCELLID, 1, 1, 1, &this->id, &status);
     48
     49    int8_t filterID = -1;
     50    if (!ippToPspsConfig_getFilterId(this->config, filterType, &filterID)) {return PS_EXIT_DATA_ERROR;}
     51    fits_write_col(this->fitsOut, TBYTE, STACKMETA_FILTERID, 1, 1, 1, &filterID, &status);
     52
     53    fits_write_col(this->fitsOut, TBYTE, STACKMETA_SURVEYID, 1, 1, 1, &this->surveyID, &status);
     54
     55
     56    // psf detections
     57    char extensionName[15];
     58    sprintf(extensionName, "SkyChip.psf");
     59    if (fits_movnam_hdu(fitsIn, BINARY_TBL, extensionName, 0, &status)) {
     60        psError(PS_ERR_IO, false, "Can't move to extension: %s\n", extensionName);
     61       
    2562    }
     63    else {
     64   
     65        // some stuff is the same for all detections so we can populate here
     66        for (long s = 0; s<MAXDETECT; s++) {
     67
     68            filterIDs[s] = filterID;
     69            surveyIDs[s] = this->surveyID;
     70            strcpy(assocDate[s], this->todaysDate);
     71        }
     72
     73
     74        long nDet = 0;
     75        if (fits_get_num_rows(fitsIn, &nDet, &status)) {
     76            fits_report_error(stderr, status);
     77        }
     78
     79        int instMagNum;
     80        status=0;fits_get_colnum(fitsIn, CASESEN, "PSF_INST_MAG", &instMagNum, &status);
     81        if (status) psError(PS_ERR_IO, false, "Unable to read col num for PSF_INST_MAG");
     82        fits_read_col(fitsIn, TFLOAT, instMagNum, 1, 1, nDet, &floatnull, instMag, &anynull, &status);
     83
     84
     85        printf("Looping through %ld psf detections\n", nDet);
     86        float mag;
     87        long unmatched = 0, totalDetections = 0, numOfDuplicates = 0, numInvalidFlux = 0, numDetectionsOut = 0;
     88
     89        for (long s = 0; s<nDet; s++) {
     90
     91            // TODO implement this match in DVO
     92            if (1) {
     93
     94                mag = instMag[s];
     95                if (!isfinite(mag) || mag < -998.0) {
     96
     97                    removeList[numOfDuplicates+numInvalidFlux] = s+1;
     98                    numInvalidFlux++;
     99                }
     100
     101                totalDetections++;
     102            }
     103            else {
     104
     105                unmatched++;
     106                continue;
     107            }
     108        }
     109
     110        numDetectionsOut = totalDetections - numInvalidFlux;
     111
     112        if (numDetectionsOut > 0) {
     113
     114            ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, nDet, "StackDetection", false);
     115            fits_write_col(this->fitsOut, TLONG, STACKDETECTION_SKYCELLID, 1, 1, 1, &this->id, &status);
     116            fits_write_col(this->fitsOut, TBYTE, STACKDETECTION_FILTERID, 1, 1, nDet, filterIDs, &status);
     117            fits_write_col(this->fitsOut, TBYTE, STACKDETECTION_SURVEYID, 1, 1, nDet, surveyIDs, &status);
     118            fits_write_col(this->fitsOut, TSTRING, STACKDETECTION_ASSOCDATE, 1, 1, nDet, assocDate, &status);
     119
     120            if (numInvalidFlux) fits_delete_rowlist(this->fitsOut, removeList, numInvalidFlux, &status);
     121
     122        }
     123        psLogMsg("ippToPsps", PS_LOG_INFO,
     124                "+---------------+---------+----------+------------------+---------------+--------------+\n"
     125                "|   Extension   | Rows in | Rows out | Missing from DVO | Duplicate IDs | Invalid Flux |\n"
     126                "|  %12s |  %5ld  |   %5ld  |      %5ld       |    %5ld      |    %5ld     |\n",
     127                extensionName, nDet, numDetectionsOut, unmatched, numOfDuplicates, numInvalidFlux);
     128
     129    }
     130
     131
     132
     133    // extended source
     134    sprintf(extensionName, "SkyChip.xsrc");
     135    if (fits_movnam_hdu(fitsIn, BINARY_TBL, extensionName, 0, &status)) {
     136        psError(PS_ERR_IO, false, "Can't move to extension: %s\n", extensionName);
     137
     138    }
     139    else {
     140
     141        long nDet = 0;
     142        if (fits_get_num_rows(fitsIn, &nDet, &status)) {
     143            fits_report_error(stderr, status);
     144        }
     145
     146        printf("Looping through %ld extended source detections\n", nDet);
     147        for (long s = 0; s<nDet; s++) {
     148
     149
     150
     151        }
     152
     153        ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, nDet, "StackApFlx", false);
     154        fits_write_col(this->fitsOut, TBYTE, STACKAPFLX_FILTERID, 1, 1, nDet, filterIDs, &status);
     155        fits_write_col(this->fitsOut, TBYTE, STACKAPFLX_SURVEYID, 1, 1, nDet, surveyIDs, &status);
     156    }
     157
     158    status=0;
     159    if (fits_close_file(fitsIn, &status)) fits_report_error(stderr, status);
     160
    26161
    27162    return PS_EXIT_SUCCESS;

  • branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchTest.c

    r28759 r29515  
    1313#include "fitsio.h"
    1414
    15 // Gets uncalibrated instrumental flux from magnitude
    16 static __inline bool ippToPsps_getFlux(const float exposureTime, const float magnitude, float* flux, const float magnitudeErr, float* fluxErr) {
    17 
    18     *flux = powf(10.0, -0.4*magnitude) / exposureTime;
    19     if (!isfinite(*flux) || *flux < 0.000001) return false;
    20     if (fluxErr) *fluxErr = fabsf((magnitudeErr * *flux)/1.085736);
    21     //  if (fluxErr)    printf("Mag = %03.03f, Flux = %03.03f, Mag err = %03.03f, Flux Err = %03.03f\n", magnitude, *flux, magnitudeErr, *fluxErr);
    22 
    23     return true;
    24 }
    2515
    2616/**
    … …  
    5545
    5646    // FrameMeta values
    57 //    fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->expId, &status);
     47//    fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->id, &status);
    5848
    5949    int8_t filterID = -1;
    … …  
    187177
    188178            // create unique int from 'frameID' (aka exposure ID) and ccd number
    189             pspsImageId = (this->expId*100) + pImage->ccdnum;
     179            pspsImageId = (this->id*100) + pImage->ccdnum;
    190180
    191181            // now get DVO detections

  • branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsConfig.c

    r28559 r29515  
    701701}
    702702
     703// gets metadata about a column
     704bool ippToPspsConfig_getFitsColumnMeta(
     705        char* name,
     706        int* colNum,
     707        int* type,
     708        long* repeat,
     709        fitsfile *fitsIn ) {
     710
     711    int status = 0;
     712    fits_get_colnum(fitsIn, CASESEN, name, colNum, &status);
     713    if (status) {
     714        psError(PS_ERR_IO, false, "Unable to read col '%s'", name);
     715        return false;
     716    }
     717
     718    status = 0;
     719    fits_get_eqcoltype(fitsIn, *colNum, type, repeat, NULL, &status);
     720    if (status) {
     721        psError(PS_ERR_IO, false, "Unable to read type info for '%s'", name);
     722        return false;
     723    }
     724
     725    return true;
     726}
     727
    703728// populate with data from another FITS table into this one
    704729static bool ippToPspsConfig_populateTableFromFits(
    … …  
    730755    int readStatus = 0;
    731756    int writeStatus = 0;
    732    
     757
    733758    // first loop round all columns and get IPP col numbers for provided column names TODO only do once, first time in
    734759    if(!fromHeader) {
    … …  
    737762
    738763            if (strlen(table->columns[i].ippName) < 1) continue;
    739             readStatus = 0;
    740             fits_get_colnum(fitsIn, CASESEN, table->columns[i].ippName, &table->columns[i].ippColNum, &readStatus);
    741             if (readStatus) psError(PS_ERR_IO, false, "%d Unable to read col num for '%s' '%s' %d", i, table->columns[i].pspsName, table->columns[i].ippName, table->columns[i].ippColNum);
     764
     765            if (!ippToPspsConfig_getFitsColumnMeta(
     766                    table->columns[i].ippName,
     767                    &table->columns[i].ippColNum,
     768                    &table->columns[i].ippType,
     769                    &table->columns[i].ippRepeat,
     770                    fitsIn)) {return false;}
    742771        }
    743772    }

  • branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsConfig.h

    r28249 r29515  
    2020    char ippName[100];
    2121    int ippType;
     22    long ippRepeat;
    2223    char pspsName[100]; // TODO change to 'name'
    2324    int pspsType;

  • branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsStackEnums.h

    r27431 r29515  
    3535  STACKMETA_CRPIX1 = 30,
    3636  STACKMETA_CRPIX2 = 31,
    37   STACKMETA_PC001001 = 32,
    38   STACKMETA_PC001002 = 33,
    39   STACKMETA_PC002001 = 34,
    40   STACKMETA_PC002002 = 35,
    41   STACKMETA_POLYORDER = 36,
    42   STACKMETA_PCA1X3Y0 = 37,
    43   STACKMETA_PCA1X2Y1 = 38,
    44   STACKMETA_PCA1X1Y2 = 39,
    45   STACKMETA_PCA1X0Y3 = 40,
    46   STACKMETA_PCA1X2Y0 = 41,
    47   STACKMETA_PCA1X1Y1 = 42,
    48   STACKMETA_PCA1X0Y2 = 43,
    49   STACKMETA_PCA2X3Y0 = 44,
    50   STACKMETA_PCA2X2Y1 = 45,
    51   STACKMETA_PCA2X1Y2 = 46,
    52   STACKMETA_PCA2X0Y3 = 47,
    53   STACKMETA_PCA2X2Y0 = 48,
    54   STACKMETA_PCA2X1Y1 = 49,
    55   STACKMETA_PCA2X0Y2 = 50,
    56   STACKMETA_CALIBMODNUM = 51,
    57   STACKMETA_DATARELEASE = 52,
     37  STACKMETA_CDELT1 = 32,
     38  STACKMETA_CDELT2 = 33,
     39  STACKMETA_PC001001 = 34,
     40  STACKMETA_PC001002 = 35,
     41  STACKMETA_PC002001 = 36,
     42  STACKMETA_PC002002 = 37,
     43  STACKMETA_POLYORDER = 38,
     44  STACKMETA_PCA1X3Y0 = 39,
     45  STACKMETA_PCA1X2Y1 = 40,
     46  STACKMETA_PCA1X1Y2 = 41,
     47  STACKMETA_PCA1X0Y3 = 42,
     48  STACKMETA_PCA1X2Y0 = 43,
     49  STACKMETA_PCA1X1Y1 = 44,
     50  STACKMETA_PCA1X0Y2 = 45,
     51  STACKMETA_PCA2X3Y0 = 46,
     52  STACKMETA_PCA2X2Y1 = 47,
     53  STACKMETA_PCA2X1Y2 = 48,
     54  STACKMETA_PCA2X0Y3 = 49,
     55  STACKMETA_PCA2X2Y0 = 50,
     56  STACKMETA_PCA2X1Y1 = 51,
     57  STACKMETA_PCA2X0Y2 = 52,
     58  STACKMETA_CALIBMODNUM = 53,
     59  STACKMETA_DATARELEASE = 54,
    5860} StackMeta;
    5961
    … …  
    107109
    108110typedef enum {
    109   SKINNYOBJECT_OBJID = 1,
    110   SKINNYOBJECT_IPPOBJID = 2,
    111   SKINNYOBJECT_PROJECTIONCELLID = 3,
    112   SKINNYOBJECT_DATARELEASE = 4,
    113 } SkinnyObject;
    114 
    115 typedef enum {
    116   STACKORPHAN_PARTITIONKEY = 1,
    117   STACKORPHAN_STACKDETECTID = 2,
    118   STACKORPHAN_IPPDETECTID = 3,
    119   STACKORPHAN_FILTERID = 4,
    120   STACKORPHAN_STACKTYPEID = 5,
    121   STACKORPHAN_SURVEYID = 6,
    122   STACKORPHAN_PRIMARYF = 7,
    123   STACKORPHAN_STACKMETAID = 8,
    124   STACKORPHAN_SKYCELLID = 9,
    125   STACKORPHAN_STACKVER = 10,
    126   STACKORPHAN_XPOS = 11,
    127   STACKORPHAN_YPOS = 12,
    128   STACKORPHAN_XPOSERR = 13,
    129   STACKORPHAN_YPOSERR = 14,
    130   STACKORPHAN_INSTFLUX = 15,
    131   STACKORPHAN_INSTFLUXERR = 16,
    132   STACKORPHAN_PEAKFLUX = 17,
    133   STACKORPHAN_SKY = 18,
    134   STACKORPHAN_SKYERR = 19,
    135   STACKORPHAN_SGSEP = 20,
    136   STACKORPHAN_PSFWIDMAJOR = 21,
    137   STACKORPHAN_PSFWIDMINOR = 22,
    138   STACKORPHAN_PSFTHETA = 23,
    139   STACKORPHAN_PSFLIKELIHOOD = 24,
    140   STACKORPHAN_PSFCF = 25,
    141   STACKORPHAN_INFOFLAG = 26,
    142   STACKORPHAN_NFRAMES = 27,
    143   STACKORPHAN_WLSIGMA = 28,
    144   STACKORPHAN_EPS1 = 29,
    145   STACKORPHAN_EPS2 = 30,
    146   STACKORPHAN_PSM11 = 31,
    147   STACKORPHAN_PSM12 = 32,
    148   STACKORPHAN_PSM21 = 33,
    149   STACKORPHAN_PSM22 = 34,
    150   STACKORPHAN_PSH11 = 35,
    151   STACKORPHAN_PSH12 = 36,
    152   STACKORPHAN_PSH21 = 37,
    153   STACKORPHAN_PSH22 = 38,
    154   STACKORPHAN_ACTIVEFLAG = 39,
    155   STACKORPHAN_ASSOCDATE = 40,
    156   STACKORPHAN_HISTORYMODNUM = 41,
    157   STACKORPHAN_DATARELEASE = 42,
    158 } StackOrphan;
    159 
    160 typedef enum {
    161   OBJECTCALCOLOR_OBJID = 1,
    162   OBJECTCALCOLOR_IPPOBJID = 2,
    163   OBJECTCALCOLOR_FILTERID = 3,
    164   OBJECTCALCOLOR_CALCOLOR = 4,
    165   OBJECTCALCOLOR_CALCOLORERR = 5,
    166   OBJECTCALCOLOR_CALIBMODNUM = 6,
    167   OBJECTCALCOLOR_DATARELEASE = 7,
    168 } ObjectCalColor;
     111  STACKAPFLX_OBJID = 1,
     112  STACKAPFLX_STACKDETECTID = 2,
     113  STACKAPFLX_IPPOBJID = 3,
     114  STACKAPFLX_IPPDETECTID = 4,
     115  STACKAPFLX_FILTERID = 5,
     116  STACKAPFLX_STACKTYPEID = 6,
     117  STACKAPFLX_SURVEYID = 7,
     118  STACKAPFLX_PRIMARYF = 8,
     119  STACKAPFLX_STACKMETAID = 9,
     120  STACKAPFLX_ISOPHOTMAG = 10,
     121  STACKAPFLX_ISOPHOTMAGERR = 11,
     122  STACKAPFLX_ISOPHOTMAJAXIS = 12,
     123  STACKAPFLX_ISOPHOTMAJAXISERR = 13,
     124  STACKAPFLX_ISOPHOTMINAXIS = 14,
     125  STACKAPFLX_ISOPHOTMINAXISERR = 15,
     126  STACKAPFLX_ISOPHOTMAJAXISGRAD = 16,
     127  STACKAPFLX_ISOPHOTMINAXISGRAD = 17,
     128  STACKAPFLX_ISOPHOTPA = 18,
     129  STACKAPFLX_ISOPHOTPAERR = 19,
     130  STACKAPFLX_ISOPHOTPAGRAD = 20,
     131  STACKAPFLX_PETRADIUS = 21,
     132  STACKAPFLX_PETRADIUSERR = 22,
     133  STACKAPFLX_PETMAG = 23,
     134  STACKAPFLX_PETMAGERR = 24,
     135  STACKAPFLX_PETR50 = 25,
     136  STACKAPFLX_PETR50ERR = 26,
     137  STACKAPFLX_PETR90 = 27,
     138  STACKAPFLX_PETR90ERR = 28,
     139  STACKAPFLX_PETCF = 29,
     140  STACKAPFLX_FLXR1 = 30,
     141  STACKAPFLX_FLXR1ERR = 31,
     142  STACKAPFLX_FLXR1VAR = 32,
     143  STACKAPFLX_FLXR2 = 33,
     144  STACKAPFLX_FLXR2ERR = 34,
     145  STACKAPFLX_FLXR2VAR = 35,
     146  STACKAPFLX_FLXR3 = 36,
     147  STACKAPFLX_FLXR3ERR = 37,
     148  STACKAPFLX_FLXR3VAR = 38,
     149  STACKAPFLX_FLXR4 = 39,
     150  STACKAPFLX_FLXR4ERR = 40,
     151  STACKAPFLX_FLXR4VAR = 41,
     152  STACKAPFLX_FLXR5 = 42,
     153  STACKAPFLX_FLXR5ERR = 43,
     154  STACKAPFLX_FLXR5VAR = 44,
     155  STACKAPFLX_FLXR6 = 45,
     156  STACKAPFLX_FLXR6ERR = 46,
     157  STACKAPFLX_FLXR6VAR = 47,
     158  STACKAPFLX_FLXR7 = 48,
     159  STACKAPFLX_FLXR7ERR = 49,
     160  STACKAPFLX_FLXR7VAR = 50,
     161  STACKAPFLX_FLXR8 = 51,
     162  STACKAPFLX_FLXR8ERR = 52,
     163  STACKAPFLX_FLXR8VAR = 53,
     164  STACKAPFLX_FLXR9 = 54,
     165  STACKAPFLX_FLXR9ERR = 55,
     166  STACKAPFLX_FLXR9VAR = 56,
     167  STACKAPFLX_FLXR10 = 57,
     168  STACKAPFLX_FLXR10ERR = 58,
     169  STACKAPFLX_FLXR10VAR = 59,
     170  STACKAPFLX_C1FLXR1 = 60,
     171  STACKAPFLX_C1FLXR1ERR = 61,
     172  STACKAPFLX_C1FLXR1VAR = 62,
     173  STACKAPFLX_C1FLXR2 = 63,
     174  STACKAPFLX_C1FLXR2ERR = 64,
     175  STACKAPFLX_C1FLXR2VAR = 65,
     176  STACKAPFLX_C1FLXR3 = 66,
     177  STACKAPFLX_C1FLXR3ERR = 67,
     178  STACKAPFLX_C1FLXR3VAR = 68,
     179  STACKAPFLX_C1FLXR4 = 69,
     180  STACKAPFLX_C1FLXR4ERR = 70,
     181  STACKAPFLX_C1FLXR4VAR = 71,
     182  STACKAPFLX_C1FLXR5 = 72,
     183  STACKAPFLX_C1FLXR5ERR = 73,
     184  STACKAPFLX_C1FLXR5VAR = 74,
     185  STACKAPFLX_C1FLXR6 = 75,
     186  STACKAPFLX_C1FLXR6ERR = 76,
     187  STACKAPFLX_C1FLXR6VAR = 77,
     188  STACKAPFLX_C1FLXR7 = 78,
     189  STACKAPFLX_C1FLXR7ERR = 79,
     190  STACKAPFLX_C1FLXR7VAR = 80,
     191  STACKAPFLX_C1FLXR8 = 81,
     192  STACKAPFLX_C1FLXR8ERR = 82,
     193  STACKAPFLX_C1FLXR8VAR = 83,
     194  STACKAPFLX_C1FLXR9 = 84,
     195  STACKAPFLX_C1FLXR9ERR = 85,
     196  STACKAPFLX_C1FLXR9VAR = 86,
     197  STACKAPFLX_C1FLXR10 = 87,
     198  STACKAPFLX_C1FLXR10ERR = 88,
     199  STACKAPFLX_C1FLXR10VAR = 89,
     200  STACKAPFLX_C2FLXR1 = 90,
     201  STACKAPFLX_C2FLXR1ERR = 91,
     202  STACKAPFLX_C2FLXR1VAR = 92,
     203  STACKAPFLX_C2FLXR2 = 93,
     204  STACKAPFLX_C2FLXR2ERR = 94,
     205  STACKAPFLX_C2FLXR2VAR = 95,
     206  STACKAPFLX_C2FLXR3 = 96,
     207  STACKAPFLX_C2FLXR3ERR = 97,
     208  STACKAPFLX_C2FLXR3VAR = 98,
     209  STACKAPFLX_C2FLXR4 = 99,
     210  STACKAPFLX_C2FLXR4ERR = 100,
     211  STACKAPFLX_C2FLXR4VAR = 101,
     212  STACKAPFLX_C2FLXR5 = 102,
     213  STACKAPFLX_C2FLXR5ERR = 103,
     214  STACKAPFLX_C2FLXR5VAR = 104,
     215  STACKAPFLX_C2FLXR6 = 105,
     216  STACKAPFLX_C2FLXR6ERR = 106,
     217  STACKAPFLX_C2FLXR6VAR = 107,
     218  STACKAPFLX_C2FLXR7 = 108,
     219  STACKAPFLX_C2FLXR7ERR = 109,
     220  STACKAPFLX_C2FLXR7VAR = 110,
     221  STACKAPFLX_C2FLXR8 = 111,
     222  STACKAPFLX_C2FLXR8ERR = 112,
     223  STACKAPFLX_C2FLXR8VAR = 113,
     224  STACKAPFLX_C2FLXR9 = 114,
     225  STACKAPFLX_C2FLXR9ERR = 115,
     226  STACKAPFLX_C2FLXR9VAR = 116,
     227  STACKAPFLX_C2FLXR10 = 117,
     228  STACKAPFLX_C2FLXR10ERR = 118,
     229  STACKAPFLX_C2FLXR10VAR = 119,
     230  STACKAPFLX_C3FLXR1 = 120,
     231  STACKAPFLX_C3FLXR1ERR = 121,
     232  STACKAPFLX_C3FLXR1VAR = 122,
     233  STACKAPFLX_C3FLXR2 = 123,
     234  STACKAPFLX_C3FLXR2ERR = 124,
     235  STACKAPFLX_C3FLXR2VAR = 125,
     236  STACKAPFLX_C3FLXR3 = 126,
     237  STACKAPFLX_C3FLXR3ERR = 127,
     238  STACKAPFLX_C3FLXR3VAR = 128,
     239  STACKAPFLX_C3FLXR4 = 129,
     240  STACKAPFLX_C3FLXR4ERR = 130,
     241  STACKAPFLX_C3FLXR4VAR = 131,
     242  STACKAPFLX_C3FLXR5 = 132,
     243  STACKAPFLX_C3FLXR5ERR = 133,
     244  STACKAPFLX_C3FLXR5VAR = 134,
     245  STACKAPFLX_C3FLXR6 = 135,
     246  STACKAPFLX_C3FLXR6ERR = 136,
     247  STACKAPFLX_C3FLXR6VAR = 137,
     248  STACKAPFLX_C3FLXR7 = 138,
     249  STACKAPFLX_C3FLXR7ERR = 139,
     250  STACKAPFLX_C3FLXR7VAR = 140,
     251  STACKAPFLX_C3FLXR8 = 141,
     252  STACKAPFLX_C3FLXR8ERR = 142,
     253  STACKAPFLX_C3FLXR8VAR = 143,
     254  STACKAPFLX_C3FLXR9 = 144,
     255  STACKAPFLX_C3FLXR9ERR = 145,
     256  STACKAPFLX_C3FLXR9VAR = 146,
     257  STACKAPFLX_C3FLXR10 = 147,
     258  STACKAPFLX_C3FLXR10ERR = 148,
     259  STACKAPFLX_C3FLXR10VAR = 149,
     260  STACKAPFLX_LOGC = 150,
     261  STACKAPFLX_LOGA = 151,
     262  STACKAPFLX_ACTIVEFLAG = 152,
     263  STACKAPFLX_DATARELEASE = 153,
     264} StackApFlx;
     265
     266typedef enum {
     267  STACKMODELFIT_OBJID = 1,
     268  STACKMODELFIT_STACKDETECTID = 2,
     269  STACKMODELFIT_IPPOBJID = 3,
     270  STACKMODELFIT_IPPDETECTID = 4,
     271  STACKMODELFIT_FILTERID = 5,
     272  STACKMODELFIT_STACKTYPEID = 6,
     273  STACKMODELFIT_SURVEYID = 7,
     274  STACKMODELFIT_PRIMARYF = 8,
     275  STACKMODELFIT_STACKMETAID = 9,
     276  STACKMODELFIT_DEVRADIUS = 10,
     277  STACKMODELFIT_DEVRADIUSERR = 11,
     278  STACKMODELFIT_DEVMAG = 12,
     279  STACKMODELFIT_DEVMAGERR = 13,
     280  STACKMODELFIT_DEVAB = 14,
     281  STACKMODELFIT_DEVABERR = 15,
     282  STACKMODELFIT_RADEVOFF = 16,
     283  STACKMODELFIT_DECDEVOFF = 17,
     284  STACKMODELFIT_RADEVOFFERR = 18,
     285  STACKMODELFIT_DECDEVOFFERR = 19,
     286  STACKMODELFIT_DEVCF = 20,
     287  STACKMODELFIT_DEVLIKELIHOOD = 21,
     288  STACKMODELFIT_DEVCOVAR11 = 22,
     289  STACKMODELFIT_DEVCOVAR12 = 23,
     290  STACKMODELFIT_DEVCOVAR13 = 24,
     291  STACKMODELFIT_DEVCOVAR14 = 25,
     292  STACKMODELFIT_DEVCOVAR15 = 26,
     293  STACKMODELFIT_DEVCOVAR16 = 27,
     294  STACKMODELFIT_DEVCOVAR22 = 28,
     295  STACKMODELFIT_DEVCOVAR23 = 29,
     296  STACKMODELFIT_DEVCOVAR24 = 30,
     297  STACKMODELFIT_DEVCOVAR25 = 31,
     298  STACKMODELFIT_DEVCOVAR26 = 32,
     299  STACKMODELFIT_DEVCOVAR33 = 33,
     300  STACKMODELFIT_DEVCOVAR34 = 34,
     301  STACKMODELFIT_DEVCOVAR35 = 35,
     302  STACKMODELFIT_DEVCOVAR36 = 36,
     303  STACKMODELFIT_DEVCOVAR44 = 37,
     304  STACKMODELFIT_DEVCOVAR45 = 38,
     305  STACKMODELFIT_DEVCOVAR46 = 39,
     306  STACKMODELFIT_DEVCOVAR55 = 40,
     307  STACKMODELFIT_DEVCOVAR56 = 41,
     308  STACKMODELFIT_DEVCOVAR66 = 42,
     309  STACKMODELFIT_EXPRADIUS = 43,
     310  STACKMODELFIT_EXPRADIUSERR = 44,
     311  STACKMODELFIT_EXPMAG = 45,
     312  STACKMODELFIT_EXPMAGERR = 46,
     313  STACKMODELFIT_EXPAB = 47,
     314  STACKMODELFIT_EXPABERR = 48,
     315  STACKMODELFIT_RAEXPOFF = 49,
     316  STACKMODELFIT_DECEXPOFF = 50,
     317  STACKMODELFIT_RAEXPOFFERR = 51,
     318  STACKMODELFIT_DECEXPOFFERR = 52,
     319  STACKMODELFIT_EXPCF = 53,
     320  STACKMODELFIT_EXPLIKELIHOOD = 54,
     321  STACKMODELFIT_EXPCOVAR11 = 55,
     322  STACKMODELFIT_EXPCOVAR12 = 56,
     323  STACKMODELFIT_EXPCOVAR13 = 57,
     324  STACKMODELFIT_EXPCOVAR14 = 58,
     325  STACKMODELFIT_EXPCOVAR15 = 59,
     326  STACKMODELFIT_EXPCOVAR16 = 60,
     327  STACKMODELFIT_EXPCOVAR22 = 61,
     328  STACKMODELFIT_EXPCOVAR23 = 62,
     329  STACKMODELFIT_EXPCOVAR24 = 63,
     330  STACKMODELFIT_EXPCOVAR25 = 64,
     331  STACKMODELFIT_EXPCOVAR26 = 65,
     332  STACKMODELFIT_EXPCOVAR33 = 66,
     333  STACKMODELFIT_EXPCOVAR34 = 67,
     334  STACKMODELFIT_EXPCOVAR35 = 68,
     335  STACKMODELFIT_EXPCOVAR36 = 69,
     336  STACKMODELFIT_EXPCOVAR44 = 70,
     337  STACKMODELFIT_EXPCOVAR45 = 71,
     338  STACKMODELFIT_EXPCOVAR46 = 72,
     339  STACKMODELFIT_EXPCOVAR55 = 73,
     340  STACKMODELFIT_EXPCOVAR56 = 74,
     341  STACKMODELFIT_EXPCOVAR66 = 75,
     342  STACKMODELFIT_SERRADIUS = 76,
     343  STACKMODELFIT_SERRADIUSERR = 77,
     344  STACKMODELFIT_SERMAG = 78,
     345  STACKMODELFIT_SERMAGERR = 79,
     346  STACKMODELFIT_SERAB = 80,
     347  STACKMODELFIT_SERABERR = 81,
     348  STACKMODELFIT_SERNU = 82,
     349  STACKMODELFIT_SERNUERR = 83,
     350  STACKMODELFIT_RASEROFF = 84,
     351  STACKMODELFIT_DECSEROFF = 85,
     352  STACKMODELFIT_RASEROFFERR = 86,
     353  STACKMODELFIT_DECSEROFFERR = 87,
     354  STACKMODELFIT_SERCF = 88,
     355  STACKMODELFIT_SERLIKELIHOOD = 89,
     356  STACKMODELFIT_SERSICCOVAR11 = 90,
     357  STACKMODELFIT_SERSICCOVAR12 = 91,
     358  STACKMODELFIT_SERSICCOVAR13 = 92,
     359  STACKMODELFIT_SERSICCOVAR14 = 93,
     360  STACKMODELFIT_SERSICCOVAR15 = 94,
     361  STACKMODELFIT_SERSICCOVAR16 = 95,
     362  STACKMODELFIT_SERSICCOVAR17 = 96,
     363  STACKMODELFIT_SERSICCOVAR22 = 97,
     364  STACKMODELFIT_SERSICCOVAR23 = 98,
     365  STACKMODELFIT_SERSICCOVAR24 = 99,
     366  STACKMODELFIT_SERSICCOVAR25 = 100,
     367  STACKMODELFIT_SERSICCOVAR26 = 101,
     368  STACKMODELFIT_SERSICCOVAR27 = 102,
     369  STACKMODELFIT_SERSICCOVAR33 = 103,
     370  STACKMODELFIT_SERSICCOVAR34 = 104,
     371  STACKMODELFIT_SERSICCOVAR35 = 105,
     372  STACKMODELFIT_SERSICCOVAR36 = 106,
     373  STACKMODELFIT_SERSICCOVAR37 = 107,
     374  STACKMODELFIT_SERSICCOVAR44 = 108,
     375  STACKMODELFIT_SERSICCOVAR45 = 109,
     376  STACKMODELFIT_SERSICCOVAR46 = 110,
     377  STACKMODELFIT_SERSICCOVAR47 = 111,
     378  STACKMODELFIT_SERSICCOVAR55 = 112,
     379  STACKMODELFIT_SERSICCOVAR56 = 113,
     380  STACKMODELFIT_SERSICCOVAR57 = 114,
     381  STACKMODELFIT_SERSICCOVAR66 = 115,
     382  STACKMODELFIT_SERSICCOVAR67 = 116,
     383  STACKMODELFIT_SERSICCOVAR77 = 117,
     384  STACKMODELFIT_ACTIVEFLAG = 118,
     385  STACKMODELFIT_DATARELEASE = 119,
     386} StackModelFit;
     387
     388typedef enum {
     389  STACKTOIMAGE_STACKMETAID = 1,
     390  STACKTOIMAGE_IMAGEID = 2,
     391} StackToImage;
    169392
    170393#endif

  • branches/eam_branches/ipp-20100823/ippTools/configure.ac

    r29124 r29515  
    11AC_PREREQ(2.61)
    22
    3 AC_INIT([ipptools], [1.1.63], [ipp-support@ifa.hawaii.edu])
     3AC_INIT([ipptools], [1.1.65], [ipp-support@ifa.hawaii.edu])
    44AC_CONFIG_SRCDIR([autogen.sh])
    55
    … …  
    1818PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
    1919PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
    20 PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.63])
     20PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.65])
    2121PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1])
    2222

  • branches/eam_branches/ipp-20100823/ippTools/share/Makefile.am

    r28959 r29515  
    6161        chiptool_pendingimfile.sql \
    6262        chiptool_processedimfile.sql \
     63        chiptool_revertcleanup.sql \
    6364        chiptool_revertprocessedimfile.sql \
    6465        chiptool_revertupdatedimfile.sql \
    … …  
    138139        difftool_inputskyfile.sql \
    139140        difftool_listrun.sql \
     141        difftool_listssrun.sql \
    140142        difftool_pendingcleanuprun.sql \
    141143        difftool_pendingcleanupskyfile.sql \
    142144        difftool_revertdiffskyfile_delete.sql \
    143145        difftool_revertdiffskyfile_updated.sql \
     146        difftool_revertcleanup.sql \
    144147        difftool_setskyfiletoupdate.sql \
    145148        difftool_skyfile.sql \
    … …  
    274277        pstamptool_revertreq.sql \
    275278        pstamptool_revertreq_deletejobs.sql \
     279        pstamptool_stopdependentjob.sql \
    276280        pstamptool_updatejob.sql \
    277281        pxadmin_create_tables.sql \
    … …  
    356360        warptool_revertwarped_delete.sql \
    357361        warptool_revertwarped_updated.sql \
     362        warptool_revertcleanup.sql \
    358363        warptool_runstate.sql \
    359364        warptool_scmap.sql \

  • branches/eam_branches/ipp-20100823/ippTools/share/addtool_find_cam_id.sql

    r26915 r29515  
    22    camRun.*
    33FROM camRun
     4JOIN camProcessedExp
     5    USING(cam_id)
    46JOIN chipRun
    57    USING(chip_id)
    … …  
    1012           FROM addRun
    1113           JOIN camRun USING(cam_id)
    12            JOIN chipRun USING(chip_id)
     14           JOIN chipRun USING(chip_id)
    1315          ) as foo
    1416     ON exp_id = added_exp_id
    … …  
    1719WHERE
    1820    camRun.state = 'full'
     21    AND camProcessedExp.quality = 0
    1922    AND added_exp_id IS NULL
    2023    -- addtool adds checks on exposure being added to the dvodb previously

  • branches/eam_branches/ipp-20100823/ippTools/share/addtool_find_cam_id_dvo.sql

    r28856 r29515  
    11SELECT camRun.* FROM camRun
     2JOIN camProcessedExp USING(cam_id)
    23JOIN chipRun USING(chip_id)
    34JOIN rawExp USING(exp_id)
    4 WHERE camRun.state = 'full'
     5WHERE camRun.state = 'full' and camProcessedExp.quality = 0
    56    AND exp_id NOT IN (SELECT exp_id
    67       FROM addRun

  • branches/eam_branches/ipp-20100823/ippTools/share/chiptool_listrun.sql

    r28586 r29515  
    2929LEFT JOIN camProcessedExp USING(cam_id)
    3030LEFT JOIN magicDSRun
    31     ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place
     31    ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place AND magicDSRun.state != 'drop'

  • branches/eam_branches/ipp-20100823/ippTools/share/chiptool_processedimfile.sql

    r28587 r29515  
    4242    AND chipProcessedImfile.class_id = rawImfile.class_id
    4343LEFT JOIN magicDSRun
    44     ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place
     44    ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place AND magicDSRun.state != 'drop'

  • branches/eam_branches/ipp-20100823/ippTools/share/difftool_listrun.sql

    r27743 r29515  
    1212    diffRun.bothways,
    1313    warp1,
    14     stack1,
    1514    warp2,
    16     stack2,
    1715    -- The following are only valid for warps
    1816    -- XXX This needs to be more clever to handle diffs between stacks
    … …  
    2725    rawInput.exp_name AS exp_name_1,
    2826    rawInput.exp_id AS exp_id_1,
     27    rawInput.comment AS comment_1,
     28    rawInput.dateobs AS dateobs_1,
    2929    chipInput.chip_id AS chip_id_1,
    3030    camInput.cam_id AS cam_id_1,
    … …  
    3434    rawTemplate.exp_name AS exp_name_2,
    3535    rawTemplate.exp_id AS exp_id_2,
     36    rawTemplate.comment AS comment_2,
     37    rawTemplate.dateobs AS dateobs_2,
    3638    chipTemplate.chip_id AS chip_id_2,
    3739    camTemplate.cam_id AS cam_id_2,

  • branches/eam_branches/ipp-20100823/ippTools/share/difftool_todiffskyfile.sql

    r27920 r29515  
    1313    diffRun.bothways,
    1414    diffRun.diff_mode,
    15     diffSkyfile.path_base
     15    diffSkyfile.path_base,
     16    IFNULL(priority, 10000) AS priority
    1617FROM diffRun
    1718JOIN diffInputSkyfile USING(diff_id)
    … …  
    6263    ON diffInputSkyfile.diff_id = diffSkyfile.diff_id
    6364    AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id
     65LEFT JOIN Label ON Label.label = diffRun.label
    6466WHERE
    6567-- Ready to be processed
    … …  
    7072    AND diffSkyfile.data_state = 'update')
    7173    )
     74    AND (Label.active OR Label.active IS NULL)
    7275-- Ensure input warps are available
    7376    AND (diffInputSkyfile.warp1 IS NULL
    … …  
    9396    AND stackTemplateSkyfile.quality = 0))
    9497
    95 
    96 

  • branches/eam_branches/ipp-20100823/ippTools/share/disttool_pending_camera.sql

    r26567 r29515  
    3030    AND distRun.stage = 'camera'
    3131    AND distComponent.dist_id IS NULL
    32     AND (((chipRun.magicked > 0) AND (camRun.magicked > 0)) OR distRun.no_magic)
     32    AND (((clean OR (chipRun.magicked > 0)) AND (camRun.magicked > 0)) OR distRun.no_magic)
    3333    AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))

  • branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_completed_runs.sql

    r25465 r29515  
    11SELECT DISTINCT
    22    magic_ds_id,
     3    magicked,
    34    re_place,
    45    label
    … …  
    78-- raw stage
    89SELECT
    9     magicDSRun.*
     10    magicDSRun.*,
     11    rawExp.magicked
    1012    FROM magicDSRun
    1113    JOIN rawImfile ON stage_id = rawImfile.exp_id
     14    JOIN rawExp using(exp_id)
    1215    LEFT JOIN magicDSFile
    1316        ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    … …  
    2528-- chip stage
    2629SELECT
    27     magicDSRun.*
     30    magicDSRun.*,
     31    chipRun.magicked
    2832    FROM magicDSRun
    29     JOIN chipProcessedImfile ON stage_id = chip_id
     33    JOIN chipRun ON stage_id = chip_id
     34    JOIN chipProcessedImfile USING(chip_id)
    3035    LEFT JOIN magicDSFile
    3136        ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id
    … …  
    4449-- camera stage
    4550SELECT
    46     magicDSRun.*
     51    magicDSRun.*,
     52    camRun.magicked
    4753    FROM magicDSRun
    48     JOIN camProcessedExp ON stage_id = camProcessedExp.cam_id
     54    JOIN camRun ON stage_id = camRun.cam_id
     55    JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id
    4956    LEFT JOIN magicDSFile
    5057        ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id
    … …  
    6168-- warp stage
    6269SELECT
    63     magicDSRun.*
     70    magicDSRun.*,
     71    warpRun.magicked
    6472    FROM magicDSRun
    65     JOIN warpSkyfile on stage_id = warp_id
     73    JOIN warpRun on stage_id = warp_id
     74    JOIN warpSkyfile USING(warp_id)
    6675    LEFT JOIN magicDSFile
    6776        ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    … …  
    8190-- diff stage
    8291SELECT DISTINCT
    83     magicDSRun.*
     92    magicDSRun.*,
     93    diffRun.magicked
    8494    FROM magicDSRun
    8595    JOIN magicRun USING (magic_id)
    8696    JOIN magicInputSkyfile USING(magic_id)
     97    JOIN diffRun
     98        ON magicRun.diff_id = diffRun.diff_id
    8799    JOIN diffSkyfile
    88         ON magicRun.diff_id = diffSkyfile.diff_id
     100        ON diffRun.diff_id = diffSkyfile.diff_id
    89101        AND magicInputSkyfile.node = diffSkyfile.skycell_id
    90102    LEFT JOIN magicDSFile

  • branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_camera.sql

    r27945 r29515  
    77    camera,
    88    magicMask.uri AS streaks_uri,
     9    magicMask.path_base AS streaks_path_base,
    910    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     11    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    1012    stage,
    1113    stage_id,
    … …  
    1921    recoveryroot,
    2022    re_place,
    21     remove
     23    remove,
     24    IFNULL(Label.priority, 10000) AS priority
    2225FROM magicDSRun
    2326JOIN magicMask USING (magic_id)
    … …  
    2932LEFT JOIN magicDSFile
    3033    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
     34LEFT JOIN Label ON magicDSRun.label = Label.label
    3135WHERE
    3236    magicDSRun.state = 'new'
    … …  
    3842    AND camProcessedExp.quality = 0
    3943    AND magicDSFile.component IS NULL
     44    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_chip.sql

    r27945 r29515  
    77    camera,
    88    magicMask.uri AS streaks_uri,
     9    magicMask.path_base AS streaks_path_base,
    910    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     11    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    1012    stage,
    1113    stage_id,
    … …  
    1921    recoveryroot,
    2022    re_place,
    21     remove
     23    remove,
     24    IFNULL(Label.priority, 10000) AS priority
    2225FROM magicDSRun
    2326JOIN magicMask USING (magic_id)
    … …  
    3134    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    3235    AND magicDSFile.component = chipProcessedImfile.class_id
     36LEFT JOIN Label
     37    ON magicDSRun.label = Label.label
    3338WHERE
    3439    magicDSRun.state = 'new'
    … …  
    3843    AND chipProcessedImfile.quality = 0
    3944    AND magicDSFile.component IS NULL
     45    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_diff.sql

    r27945 r29515  
    77    rawExp.camera,
    88    magicMask.uri AS streaks_uri,
     9    magicMask.path_base AS streaks_path_base,
    910    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     11    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    1012    stage,
    1113    magicRun.diff_id AS stage_id,
    … …  
    2022    recoveryroot,
    2123    re_place,
    22     remove
     24    remove,
     25    IFNULL(Label.priority, 10000) AS priority
    2326FROM rawExp
    2427JOIN magicRun USING (exp_id)
    … …  
    3336    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    3437    AND magicDSFile.component = diffSkyfile.skycell_id
     38LEFT JOIN Label ON magicDSRun.label = Label.label
    3539WHERE
    3640    magicDSRun.state = 'new'
    … …  
    4044    AND diffSkyfile.quality = 0
    4145    AND magicDSFile.component IS NULL
     46    AND (Label.active OR Label.active IS NULL)
    4247-- bothways diffSkyfiles
    4348UNION
    … …  
    4954    rawExp.camera,
    5055    magicMask.uri AS streaks_uri,
     56    magicMask.path_base AS streaks_path_base,
    5157    (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri,
     58    (SELECT path_base from magicMask where magic_id = inv_magic_id) AS inv_streaks_path_base,
    5259    stage,
    5360    magicRun.diff_id AS stage_id,
    … …  
    6269    recoveryroot,
    6370    re_place,
    64     remove
     71    remove,
     72    IFNULL(Label.priority, 10000) AS priority
    6573FROM rawExp
    6674JOIN magicRun USING (exp_id)
    … …  
    7583    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    7684    AND magicDSFile.component = diffSkyfile.skycell_id
     85LEFT JOIN Label ON magicDSRun.label = Label.label
    7786WHERE
    7887    magicDSRun.state = 'new'
    … …  
    8291    AND diffSkyfile.quality = 0
    8392    AND magicDSFile.component IS NULL
     93    AND (Label.active OR Label.active IS NULL)
    8494) AS magicDSRun
    8595-- we need the following so this query is compatible with the other stages

  • branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_raw.sql

    r27945 r29515  
    66    rawExp.camera,
    77    magicMask.uri as streaks_uri,
     8    magicMask.path_base as streaks_path_base,
    89    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     10    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    911    stage,
    1012    stage_id,
    … …  
    2022    recoveryroot,
    2123    re_place,
    22     remove
     24    remove,
     25    10000 AS priority
    2326FROM magicDSRun
    2427JOIN magicMask USING (magic_id)

  • branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_warp.sql

    r27945 r29515  
    66    camera,
    77    magicMask.uri as streaks_uri,
     8    magicMask.path_base as streaks_path_base,
    89    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     10    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    911    stage,
    1012    stage_id,
    … …  
    1921    recoveryroot,
    2022    re_place,
    21     remove
     23    remove,
     24    IFNULL(Label.priority, 10000) AS priority
    2225FROM magicDSRun
    2326JOIN magicMask USING (magic_id)
    … …  
    2932    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    3033    AND magicDSFile.component = warpSkyfile.skycell_id
     34LEFT JOIN Label ON magicDSRun.label = Label.label
    3135WHERE
    3236    magicDSRun.state = 'new'
    … …  
    3640    AND warpSkyfile.quality = 0
    3741    AND magicDSFile.component IS NULL
     42    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20100823/ippTools/share/magictool_definebyquery_select.sql

    r28765 r29515  
    1515    FROM diffRun
    1616    JOIN diffInputSkyfile USING(diff_id)
     17    JOIN diffSkyfile USING(diff_id, skycell_id)
    1718    WHERE diffInputSkyfile.warp1 IS NOT NULL
    1819        AND diffRun.exposure = 1
    1920        AND diffRun.magicked = 0
     21        AND diffSkyfile.quality = 0
    2022    -- diff WHERE hook %s
    2123    UNION
    … …  
    2830    FROM diffRun
    2931    JOIN diffInputSkyfile USING(diff_id)
     32    JOIN diffSkyfile USING(diff_id, skycell_id)
    3033    WHERE diffInputSkyfile.warp2 IS NOT NULL
    3134        AND diffRun.exposure = 1
    3235        AND diffRun.bothways = 1
    3336        AND diffRun.magicked = 0
     37        AND diffSkyfile.quality = 0
    3438    -- diff WHERE hook %s
    3539    ) AS diffWarps

  • branches/eam_branches/ipp-20100823/ippTools/share/magictool_revertnode.sql

    r23660 r29515  
    33JOIN magicRun USING(magic_id)
    44WHERE magicNodeResult.fault != 0
     5    AND magicRun.state = 'new'

  • branches/eam_branches/ipp-20100823/ippTools/share/magictool_toprocess_inputs.sql

    r28768 r29515  
    33    magicRun.workdir,
    44    rawExp.exp_id,
     5    rawExp.exp_name,
     6    rawExp.workdir AS raw_workdir,
    57    rawExp.camera,
    68    -- convert magic_id into a boolean value (1 or 0)

  • branches/eam_branches/ipp-20100823/ippTools/share/magictool_toprocess_tree.sql

    r28768 r29515  
    33    magicRun.workdir,
    44    rawExp.exp_id,
     5    rawExp.exp_name,
     6    rawExp.workdir AS raw_workdir,
    57    rawExp.camera,
    68    -- convert magic_id into a boolean value (1 or 0)
    … …  
    1618WHERE
    1719    magicRun.state = 'new'
     20    AND (Label.active OR Label.active IS NULL)
    1821-- WHERE hook %s
    1922ORDER BY

  • branches/eam_branches/ipp-20100823/ippTools/share/magictool_totree.sql

    r28768 r29515  
    1818    AND magicTree.node IS NULL
    1919    AND magicRun.fault = 0
     20    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_revertdependent.sql

    r28146 r29515  
    33    JOIN pstampRequest USING(req_id)
    44SET pstampDependent.fault = 0
     5-- fault count hook %s
    56WHERE pstampDependent.state = 'new'
    67    AND (pstampDependent.fault > 0)

  • branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_revertjob.sql

    r26213 r29515  
    22    JOIN pstampRequest USING(req_id)
    33SET pstampJob.fault = 0
     4 -- clear fault count clause goes here %s
    45WHERE  pstampRequest.state = 'run'
    56    AND pstampJob.state = 'run'

  • branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_updatejob.sql

    r27874 r29515  
    1 UPDATE pstampJob LEFT JOIN pstampDependent USING(dep_id)
     1UPDATE pstampJob
     2    JOIN pstampRequest USING(req_id)
     3    LEFT JOIN pstampDependent USING(dep_id)
    24SET

  • branches/eam_branches/ipp-20100823/ippTools/share/pubtool_revert.sql

    r25929 r29515  
    22USING publishDone, publishRun, publishClient
    33WHERE publishDone.pub_id = publishRun.pub_id
     4    AND publishRun.state = 'new'
    45    AND publishRun.client_id = publishClient.client_id
    56    AND publishClient.active = 1

  • branches/eam_branches/ipp-20100823/ippTools/share/pxadmin_create_tables.sql

    r28874 r29515  
    12211221CREATE TABLE magicInputSkyfile (
    12221222        magic_id BIGINT,
    1223         diff_id BIGINT,
    12241223        node VARCHAR(64),
    1225         PRIMARY KEY(magic_id, diff_id, node),
    1226         FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id),
    1227         FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
     1224        PRIMARY KEY(magic_id, node),
     1225        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id)
    12281226) ENGINE=innodb DEFAULT CHARSET=latin1;
    12291227
    … …  
    12531251        magic_id BIGINT,
    12541252        uri VARCHAR(255),
     1253        path_base VARCHAR(255),
    12551254        streaks INT,
    12561255        fault SMALLINT,
    … …  
    14411440        imagedb    VARCHAR(64),
    14421441        rlabel     VARCHAR(64),
     1442        need_magic TINYINT,
    14431443        outdir     VARCHAR(255),
    1444         need_magic TINYINT,
     1444        fault      SMALLINT,
     1445        fault_count INT,
    14451446        PRIMARY KEY(dep_id),
    14461447        KEY(state),
    14471448        KEY(stage),
    1448         KEY(stage_id)
     1449        KEY(stage_id),
     1450        KEY(fault)
    14491451) ENGINE=innodb DEFAULT CHARSET=latin1;
    14501452
    … …  
    18371839    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
    18381840    note VARCHAR(255),             -- note
     1841    magicked BIGINT NOT NULL DEFAULT 0, -- magic mask applied
    18391842    PRIMARY KEY(diff_phot_id),
    18401843    KEY(diff_id),

  • branches/eam_branches/ipp-20100823/ippTools/share/stacktool_tosum.sql

    r27929 r29515  
    77    stackRun.label,
    88    stackRun.state,
    9     stackSumSkyfile.path_base
     9    stackSumSkyfile.path_base,
     10    IFNULL(Label.priority, 10000) AS priority
    1011FROM stackRun
    1112JOIN stackInputSkyfile USING(stack_id)
    1213JOIN warpRun USING(warp_id)
    1314LEFT JOIN stackSumSkyfile USING(stack_id)
     15LEFT JOIN Label ON Label.label = stackRun.label
    1416WHERE
    1517    ((stackRun.state = 'new' AND stackSumSkyfile.stack_id IS NULL)
    1618    OR (stackRun.state = 'update' AND stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0))
     19    AND (Label.active OR Label.active IS NULL)
    1720    -- WHERE hook %s
    1821GROUP BY stack_id

  • branches/eam_branches/ipp-20100823/ippTools/share/warptool_scmap.sql

    r27737 r29515  
    77    chipRun.state,
    88    chipProcessedImfile.data_state,
     9    chipProcessedImfile.fault AS chip_fault,
    910    chipProcessedImfile.magicked,
    10     rawImfile.magicked AS raw_magicked
     11    rawImfile.magicked AS raw_magicked,
     12    IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id,
     13    IFNULL(magicDSRun.state, 0) AS dsRun_state,
     14    IFNULL(magicDSFile.fault, 0) as dsFile_fault
    1115FROM warpRun
    1216JOIN warpSkyCellMap
    … …  
    2630    ON chipRun.exp_id = rawImfile.exp_id
    2731    AND chipProcessedImfile.class_id = rawImfile.class_id
     32LEFT JOIN magicDSRun
     33    ON chipRun.chip_id = magicDSRun.stage_id AND magicDSRun.stage = 'chip'
     34LEFT JOIN magicDSFile
     35    ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id
     36        AND chipProcessedImfile.class_id = magicDSFile.component
    2837WHERE
    2938    1

  • branches/eam_branches/ipp-20100823/ippTools/share/warptool_towarped.sql

    r28082 r29515  
    9797        AND warpSkyfile.fault = 0
    9898        AND camRun.state = 'full'
    99         AND chipProcessedImfile.data_state = 'full'
    100         -- if warpSkyfile was magicked previously require inputs to be magicked
    101         -- this blocks processing until all the chip inputs have been destreaked
    102         AND (warpSkyfile.magicked = 0 OR chipProcessedImfile.magicked >= 0)
    10399        AND (Label.active OR Label.active IS NULL)
    104100        -- where hook 2 %s
    105101    GROUP BY warp_id, skycell_id
    106     HAVING COUNT(warpSkyCellMap.class_id) = COUNT(chipProcessedImfile.class_id)
     102    -- if warpSkyfile was magicked previously require inputs to be magicked
     103    -- this blocks processing until all the chip inputs have been destreaked
     104    HAVING COUNT(warpSkyCellMap.class_id) = SUM(IF(chipProcessedImfile.data_state ='full' and (chipRun.magicked = 0 OR chipProcessedImfile.magicked > 0), 1, 0))
    107105    -- limit hook 2 %s
    108106    )

  • branches/eam_branches/ipp-20100823/ippTools/src/addtool.c

    r28856 r29515  
    493493
    494494    // since there is only one exp per 'new' set addRun.state = 'full'
    495     if (!pxaddRunSetState(config, row->add_id, "full")) {
     495    // but check to make sure there are no faults
     496
     497    if (!fault) {
     498      if (!pxaddRunSetState(config, row->add_id, "full")) {
    496499        psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id);
    497500        psFree(row);
    498501        psFree(pendingRow);
    499502        return false;
     503      }
    500504    }
    501505    psFree(row);

  • branches/eam_branches/ipp-20100823/ippTools/src/bgtool.c

    r28941 r29515  
    658658    }
    659659
    660     if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script,
     660    if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, "full", magicked, dtime_script,
    661661                                    hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
    662662                                    maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
    … …  
    14971497    }
    14981498
    1499     if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script,
     1499    if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, "full", magicked, dtime_script,
    15001500                                    hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
    15011501                                    maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {

  • branches/eam_branches/ipp-20100823/ippTools/src/chiptool.c

    r28716 r29515  
    5050static bool pendingcleanuprunMode(pxConfig *config);
    5151static bool pendingcleanupimfileMode(pxConfig *config);
     52static bool revertcleanupMode(pxConfig *config);
    5253static bool donecleanupMode(pxConfig *config);
    5354static bool runMode(pxConfig *config);
    … …  
    9495        MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPRUN,       pendingcleanuprunMode);
    9596        MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPIMFILE,    pendingcleanupimfileMode);
     97        MODECASE(CHIPTOOL_MODE_REVERTCLEANUP,           revertcleanupMode);
    9698        MODECASE(CHIPTOOL_MODE_DONECLEANUP,             donecleanupMode);
    9799        MODECASE(CHIPTOOL_MODE_RUN,                     runMode);
    … …  
    859861    }
    860862    psString query_update = pxDataGet("chiptool_revertupdatedimfile.sql");
    861     if (!query) {
     863    if (!query_update) {
    862864        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
    863865        psFree(where);
    864866        return false;
    865867    }
     868
    866869
    867870    if (where && psListLength(where->list)) {
    … …  
    886889    }
    887890    psFree(query_update);
     891
     892    return true;
     893}
     894static bool revertcleanupMode(pxConfig *config)
     895{
     896    PS_ASSERT_PTR_NON_NULL(config, false);
     897
     898    psMetadata *where = psMetadataAlloc();
     899    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
     900    pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE");
     901    pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE");
     902    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
     903
     904    char* newState = NULL;
     905    if (!state) {
     906        state = "error_cleaned";
     907    }
     908    if (!strcmp(state, "error_cleaned")) {
     909        newState = "goto_cleaned";
     910    } else if (!strcmp(state, "error_purged")) {
     911        newState = "goto_purged";
     912    } else if (!strcmp(state, "error_scrubbed")) {
     913        newState = "goto_scrubbed";
     914    } else {
     915        psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed");
     916        return false;
     917    }
     918
     919    if (!psListLength(where->list)) {
     920        psFree(where);
     921        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
     922        return false;
     923    }
     924
     925    psString query = pxDataGet("chiptool_revertcleanup.sql");
     926    if (!query) {
     927        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
     928        psFree(where);
     929        return false;
     930    }
     931
     932    if (where && psListLength(where->list)) {
     933        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     934        psStringAppend(&query, " AND %s", whereClause);
     935        psFree(whereClause);
     936    }
     937
     938    psFree(where);
     939
     940    if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) {
     941        psError(PS_ERR_UNKNOWN, false, "database error");
     942        psFree(query);
     943        return false;
     944    }
     945    psFree(query);
    888946
    889947    return true;

  • branches/eam_branches/ipp-20100823/ippTools/src/chiptool.h

    r27742 r29515  
    4242    CHIPTOOL_MODE_PENDINGCLEANUPRUN,
    4343    CHIPTOOL_MODE_PENDINGCLEANUPIMFILE,
     44    CHIPTOOL_MODE_REVERTCLEANUP,
    4445    CHIPTOOL_MODE_DONECLEANUP,
    4546    CHIPTOOL_MODE_RUN,

  • branches/eam_branches/ipp-20100823/ippTools/src/chiptoolConfig.c

    r28716 r29515  
    290290    psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    291291
     292    // -revertcleanup
     293    psMetadata *revertcleanupArgs = psMetadataAlloc();
     294    psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-chip_id", 0,            "search by chip ID", 0);
     295    psMetadataAddStr(revertcleanupArgs,  PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL);
     296    psMetadataAddStr(revertcleanupArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL);
     297    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state", 0,             "search by current state", NULL);
     298    psMetadataAddS16(revertcleanupArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
     299
    292300    // -run
    293301    psMetadata *runArgs = psMetadataAlloc();
    … …  
    371379    PXOPT_ADD_MODE("-pendingcleanupimfile", "show runs that need to be cleaned up", CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileArgs);
    372380    PXOPT_ADD_MODE("-donecleanup",          "show runs that have been cleaned",     CHIPTOOL_MODE_DONECLEANUP,          donecleanupArgs);
     381    PXOPT_ADD_MODE("-revertcleanup",        "show runs that have been cleaned",     CHIPTOOL_MODE_REVERTCLEANUP,        revertcleanupArgs);
    373382    PXOPT_ADD_MODE("-run",                  "show runs",                            CHIPTOOL_MODE_RUN,                  runArgs);
    374383    PXOPT_ADD_MODE("-tocleanedimfile",      "set imfile state to cleaned",          CHIPTOOL_MODE_TOCLEANEDIMFILE,      tocleanedimfileArgs);

  • branches/eam_branches/ipp-20100823/ippTools/src/difftool.c

    r29324 r29515  
    4848static bool pendingcleanuprunMode(pxConfig *config);
    4949static bool pendingcleanupskyfileMode(pxConfig *config);
     50static bool revertcleanupMode(pxConfig *config);
    5051static bool donecleanupMode(pxConfig *config);
    5152static bool updatediffskyfileMode(pxConfig *config);
    … …  
    6061static bool tofullskyfileMode(pxConfig *config);
    6162static bool listrunMode(pxConfig *config);
     63static bool listssrunMode(pxConfig *config);
    6264static bool setskyfiletoupdateMode(pxConfig *config);
    6365
    … …  
    99101        MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
    100102        MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
     103        MODECASE(DIFFTOOL_MODE_REVERTCLEANUP,         revertcleanupMode);
    101104        MODECASE(DIFFTOOL_MODE_DONECLEANUP,           donecleanupMode);
    102105        MODECASE(DIFFTOOL_MODE_UPDATEDIFFSKYFILE,     updatediffskyfileMode);
    … …  
    108111        MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE,         tofullskyfileMode);
    109112        MODECASE(DIFFTOOL_MODE_LISTRUN,               listrunMode);
     113        MODECASE(DIFFTOOL_MODE_LISTSSRUN,             listssrunMode);
    110114        MODECASE(DIFFTOOL_MODE_SETSKYFILETOUPDATE,    setskyfiletoupdateMode);
    111115
    … …  
    478482    }
    479483    psFree(where);
     484
     485     psStringAppend(&query, "\nORDER by priority DESC, diff_id");
    480486
    481487    // treat limit == 0 as "no limit"
    … …  
    12361242    PXOPT_COPY_F32(config->args, warp2Where,  "-good_frac", "warpSkyfile.good_frac", ">=");
    12371243    PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
     1244    PXOPT_COPY_STR(config->args, stackWhere, "-stack_data_group", "stackRun.data_group", "==");
    12381245
    12391246    PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
    … …  
    16091616    PXOPT_COPY_S64(config->args, selectWhere, "-warp_id", "inputWarpRun.warp_id", "==");
    16101617    PXOPT_COPY_S64(config->args, selectWhere, "-exp_id", "inputRawExp.exp_id", "==");
     1618    PXOPT_COPY_S64(config->args, selectWhere, "-template_exp_id", "templateRawExp.exp_id", "==");
    16111619    PXOPT_COPY_STR(config->args, selectWhere, "-filter", "inputRawExp.filter", "==");
    16121620    PXOPT_COPY_STR(config->args, selectWhere, "-obs_mode", "inputRawExp.obs_mode", "==");
    … …  
    16471655    PXOPT_COPY_F32(config->args,   selectWhere, "-sun_angle_min",      "inputRawExp.sun_angle",      ">=");
    16481656    PXOPT_COPY_F32(config->args,   selectWhere, "-sun_angle_max",      "inputRawExp.sun_angle",      "<");
    1649     PXOPT_COPY_STR(config->args,   selectWhere, "-comment",            "inputRawExp.comment",        "LIKE");
     1657    PXOPT_COPY_STR(config->args,   selectWhere, "-input_comment",      "inputRawExp.comment",        "LIKE");
     1658    PXOPT_COPY_STR(config->args,   selectWhere, "-template_comment",   "templateRawExp.comment",     "LIKE");
    16501659
    16511660    PXOPT_LOOKUP_BOOL(not_bothways, config->args, "-not-bothways", false);
    … …  
    16801689
    16811690    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     1691    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    16821692    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
    16831693
    … …  
    18391849    long numGood = 0;                   // Number of good rows added
    18401850    for (long i = 0; i < results->n; i++) {
     1851        if (limit && numGood >= limit) {
     1852            break;
     1853        }
    18411854        psMetadata *row = results->data[i]; // Result row from query
    18421855
    … …  
    25352548
    25362549
     2550static bool revertcleanupMode(pxConfig *config)
     2551{
     2552    PS_ASSERT_PTR_NON_NULL(config, false);
     2553
     2554    psMetadata *where = psMetadataAlloc();
     2555    PXOPT_COPY_S64(config->args, where, "-diff_id",    "diffSkyfile.diff_id", "==");
     2556    pxAddLabelSearchArgs (config, where, "-label",     "diffRun.label", "LIKE");
     2557    pxAddLabelSearchArgs (config, where, "-data_group",   "diffRun.data_group", "LIKE");
     2558
     2559    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
     2560
     2561    if (!psListLength(where->list)) {
     2562        psFree(where);
     2563        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
     2564        return false;
     2565    }
     2566    if (!state) {
     2567        state = "error_cleaned";
     2568    }
     2569    char *newState = NULL;
     2570    if (!strcmp(state, "error_cleaned")) {
     2571        newState = "goto_cleaned";
     2572    } else if (!strcmp(state, "error_purged")) {
     2573        newState = "goto_purged";
     2574    } else if (!strcmp(state, "error_scrubbed")) {
     2575        newState = "goto_scrubbed";
     2576    } else {
     2577        psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed");
     2578        return false;
     2579    }
     2580
     2581    psString query = pxDataGet("difftool_revertcleanup.sql");
     2582    if (!query) {
     2583        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
     2584        return false;
     2585    }
     2586
     2587    if (psListLength(where->list)) {
     2588        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     2589        psStringAppend(&query, " AND %s", whereClause);
     2590        psFree(whereClause);
     2591    }
     2592    psFree(where);
     2593
     2594    if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) {
     2595        psError(PS_ERR_UNKNOWN, false, "database error");
     2596        psFree(query);
     2597        return false;
     2598    }
     2599    psFree(query);
     2600
     2601    int numDeleted = psDBAffectedRows(config->dbh);
     2602
     2603    psLogMsg("difftool", PS_LOG_INFO, "Reverted %d diffRuns and diffSkyfiles", numDeleted);
     2604
     2605    return true;
     2606}
    25372607static bool donecleanupMode(pxConfig *config)
    25382608{
    … …  
    28772947
    28782948    psMetadata *where = psMetadataAlloc();
    2879     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffRun.diff_id", "==");
    2880     PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "==");
    2881     PXOPT_COPY_S64(config->args, where,  "-magicked", "diffRun.magicked", "==");
    2882     pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE");
     2949    PXOPT_COPY_S64(config->args, where,  "-diff_id",    "diffRun.diff_id",  "==");
     2950    PXOPT_COPY_STR(config->args, where,  "-tess_id",    "diffRun.tess_id",  "LIKE");
     2951    PXOPT_COPY_S64(config->args, where,  "-magicked",   "diffRun.magicked", "==");
     2952    PXOPT_COPY_STR(config->args, where,  "-state",      "diffRun.state",    "==");
     2953    pxAddLabelSearchArgs (config, where, "-label",      "diffRun.label", "LIKE");
    28832954    pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE");
    28842955    pxAddLabelSearchArgs (config, where, "-dist_group", "diffRun.dist_group", "LIKE");
    2885 
     2956   
    28862957    PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
    28872958    if (!template) {
    … …  
    29052976    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    29062977    PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false);
    2907 
     2978    PXOPT_LOOKUP_S16(diff_mode, config->args, "-diff_mode", false, false);
     2979    if (diff_mode) {
     2980      PXOPT_COPY_S16(config->args, where, "-diff_mode", "diffRun.diff_mode", "==");
     2981    }
    29082982
    29092983    psString where2 = NULL;
    … …  
    29833057    return true;
    29843058}
     3059static bool listssrunMode(pxConfig *config)
     3060{
     3061    PS_ASSERT_PTR_NON_NULL(config, false);
     3062
     3063    psMetadata *where = psMetadataAlloc();
     3064    PXOPT_COPY_S64(config->args, where,  "-diff_id",    "diffRun.diff_id",  "==");
     3065    PXOPT_COPY_S64(config->args, where,  "-stack_id",   "stackInput.stack_id",  "==");
     3066    PXOPT_COPY_S64(config->args, where,  "-template_stack_id",  "stackTemplate.stack_id",  "==");
     3067    PXOPT_COPY_STR(config->args, where,  "-tess_id",    "diffRun.tess_id",  "LIKE");
     3068    PXOPT_COPY_STR(config->args, where,  "-state",      "diffRun.state",    "==");
     3069    pxAddLabelSearchArgs (config, where, "-label",      "diffRun.label", "LIKE");
     3070    pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE");
     3071    pxAddLabelSearchArgs (config, where, "-dist_group", "diffRun.dist_group", "LIKE");
     3072
     3073    // lookup these so we don't compare to zero if they are not supplied
     3074    PXOPT_LOOKUP_F64(mjd_obs_begin, config->args, "-mjd_obs_begin", false, false);
     3075    PXOPT_LOOKUP_F64(mjd_obs_end, config->args, "-mjd_obs_end", false, false);
     3076   
     3077    PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
     3078
     3079    if (!template) {
     3080        if (mjd_obs_begin) {
     3081            PXOPT_COPY_F64(config->args, where, "-mjd_obs_begin", "stackInput.mjd_obs",  ">=");
     3082        }
     3083        if (mjd_obs_end) {
     3084            PXOPT_COPY_F64(config->args, where, "-mjd_obs_end",   "stackInput.mjd_obs",  "<=");
     3085        }
     3086        PXOPT_COPY_STR(config->args, where, "-filter",        "stackInputRun.filter", "LIKE");
     3087    } else {
     3088        if (mjd_obs_begin) {
     3089            PXOPT_COPY_F64(config->args, where, "-mjd_obs_begin", "stackTemplate.mjd_obs",  ">=");
     3090        }
     3091        if (mjd_obs_end) {
     3092            PXOPT_COPY_F64(config->args, where, "-mjd_obs_end",   "stackTemplate.mjd_obs",  "<=");
     3093        }
     3094        PXOPT_COPY_STR(config->args, where, "-filter",        "stackTemplateRun.filter", "LIKE");
     3095    }
     3096
     3097    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
     3098    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     3099    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     3100    PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false);
     3101
     3102    psString where2 = NULL;
     3103    psString query = pxDataGet("difftool_listssrun.sql");
     3104    if (!query) {
     3105        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
     3106        return false;
     3107    }
     3108
     3109    if (psListLength(where->list)) {
     3110        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     3111        psStringAppend(&query, " AND %s", whereClause);
     3112        psFree(whereClause);
     3113    } else if (where2) {
     3114        psStringAppend(&query, " AND %s", where2);
     3115    } else if (!all) {
     3116        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
     3117        return false;
     3118    }
     3119    psFree(where);
     3120
     3121    if (pstamp_order) {
     3122        if (template) {
     3123            psStringAppend(&query, " ORDER BY stackTemplate.stack_id, diff_id DESC");
     3124        } else {
     3125            psStringAppend(&query, " ORDER BY stackInput.stack_id, diff_id DESC");
     3126        }
     3127    }
     3128           
     3129    // treat limit == 0 as "no limit"
     3130    if (limit) {
     3131        psString limitString = psDBGenerateLimitSQL(limit);
     3132        psStringAppend(&query, " %s", limitString);
     3133        psFree(limitString);
     3134    }
     3135
     3136    if (!p_psDBRunQuery(config->dbh, query)) {
     3137        psError(PS_ERR_UNKNOWN, false, "database error");
     3138        psFree(query);
     3139        return false;
     3140    }
     3141    psFree(query);
     3142
     3143    psArray *output = p_psDBFetchResult(config->dbh);
     3144    if (!output) {
     3145        psErrorCode err = psErrorCodeLast();
     3146        switch (err) {
     3147            case PS_ERR_DB_CLIENT:
     3148                psError(PXTOOLS_ERR_SYS, false, "database error");
     3149            case PS_ERR_DB_SERVER:
     3150                psError(PXTOOLS_ERR_PROG, false, "database error");
     3151            default:
     3152                psError(PXTOOLS_ERR_PROG, false, "unknown error");
     3153        }
     3154
     3155        return false;
     3156    }
     3157    if (!psArrayLength(output)) {
     3158        psTrace("difftool", PS_LOG_INFO, "no rows found");
     3159        psFree(output);
     3160        return true;
     3161    }
     3162
     3163    if (psArrayLength(output)) {
     3164        // negative simple so the default is true
     3165        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
     3166            psError(PS_ERR_UNKNOWN, false, "failed to print array");
     3167            psFree(output);
     3168            return false;
     3169        }
     3170    }
     3171
     3172    psFree(output);
     3173
     3174    return true;
     3175}
    29853176// a very specfic function to queue a cleaned warpSkyfile to be updated
    29863177static bool setskyfiletoupdateMode(pxConfig *config)

  • branches/eam_branches/ipp-20100823/ippTools/src/difftool.h

    r28375 r29515  
    4242    DIFFTOOL_MODE_PENDINGCLEANUPRUN,
    4343    DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE,
     44    DIFFTOOL_MODE_REVERTCLEANUP,
    4445    DIFFTOOL_MODE_DONECLEANUP,
    4546    DIFFTOOL_MODE_UPDATEDIFFSKYFILE,
    … …  
    5152    DIFFTOOL_MODE_TOFULLSKYFILE,
    5253    DIFFTOOL_MODE_LISTRUN,
     54    DIFFTOOL_MODE_LISTSSRUN,
    5355    DIFFTOOL_MODE_SETSKYFILETOUPDATE,
    5456} difftoolMode;

  • branches/eam_branches/ipp-20100823/ippTools/src/difftoolConfig.c

    r29324 r29515  
    173173    psMetadata *listrunArgs = psMetadataAlloc();
    174174    psMetadataAddS64(listrunArgs, PS_LIST_TAIL,  "-diff_id", 0,           "search by diff ID", 0);
    175     psMetadataAddStr(listrunArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID", NULL);
     175    psMetadataAddStr(listrunArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID (LIKE comparison)", NULL);
    176176    psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-warp_id",  0,         "search by warp_id", NULL);
    177177    psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-template",  0,        "apply exposure args to template of bothways diff", false);
    … …  
    184184    psMetadataAddStr(listrunArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL);
    185185    psMetadataAddStr(listrunArgs,  PS_LIST_TAIL, "-dist_group",  PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL);
     186    psMetadataAddS16(listrunArgs,  PS_LIST_TAIL, "-diff_mode", 0,        "search for diff_mode", 0);
     187    psMetadataAddStr(listrunArgs, PS_LIST_TAIL,  "-state",  0,           "search by state", NULL);
    186188    pxmagicAddArguments(listrunArgs);
    187189    pxspaceAddArguments(listrunArgs);
    … …  
    192194    psMetadataAddU64(listrunArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
    193195    psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
     196
     197
     198    // -listssrun
     199    psMetadata *listssrunArgs = psMetadataAlloc();
     200    psMetadataAddS64(listssrunArgs, PS_LIST_TAIL,  "-diff_id", 0,          "search by diff ID", 0);
     201    psMetadataAddStr(listssrunArgs, PS_LIST_TAIL,  "-tess_id",  0,         "search by tessellation ID (LIKE comparison)", NULL);
     202    psMetadataAddS64(listssrunArgs , PS_LIST_TAIL, "-stack_id",  0,        "search by input stack_id", 0);
     203    psMetadataAddS64(listssrunArgs , PS_LIST_TAIL, "-template_stack_id",0, "search by template stack_id", 0);
     204    psMetadataAddBool(listssrunArgs, PS_LIST_TAIL, "-template",  0,        "apply stack selectors to template", false);
     205    psMetadataAddF64(listssrunArgs, PS_LIST_TAIL,  "-mjd_obs_begin", 0,    "search by stack MJD-OBS (>=)", 0);
     206    psMetadataAddF64(listssrunArgs, PS_LIST_TAIL,  "-mjd_obs_end", 0,      "search by stack MJD-OBS(<=)", 0);
     207    psMetadataAddStr(listssrunArgs, PS_LIST_TAIL,  "-filter", 0,           "search by stack filter", NULL);
     208    psMetadataAddStr(listssrunArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL);
     209    psMetadataAddStr(listssrunArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL);
     210    psMetadataAddStr(listssrunArgs,  PS_LIST_TAIL, "-dist_group",  PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL);
     211    psMetadataAddS16(listssrunArgs,  PS_LIST_TAIL, "-diff_mode", 0,        "search for diff_mode", 0);
     212    psMetadataAddStr(listssrunArgs, PS_LIST_TAIL,  "-state",  0,           "search by state", NULL);
     213    pxmagicAddArguments(listssrunArgs);
     214    pxspaceAddArguments(listssrunArgs);
     215
     216    psMetadataAddBool(listssrunArgs, PS_LIST_TAIL, "-pstamp_order",  0,    "order results for postage stamp server", false);
     217
     218    psMetadataAddBool(listssrunArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
     219    psMetadataAddU64(listssrunArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
     220    psMetadataAddBool(listssrunArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
    194221
    195222
    … …  
    226253    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-comment", 0, "search by comment (LIKE)", NULL);
    227254    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
     255    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_data_group", 0, "search by stack data_group", NULL);
    228256    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
    229257    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL);
    … …  
    247275    psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
    248276    psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-not-bothways",  0, "only do the single-direction subtraction?", false);
     277    psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL,  "-template_exp_id",  0,  "search by template exposure ID", 0);
    249278    psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
    250279    psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-distance", 0, "limit distance between input and template (deg)", NAN);
    … …  
    292321    psMetadataAddF64(definewarpwarpArgs,  PS_LIST_TAIL, "-posang_min",         0, "search by min rotator position angle", NAN);
    293322    psMetadataAddF64(definewarpwarpArgs,  PS_LIST_TAIL, "-posang_max",         0, "search by max rotator position angle", NAN);
    294     psMetadataAddF32(definewarpwarpArgs,  PS_LIST_TAIL, "-sun_angle_min",         0, "search by min solar angle", NAN);
    295     psMetadataAddF32(definewarpwarpArgs,  PS_LIST_TAIL, "-sun_angle_max",         0, "search by max solar angle", NAN);
    296     psMetadataAddStr(definewarpwarpArgs,  PS_LIST_TAIL, "-comment",            0, "search by comment field (LIKE comparison)", NULL);
     323    psMetadataAddF32(definewarpwarpArgs,  PS_LIST_TAIL, "-sun_angle_min",      0, "search by min solar angle", NAN);
     324    psMetadataAddF32(definewarpwarpArgs,  PS_LIST_TAIL, "-sun_angle_max",      0, "search by max solar angle", NAN);
     325    psMetadataAddStr(definewarpwarpArgs,  PS_LIST_TAIL, "-input_comment",      0, "search by comment field for input exposure(LIKE comparison)", NULL);
     326    psMetadataAddStr(definewarpwarpArgs,  PS_LIST_TAIL, "-template_comment",   0, "search by comment field for template exposure(LIKE comparison)", NULL);
     327    psMetadataAddU64(definewarpwarpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    297328
    298329    // -definestackstack
    … …  
    362393    psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    363394
     395    // -revertcleanup
     396    psMetadata *revertcleanupArgs = psMetadataAlloc();
     397    psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-diff_id", 0,            "search by difftool ID", 0);
     398    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by diffRun label", NULL);
     399    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by diffRun data_group", NULL);
     400    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state",  0,            "search by state", NULL);
     401
     402
    364403    // -donecleanup
    365404    psMetadata *donecleanupArgs = psMetadataAlloc();
    … …  
    427466    PXOPT_ADD_MODE("-advance",          "", DIFFTOOL_MODE_ADVANCE,           advanceArgs);
    428467    PXOPT_ADD_MODE("-diffskyfile",      "", DIFFTOOL_MODE_DIFFSKYFILE,       diffskyfileArgs);
    429     PXOPT_ADD_MODE("-listrun",          "", DIFFTOOL_MODE_LISTRUN,           listrunArgs);
    430468    PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs);
    431469    PXOPT_ADD_MODE("-definepoprun",     "", DIFFTOOL_MODE_DEFINEPOPRUN,      definepoprunArgs);
    … …  
    433471    PXOPT_ADD_MODE("-definewarpwarp",   "", DIFFTOOL_MODE_DEFINEWARPWARP,    definewarpwarpArgs);
    434472    PXOPT_ADD_MODE("-definestackstack", "", DIFFTOOL_MODE_DEFINESTACKSTACK,  definestackstackArgs);
     473    PXOPT_ADD_MODE("-listrun",          "list diff runs", DIFFTOOL_MODE_LISTRUN,           listrunArgs);
     474    PXOPT_ADD_MODE("-listssrun",        "list stack-stack diff runs", DIFFTOOL_MODE_LISTSSRUN,           listssrunArgs);
    435475    PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
    436476    PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);
     477    PXOPT_ADD_MODE("-revertcleanup",           "revert cleanup runs with errors",     DIFFTOOL_MODE_REVERTCLEANUP,          revertcleanupArgs);
    437478    PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     DIFFTOOL_MODE_DONECLEANUP,          donecleanupArgs);
    438479    PXOPT_ADD_MODE("-updatediffskyfile",     "update fault code for a diffskyfile",  DIFFTOOL_MODE_UPDATEDIFFSKYFILE,          updatediffskyfileArgs);

  • branches/eam_branches/ipp-20100823/ippTools/src/magicdstool.c

    r28936 r29515  
    637637    psFree(where);
    638638
     639    psStringAppend(&query, "\nORDER BY priority DESC, magic_ds_id");
     640
    639641    // treat limit == 0 as "no limit"
    640642    if (limit) {
    … …  
    933935    }
    934936
     937    // treat limit == 0 as "no limit"
     938    if (limit) {
     939        psString limitString = psDBGenerateLimitSQL(limit);
     940        psStringAppend(&query, " %s", limitString);
     941        psFree(limitString);
     942    }
     943
    935944    if (!p_psDBRunQuery(config->dbh, query)) {
    936945        psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    950959        return true;
    951960    }
    952     if (!psDBTransaction(config->dbh)) {
    953         psError(PS_ERR_UNKNOWN, false, "database error");
    954         return false;
    955     }
    956961    for (long i = 0; i < psArrayLength(output); i++) {
    957962        psMetadata *row = output->data[i];
    958963
    959964        psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id");
     965        psS64 magicked =  psMetadataLookupS64(NULL, row, "magicked");
     966        if (!psDBTransaction(config->dbh)) {
     967            psError(PS_ERR_UNKNOWN, false, "database error");
     968            return false;
     969        }
    960970
    961971        // if re_place, set stageRun.magicked
    962972        bool setmagicked = psMetadataLookupBool(NULL, row, "re_place");
    963         if (setmagicked && !setRunMagicked(config, magic_ds_id)) {
    964             psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64,
    965                 magic_ds_id);
    966             if (!psDBRollback(config->dbh)) {
    967                 psError(PS_ERR_UNKNOWN, false, "database error");
     973        if (setmagicked) {
     974            if (magicked <= 0) {
     975                if (!setRunMagicked(config, magic_ds_id)) {
     976                    psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64,
     977                        magic_ds_id);
     978                    if (!psDBRollback(config->dbh)) {
     979                        psError(PS_ERR_UNKNOWN, false, "database error");
     980                    }
     981                    return false;
     982                }
     983            } else {
     984                fprintf(stderr, "run is already marked as destreaked for magic_ds_id %" PRId64 "\n", magic_ds_id);
    968985            }
    969             return false;
    970986        }
    971987
    … …  
    980996            return false;
    981997        }
    982     }
    983     if (!psDBCommit(config->dbh)) {
    984         psError(PS_ERR_UNKNOWN, false, "database error");
    985         return false;
     998        if (!psDBCommit(config->dbh)) {
     999            psError(PS_ERR_UNKNOWN, false, "database error");
     1000            return false;
     1001        }
    9861002    }
    9871003

  • branches/eam_branches/ipp-20100823/ippTools/src/magictool.c

    r28768 r29515  
    5050static bool exposureMode(pxConfig *config);
    5151
    52 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
     52static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString);
    5353static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename);
    5454
    … …  
    131131
    132132    psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else
    133     PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "==");
     133    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "rawExp.exp_id", "==");
    134134    PXOPT_COPY_STR(config->args, queryWhere, "-select_filter", "rawExp.filter", "==");
    135135
    … …  
    394394    // required
    395395    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
    396     PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
     396    PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
     397    PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false);
     398    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
     399    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false);
     400
     401    psString setString = NULL;
     402    if (fault || clearfault) {
     403        psStringAppend(&setString, ", fault = %d", fault);
     404    }
     405    if (note) {
     406        psStringAppend(&setString, ", note = '%s'", note);
     407    }
    397408
    398409    if (state) {
    399410        // set detRun.state to state
    400         return setmagicRunState(config, magic_id, state);
     411        return setmagicRunState(config, magic_id, state, setString);
    401412    }
    402413
    … …  
    612623    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
    613624
    614     psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0");
     625    psString query = psStringCopy("UPDATE magicRun SET fault = 0 WHERE state = 'new' AND fault != 0");
    615626
    616627    if (psListLength(where->list)) {
    … …  
    11711182
    11721183    // optional
    1173     PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
     1184//    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
     1185    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
    11741186    PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false);
    11751187
    … …  
    11841196    if (!magicMaskInsert(config->dbh,
    11851197                         magic_id,
    1186                          uri,
     1198                         NULL,
     1199                         path_base,
    11871200                         streaks,
    11881201                         fault
    … …  
    13641377}
    13651378
    1366 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state)
     1379static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString)
    13671380{
    13681381    PS_ASSERT_PTR_NON_NULL(state, false);
    … …  
    13801393        return false;
    13811394    }
    1382 
    1383     char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
    1384     if (!p_psDBRunQueryF(config->dbh, query, state, magic_id)) {
     1395    psString query = NULL;
     1396    psStringAppend(&query, "UPDATE magicRun SET state = '%s'", state);
     1397    if (setString) {
     1398        psStringAppend(&query, setString);
     1399    }
     1400    psStringAppend(&query, " WHERE magic_id = %" PRId64, magic_id);;
     1401
     1402//    char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
     1403    if (!p_psDBRunQuery(config->dbh, query)) {
    13851404        psError(PS_ERR_UNKNOWN, false,
    13861405                "failed to change state for magic_id %" PRId64, magic_id);

  • branches/eam_branches/ipp-20100823/ippTools/src/magictoolConfig.c

    r28039 r29515  
    7979    psMetadata *updaterunArgs = psMetadataAlloc();
    8080    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
    81     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL);
     81    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL);
     82    psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0);
     83    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note",  0, "define note", NULL);
     84    psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-clearfault",  0, "set fault to zero", NULL);
    8285
    8386    // -addinputskyfile
    … …  
    149152    psMetadata *addmaskArgs = psMetadataAlloc();
    150153    psMetadataAddS64(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
    151     psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL);
     154    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL);
    152155    psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks", 0, "define number of streaks", 0);
    153156    psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);

  • branches/eam_branches/ipp-20100823/ippTools/src/pstamptool.c

    r28937 r29515  
    4646static bool pendingjobMode(pxConfig *config);
    4747static bool updatejobMode(pxConfig *config);
     48static bool stopdependentjobMode(pxConfig *config);
    4849static bool revertjobMode(pxConfig *config);
    4950static bool addprojectMode(pxConfig *config);
    … …  
    6364    break;
    6465
     66// XXX make this a configurable parameter
     67#define PSTAMP_MAX_JOB_FAULTS 5
     68#define PSTAMP_MAX_DEP_FAULTS 5
    6569
    6670int main(int argc, char **argv)
    … …  
    8993        MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingjobMode);
    9094        MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updatejobMode);
     95        MODECASE(PSTAMPTOOL_MODE_STOPDEPENDENTJOB, stopdependentjobMode);
    9196        MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode);
    9297        MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode);
    … …  
    715720            outputBase,
    716721            options,
    717             dep_id
     722            dep_id,
     723            0   // fault_count
    718724            )) {
    719725        psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    877883    PXOPT_LOOKUP_S64(req_id,    config->args, "-req_id", false, false);
    878884    PXOPT_LOOKUP_S64(dep_id,    config->args, "-dep_id", false, false);
    879 
    880     if (!job_id && !req_id && !dep_id) {
    881         psError(PS_ERR_UNKNOWN, true, "at least one of -job_id -req_id or -dep_id is required");
     885    PXOPT_LOOKUP_S32(fault_count, config->args, "-fault_count",  false, false);
     886
     887    if (!job_id && !req_id && !dep_id && !fault_count) {
     888        psError(PS_ERR_UNKNOWN, true, "at least one of -job_id -req_id -dep_id or -fault_count is required");
    882889        return false;
    883890    }
    … …  
    898905    PXOPT_COPY_S32(config->args, where, "-fault",  "pstampJob.fault", "==");
    899906    PXOPT_COPY_STR(config->args, where, "-state",  "pstampJob.state", "==");
     907    PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampJob.fault_count", ">=");
     908    pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE");
    900909
    901910    psString query = pxDataGet("pstamptool_updatejob.sql");
    … …  
    909918        psStringAppend(&query, "\n %c pstampJob.fault = %d", c, fault);
    910919        c = ',';
     920        psStringAppend(&query, ", pstampJob.fault_count = pstampJob.fault_count+ 1");
    911921    }
    912922   
    … …  
    925935    psU64 affected = psDBAffectedRows(config->dbh);
    926936    psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampJobs", affected);
     937
     938    return true;
     939}
     940// Terminate jobs which have dependents setting both the pstampDependent and pstampJob.fault
     941static bool stopdependentjobMode(pxConfig *config)
     942{
     943    PS_ASSERT_PTR_NON_NULL(config, false);
     944
     945    PXOPT_LOOKUP_S32(fault,  config->args, "-set_fault",  true, false);
     946
     947    psMetadata *where = psMetadataAlloc();
     948
     949    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
     950    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
     951    PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
     952    PXOPT_COPY_S32(config->args, where, "-fault",  "pstampDependent.fault", "==");
     953    pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE");
     954    // if (fault_count) {
     955        PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">=");
     956    // }
     957
     958    // XXX: How about selecting by pstampRequest.label? No. That is too dangerous by itself.
     959
     960    psString query = pxDataGet("pstamptool_stopdependentjob.sql");
     961
     962    if (psListLength(where->list)) {
     963        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     964        psStringAppend(&query, " AND %s", whereClause);
     965        psFree(whereClause);
     966    } else {
     967        psFree(where);
     968        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
     969        return false;
     970    }
     971    psFree(where);
     972
     973    if (!p_psDBRunQueryF(config->dbh, query, fault, fault)) {
     974        psError(PS_ERR_UNKNOWN, false, "database error");
     975        psFree(query);
     976        return false;
     977    }
     978    psFree(query);
     979
     980    psU64 affected = psDBAffectedRows(config->dbh);
     981    psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampJobs and pstampDependents", affected);
    927982
    928983    return true;
    … …  
    941996    pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE");
    942997
     998    PXOPT_LOOKUP_BOOL(clear_fault_count, config->args, "-clear_fault_count", false);
     999
    9431000    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
     1001
     1002    // XXX: we don't actually use -limit. It doesn't work for UPDATE
     1003    // it's an allowed arg because add_poll_args adds it
    9441004    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    9451005    PXOPT_LOOKUP_S16(fault, config->args, "-fault",  false, false);
    … …  
    9471007    // By default only revert faults < PSTAMP_FIRST_ERROR_CODE which are our "ipp exit codes"
    9481008    // codes larger than that are the pstamp request interface.
    949     // Don't fault those unless -fault waa provided
     1009    // Don't fault those unless -fault was explicitly provided
    9501010    psString faultClause = psStringCopy("");
    9511011    if (!fault) {
    … …  
    9661026    psFree(where);
    9671027
    968     if (!p_psDBRunQueryF(config->dbh, query, faultClause)) {
     1028    // don't keep reverting once the number of faults reaches some value unless
     1029    // a parameter asking us to clear that count is provided
     1030    psString faultCountClause = NULL;
     1031    if (clear_fault_count) {
     1032        psStringAppend(&faultCountClause, "\n, pstampJob.fault_count = 0");
     1033    } else {
     1034        psStringAppend(&query, " AND pstampJob.fault_count < %d", PSTAMP_MAX_JOB_FAULTS);
     1035    }
     1036
     1037    if (!p_psDBRunQueryF(config->dbh, query, faultCountClause, faultClause)) {
     1038        psFree(faultCountClause);
    9691039        psFree(faultClause);
    9701040        psFree(query);
    … …  
    9741044
    9751045    psFree(faultClause);
     1046    psFree(faultCountClause);
     1047    psFree(query);
    9761048
    9771049    return true;
    … …  
    11781250        need_magic,
    11791251        outdir,
    1180         0               // fault
     1252        0,              // fault
     1253        0               // fault_count
    11811254        )) {
    11821255        if (!psDBRollback(config->dbh)) {
    … …  
    13001373        psStringAppend(&query, "%s fault = %d", needComma ? ", " : "", fault);
    13011374        needComma = true;
     1375        psStringAppend(&query, ", pstampDependent.fault_count = pstampDependent.fault_count + 1");
    13021376    }
    13031377    psStringAppend(&query, " WHERE dep_id = %" PRId64, dep_id);
    … …  
    13251399    PXOPT_COPY_S64(config->args, where, "-fault", "pstampDependent.fault", "==");
    13261400    PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
     1401    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
     1402    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
    13271403    pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "==");
     1404    PXOPT_LOOKUP_BOOL(clear_fault_count, config->args, "-clear_fault_count", false);
     1405
     1406    // XXX: we don't actually use -limit. It doesn't work for UPDATE
     1407    // it's an allowed arg because add_poll_args adds it
     1408    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    13281409
    13291410    if (!psListLength(where->list)) {
    … …  
    13521433    }
    13531434
    1354     if (!p_psDBRunQuery(config->dbh, query)) {
     1435    // don't keep reverting once the number of faults reaches some value unless
     1436    // a parameter asking us to clear that count is provided
     1437    psString faultCountClause = NULL;
     1438    if (clear_fault_count) {
     1439        psStringAppend(&faultCountClause, "\n, pstampDependent.fault_count = 0");
     1440    } else {
     1441        psStringAppend(&faultCountClause, " ");
     1442        psStringAppend(&query, " AND pstampDependent.fault_count < %d", PSTAMP_MAX_DEP_FAULTS);
     1443    }
     1444
     1445    if (!p_psDBRunQueryF(config->dbh, query, faultCountClause)) {
    13551446        psError(PS_ERR_UNKNOWN, false, "database error");
    13561447        psFree(query);

  • branches/eam_branches/ipp-20100823/ippTools/src/pstamptool.h

    r28183 r29515  
    3838    PSTAMPTOOL_MODE_LISTJOB,
    3939    PSTAMPTOOL_MODE_PENDINGJOB,
     40    PSTAMPTOOL_MODE_STOPDEPENDENTJOB,
    4041    PSTAMPTOOL_MODE_JOBRESULT,
    4142    PSTAMPTOOL_MODE_UPDATEJOB,

  • branches/eam_branches/ipp-20100823/ippTools/src/pstamptoolConfig.c

    r28937 r29515  
    167167    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0,             "current state of jobs to update", 0);
    168168    psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-fault", 0,             "current value for job fault", 0);
     169    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
     170    psMetadataAddS32(updatejobArgs, PS_LIST_TAIL, "-fault_count", 0,       "select by fault_count (>=)", 0);
    169171    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-set_state", 0,            "new state", NULL);
    170172    psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-set_fault", 0,            "new result", 0);
     173    psMetadataAddU64(updatejobArgs, PS_LIST_TAIL, "-limit", 0,      "not used", 0);
     174
     175    // -stopdependentjob
     176    psMetadata *stopdependentjobArgs = psMetadataAlloc();
     177    psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL,  "-req_id", 0,       "req_id of jobs to update", 0);
     178    psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL,  "-job_id", 0,       "job_id of jobs to update", 0);
     179    psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL,  "-dep_id", 0,       "dep_id of jobs to update", 0);
     180    psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL,  "-fault", 0,        "current value for dependent fault", 0);
     181    psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL,  "-fault_count", 0,   "select by fault_count (>=)", 0);
     182    psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL,  "-set_fault", 0,    "new fault value for job and dependent (required)", 0);
     183    psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
     184    psMetadataAddU64(stopdependentjobArgs, PS_LIST_TAIL, "-limit", 0,      "not used", 0);
    171185
    172186    // -revertjob
    … …  
    177191    psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault",  0,     "fault to revert", 0);
    178192    psMetadataAddStr(revertjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL);
     193    psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-clear_fault_count", 0,       "clear job fault count", false);
    179194    psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0,       "revert all faulted jobs", false);
    180     psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0,      "limit result set to N items", 0);
     195    psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0,      "not used", 0);
    181196
    182197    // -getdependent
    … …  
    201216    // -revertdependent
    202217    psMetadata *revertdependentArgs = psMetadataAlloc();
    203     psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent", 0);
     218    psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "search by dep_id for dependent", 0);
     219    psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-job_id", 0, "search by job_ idfor dependent", 0);
     220    psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-req_id", 0, "search by req_id for dependent", 0);
    204221    psMetadataAddStr(revertdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK,   "define label for dependent ", NULL);
    205     psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault",  0,   "define fault", 0);
     222    psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault",  0, "search by dependent fault", 0);
     223    psMetadataAddBool(revertdependentArgs, PS_LIST_TAIL, "-clear_fault_count", 0,       "clear job fault count", false);
     224    psMetadataAddU64(revertdependentArgs, PS_LIST_TAIL, "-limit", 0,  "limit result set to N items", 0);
    206225
    207226    // -pendingdependent
    … …  
    262281    PXOPT_ADD_MODE("-pendingjob",      "", PSTAMPTOOL_MODE_PENDINGJOB,   pendingjobArgs);
    263282    PXOPT_ADD_MODE("-updatejob",       "", PSTAMPTOOL_MODE_UPDATEJOB,    updatejobArgs);
     283    PXOPT_ADD_MODE("-stopdependentjob", "", PSTAMPTOOL_MODE_STOPDEPENDENTJOB,  stopdependentjobArgs);
    264284    PXOPT_ADD_MODE("-revertjob",       "", PSTAMPTOOL_MODE_REVERTJOB,    revertjobArgs);
    265285

  • branches/eam_branches/ipp-20100823/ippTools/src/pubtool.c

    r28494 r29515  
    340340    }
    341341
    342     if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) {
     342    if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause)) {
    343343        psError(PS_ERR_UNKNOWN, false, "Database error");
    344344        psFree(whereClause);

  • branches/eam_branches/ipp-20100823/ippTools/src/stacktool.c

    r28681 r29515  
    901901    psFree(where);
    902902
     903    psStringAppend(&query, "\nORDER by priority DESC, stack_id");
     904
    903905    // treat limit == 0 as "no limit"
    904906    if (limit) {
    … …  
    982984    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
    983985    PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
     986    PXOPT_LOOKUP_F64(mjd_obs, config->args, "-mjd_obs", false, false);
    984987
    985988    PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false);
    … …  
    10471050                               hostname,
    10481051                               good_frac,
     1052                               mjd_obs,
    10491053                               fault,
    10501054                               software_ver,
    … …  
    10941098    psMetadata *where = psMetadataAlloc();
    10951099    PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
    1096     PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "==");
     1100    PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE");
    10971101    PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "==");
    10981102    PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE");
    … …  
    11001104    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE");
    11011105    PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
     1106    PXOPT_COPY_F64(config->args, where, "-mjd_obs_begin", "stackSumSkyfile.mjd_obs", ">=");
     1107    PXOPT_COPY_F64(config->args, where, "-mjd_obs_end", "stackSumSkyfile.mjd_obs", "<=");
     1108
    11021109
    11031110//  The following three selectors are incompatible with the sql so omit them

  • branches/eam_branches/ipp-20100823/ippTools/src/stacktoolConfig.c

    r28681 r29515  
    172172    psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", 0);
    173173    psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
     174    psMetadataAddF64(addsumskyfileArgs, PS_LIST_TAIL, "-mjd_obs",  0,            "define mjd_obs (average mjd_obs of inputs)", NAN);
    174175    psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
    175176    psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
    … …  
    185186    psMetadata *sumskyfileArgs= psMetadataAlloc();
    186187    psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,            "search by stack ID", 0);
    187     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", 0);
     188    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID (LIKE comparison)", 0);
    188189    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,         "search by skycell ID", 0);
    189190#ifdef notdef
    … …  
    196197    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-data_group", 0,        "search by stackRun.data_group (LIKE comparison)", NULL);
    197198    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-filter", 0,            "search by filter (LIKE comparison)", NULL);
     199    psMetadataAddF64(sumskyfileArgs, PS_LIST_TAIL, "-mjd_obs_begin",  0,      "search by mjd_obs (average mjd_obs of inputs <=)", NAN);
     200    psMetadataAddF64(sumskyfileArgs, PS_LIST_TAIL, "-mjd_obs_end",  0,      "search by mjd_obs (average mjd_obs of inputs>=)", NAN);
    198201    psMetadataAddS16(sumskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
    199202    psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    … …  
    204207    psMetadata *sassskyfileArgs = psMetadataAlloc();
    205208    psMetadataAddS64(sassskyfileArgs, PS_LIST_TAIL, "-sass_id", 0,           "search by stack association ID", 0);
    206     psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", 0);
     209    psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID (LIKE comparison)", 0);
    207210    psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-projection_cell", 0,         "search by projection cell", 0);
    208211

  • branches/eam_branches/ipp-20100823/ippTools/src/warptool.c

    r28375 r29515  
    5252static bool pendingcleanuprunMode(pxConfig *config);
    5353static bool pendingcleanupwarpMode(pxConfig *config);
     54static bool revertcleanupMode(pxConfig *config);
    5455static bool donecleanupMode(pxConfig *config);
    5556static bool tocleanedskyfileMode(pxConfig *config);
    … …  
    107108        MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN,  pendingcleanuprunMode);
    108109        MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode);
     110        MODECASE(WARPTOOL_MODE_REVERTCLEANUP,      revertcleanupMode);
    109111        MODECASE(WARPTOOL_MODE_DONECLEANUP,        donecleanupMode);
    110112        MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE,   tocleanedskyfileMode);
    … …  
    18051807
    18061808
     1809static bool revertcleanupMode(pxConfig *config)
     1810{
     1811    PS_ASSERT_PTR_NON_NULL(config, false);
     1812
     1813    psMetadata *where = psMetadataAlloc();
     1814    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
     1815    pxAddLabelSearchArgs (config, where, "-label",     "warpRun.label", "LIKE");
     1816    pxAddLabelSearchArgs (config, where, "-data_group",   "warpRun.data_group", "LIKE");
     1817
     1818    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
     1819
     1820    if (!psListLength(where->list)) {
     1821        psFree(where);
     1822        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
     1823        return false;
     1824    }
     1825    if (!state) {
     1826        state = "error_cleaned";
     1827    }
     1828    char *newState = NULL;
     1829    if (!strcmp(state, "error_cleaned")) {
     1830        newState = "goto_cleaned";
     1831    } else if (!strcmp(state, "error_purged")) {
     1832        newState = "goto_purged";
     1833    } else if (!strcmp(state, "error_scrubbed")) {
     1834        newState = "goto_scrubbed";
     1835    } else {
     1836        psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed");
     1837        return false;
     1838    }
     1839
     1840    psString query = pxDataGet("warptool_revertcleanup.sql");
     1841    if (!query) {
     1842        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
     1843        return false;
     1844    }
     1845
     1846    if (psListLength(where->list)) {
     1847        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     1848        psStringAppend(&query, " AND %s", whereClause);
     1849        psFree(whereClause);
     1850    }
     1851    psFree(where);
     1852
     1853    if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) {
     1854        psError(PS_ERR_UNKNOWN, false, "database error");
     1855        psFree(query);
     1856        return false;
     1857    }
     1858    psFree(query);
     1859
     1860    int numDeleted = psDBAffectedRows(config->dbh);
     1861
     1862    psLogMsg("warptool", PS_LOG_INFO, "Reverted %d warpRuns and warpSkyfiles", numDeleted);
     1863
     1864    return true;
     1865}
    18071866static bool donecleanupMode(pxConfig *config)
    18081867{

  • branches/eam_branches/ipp-20100823/ippTools/src/warptool.h

    r28375 r29515  
    4848    WARPTOOL_MODE_PENDINGCLEANUPRUN,
    4949    WARPTOOL_MODE_PENDINGCLEANUPSKYFILE,
     50    WARPTOOL_MODE_REVERTCLEANUP,
    5051    WARPTOOL_MODE_DONECLEANUP,
    5152    WARPTOOL_MODE_TOCLEANEDSKYFILE,

  • branches/eam_branches/ipp-20100823/ippTools/src/warptoolConfig.c

    r28375 r29515  
    362362    psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    363363
     364    // -revertcleanup
     365    psMetadata *revertcleanupArgs = psMetadataAlloc();
     366    psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warptool ID", 0);
     367    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by warpRun label", NULL);
     368    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL);
     369    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state",  0,            "search by state", NULL);
     370
    364371    // -donecleanup
    365372    psMetadata *donecleanupArgs = psMetadataAlloc();
    … …  
    451458    PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
    452459    PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);
     460    PXOPT_ADD_MODE("-revertcleanup",         "revert cleanup runs with errors",     WARPTOOL_MODE_REVERTCLEANUP,          revertcleanupArgs);
    453461    PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     WARPTOOL_MODE_DONECLEANUP,          donecleanupArgs);
    454462    PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs);

  • branches/eam_branches/ipp-20100823/magic/remove/src/streaksio.c

    r28310 r29515  
    12661266        return;
    12671267    }
    1268     if (sfiles->inMask && sfiles->inMask->header) {
    1269         if (!pmConfigMaskReadHeader(sfiles->config, sfiles->inMask->header)) {
    1270             streaksExit("failed to read mask values from file", PS_EXIT_CONFIG_ERROR);
    1271         }
    1272     }
    1273 
    1274     bool status;
    1275     psMetadata *masks = psMetadataLookupMetadata(&status, sfiles->config->recipes, "MASKS");
    1276     if (!status) {
    1277         psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n");
    1278         streaksExit("", PS_EXIT_CONFIG_ERROR);
    1279     }
    1280     sfiles->maskStreak = psMetadataLookupU32(&status, masks, "STREAK");
    1281     if (!status) {
    1282         psError(PM_ERR_CONFIG, false, "failed to lookup mask value for STREAK in recipes\n");
    1283         streaksExit("", PS_EXIT_CONFIG_ERROR);
    1284     }
    1285 
    1286     // optionally setting pixels with any mask bits execpt CONV.POOR to NAN
    1287     // check old name first
    1288     psU32 convPoor = (double) psMetadataLookupU32(&status, masks, "POOR.WARP");
    1289     if (!status) {
    1290         convPoor = (double) psMetadataLookupU32(&status, masks, "CONV.POOR");
    1291         if (!status) {
    1292             psError(PM_ERR_CONFIG, false, "failed to lookup mask value for CONV.POOR in recipes\n");
    1293             streaksExit("", PS_EXIT_CONFIG_ERROR);
    1294         }
    1295     }
    1296     // preserve pixels that are only suspect
    1297     psU32 suspect = (double) psMetadataLookupU32(&status, masks, "SUSPECT");
    1298     if (!status) {
    1299         psError(PM_ERR_CONFIG, false, "failed to lookup mask value for SUSPECT in recipes\n");
    1300         streaksExit("", PS_EXIT_CONFIG_ERROR);
    1301     }
    1302 
    1303     sfiles->maskMask = ~(convPoor | suspect);
     1268    if (!pmCensorGetMasks(sfiles->config, &sfiles->maskMask, &sfiles->maskStreak)) {
     1269        streaksExit("failed to find mask values", PS_EXIT_CONFIG_ERROR);
     1270    }
    13041271}
    13051272

  • branches/eam_branches/ipp-20100823/pstamp/scripts/Makefile.am

    r29124 r29515  
    2020        pstamp_get_image_job.pl \
    2121        psmkreq \
     22        pstampstopfaulted \
    2223        pstamp_checkdependent.pl \
    2324        request_finish.pl \

  • branches/eam_branches/ipp-20100823/pstamp/scripts/detect_query_create

    r28806 r29515  
    307307    while (<$in>) {
    308308        chomp;
     309#       print STDERR "$line_num $#keywords $_\n";
     310        if ($_ =~ /^\s*$/) {
     311            next;
     312        }
    309313        if ($line_num == 0) {
     314            if ($_ !~ /EXTVER/) {
     315                next;
     316            }
    310317            # Parse header information keywords
    311318            $_ =~ s/#//g;
    … …  
    319326            my @values = split /\s+/;
    320327            if ($#values != $#keywords) {
    321                 die "Number of header columns in input does not equal expected number of header words";
     328                die "Number of header columns in input does not equal expected number of header words $#values $#keywords";
    322329            }
    323330            for (my $i = 0; $i <= $#values; $i++) {
    … …  
    326333        }
    327334        elsif ($line_num == 2) {
     335            if ($_ !~ /ROWNUM/) {
     336                next;
     337            }
    328338            # Parse table information keywords, dumping old keywords
    329339            $_ =~ s/#//g;
    … …  
    338348                my @values = split /\s+/;
    339349                if ($#values != $#keywords) {
    340                     die "Number of header columns in input does not equal expected number of header words";
     350                    die "Number of header columns in input does not equal expected number of header words $#values $#keywords";
    341351                }
    342352                for (my $i = 0; $i <= $#values; $i++) {

  • branches/eam_branches/ipp-20100823/pstamp/scripts/detect_query_read

    r28808 r29515  
    3636
    3737my $regtool = can_run('regtool') or (die "Can't find regtool");
    38 my $camtool = can_run('camtool') or (die "Can't find regtool");
     38my $camtool = can_run('camtool') or (die "Can't find camtool");
     39my $difftool = can_run('difftool') or (die "Can't find difftool");
    3940
    4041
    … …  
    164165    }
    165166    elsif ($status == 219) {
    166         @{ $colData{$col->{name}} } = map { "Not Set" } (0 .. $numRows - 1);
     167        @{ $colData{$col->{name}} } = map { "Not_Set" } (0 .. $numRows - 1);
    167168        $status = 0;
    168169    }
    … …  
    173174# Simple stuff first.
    174175    my %known_filters = ();
    175     if ($colData{STAGE}[$i] eq "Not Set") {
     176    if ($colData{STAGE}[$i] eq "Not_Set") {
    176177        $colData{STAGE}[$i] = 'diff';
    177178    }
    178     if ($colData{OBSCODE}[$i] eq "Not Set") {
     179    if ($colData{OBSCODE}[$i] eq "Not_Set") {
    179180        $colData{OBSCODE}[$i] = 566;
    180181    }
    181182# Define filter and MJD from FPA_ID
    182     if (($colData{FILTER}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
     183    if (($colData{FILTER}[$i] eq "Not_Set")&&($colData{FPA_ID}[$i] ne "Not_Set")) {
    183184        if (exists($known_filters{$colData{FPA_ID}[$i]})) {
    184185            $colData{FILTER}[$i] = $known_filters{$colData{FPA_ID}[$i]};
    … …  
    210211        }
    211212    }
    212     if (($colData{'MJD-OBS'}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
     213    # Calculate a MJD
     214    if (($colData{'MJD-OBS'}[$i] eq "Not_Set")&&($colData{FPA_ID}[$i] ne "Not_Set")) {
    213215        # HACK!
    214216        my $mjd = $colData{FPA_ID}[$i];
    … …  
    217219        $colData{'MJD-OBS'}[$i] = $mjd;
    218220    }
    219     if (($colData{FPA_ID}[$i] eq "Not Set")&&(($colData{'FILTER'}[$i] ne "Not Set")&&
    220                                               ($colData{'MJD-OBS'}[$i] ne "Not Set"))) {
     221    # Calculate an exp_id
     222    if (($colData{FPA_ID}[$i] eq "Not_Set")&&(($colData{'FILTER'}[$i] ne "Not_Set")&&
     223                                              ($colData{'MJD-OBS'}[$i] ne "Not_Set"))) {
    221224        my $dateobs_begin = mjd_to_dateobs($colData{'MJD-OBS'}[$i]);
    222225        my $dateobs_end = mjd_to_dateobs($colData{'MJD-OBS'}[$i] + 1);
    … …  
    224227        my $dec = $colData{'DEC1_DEG'}[$i];
    225228        my $filter = $colData{'FILTER'}[$i] . ".00000";
    226         my $cmd = "$camtool -processedexp -dbname $dbname -filter $filter ";
    227         $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
    228         $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
     229        my $cmd;
     230        if ($colData{STAGE}[$i] eq 'SSdiff') {
     231            $cmd = "$difftool -listrun -dbname $dbname -filter $filter ";
     232            $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
     233            $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1 -diff_mode 4";
     234        }
     235        elsif ($colData{STAGE}[$i] eq 'WSdiff') {
     236            $cmd = "$difftool -listrun -dbname $dbname -filter $filter ";
     237            $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
     238            $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1 -diff_mode 2";
     239        }
     240        elsif ($colData{STAGE}[$i] eq 'diff') {
     241            $cmd = "$difftool -listrun -dbname $dbname -filter $filter ";
     242            $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
     243            $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
     244        }
     245        else {
     246            $cmd = "$camtool -processedexp -dbname $dbname -filter $filter ";
     247            $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
     248            $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
     249        }
    229250        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    230             run(command => $cmd, verbose => 0);
     251            run(command => $cmd, verbose => 1);
    231252        unless ($success) {
    232253            # This is a problem, because I'm not sure how we handle a failure to read something.
    … …  
    243264            my ($key,$type,$value) = @line;
    244265#               print "$entry => $key / $type / $value \n";
    245             if ($key =~ /exp_name/i) {
    246                 $colData{FPA_ID}[$i] = $value;
    247             }
    248         }
     266            if ($colData{STAGE}[$i] =~ /diff/) {
     267                if ($key =~ /diff_id/i) {
     268                    $colData{FPA_ID}[$i] = $value;
     269                }
     270            }
     271            else {
     272                if ($key =~ /exp_name/i) {
     273                    $colData{FPA_ID}[$i] = $value;
     274                }
     275            }
     276        }
     277    }
     278    # Correct diff modes
     279    if ($colData{STAGE}[$i] eq 'WSdiff') {
     280        $colData{STAGE}[$i] = 'diff';
     281    }
     282    elsif ($colData{STAGE}[$i] eq 'SSdiff') {
     283        $colData{STAGE}[$i] = 'diff';
    249284    }
    250285}

  • branches/eam_branches/ipp-20100823/pstamp/scripts/detectability_respond.pl

    r29124 r29515  
    9191    print "Reading wisdom file $wisdom_file instead of parsing...\n";
    9292    open(WISDOM,"$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
    93     my $i = 0;
    9493    while(<WISDOM>) {
    9594        chomp;
    9695        my ($fpa_id,@key_values) = split /\s+/;
     96        my $index = $#{ $query{$fpa_id}{ROWNUM} } + 1;
    9797        while ($#key_values > -1) {
    9898            my $key = shift(@key_values);
    … …  
    102102                $val = 0;
    103103            }
    104             $query{$fpa_id}{$key}[$i] = $val;
    105         }
    106         $i++;
     104            $query{$fpa_id}{$key}[$index] = $val;
     105        }
    107106    }
    108107    close(WISDOM);
    … …  
    150149        my $filter;
    151150        my $mjd;
    152         # Determine the query style for this fpa_id
    153         if ($fpa_id =~ /o.*g.*o/) {
    154             $query_style = 'byexp';
    155         }
    156         elsif ($fpa_id =~ /\d+/) {
    157             $query_style = 'byid';
    158         }
    159         else {
    160             exit_with_failure(21,"Parse error in request file");
    161         }
    162151        # Confirm that we only have one stage/filter/mjd
    163         for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
    164             $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
    165             $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
    166             $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
    167         }
    168         if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
    169             $stage = (keys(%{ $temp_hash{STAGE} }))[0];
    170         }
    171         else {
    172             exit_with_failure(21,"Too many STAGEs specified");
    173         }
    174         if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
    175             $filter = (keys(%{ $temp_hash{FILTER} }))[0];
    176         }
    177         else {
    178             exit_with_failure(21,"Too many FILTERs specified");
    179         }
    180         if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
    181             $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
    182         }
    183         else {
    184             exit_with_failure(21,"Too many MJD-OBS specified");
    185         }
    186         # Set common request components
    187         my $option_mask |= 1;
    188         $option_mask |= $PSTAMP_SELECT_IMAGE;
    189         $option_mask |= $PSTAMP_SELECT_MASK;
    190         $option_mask |= $PSTAMP_SELECT_VARIANCE;
    191         $option_mask |= $PSTAMP_SELECT_PSF;
    192         my $need_magic = 1;
    193         if ($stage eq 'stack') {
    194             $need_magic = 0;
    195         }
    196         my $mjd_min = $mjd;
    197         my $mjd_max = $mjd + 1;
    198        
    199         # Construct a row list.
     152        if ($fpa_id ne 'Not_Set') {   # We only need to check things that aren't the known odd case.
     153            for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
     154                $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
     155                $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
     156                $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
     157            }
     158            if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
     159                $stage = (keys(%{ $temp_hash{STAGE} }))[0];
     160            }
     161            else {
     162                exit_with_failure(21,"Too many STAGEs specified");
     163            }
     164            if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
     165                $filter = (keys(%{ $temp_hash{FILTER} }))[0];
     166            }
     167            else {
     168                exit_with_failure(21,"Too many FILTERs specified");
     169            }
     170            if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
     171                $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
     172            }
     173            else {
     174                exit_with_failure(21,"Too many MJD-OBS specified");
     175            }
     176        }
     177
     178        # Set up a rowList with default values
    200179        my @rowList;
    201180        for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
    … …  
    223202            $query{$fpa_id}{FAULT}[$i] = 'no_fault';
    224203            $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate';
    225 
    226         }
     204        }
     205
     206        # Determine the query style for this fpa_id
     207        if ($fpa_id =~ /o.*g.*o/) {
     208            $query_style = 'byexp';
     209        }
     210        elsif ($fpa_id =~ /\d+/) {
     211            $query_style = 'byid';
     212        }
     213        elsif ($fpa_id eq 'Not_Set') {
     214            next;
     215        }
     216        else {
     217            exit_with_failure(21,"Parse error in request file");
     218        }
     219        # Set common request components
     220        my $option_mask |= 1;
     221        $option_mask |= $PSTAMP_SELECT_IMAGE;
     222        $option_mask |= $PSTAMP_SELECT_MASK;
     223        $option_mask |= $PSTAMP_SELECT_VARIANCE;
     224        $option_mask |= $PSTAMP_SELECT_PSF;
     225        my $need_magic = 1;
     226        if ($stage eq 'stack') {
     227            $need_magic = 0;
     228        }
     229        my $mjd_min = $mjd;
     230        my $mjd_max = $mjd + 1;
     231       
    227232       
    228233        # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
    … …  
    321326open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
    322327foreach my $fpa_id (keys %query) {
    323     for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
     328    for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
    324329        print WISDOM "$fpa_id\t";
    325330        foreach my $key (keys %{ $query{$fpa_id} }) {
    … …  
    639644        $columns = [
    640645            # matching rownum from detectability original request
    641             { name => 'ROWNUM',   type => 'V', writetype => TULONG },
     646            { name => 'ROWNUM',   type => '20A', writetype => TULONG },
    642647            # any errors that occurred during processing
    643648            { name => 'ERROR_CODE',   type => 'V', writetype => TULONG },
    … …  
    690695            $inHeader->{FPA_ID}->{value} = $fpa_id;
    691696        }
    692 
     697       
    693698        push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} };
    694699        push @{$colData{'ERROR_CODE'}}, @{ $query{$fpa_id}{PROC_ERROR} };

  • branches/eam_branches/ipp-20100823/pstamp/scripts/psmkreq

    r29124 r29515  
    2424
    2525# list file columns
    26 # RA DEC FILTER MJD_MIN MJD_MAX
     26# RA DEC FILTER MJD_MIN MJD_MAX | COMMENT
    2727
    2828my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base);
    … …  
    6262GetOptions(
    6363    'list=s'            => \$list,          # list of coordinates if undef ra and dec or x and y are required
    64     'ra=s'              => \$ra,             #
     64    'ra=s'              => \$ra,            #
    6565    'dec=s'             => \$dec,
    6666    'x=s'               => \$x,

  • branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_checkdependent.pl

    r29124 r29515  
    239239                    print "skipping $command\n";
    240240                }
     241            } elsif ($chip->{fault}) {
     242                # fault the dependent
     243                my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault});
     244            } elsif ($chip->{dsFile_fault}) {
     245                # fault the dependent
     246                my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault});
    241247            }
    242248        }
    … …  
    317323            if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
    318324                $chips_ready = 0;
     325                $chip->{fault} = $chip->{chip_fault};
    319326                push @chipsToUpdate, $chip;
    320327            } else {

  • branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_finish.pl

    r28229 r29515  
    191191            my $filter = $job_params->{filter};
    192192            $filter = "0" if !$filter;
    193             $exp_info = "0|0|0|$filter|0|0";
     193            my $mjd_obs = $job_params->{mjd_obs};
     194            $mjd_obs = "0" if !$mjd_obs;
     195            $exp_info = "$mjd_obs|0|0|$filter|0|0";
    194196        }
    195197
    … …  
    477479    my $ticks = timegm($sec, $min, $hr, $day, $mon-1, $year-1900);
    478480
     481    # dateobs is UTC convert to TAI
     482    # XXX: Do this properly
     483    if ($year >= 2009) {
     484        $ticks += 34;
     485    } else {
     486        $ticks += 33;
     487    }
     488
    479489    return 40587.0 + ($ticks/86400.);
    480490}

  • branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_insert_request.pl

    r28106 r29515  
    4444
    4545my $pstamptool = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
    46 my $pstampdump = can_run('pstampdump')  or (warn "Can't find pstampdump"  and $missing_tools = 1);
     46my $fields = can_run('fields')  or (warn "Can't find fields"  and $missing_tools = 1);
    4747
    4848if ($missing_tools) {
    … …  
    5353my ($extname, $extver, $req_name);
    5454{
    55     my $command = "$pstampdump -headeronly $tmp_req_file";
     55    # get the header keywords of interest.
     56    # Note that if it's a pstamp request then REQ_NAME should be defined.
     57    # if it's a detectability query it will not have a REQ_NAME but will have a QUERY_ID
     58    my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
    5659    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    5760        run(command => $command, verbose => $verbose);
     61if (0) {
     62    # stoopid fields doesn't set exit status to zero when it works
    5863    unless ($success) {
    5964        print STDERR @$stderr_buf;
    6065        exit $error_code >> 8;
    6166    }
     67}
    6268    my $output = join "", @$stdout_buf;
    63     ($extname, $extver, $req_name) = split " ", $output;
    64     if ($extname and ($extname ne "PS1_PS_REQUEST")) {
     69    (undef, $extname, $extver, $req_name) = split " ", $output;
     70    if (!$extname or ! (($extname eq "PS1_PS_REQUEST") or ($extname eq "MOPS_DETECTABILITY_QUERY"))) {
    6571        print STDERR "invalid request file\n";
    6672        exit $PS_EXIT_DATA_ERROR;
    6773    }
    6874    if (!defined $req_name) {
    69         print STDERR "invalid request file no REQ_NAME\n";
     75        print STDERR "invalid request file no REQ_NAME or QUERY_ID\n";
    7076        exit $PS_EXIT_DATA_ERROR;
    7177    }
    … …  
    9197my $req_id = 0;
    9298{
    93 
    94     my $command = "$pstamptool -addreq -name $req_name -uri $req_file -ds_id 0";
     99    my $command = "$pstamptool -addreq -uri $req_file -ds_id 0 -name $req_name";
    95100    $command .= " -dbname $dbname" if $dbname;
    96101    $command .= " -dbserver $dbserver" if $dbserver;

  • branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_job_run.pl

    r28927 r29515  
    118118    $command .= " -dbname $dbname" if $dbname;
    119119    $command .= " -dbserver $dbserver" if $dbserver;
     120    $command .= " -stage $params->{stage}" if $params->{stage};
    120121    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    121122        run(command => $command, verbose => $verbose);

  • branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_server_status

    r27211 r29515  
    1616
    1717my ($rundir, $verbose, $save_temps);
     18my $br = '<br />';
    1819
    1920GetOptions(
    … …  
    99100    print "Pantasks Scheduler $scheduler_state.\n";
    100101    if ($controller_state) {
    101         print "Pantasks Controller $controller_state.\n";
     102        print $br . "Pantasks Controller $controller_state.\n";
    102103    } else {
    103104        print STDERR "Controller state not found";
    104105        exit 1;
    105106    }
    106     print "\n";
     107    print "$br$br\n";
    107108} else {
    108109    print STDERR "Scheduler state not found";
    … …  
    116117
    117118if ($request_run) {
    118     print "<b>Parser Status:</b> $request_run->{nrun} running. $request_run->{ngood} parsed successfully.&nbsp; $request_run->{nfail} failed to parse.";
     119    print "<br /><b>Parser Status:</b> $request_run->{nrun} running. $request_run->{ngood} parsed successfully.&nbsp; $request_run->{nfail} failed to parse.";
    119120    if ($request_fin) {
    120121        print "  $request_fin->{ngood} Requests finished.";
    … …  
    125126    print "\n";
    126127    if ($job_run) {
    127         print "<b>Job Status:</b> &nbsp;&nbsp; $job_run->{nrun} running. &nbsp; $job_run->{ngood} completed successfully. &nbsp; $job_run->{nfail} failed";
    128         print "\n";
     128        print "<br /><b>Job Status:</b> &nbsp;&nbsp; $job_run->{nrun} running. &nbsp; $job_run->{ngood} completed successfully. &nbsp; $job_run->{nfail} failed";
     129        print "<br />\n";
    129130    } else {
    130         print "Task pstamp.job.run not found.\n";
     131        print "Task pstamp.job.run not found.<br />\n";
    131132    }
     133    print "<br /><b>Unfinished Requests</b><br />\n";
     134    print "<pre>\n";
     135    system "/home/panstarrs/bills/ipp/tools/psstatus -r ";
     136    print "</pre>\n";
     137    print "<br /><b>Requests completed in last 24 hours (most recently completed first)</b><br />\n";
     138    print "<pre>\n";
     139    # finished requests
     140    system "/home/panstarrs/bills/ipp/tools/psstatus -f";
     141    print "</pre>\n";
    132142} else {
    133143    print "Task pstamp.request.run not found.\n";
    … …  
    135145
    136146
     147
     148
    137149exit 0;

  • branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_webrequest.pl

    r29124 r29515  
    2626my $dbname;
    2727my $dbserver;
    28 my $project;
    29 my $job_type;
    3028
    3129GetOptions(
    32     'job_type=s'    =>  \$job_type,
    3330    'dbname=s'      =>  \$dbname,
    3431    'dbserver=s'    =>  \$dbserver,
    35     'project=s'     => \$project,
    3632    'verbose'       => \$verbose,
    3733);
    … …  
    4238    print "\n\n";
    4339    print "Starting script $0 on $host\n\n";
    44 }
    45 
    46 my $listMode;
    47 if ($job_type and ($job_type eq 'list_uri')) {
    48     $listMode=1;
    49 } else  {
    50     $listMode=0;
    5140}
    5241
    … …  
    10594}
    10695
    107 # ok at this point we have a request file add it to the database (unless we're in listMode)
    108 if ($listMode == 1 ) {
    109     ###
    110     ### In list mode just parse the file print the output and we're done
    111     ###
    112     my $command = "$pstampparse --mode list_uri --file $request_file";
    113     $command .= " --dbname $dbname" if $dbname;
    114     $command .= " --dbserver $dbserver" if $dbserver;
    115     $command .= " --verbose" if $verbose;
    116     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    117         run(command => $command, verbose => $verbose);
    118 
    119     if ($success) {
    120         ### print "Matching Images:\n";
    121         print @$stdout_buf;
    122         exit 0;
    123     } else {
    124         # we send the output to STDOUT because that's where PHP finds it
    125         print @$stdout_buf;
    126         print @$stderr_buf;
    127         exit 1;
    128     }
    129 }
    130 
    13196# Queue the request
    13297my $req_id = 0;

  • branches/eam_branches/ipp-20100823/pstamp/scripts/pstampparse.pl

    r28878 r29515  
    805805{
    806806    my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
    807 
    808     # XXX: The update process for warp and subsequent stages requires # destreaking to be performed
    809     # because the -pending queries require the inputs to have magicked >= 0
    810     # The case of stack-stack diffs not needing to be destreaked is taken care of in pstamp_checkdependent
    811     $need_magic = 1 if $imagedb eq 'gpc1';
    812807
    813808    # chipRun's can be in full state if destreaking is necessary
    … …  
    900895                    $$r_newState = 'stop';
    901896                    $$r_fault = $PSTAMP_NOT_AVAILABLE;
    902                 } elsif (!$image->{magicked}) {
     897                } elsif ($need_magic and !$image->{magicked}) {
    903898                    $$r_newState = 'stop';
    904899                    $$r_fault = $PSTAMP_NOT_DESTREAKED;
    … …  
    906901                    # cause the image to be re-made
    907902                    # set up to queue an update run
     903                    my $require_magic = ($need_magic or $image->{magicked});
    908904                    get_dependent(\$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb},
    909                         $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic);
     905                        $run_state, $stage, $image->{stage_id}, $image->{component}, $require_magic );
    910906                }
    911907            }

  • branches/eam_branches/ipp-20100823/pstamp/src/ppstamp.c

    r25197 r29515  
    2222
    2323    // define the active I/O files
    24     if (!ppstampParseCamera(config)) {
     24    if (!ppstampParseCamera(config, options)) {
    2525        psErrorStackPrint(stderr, "Unable to parse camera.");
    2626        ppstampCleanup(config, options);

  • branches/eam_branches/ipp-20100823/pstamp/src/ppstamp.h

    r25709 r29515  
    1818
    1919// Determine what type of camera, and initialise
    20 bool ppstampParseCamera(pmConfig *config);
     20bool ppstampParseCamera(pmConfig *config, ppstampOptions *options);
    2121
    2222int ppstampMakeStamp(pmConfig *config, ppstampOptions *);

  • branches/eam_branches/ipp-20100823/pstamp/src/ppstampArguments.c

    r24966 r29515  
    7878        psArgumentRemove(argnum, &argc, argv);
    7979    }
     80    if ((argnum = psArgumentGet(argc, argv, "-stage"))) {
     81        psArgumentRemove(argnum, &argc, argv);
     82        options->stage = psStringCopy(argv[argnum]);
     83        psArgumentRemove(argnum, &argc, argv);
     84    }
    8085
    8186    pmConfigFileSetsMD(config->arguments, &argc, argv, "ASTROM", "-astrom", "-astromlist");

  • branches/eam_branches/ipp-20100823/pstamp/src/ppstampMakeStamp.c

    r28882 r29515  
    233233    int status = false;
    234234
    235     pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSTAMP.OUTPUT");
     235    pmFPAfile *output;
     236    if (!options->stage || (strcmp(options->stage, "diff") != 0)) {
     237        output = psMetadataLookupPtr(NULL, config->files, "PPSTAMP.OUTPUT");
     238    } else {
     239        output = psMetadataLookupPtr(NULL, config->files, "PPSTAMP.OUTPUT.DIFF");
     240    }
    236241    if (!output) {
    237242        psError(PS_ERR_UNKNOWN, false, "Can't find output data\n");
    … …  
    674679static bool setMaskedToNAN(pmConfig *config, psImage *image, psImage *mask, psImage *variance)
    675680{
     681#ifdef notdef
    676682    bool status;
    677683    psMetadata *masks = psMetadataLookupMetadata(&status, config->recipes, "MASKS");
    … …  
    719725        }
    720726    }
    721     fprintf(stderr, "excised %ld masked pixels\n", numExcised);
     727#endif
     728
     729    long numCensored;
     730    if (!pmCensorMasked(config, image, mask, variance, &numCensored)) {
     731        psError(PS_ERR_UNKNOWN, false, "Failed to censor masked pixels.\n");
     732        return false;
     733    }
     734
     735    // XXX: this shouldn't be a fprintf
     736    fprintf(stderr, "Censored %ld masked pixels\n", numCensored);
    722737
    723738    return true;

  • branches/eam_branches/ipp-20100823/pstamp/src/ppstampOptions.h

    r16132 r29515  
    1010    psString    chipName;
    1111    psString    cellName;
     12    psString    stage;
    1213    //
    1314    // Calculated Values

  • branches/eam_branches/ipp-20100823/pstamp/src/ppstampParseCamera.c

    r24966 r29515  
    88
    99// Set up the ppstamp output Image file
    10 bool setupOutput(pmConfig *config, pmFPAfile *input, bool doMask, bool doWeight)
     10bool setupOutput(pmConfig *config, pmFPAfile *input, psString stage, bool doMask, bool doWeight)
    1111{
    12     pmFPAfile *output = pmFPAfileDefineSkycell(config, NULL, "PPSTAMP.OUTPUT");
     12    pmFPAfile *output;
     13   
     14    if (!stage || (strcmp(stage, "diff") != 0)) {
     15        output = pmFPAfileDefineSkycell(config, NULL, "PPSTAMP.OUTPUT");
     16    } else {
     17        // need special filerule for diff stage image to allow for negative values
     18        output = pmFPAfileDefineSkycell(config, NULL, "PPSTAMP.OUTPUT.DIFF");
     19    }
     20    if (!output) {
     21        psError(PS_ERR_UNKNOWN, false, "Unable to setup output.");
     22        return false;
     23    }
     24
    1325    output->save = true;
    1426
    … …  
    2941// something else that I'm missing?
    3042
    31 bool ppstampParseCamera(pmConfig *config)
     43bool ppstampParseCamera(pmConfig *config, ppstampOptions *options)
    3244{
    3345    bool status = false;
    … …  
    8395
    8496    // Set up the output target
    85     if (!setupOutput(config, input, doMask, doWeight)) {
     97    if (!setupOutput(config, input, options->stage, doMask, doWeight)) {
    8698        psError(PS_ERR_UNKNOWN, false, "Unable to setup output.");
    8799        return false;

  • branches/eam_branches/ipp-20100823/pswarp/src/pswarpTransformTile.c

    r28006 r29515  
    7878    int yMax = PS_MIN(maxPt.y, outNumRows);
    7979
    80     double jacobian = map->Xx * map->Yy - map->Yx * map->Xy; // Jacobian of transformation
     80    double jacobian = fabs(map->Xx * map->Yy - map->Yx * map->Xy); // Jacobian of transformation
    8181    double jacobian2 = PS_SQR(jacobian);                     // Square Jacobian
    8282

  • branches/eam_branches/ipp-20100823/tools/checkexp

    r27928 r29515  
    2020my $endDate;
    2121my $verbose;
    22 my $dbname = "gpc1";
     22my $dbname;
    2323
    2424GetOptions(
    … …  
    3333#           -exitval => 3)
    3434#    unless defined $startDate;
     35
     36if (!$dbname) {
     37    my $dbname = "gpc1";
     38}
    3539
    3640my $dbh  = getDBHandle($ipprc, $dbname);

  • branches/eam_branches/ipp-20100823/tools/czarclean.pl

    r29124 r29515  
    99
    1010my $czarDbName = undef;
    11 my $begin = undef;
    12 my $end = undef;
     11my $from = undef;
     12my $to = undef;
    1313my $interval = undef;
    1414my $verbose = undef;
     15my $optimize = undef;
    1516
    1617GetOptions (
    1718        "dbname|d=s" => \$czarDbName,
    1819        "interval|i=s" => \$interval,
    19         "begin|b=s" => \$begin,
    20         "end|e=s" => \$end,
     20        "from|f=s" => \$from,
     21        "to|t=s" => \$to,
     22        "optimize|o" => \$optimize,
    2123        "verbose|v" => \$verbose,
    2224        );
    … …  
    3032    print "* UNKNKOWN: option                          @ARGV\n";
    3133}
    32 if (!$begin) {
     34if (!$from) {
    3335    $quit=1;
    34     print "* REQUIRED: choose a begin date             -b <datetime>                (default=7am this morning)\n";
     36    print "* REQUIRED: choose a from date                 -b <datetime>\n";
    3537}
    36 if (!$end) {
    37     $end=$begin;
    38     print "* OPTIONAL: choose an end date              -e <datetime>                (default=$end)\n";
     38if (!$to) {
     39    if($from) {$to=$from;} else {$to="NULL";}
     40    print "* OPTIONAL: choose a to date                   -e <datetime>                   (default=$to)\n";
    3941}
    4042if (!$czarDbName) {
    4143    $czarDbName = "czardb";
    42     print "* OPTIONAL: choose czar Db name             -d <name>                    (default=$czarDbName\n";
     44    print "* OPTIONAL: choose czar Db name                -d <name>                       (default=$czarDbName\n";
    4345}
    4446if (!$interval) {
    4547    $interval = "30 MINUTE";
    46     print "* OPTIONAL: choose time interval            -i <'1 hour'|'2 hour'|etc>   (default=$interval)\n";
     48    print "* OPTIONAL: choose time interval               -i <'30 MINUTE'|'1 hour'|etc>   (default=$interval)\n";
     49}
     50if (!$optimize) {
     51    $optimize = 0;
     52    print "* OPTIONAL: optimize database after cleanup    -o <'30 MINUTE'|'1 hour'|etc>   (default=$interval)\n";
    4753}
    4854print "*\n*******************************************************************************\n";
    … …  
    5965
    6066
    61 $czarDb->cleanupDateRange($begin, $end, $interval);
    62 
     67$czarDb->cleanupDateRange($from, $to, $interval);
     68if ($optimize) {$czarDb->optimize();}

  • branches/eam_branches/ipp-20100823/tools/czarplot.pl

    r29124 r29515  
    1717my $begin = undef;
    1818my $end = undef;
     19my $day = undef;
    1920my $path = undef;
    2021my $verbose = undef;
    2122my $histogram = undef;
    2223my $timeSeries = undef;
     24my $rate = undef;
     25my $deriv = undef;
     26my $showCleanup = undef;
    2327my $nebulous = undef;
    2428my $savingToFile = undef;
    … …  
    2630
    2731GetOptions (
    28         "dbname|d=s" => \$czarDbName,
     32        "dbname=s" => \$czarDbName,
    2933        "label|l=s" => \$label,
    3034        "stage|s=s" => \$stage,
    … …  
    3236        "begin|b=s" => \$begin,
    3337        "end|e=s" => \$end,
     38        "day|y=s" => \$day,
    3439        "output|o=s" => \$path,
    3540        "histogram|h" => \$histogram,
    3641        "nebulous|n" => \$nebulous,
     42        "cleanup|c" => \$showCleanup,
     43        "rate|r" => \$rate,
     44        "deriv|d" => \$deriv,
    3745        "timeseries|t" => \$timeSeries,
    3846        "verbose|v" => \$verbose,
    … …  
    5159if (!$timeSeries) {
    5260    print "* OPTIONAL: plot timeseries                 -t                          (default=on)\n";}
     61if (!$rate) {
     62    print "* OPTIONAL: plot timeseries of rate         -r                          (default=off)\n";}
     63if (!$deriv) {
     64    $deriv = 0;
     65    print "* OPTIONAL: plot first derivative           -d                          (default=$deriv)\n";}
     66if (!$showCleanup) {
     67    $showCleanup = 0;
     68    print "* OPTIONAL: include cleanup in timeseries   -c                          (default=$showCleanup)\n";}
    5369if (!$log) {
    5470    $log = 0;
    … …  
    6480    print "* OPTIONAL: choose time interval in past    -i <'1 hour'|'1 day'|etc>   (default=none\n";}
    6581if (!$begin) {
    66     print "* OPTIONAL: choose a begin time             -b <datetime>               (default=7am this morning)\n";}
     82    print "* OPTIONAL: choose a begin time             -b <datetime>               (default=6:35am this morning)\n";}
    6783if (!$end) {
    6884    print "* OPTIONAL: choose an end time              -e <datetime>               (default=now)\n";}
     85if (!$day) {
     86    print "* OPTIONAL: choose a single day to plot     -y <date>                   (default=today)\n";}
    6987if (!$path) {
    7088    print "* OPTIONAL: choose an output location       -o <path>                   (default=none)\n";
    … …  
    85103
    86104
    87 # GENE PLOTS my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png size 1280,960 font \"/usr/share/fonts/corefonts/arial.ttf\" 12" : "X11", $path, $save_temps);
    88 my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11", $path, $save_temps);
     105my $plotter = new czartool::Plotter(
     106        $czarDb,
     107        "%Y%m%d-%H%M%S",
     108        $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11",
     109        $path,
     110        $save_temps);
    89111
    90112# sort out times
    91 if (!$end) {$end = $czarDb->getNowTimestamp();}
    92 if (!$begin) {
    93     if ($interval) {$begin = $czarDb->subtractInterval($end, $interval);}
    94     else {$begin =  strftime('%Y-%m-%d 06:35',localtime);}
     113if($day) {
     114
     115    # day plots should run from about 6:30am until midnight
     116    $begin =  "$day 06:35";
     117    $end = "$day 23:59";
    95118}
     119else {
    96120
     121    if (!$end) {$end = $czarDb->getNowTimestamp();}
     122    if (!$begin) {
     123
     124        if ($interval) {$begin = $czarDb->subtractInterval($end, $interval);}
     125        else {$begin =  strftime('%Y-%m-%d 06:35',localtime);}
     126    }
     127}
    97128my $diskUsage = 1; # TODO
    98129
    99 if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log);}
     130if ($rate) {
     131
     132    if (!$interval) {$interval = "1 HOUR";}
     133    $plotter->createRateTimeSeries($label, $stage, $begin, $end, $interval, $log);
     134    exit;
     135}
     136if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log, $showCleanup, $deriv);}
    100137if ($histogram) {$plotter->createHistogram($label, $begin, $end);}
    101138if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);}

  • branches/eam_branches/ipp-20100823/tools/czartool/Burntool.pm

    r29124 r29515  
    5454
    5555        chomp($line);
    56         if($line =~ m/([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)/) {
     56        if($line =~ m/BTSTATS: $today $target ([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)/) {
    5757            if ($self->{_verbose}) {print "Output =  $1 $2 $3 $4\n";}
    5858            ${$pending} = ($2 - $3)/60;

  • branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm

    r29124 r29515  
    1111our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
    1212
    13 # Override constructor
     13###########################################################################
     14#
     15# Constructor (overrides superclass)
     16#
     17###########################################################################
    1418sub new {
    1519    my ($class) = @_;
    1620
    17     # Call the constructor of the parent class, Person.
    18     my $self = $class->SUPER::new($_[1], $_[2], $_[3], $_[4],  $_[5], $_[6]);
     21    # Call the constructor of the parent class
     22    my $self = $class->SUPER::new($_[1], $_[2], $_[3], $_[4], $_[5], $_[6]);
    1923
    2024    bless $self, $class;
    … …  
    3943SQL
    4044
    41 
    42     if (!$query->execute) {
    43 
    44         return 0;
    45     }
     45    if (!$query->execute) {return 0;}
    4646
    4747    ${$labels} = $query->fetchall_arrayref();
    … …  
    238238###########################################################################
    239239#
    240 # Gets time series data and stores it to temp file
    241 #
    242 ###########################################################################
    243 sub createTimeSeriesData {
    244     my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff, $dataFile, $isLog) = @_;
    245 
    246     ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
    247 
    248     open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
     240# Gets time min/max/diff for a given period
     241#
     242###########################################################################
     243sub getTimeMinMaxDiff {
     244    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff) = @_;
    249245
    250246    my $query = $self->{_db}->prepare(<<SQL);
    251247    SELECT
    252         MIN(processed), GREATEST(MAX(pending), MAX(faults), max(processed)-min(processed)) AS maxY,
    253         LEAST(MIN(pending), MIN(faults), max(processed)-min(processed)) AS minY,
    254248        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'),
    255249        DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
    … …  
    262256    if (!$query->execute) {return 0;}
    263257
    264     my $minProcessed = undef;
    265     my ($_maxY, $_minY, $_timeDiff);
    266     ($minProcessed, $_maxY, $_minY, ${$maxX}, ${$minX}, $_timeDiff) = $query->fetchrow_array();
    267 
    268     if ($_maxY > ${$maxY}) {${$maxY} = $_maxY;}
    269     if ($_minY < ${$minY}) {${$minY} = $_minY;}
     258    my ($_timeDiff);
     259    (${$maxX}, ${$minX}, $_timeDiff) = $query->fetchrow_array();
     260
    270261    if ($_timeDiff > ${$timeDiff}) {${$timeDiff} = $_timeDiff;}
    271262
     263return 1;
     264}
     265
     266
     267###########################################################################
     268#
     269# Gets time series data and stores it to temp file
     270#
     271###########################################################################
     272sub createTimeSeriesData {
     273    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup, $firstDeriv) = @_;
     274
     275    my $query = $self->{_db}->prepare(<<SQL);
     276    SELECT
     277        MIN(processed)
     278        FROM $stage
     279        WHERE label LIKE '$label'
     280        AND timestamp >= '$fromTime' AND timestamp <= '$toTime';
     281SQL
     282
     283    if (!$query->execute) {return 0;}
     284    my $minProcessed = scalar  $query->fetchrow_array(); # TODO remove
    272285    if (!defined $minProcessed) {return 0;}
     286
     287    $self->getTimeMinMaxDiff($label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff);
    273288
    274289    $query = $self->{_db}->prepare(<<SQL);
    275290    SELECT
    276         DATE_FORMAT(timestamp, '$self->{_dateFormat}'), pending, faults, processed-$minProcessed
     291        timestamp, pending, faults, processed
    277292        FROM $stage
    278293        WHERE label LIKE '$label'
    … …  
    283298    $query->execute;
    284299
    285     # loop round results
     300    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
     301    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
     302
     303    my $lastProcessed = -1;
     304    my $lastPending = -1;
     305    my $lastFaults = -1;
     306    my $lastTimestamp = undef;
     307    my $processed = 0;
     308    my $runningProcessed = 0;
     309    my $pending = 0;
     310    my $faults = 0;
     311    my $timestamp = undef;
    286312    while (my @row = $query->fetchrow_array()) {
    287         my ($timestamp, $pending, $faults, $processed) = @row;
    288 
    289         if ($isLog) {
    290 
    291             $processed = log($processed + 1);
    292             $pending = log($pending + 1);
    293             $faults = log($faults + 1);
    294         }
     313        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
     314
     315        if ($showCleanup ) {
     316       
     317            $runningProcessed = $thisProcessed - $minProcessed;
     318        }
     319        elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
     320           
     321            $runningProcessed = $runningProcessed + ($thisProcessed - $lastProcessed);
     322        }
     323
     324
     325        if ($firstDeriv) {
     326
     327            if (defined $lastTimestamp) {
     328
     329                my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
     330               
     331                if ($thisProcessed > $lastProcessed){
     332                $processed = abs($thisProcessed - $lastProcessed)/$timeDiff;
     333#print "$processed = ($thisProcessed - $lastProcessed)/$timeDiff\n";
     334                }
     335                else {$processed = 0;}
     336                #$pending = abs($thisPending - $lastPending)/$timeDiff;   
     337                #$faults = abs($thisFaults - $lastFaults)/$timeDiff;   
     338            }
     339            else {
     340                $processed = 0;
     341                $pending = 0;
     342                $faults = 0;
     343            }
     344        }
     345        elsif ($isLog) {
     346
     347            $processed = ($runningProcessed < 0) ? 0 : $runningProcessed;
     348            $pending =  ($thisPending < 0) ? 0 : $thisPending;
     349            $faults =  ($thisFaults < 0) ? 0 : $thisFaults;
     350
     351            $processed = log($runningProcessed + 1)/log(10);
     352            $pending = log($thisPending + 1)/log(10);
     353            $faults = log($faults + 1)/log(10);
     354        }
     355        else {
     356       
     357            $processed = $runningProcessed;
     358            $pending = $thisPending;
     359            $faults = $thisFaults;
     360        }
     361
     362        $timestamp = $self->getFormattedDate($thisTimestamp);
    295363
    296364        print GNUDAT "$timestamp $pending $faults $processed\n";
     365
     366        $lastProcessed = $thisProcessed;
     367        $lastPending = $thisPending;
     368        $lastFaults = $thisFaults;
     369        $lastTimestamp = $thisTimestamp;
    297370    }
    298371
    … …  
    357430sub countProcessedPendingAndFaults {
    358431    my ($self, $label, $stage, $fromTime, $toTime, $processed, $pending, $faults) = @_;
    359 
    360432
    361433    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    371443    (${$pending}, ${$faults}) = $query->fetchrow_array();   
    372444
    373     $query = $self->{_db}->prepare(<<SQL);
    374     SELECT
    375         MAX(processed) - MIN(processed)
    376         FROM $stage
    377         WHERE label LIKE '$label'
    378         AND timestamp >= '$fromTime' AND timestamp <= '$toTime';
    379 SQL
    380     $query->execute;
    381     (${$processed}) = $query->fetchrow_array();
     445    ${$processed} = $self->countProcessed($label, $stage, $fromTime, $toTime);
     446
     447    return 1;
    382448}
    383449
    … …  
    502568        print GNUDAT "$timestamp $available\n";
    503569    }
     570
    504571    close(GNUDAT);
    505572
    506573    return $dataFile;
    507574}
    508 ###########################################################################
    509 #
    510 # Determines how much has been processed in the provided interval of time
    511 # (format: 1 HOUR, 1 MINUTE 1 DAY etc)
    512 #
    513 ###########################################################################
    514 sub countProcessed { # TODO use time not interval
    515     my ($self, $label, $stage, $interval) = @_;
     575
     576###########################################################################
     577#
     578# TODO implement isLog
     579#
     580###########################################################################
     581sub createProcessingRateData {
     582    my ($self, $stage, $label, $startDay, $endDay, $interval, $dataFile, $isLog) = @_;
     583
     584    my $startTime = $startDay;
     585    my $endTime;
     586    my $quit = 0;
     587    my $processed;
     588    my $pending;
     589    my $faults;
     590    my $timestamp;
     591    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_r.dat";
     592    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
     593    my $cleanupCarry = 0;
     594    while(1) {
     595
     596        if (!$self->isBefore($startTime, $endDay)) {last;}
     597        $endTime = $self->addInterval($startTime, $interval);
     598        $self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults);
     599        $timestamp = $self->getFormattedDate($endTime);
     600        print GNUDAT "$timestamp $processed 0 0\n";
     601
     602        $startTime = $endTime;
     603    }
     604
     605    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n"
     606}
     607
     608###########################################################################
     609#
     610# When did this stage finish?
     611#
     612###########################################################################
     613sub getFinishTime {
     614    my ($self, $label, $stage, $begin, $end) = @_;
    516615
    517616    my $query = $self->{_db}->prepare(<<SQL);
    518617    SELECT
    519         MAX(processed) - MIN(processed)
    520         FROM $stage 
     618        timestamp, pending, faults
     619        FROM $stage
    521620        WHERE label LIKE '$label'
    522         AND timestamp > (now() - INTERVAL $interval);
    523 SQL
    524         $query->execute;
    525 
    526     return scalar $query->fetchrow_array();
    527 }
    528 
     621        AND timestamp >= '$begin'
     622        AND timestamp <= '$end'
     623SQL
     624    $query->execute;
     625
     626    my $array = $query->fetchall_arrayref();
     627
     628    my $row;
     629    my $thisPending = -1;
     630    my $lastPending = -1;
     631    my $thisFaults = -1;
     632    my $lastFaults = -1;
     633    my $timestamp;
     634    my $started = undef;
     635    my $finished = undef;
     636    my $thisLeft = 0;
     637    my $lastLeft = 0;
     638    my $finishTimeout = "01:00:00";
     639    my $numRows = @{$array};
     640    print "$numRows rows\n";
     641    my $n = 0;
     642    foreach $row ( @{$array} ){
     643
     644        ($timestamp, $thisPending, $thisFaults) = @{$row};
     645
     646#        print "$timestamp, $thisPending, $thisFaults\n";
     647
     648        $thisLeft = $thisPending - $thisFaults;
     649        $lastLeft = $lastPending - $lastFaults;
     650
     651
     652        if ($n == 0 && $thisLeft > 0) {
     653
     654            print "Starting this time period with stuff pending ($thisLeft)\n";
     655            $started = $timestamp;
     656        }
     657
     658        if (!defined $started && $lastLeft == 0 && $thisLeft > 0) {
     659
     660            $started = $timestamp;
     661            print "STARTED at $started\n";
     662        }
     663
     664        if ($started && !defined $finished && $thisLeft == 0) {
     665
     666            $finished = $timestamp;
     667            print "setting FINISHED to $finished\n";
     668        }
     669
     670        if (defined $finished) {
     671
     672            if ($thisLeft != 0) {
     673               
     674                $finished = undef;
     675           
     676                print "NEW pending  at $timestamp\n";
     677           
     678            }
     679            elsif ($thisLeft == 0) {
     680               
     681                my $howLongFinished = $self->diffTimes($timestamp, $finished);
     682
     683print "$howLongFinished = $timestamp, $finished\n";
     684
     685                if ($self->isIntervalGreaterThan($howLongFinished, $finishTimeout)) {
     686                   
     687                    print "0 pending for $howLongFinished. We're done \n";
     688                    last;
     689                }
     690
     691            }
     692
     693        }
     694
     695
     696        $lastPending = $thisPending;
     697        $lastFaults = $thisFaults;
     698        $n++;
     699        if ($n == $numRows) {
     700
     701            if ($thisLeft == 0) {
     702
     703                print "exceeded interval before reaching finished timeout\n";
     704            }
     705            else {
     706                print "Reached end of interval and not finished yet. $thisLeft left\n";
     707            }
     708        }
     709    }
     710
     711    if ($finished) {
     712
     713        my $timeTaken = $self->diffTimes($finished, $started);
     714        print "FINISHED at $finished with $thisFaults remaining faults, time taken = $timeTaken\n";
     715    }
     716
     717}
     718
     719###########################################################################
     720#
     721# Figures out if a stage has plateaued
     722#
     723###########################################################################
     724sub hasPlateaued {
     725    my ($self, $label, $stage, $end, $interval) = @_;
     726
     727    my $query = $self->{_db}->prepare(<<SQL);
     728    SELECT
     729        timestamp, pending, faults
     730        FROM $stage
     731        WHERE label LIKE '$label'
     732        AND timestamp >= '$end - INTERVAL $interval'
     733        AND timestamp <= '$end'
     734        ORDER BY timestamp DESC;
     735SQL
     736    $query->execute;
     737
     738    my $array = $query->fetchall_arrayref();
     739
     740    my $thisPending = -1;
     741    my $lastPending = -1;
     742    my $thisFaults = -1;
     743    my $lastFaults = -1;
     744    my $timestamp;
     745    my $thisLeft = 0;
     746    my $lastLeft = 0;
     747    my $row;
     748    my $numRows = @{$array};
     749    my $n = 0;
     750    foreach $row ( @{$array} ){
     751
     752        ($timestamp, $thisPending, $thisFaults) = @{$row};
     753
     754        $thisLeft = $thisPending - $thisFaults;
     755        $lastLeft = $lastPending - $lastFaults;
     756
     757        print "$timestamp $thisPending\n";
     758
     759        if ($n>0) {
     760
     761            #if () {}
     762
     763        }
     764
     765        $lastPending = $thisPending;
     766        $lastFaults = $thisFaults;
     767        $n++;
     768    }
     769}
     770
     771###########################################################################
     772#
     773# Gets count of processed stuff in a given time period
     774#
     775###########################################################################
     776sub countProcessed {
     777    my ($self, $label, $stage, $begin, $end) = @_;
     778
     779    my $query = $self->{_db}->prepare(<<SQL);
     780    SELECT
     781        processed
     782        FROM $stage
     783        WHERE label LIKE '$label'
     784        AND timestamp >= '$begin'
     785        AND timestamp < '$end'
     786SQL
     787        $query->execute;
     788
     789    my $processedArray = $query->fetchall_arrayref();
     790
     791    my $processed;
     792    my $thisCount = -1;
     793    my $lastCount = -1;
     794    my $count = 0;
     795    foreach $processed ( @{$processedArray} ){
     796
     797        ($thisCount) = @{$processed};
     798
     799        if ($thisCount > $lastCount && $lastCount != -1) {$count = $count + ($thisCount -  $lastCount);}
     800        $lastCount = $thisCount;
     801    }
     802
     803    return $count;
     804}
    529805
    530806###########################################################################
    … …  
    541817
    542818        if (!$self->isBefore($thisDay, $endDay)) {
    543        
     819
    544820            $quit = 1;
    545821        }
    … …  
    553829###########################################################################
    554830#
    555 # Deletes all but one row per interval from all stage tables for all labels between the provided day
     831# Deletes all but one row per interval from all tables for the provided date range
     832# TODO this is very clumsy, I just have time to thinmk of something more elegant
    556833#
    557834###########################################################################
    … …  
    566843    my $totalDeleted = undef;
    567844    my $quit = 0;
     845    my $query = undef;
    568846    while(!$quit) {
    569847
    570848        $toTime = $self->addInterval($fromTime, $interval);
    571849        if (!$self->isBefore($toTime, $endDay)) {
    572        
     850
    573851            $toTime = $endDay;
    574852            $quit = 1;
    575853        }
    576    
     854
    577855        my $stage = undef;
    578856        $totalDeleted = 0;
    … …  
    586864                my ($label) = @{$row};
    587865
    588                 my $query = $self->{_db}->prepare(<<SQL);
     866                $query = $self->{_db}->prepare(<<SQL);
    589867                SELECT COUNT(*)
    590868                    FROM $stage
    … …  
    609887                $totalDeleted += $toDelete;
    610888            }
    611         }
     889
     890        }
     891
     892        # servers table
     893        my $servers = undef;
     894        if ($self->getServers(\$servers)) {
     895
     896            my $server = undef;
     897            my $row = undef;
     898            foreach $row ( @{$servers} ) {
     899                my ($server) = @{$row};
     900
     901                $query = $self->{_db}->prepare(<<SQL);
     902                SELECT COUNT(*)
     903                    FROM servers
     904                    WHERE timestamp > '$fromTime'
     905                    AND timestamp <= '$toTime'
     906                    AND server = '$server'
     907SQL
     908                    $query->execute;
     909
     910                my $toDelete = scalar $query->fetchrow_array() - 1;
     911                if ($toDelete > 0) {
     912
     913                    $query = $self->{_db}->prepare(<<SQL);
     914                    DELETE FROM servers
     915                        WHERE timestamp > '$fromTime'
     916                        AND timestamp <= '$toTime'
     917                        AND server = '$server' ORDER BY timestamp DESC LIMIT $toDelete
     918SQL
     919                        $query->execute;
     920
     921                    $totalDeleted += $toDelete;
     922                }
     923            }
     924        }
     925
     926        # now deal with cluster_space table
     927        $query = $self->{_db}->prepare(<<SQL);
     928        SELECT COUNT(*)
     929            FROM cluster_space
     930            WHERE timestamp > '$fromTime'
     931            AND timestamp <= '$toTime'
     932SQL
     933            $query->execute;
     934
     935        my $toDelete = scalar $query->fetchrow_array() - 1;
     936        if ($toDelete > 0) {
     937
     938            $query = $self->{_db}->prepare(<<SQL);
     939            DELETE FROM cluster_space
     940                WHERE timestamp > '$fromTime'
     941                AND timestamp <= '$toTime'
     942                ORDER BY timestamp DESC LIMIT $toDelete
     943SQL
     944                $query->execute;
     945
     946            $totalDeleted += $toDelete;
     947        }
     948
    612949        print "   * Deleted $totalDeleted between $fromTime and  $toTime\n";
    613950        $fromTime = $toTime;
    614951    }
     952}
     953
     954###########################################################################
     955#
     956# Optimizes all tables that need to be optimized
     957#
     958###########################################################################
     959sub optimize {
     960    my ($self) = @_;
     961
     962    my $stage = undef;
     963    foreach $stage (@stages) {
     964
     965        $self->optimizeTable($stage);
     966    }
     967
     968    $self->optimizeTable("cluster_space");
     969    $self->optimizeTable("servers");
     970}
     971
     972###########################################################################
     973#
     974# Returns if a particular server has been down for a certain interval
     975#
     976###########################################################################
     977sub isServerDown {
     978    my ($self, $server, $interval) = @_;
     979
     980    my $query = $self->{_db}->prepare(<<SQL);
     981    SELECT COUNT(*)
     982        FROM servers
     983        WHERE server = "$server"
     984        AND timestamp > now() - INTERVAL $interval
     985SQL
     986
     987        $query->execute;
     988    my $numOfReadings = scalar $query->fetchrow_array();
     989
     990    if ($numOfReadings == 0) {return 0;}
     991
     992    $query = $self->{_db}->prepare(<<SQL);
     993    SELECT COUNT(*)
     994        FROM servers
     995        WHERE server = "$server"
     996        AND timestamp > now() - INTERVAL $interval
     997        AND !running
     998SQL
     999
     1000        $query->execute;
     1001    my $numNotRunning = scalar $query->fetchrow_array();
     1002
     1003    if ($numOfReadings == $numNotRunning) {return 1;}
     1004
     1005    return 0;
     1006}
     1007
     1008###########################################################################
     1009#
     1010# Returns an array of servers
     1011#
     1012###########################################################################
     1013sub getServers {
     1014    my ($self, $servers) = @_;
     1015
     1016    my $query = $self->{_db}->prepare(<<SQL);
     1017    SELECT DISTINCT server
     1018        FROM servers
     1019SQL
     1020
     1021        if (!$query->execute) {
     1022
     1023            return 0;
     1024        }
     1025
     1026    ${$servers} = $query->fetchall_arrayref();
     1027
     1028    return 1;
    6151029}
    6161030
    … …  
    6821096    INSERT INTO revision (revision) VALUES ($revision);
    6831097SQL
    684     $query->execute;
     1098        $query->execute;
    6851099}
    6861100
    … …  
    7011115SQL
    7021116
    703     $query->execute;
     1117        $query->execute;
    7041118    my @row = $query->fetchrow_array();
    7051119
    … …  
    7291143    foreach $stage (@stages) {
    7301144
    731     my $query = $self->{_db}->prepare(<<SQL);
     1145        my $query = $self->{_db}->prepare(<<SQL);
    7321146        CREATE TABLE $stage (
    7331147                timestamp TIMESTAMP DEFAULT NOW(),
    … …  
    7581172    foreach $stage (@stages) {
    7591173
    760     my $query = $self->{_db}->prepare(<<SQL);
     1174        my $query = $self->{_db}->prepare(<<SQL);
    7611175        ALTER TABLE $stage
    7621176            ADD COLUMN reverting TINYINT NOT NULL DEFAULT 0;
    … …  
    7831197    foreach $stage (@stages) {
    7841198
    785     my $query = $self->{_db}->prepare(<<SQL);
     1199        my $query = $self->{_db}->prepare(<<SQL);
    7861200        ALTER TABLE $stage
    7871201            ADD COLUMN processed BIGINT NOT NULL DEFAULT 0;
    7881202SQL
    7891203
    790       $query->execute;
     1204            $query->execute;
    7911205    }
    7921206
    … …  
    8081222SQL
    8091223
    810       $query->execute;
     1224        $query->execute;
    8111225
    8121226    $self->setRevision(4);
    … …  
    8301244SQL
    8311245
    832       $query->execute;
     1246        $query->execute;
    8331247
    8341248    $self->setRevision(5);
    … …  
    8541268SQL
    8551269
    856       $query->execute;
    857     }
    858 
    859         my $query = $self->{_db}->prepare(<<SQL);
    860         CREATE INDEX serverIndex ON servers (timestamp, server);
    861 SQL
    862 
    863       $query->execute;
     1270            $query->execute;
     1271    }
     1272
     1273    my $query = $self->{_db}->prepare(<<SQL);
     1274    CREATE INDEX serverIndex ON servers (timestamp, server);
     1275SQL
     1276
     1277        $query->execute;
    8641278
    8651279
    … …  
    8821296SQL
    8831297
    884     $query->execute;
     1298        $query->execute;
    8851299
    8861300    $self->setRevision(7);
    … …  
    9061320SQL
    9071321
    908     $query->execute;
     1322            $query->execute;
    9091323    }
    9101324
    … …  
    9161330SQL
    9171331
    918     $query->execute;
     1332        $query->execute;
    9191333
    9201334    # insert stages into revert table
    … …  
    9261340            ('$stage', 0);
    9271341SQL
    928        $query->execute;
     1342            $query->execute;
    9291343    }
    9301344
    … …  
    9511365SQL
    9521366
    953       $query->execute;
     1367        $query->execute;
    9541368    $query = $self->{_db}->prepare(<<SQL);
    9551369    CREATE TABLE hosts (
    … …  
    9601374SQL
    9611375
    962       $query->execute;
    963 
    964         $query = $self->{_db}->prepare(<<SQL);
    965         CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
    966 SQL
    967 
    968       $query->execute;
     1376        $query->execute;
     1377
     1378    $query = $self->{_db}->prepare(<<SQL);
     1379    CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
     1380SQL
     1381
     1382        $query->execute;
    9691383
    9701384    $self->setRevision(9);
    … …  
    9841398    ALTER TABLE hosts
    9851399        ADD COLUMN writable TINYINT NOT NULL DEFAULT 0,
    986         ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
    987 SQL
    988 
    989     $query->execute;
     1400            ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
     1401SQL
     1402
     1403        $query->execute;
    9901404
    9911405    $self->setRevision(10);
    … …  
    10121426SQL
    10131427
    1014       $query->execute;
    1015         $query = $self->{_db}->prepare(<<SQL);
    1016         CREATE INDEX burntoolIndex ON burntool (timestamp, label);
    1017 SQL
    1018 
    1019       $query->execute;
     1428        $query->execute;
     1429    $query = $self->{_db}->prepare(<<SQL);
     1430    CREATE INDEX burntoolIndex ON burntool (timestamp, label);
     1431SQL
     1432
     1433        $query->execute;
    10201434
    10211435    $self->setRevision(11);
    … …  
    10391453SQL
    10401454
    1041     $query->execute;
     1455        $query->execute;
    10421456
    10431457    $self->setRevision(12);

  • branches/eam_branches/ipp-20100823/tools/czartool/MySQLDb.pm

    r29124 r29515  
    6464    return $self->{_dbHost};                                                 
    6565}                                                                               
     66###########################################################################
     67#
     68# Adds the provided interval to the provided time
     69#
     70###########################################################################
     71sub getFormattedDate {
     72    my ($self, $time) = @_;
     73
     74    my $query = $self->{_db}->prepare(<<SQL);
     75    SELECT DATE_FORMAT('$time','$self->{_dateFormat}');
     76SQL
     77    $query->execute;
     78
     79    return scalar $query->fetchrow_array();
     80}
    6681
    6782###########################################################################
    … …  
    8398###########################################################################
    8499#
     100# Finds the difference of two times
     101#
     102###########################################################################
     103sub diffTimes {
     104    my ($self, $time1, $time2) = @_;
     105
     106      my $query = $self->{_db}->prepare(<<SQL);
     107         SELECT TIMEDIFF('$time1','$time2');
     108SQL
     109
     110    $query->execute;
     111    return scalar $query->fetchrow_array();
     112}
     113
     114###########################################################################
     115#
     116# Finds the difference of two times in seconds
     117#
     118###########################################################################
     119sub diffTimesInSecs {
     120    my ($self, $time1, $time2) = @_;
     121
     122      my $query = $self->{_db}->prepare(<<SQL);
     123         SELECT TIME_TO_SEC(TIMEDIFF('$time1','$time2'));
     124SQL
     125
     126    $query->execute;
     127    return scalar $query->fetchrow_array();
     128}
     129
     130###########################################################################
     131#
     132# Returns whether the first interval is larger than the second
     133#
     134###########################################################################
     135sub isIntervalGreaterThan {
     136    my ($self, $interval1, $interval2) = @_;
     137
     138      my $query = $self->{_db}->prepare(<<SQL);
     139          SELECT INTERVAL('$interval1', '$interval2');
     140SQL
     141    $query->execute;
     142
     143return scalar $query->fetchrow_array();
     144}
     145
     146###########################################################################
     147#
    85148# Subtracts the provided interval from the provided time
    86149#
    … …  
    127190
    128191return scalar $query->fetchrow_array();
     192}
     193
     194#######################################################################################
     195#
     196# Optimizes a table
     197#
     198#######################################################################################
     199sub optimizeTable {
     200    my ($self, $table) = @_;
     201
     202    my $query = $self->{_db}->prepare(<<SQL);
     203    OPTIMIZE TABLE $table;
     204SQL
     205
     206    my $success = $query->execute;
     207
     208    print "* ";
     209    if (!$success) {print "UN";}
     210    print "successfully optimized '$table' table\n";
     211
     212    return $success;
    129213}
    130214

  • branches/eam_branches/ipp-20100823/tools/czartool/Plotter.pm

    r29124 r29515  
    3333###########################################################################
    3434sub createImageFileName {
    35     my ($self, $label, $stage, $suffix, $isLog) = @_;
     35    my ($self, $label, $stage, $suffix, $isLog, $isRate) = @_;
    3636
    3737    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : ".";
    … …  
    5555###########################################################################
    5656#
     57# Plots a time series of processing rate for all stages for this label TODO duplication below. combine
     58#
     59###########################################################################
     60sub createRateTimeSeries {
     61    my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $isLog) = @_;
     62
     63    my $minX = 999999999;     
     64    my $maxX = -9999999999;
     65    my $timeDiff = -1;
     66
     67    my $stages = undef;                 
     68
     69    if (!$selectedStage) {$stages = \@allStages;}
     70    else {$stages = ["$selectedStage"]};       
     71
     72    $self->{_czarDb}->getTimeMinMaxDiff($label, $selectedStage, $beginTime, $endTime, \$minX, \$maxX, \$timeDiff);
     73
     74    my $divX;
     75    my $timeFormat;
     76    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
     77
     78    my %gnuplotFiles;
     79    my $stage = undef;
     80    my $gnuplotFile = undef;
     81    my $outputFile = createImageFileName($self, $label, $selectedStage, "r", $isLog);
     82    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
     83    use FileHandle;
     84    GP->autoflush(1);
     85    print GP
     86        "set term $self->{_outputFormat};" .
     87        "set output \"$outputFile\";" .
     88        "set title \"'$label', '$selectedStage' during '$beginTime' to '$endTime'\";" .
     89        #"set xdata time;" .
     90        "set timefmt \"$self->{_dateFormat}\";" .
     91       "set format x \"$timeFormat\";" .
     92        "set xtics \"$minX\", $divX, \"$maxX\";" .
     93#        "set xrange [\"$minX\":\"$maxX\"];" .
     94        "set grid;" .
     95        "set boxwidth;" .
     96        "set style data histogram;" .
     97        "set style histogram rowstacked;" .
     98        "set style fill solid border -1;" .
     99        "set ylabel \"Exposures processed per $interval\";" .
     100        "set boxwidth 0.75;" .
     101        "plot ";
     102
     103    my $first = 1;
     104    foreach $stage (@{$stages}) {
     105
     106        if ($self->{_czarDb}->createProcessingRateData(
     107                    $stage,
     108                    $label,
     109                    $beginTime,
     110                    $endTime,
     111                    $interval,
     112                    \$gnuplotFile,
     113                    $isLog)) {
     114
     115            $gnuplotFiles{$stage} = $gnuplotFile;
     116
     117            if (!$first) { print GP ","; }
     118            print GP "'$gnuplotFile' using 2:xtic(1) title \"$stage\" ";
     119            $first = 0;
     120        }
     121    }
     122
     123    print GP ";\n";
     124    close GP;
     125}                                                   
     126###########################################################################
     127#
    57128# Plots a time series for all stages for this label
    58129#
    59130###########################################################################
    60131sub createTimeSeries {
    61     my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog) = @_;
    62 
    63     my ($minX, $maxX, $minY, $maxY, $timeDiff);
    64     $minX = 999999999;     
    65     $maxX = -9999999999;
    66     $minY = 999999999;         
    67     $maxY = -99999999;
    68     $timeDiff = 0;
     132    my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog, $showCleanup, $deriv) = @_;
     133
     134    my $minX = 999999999;     
     135    my $maxX = -9999999999;
     136    my $timeDiff = 0;
    69137
    70138    my $stages = undef;                 
    … …  
    85153                    \$minX,
    86154                    \$maxX,
    87                     \$minY,
    88                     \$maxY,
    89155                    \$timeDiff,
    90156                    \$gnuplotFile,
    91                     $isLog)) {
     157                    $isLog,
     158                    $showCleanup,
     159                    $deriv)) {
    92160
    93161            $gnuplotFiles{$stage} = $gnuplotFile;
    … …  
    118186            $maxX,
    119187            $minX,
    120             $maxY,
    121             $minY,
    122188            $timeDiff,
    123             $isLog);
     189            $isLog,
     190            $deriv);
    124191}                                                   
    125192
    … …  
    161228    close(GNUDAT);
    162229
    163     plotHistogram($self, $inputFile, $outputFile, $label, $beginTime, $endTime, $maxY);
     230    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
     231    use FileHandle;
     232    GP->autoflush(1);
     233
     234    if ($maxY == 0) {$maxY = 1;}
     235    else {$maxY = $maxY*1.1;}
     236
     237    print GP
     238        "set term $self->{_outputFormat};" .
     239        "set output \"$outputFile\";" .
     240        "set title \"'$label', '$beginTime' to '$endTime'\";" .
     241        "set grid;" .
     242        "set boxwidth;" .
     243        "set yrange [\"0\":\"$maxY\"];" .
     244        "set style data histogram;" .
     245        "set style histogram rowstacked;" .
     246        "set style fill solid border -1;" .
     247        "set ylabel \"Exposures\";" .
     248        "set boxwidth 0.75;" .
     249        "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" .
     250        "\n";
     251
     252    close GP;
    164253    unlink($inputFile);
    165254}
    166 
    167255
    168256###########################################################################
    … …  
    239327###########################################################################
    240328sub plotTimeSeries {
    241     my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff, $isLog) = @_;
     329    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $timeDiff, $isLog, $isDeriv) = @_;
    242330
    243331    my $timeFormat = undef;
    244332    my $divX = undef;
    245333    my $yTitle = undef;
    246     if ($isLog) {$yTitle = "Log( numExposures )";}
    247     else {$yTitle = "numExposures";}
     334    if ($isLog) {$yTitle = "Log( Exposures )";}
     335    elsif ($isDeriv) {$yTitle = "dExposures/dTime";}
     336    else {$yTitle = "Exposures";}
    248337
    249338    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
    250339
    251340    my $numOfPlots = keys %$gnuplotFiles;
    252     my $title = undef;
    253341
    254342    # sort out plot title
    255     if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title = "'".$stage."'";}}
    256     else {$title = "'All stages'"}
     343    my $title = "";
     344    if ($isDeriv) {$title .= "First derivatives of "}
     345    if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title .= "'".$stage."'";}}
     346    else {$title .= "'all stages'"}
    257347
    258348    $title .= " for '$label', '$fromTime' to '$toTime'";
    … …  
    283373        if ($numOfPlots == 1) {
    284374
     375            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
    285376            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2 lw 2,";
    286             print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
    287377            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 7 lw 2";
    288378        }
    … …  
    346436    close GP;
    347437    unlink($gnuplotFile);
    348 }
    349 
    350 ###########################################################################
    351 #
    352 # Plots a histogram of processed stuff
    353 #
    354 ###########################################################################
    355 sub plotHistogram {
    356     my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime, $maxY) = @_;
    357 
    358     open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
    359     use FileHandle;
    360     GP->autoflush(1);
    361 
    362     if ($maxY == 0) {$maxY = 1;}
    363     else {$maxY = $maxY*1.1;}
    364 
    365     print GP
    366         "set term $self->{_outputFormat};" .
    367         "set output \"$outputFile\";" .
    368         "set title \"'$label', '$fromTime' to '$toTime'\";" .
    369         "set grid;" .
    370         "set boxwidth;" .
    371         "set yrange [\"0\":\"$maxY\"];" .
    372         "set style data histogram;" .
    373         "set style histogram rowstacked;" .
    374         "set style fill solid border -1;" .
    375         "set ylabel \"Exposures\";" .
    376         "set boxwidth 0.75;" .
    377         "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" .
    378         "\n";
    379 
    380     close GP;
    381438}
    382439

  • branches/eam_branches/ipp-20100823/tools/definetargets

    r26097 r29515  
    3333
    3434my @all_filters = qw( g r i z y );
    35 my @all_stages = qw( raw chip camera fake warp stack diff );
     35my @all_stages = qw( raw chip camera fake warp stack diff SSdiff );
    3636
    3737my @filters;

  • branches/eam_branches/ipp-20100823/tools/roboczar.pl

    r29124 r29515  
    1414use czartool::Burntool;
    1515
    16 my $period = 60;
    1716my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
    1817my $save_temps = 0;
    1918
    2019GetOptions (
    21         "period|p=s" => \$period, # TODO more Db args
    2220        "dbname|d=s" => \$czarDbName,
    2321        );
    2422
    2523my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
    26 my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
    27 my $nebulous = new czartool::Nebulous($czarDb);
    28 my $pantasks = new czartool::Pantasks();
    29 my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
    30 my $burntool = new czartool::Burntool();
    3124
    3225$czarDb->setDateFormat("%Y%m%d-%H%i%s");
    3326
    3427my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
     28my @serversWeCareAbout = ("stdscience", "distribution", "summitcopy", "registration");
    3529
    3630
    37 timePoll($period);
     31while(1) {
    3832
    39 ###########################################################################
    40 #
    41 # Updates the labels from pantasks for all interested servers
    42 #
    43 ###########################################################################
    44 sub updateLabels {
    45 
    46     print "* Updating labels\n";
    47     my @servers = ("stdscience", "distribution", "publishing", "update");
    48 
    49     my $server = undef;
    50     foreach $server (@servers) {
    51 
    52         my @labels = @{$pantasks->getLabels($server)};
    53         if (@labels) {
    54        
    55             $czarDb->updateCurrentLabels($server, \@labels);
    56         }
    57         else {
    58        
    59              print "WARNING: No labels to update for '$server'\n";
    60         }
    61     }
     33    checkServers("20 MINUTE");
     34    sleep(1200);
    6235}
    6336
    6437###########################################################################
    6538#
    66 # Updates pantasks server status TODO should really get info for all servers at once
     39# Checks tha the important servers are running
    6740#
    6841###########################################################################
    69 sub updateServerStatus {
    70     print "* Checking all pantasks servers\n";
     42sub checkServers {
     43    my ($interval) = @_;
    7144
    72     my $servers = $pantasks->getServerList();
     45    my $server;
     46    foreach $server (@serversWeCareAbout) {
    7347
    74     my $server = undef;
    75     my $alive = undef;
    76     my $running = undef;
    77     foreach $server (@{$servers}) {
     48        if ($czarDb->isServerDown($server, $interval)) {
    7849
    79         $pantasks->getServerStatus($server, \$alive, \$running);
    80         $czarDb->updateServerStatus($server, $alive, $running);
    81     }
    82 }
    83 
    84 ###########################################################################
    85 #
    86 # Polls with provided period (seconds)
    87 #
    88 ###########################################################################
    89 sub timePoll {
    90     my ($period) = @_;
    91 
    92     my $label;
    93     my $new;
    94     my $full;
    95     my $faults;
    96     my $stage;
    97     my $query = undef;
    98     my $str = undef;
    99     my $labels = undef;
    100     my $updateLabels = undef;
    101     my $row = undef;
    102     my $begin = undef;
    103     my $end = undef;
    104     my $priority = undef;
    105     my $newState = undef;
    106     my $nsStatus = undef;
    107 
    108     while (1) {
    109 
    110         # sort out times
    111         $begin =  strftime('%Y-%m-%d 06:35',localtime);
    112         $end = $czarDb->getNowTimestamp();
    113 
    114         if ($czarDb->isBefore($end, $begin)) {
    115 
    116             $begin = $czarDb->subtractInterval($begin, "1 DAY");
     50            print "$server has been down for the last $interval\n";
     51            sendEmail(
     52                    "roydhenderson\@gmail.com",
     53                    "roboczar\@ipp.com",
     54                    "Roboczar update",
     55                    "\n\n* '$server' server has been down for the last $interval\n\n");
    11756        }
    118 
    119         # check nightly science status
    120         print "* Checking nightly science status\n";
    121         if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";}
    122         $czarDb->updateNightlyScience($nsStatus);
    123 
    124         # check nebulous
    125         print "* Checking Nebulous\n";
    126         $nebulous->updateClusterSpaceInfo();
    127         $plotter->plotDiskUsageHistogram();
    128         updateServerStatus();
    129 
    130         # check labels
    131         updateLabels();
    132 
    133         # servers to check
    134         my @serversToCheck = ("stdscience", "update");
    135 
    136         my $thisServer = undef;
    137         foreach $thisServer (@serversToCheck) {
    138 
    139             if ($thisServer eq "update") {$newState = "update";}
    140             else {$newState = "new";}
    141 
    142             # deal with stdscience labels
    143             if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;}
    144             my $size = @{$labels};
    145             if($size > 0) {
    146 
    147                 # get priority
    148                 foreach $row ( @{$labels} ) {
    149                     my ($label) = @{$row};
    150                     $priority = $gpc1Db->getPriority($label);
    151                     $czarDb->setLabelPriority($label, $priority);
    152                 }
    153 
    154                 updateAllStages($thisServer, $newState, $labels, $begin, $end);
    155                 createPlots($thisServer, $labels, $begin, $end);
    156             }
    157             else { print "* WARNING: no $thisServer labels found in Db\n";}
    158         }
    159 
    160         print "--------------------------------------------------------------------------\n";
    161         print "* Going to sleep\n";
    162         sleep($period);
    163         print "* Waking up\n";
    164 
    165         #sendEmail("roydhenderson\@gmail.com", "roboczar\@ipp.com", "Roboczar update", "Some content");
    166     };
    167 }
    168 
    169 ###########################################################################
    170 #
    171 # Loops through labels and creates time series and histogram plots
    172 #
    173 ###########################################################################
    174 sub createPlots {
    175     my ($server, $rows, $begin, $end) = @_;
    176 
    177     my $stage = undef;
    178     my $row = undef;
    179 
    180     print "* Generating plots\n";
    181 
    182     # create plots for each label for each stage
    183     foreach $stage (@stages) {
    184         foreach $row ( @{$rows} ) {
    185             my ($label) = @{$row};
    186 
    187             chomp($label);
    188             $plotter->createLogAndLinearTimeSeries($label,  $stage, $begin, $end);
     57        else {
     58            #print "$server has been running for some of the last $interval\n";
    18959        }
    19060    }
    191 
    192     # create plots for each label for all stages
    193     foreach $row ( @{$rows} ) {
    194         my ($label) = @{$row};
    195 
    196         $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end);
    197         $plotter->createHistogram($label, $begin, $end);
    198 
    199         #routineChecks($label, "1 HOUR");
    200     }
    201     $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", undef, $begin, $end);
    202     $plotter->createHistogram("all_".$server."_labels", $begin, $end);
    203     foreach $stage (@stages) {
    204 
    205         $plotter->createLogAndLinearTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
    206     }
    207 }
    208 
    209 ###########################################################################
    210 #
    211 # Loops through some labels and updates processed/pending/faults in the Db
    212 #
    213 ###########################################################################
    214 sub updateAllStages {
    215     my ($labelServer, $newState, $rows, $begin, $end) = @_;
    216 
    217     print "* Updating stage data\n";
    218     my $totalNew = undef;
    219     my $totalFaults = undef;
    220     my $totalFull = undef;
    221     my $stage = undef;
    222     my $reverting = 0;
    223     my $row = undef;
    224     my $new = undef;
    225     my $full = undef;
    226     my $faults = undef;
    227     my $server = undef;
    228     my $state = undef;
    229    
    230     foreach $stage (@stages) {
    231 
    232         $server = $pantasks->getServerForThisStage($stage);
    233         $pantasks->getRevertStatus($stage, \$reverting);
    234         $czarDb->updateRevertStatus($stage, $reverting);
    235 
    236         print "* Checking labels for $stage stage\n";
    237 
    238         $totalNew=$totalFaults=$totalFull=0;
    239         foreach $row ( @{$rows} ) {
    240             my ($label) = @{$row};
    241 
    242             chomp($label);
    243 
    244             if ($stage eq "burntool") {
    245 
    246                 if ($labelServer eq "stdscience") {
    247 
    248                     $burntool->getPendingAndProcessed($label, \$new, \$full);
    249                     $faults = 0;
    250                 }
    251                 else { $new = $full = $faults = 0;}
    252             }
    253             else {
    254 
    255                 $new = $gpc1Db->countExposures($label, $stage, $newState);
    256                 $full = $gpc1Db->countExposures($label, $stage, "full");
    257                 $faults = $gpc1Db->countFaults($label, $stage, $newState);
    258             }
    259             #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
    260             $totalNew += $new;
    261             $totalFull += $full;
    262             $totalFaults += $faults;
    263 
    264             $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults);
    265         }
    266 
    267         $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", $totalNew, $totalFull, $totalFaults);
    268     }
    269 }
    270 
    271 ###########################################################################
    272 #
    273 # Performs some routine checks on processing status and sends alerts if it needs to
    274 #
    275 ###########################################################################
    276 sub routineChecks {
    277     my ($label, $interval) = @_;
    278 
    279     my $faultsNow;
    280     my $faultsInPast;
    281     my $newFaults;
    282     my $pendingNow;
    283     my $processedRecently;
    284     my $stage = undef;
    285 
    286     print "* Checking all stages for label $label\n";
    287 
    288     foreach $stage (@stages) {
    289 
    290         # check for increasing faults
    291         $faultsNow = $czarDb->countFaultsInPast($label, $stage, "0 MINUTE");
    292         $faultsInPast = $czarDb->countFaultsInPast($label, $stage, $interval);
    293         if ($faultsNow > $faultsInPast) {
    294             $newFaults = $faultsNow - $faultsInPast;
    295             print "There have been $newFaults new faults in the last $interval (label='$label', stage='$stage')\n";
    296         }
    297 
    298         # check for lack of processing
    299         $pendingNow =  $czarDb->countPendingNow($label, $stage);
    300         $processedRecently = $czarDb->countProcessed($label, $stage, $interval);
    301         if ($pendingNow > 0 && $processedRecently < 1) {
    302 
    303             print "Only $processedRecently exposures have processed out of $pendingNow($faultsNow) pending in the last $interval (label='$label', stage='$stage')\n";
    304 
    305         }
    306     }
    307 }
     61}
    30862
    30963###########################################################################
Note: See TracChangeset for help on using the changeset viewer.