IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29124


Ignore:
Timestamp:
Sep 7, 2010, 9:16:10 AM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/eam_branches/ipp-20100823
Files:
2 deleted
54 edited
17 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20100823

  • branches/eam_branches/ipp-20100823/Ohana/src/opihi/dvo/ImageSelection.c

    r28958 r29124  
    4747Image *MatchImage (unsigned int time, short int source, unsigned int imageID) {
    4848
    49   int m;
     49  int m = -1;
    5050
    5151  if ((imageID != 0) && (imageID < Nimage)) {
    52     m = (int) imageID - 1;
    53   } else {
     52    // imageID is in range for the array of images. If the table is still in order and
     53    // no images have been deleted the index of the image we are looking for will be imageID - 1
     54    // If this is the case, we have it. Otherwise we'll have to go search for it below
     55    int guess = (int) imageID - 1;
     56    if (image[guess].imageID == imageID) {
     57        m = guess;
     58    }
     59  }
     60  if (m == -1) {
    5461    m = match_image_subset (image, subset, Nsubset, time, source);
    5562  }

  • branches/eam_branches/ipp-20100823/Ohana/src/opihi/dvo/find_matches.c

    r27610 r29124  
    1010  off_t *N1, *N2;
    1111  double *X1, *Y1, *X2, *Y2;
    12   double dX, dY, dR;
     12  double dX, dY, dR, Doff, Dmin, Dmax, Rmin, Rmax;
    1313  int status;
    14   double RADIUS2, Rmin;
     14  double RADIUS2, RadMin;
    1515  off_t Nave;
    1616  Coords tcoords;
    … …  
    5252  strcpy (tcoords.ctype, "RA---ARC");
    5353
     54  // this region includes a boundary layer of size RADIUS
     55  if (fabs(region[0].Dmin) < fabs(region[0].Dmax)) {
     56    Doff = RAD_DEG*region[0].Dmax;
     57  } else {
     58    Doff = RAD_DEG*region[0].Dmin;
     59  }   
     60  if (Doff < 80) {
     61    Rmin = region[0].Rmin - RADIUS / 3600.0 / cos(Doff);
     62    Rmax = region[0].Rmax + RADIUS / 3600.0 / cos(Doff);
     63  } else {
     64    Rmin = 0.0;
     65    Rmax = 360.0;
     66  }
     67  Dmin = region[0].Dmin - RADIUS / 3600.0;
     68  Dmax = region[0].Dmax + RADIUS / 3600.0;
     69
    5470  // identify the entries contained by this catalog & init index
    5571  for (i = 0; i < Npoints; i++) {
    5672    index[i] = -1;
    57     if (RAvec->elements.Flt[i] < region[0].Rmin) continue;
    58     if (RAvec->elements.Flt[i] > region[0].Rmax) continue;
    59     if (DECvec->elements.Flt[i] < region[0].Dmin) continue;
    60     if (DECvec->elements.Flt[i] > region[0].Dmax) continue;
     73    if (RAvec->elements.Flt[i] < Rmin) continue;
     74    if (RAvec->elements.Flt[i] > Rmax) continue;
     75    if (DECvec->elements.Flt[i] < Dmin) continue;
     76    if (DECvec->elements.Flt[i] > Dmax) continue;
    6177    index[i] = -2;
    6278  }
    … …  
    100116    /* within match range; look for matches */
    101117    Jmin = -1;
    102     Rmin = RADIUS2;
     118    RadMin = RADIUS2;
    103119    for (J = j; (dX > -1.02*RADIUS) && (J < Nave); J++) {
    104120      /* find closest match for this detection */
    … …  
    107123      dR = dX*dX + dY*dY;
    108124      if (dR > RADIUS2) continue;
    109       if (dR < Rmin) {
    110         Rmin = dR;
     125      if (dR < RadMin) {
     126        RadMin = dR;
    111127        Jmin  = J;
    112128      }

  • branches/eam_branches/ipp-20100823/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

    r28562 r29124  
    11301130                    }
    11311131                    $cam_path_base = $run->{cam_path_base};
    1132                     $astrom_file = $run->{astrom_file};
     1132                    $astrom_file = $run->{astrom};
    11331133                }
    11341134                my $astrom_file_resolved = $ipprc->file_resolve($astrom_file);

  • branches/eam_branches/ipp-20100823/ippMonitor/Makefile.in

    r28969 r29124  
    6161$(DESTWWW)/cleanTmpDirectory.php \
    6262$(DESTWWW)/warpProcessedExp_Images.php \
    63 $(DESTWWW)/warpProcessedExp.php
     63$(DESTWWW)/warpProcessedExp.php \
     64$(DESTWWW)/diskUsage.php
    6465
    6566DEFSRC = \

  • branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_labels.php

    r28969 r29124  
    3939if ($selectedStage == "") { $selectedStage = "all_stages"; }
    4040
    41 echo 'Some documentation can be found <a href="http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing">here</a><br>';
     41$nsStatus = getNightlyScienceStatus($czardb);
     42echo "<p  align=\"center\"> Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses). <br>Documentation can be found <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\">here</a> and cluster load monitored <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&c=IPP%2520Production\">here</a><br>Current nightly science status: $nsStatus</p>";
     43
    4244
    4345// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
    … …  
    6668
    6769$states=array("full","new","drop","wait");
    68 $stages=array("chip","cam","fake","warp","stack","diff","magic","magicDS","dist");
     70$stages=array("burntool", "chip","cam","fake","warp","stack","diff","magic","magicDS","dist");
    6971$servers=array(
    7072        "addstar",
    … …  
    8789echo "<input type=\"hidden\" name=\"menu\" value=\"$menu\">\n";
    8890echo "</form>\n";
    89 echo "<p  align=\"center\"> Current status of IPP (any faults are shown in parentheses). NOTE: This data is good as of: $lastUpdateTime </p>";
    9091
    9192echo "<table>\n";
    … …  
    231232    echo "<tr><td></td>\n";
    232233
    233     echo "<p  align=\"center\"> Current labels for $server server </p>";
     234    echo "<p  align=\"center\"> Current labels for $server server (D=distributing, P=publishing)</p>";
    234235
    235236    write_header_cell($class, "");
    … …  
    238239    foreach ($stages as &$stage) {
    239240
     241        if ($stage == "burntool") continue;
    240242        $reverting = getRevertStatus($db, $stage);
    241243        $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&revertmode=";
    … …  
    249251    echo "</tr>\n";
    250252    echo "<tr><td></td>\n";
     253    write_header_cell($class, " ");
    251254    write_header_cell($class, "Label (in order of priority)");
    252     write_header_cell($class, "Distributing?");
    253     write_header_cell($class, "Publishing?");
    254255    foreach ($stages as &$stage) {
    255256       
    … …  
    287288
    288289        echo "<tr><td></td>\n";
     290        $distPub = " ";
     291        if ($distributing) $distPub = "D";
     292        if ($publishing) $distPub = $distPub . "P";
     293        write_table_cell($class, '%s', "", $distPub);
     294
    289295        write_table_cell($class, '%s', $link, $thisLabel);
    290         write_table_cell($class, '%s', "", $distributing ? "yes" : "NO");
    291         write_table_cell($class, '%s', "", $publishing ? "yes" : "NO");
    292296
    293297        $str = "";
    294298        $anyFaults = false;
     299
     300        $link = $defaultlink;
     301        getStateAndFaults($db, $thisLabel, $selectedState, "burntool", $str, $anyFaults);
     302        write_table_cell($class, '%s', $anyFaults ? $link : "", $str);
    295303
    296304        $link = "failedChipProcessedImfile.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $thisLabel . "&chipRun.state=new";
    … …  
    337345
    338346    echo "<tr><td></td>\n";
     347    write_table_cell($class, '%s', "", " ");
    339348    write_table_cell($class, '%s', $link, "All $server labels");
    340349
    … …  
    355364    $qry = $db->query($sql);
    356365    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
     366}
     367
     368###########################################################################
     369#
     370# Returns current nightly science status
     371#
     372###########################################################################
     373function getNightlyScienceStatus($db) {
     374
     375    $sql = "SELECT status FROM nightlyscience";
     376    if ($debug) {echo "$sql<br>";}
     377
     378    $qry = $db->query($sql);
     379    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
     380    $qry->fetchInto($row);
     381
     382    return $row[0];
    357383}
    358384

  • branches/eam_branches/ipp-20100823/ippMonitor/raw/ipp.imfiles.dat

    r28969 r29124  
    3838menulink  | menuselect   | link    | Tables columns               | columns_in_db.php
    3939menulink  | menuselect   | link    | Clean /tmp directory         | cleanTmpDirectory.php
     40menutop   | menutop      | link    | disk usage                   | diskUsage.php

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl

    r28982 r29124  
    123123    defined $queue_chips or defined $queue_stacks or $queue_sweetspot or $queue_detrends or $queue_dqstats or
    124124    defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or
    125     defined $test_mode or defined $clean_old or defined $check_mode;
     125    defined $test_mode or defined $clean_old or defined $check_mode or
     126    defined $confirm_stacks;
    126127
    127128# Configurable parameters from our config file.
    … …  
    12711272                    print STDERR "confirm_stacks: Target $target on $date is not done stacking. $Nstacks $Nattempted\n"
    12721273                }
     1274                if ($metadata_out{nsState} eq 'CONFIRM_STACKING') {
     1275                    $metadata_out{nsState} = 'STACKING';
     1276                }
     1277                   
    12731278                next;
    12741279            }
    1275             if (defined($pretend)) {
    1276                 add_to_macro_list('check_confirm_stacks',$stackable_list{$target},$date,$target);
    1277             }
    1278             else {
    1279                 add_to_macro_list('confirm_stacks',$stackable_list{$target},$date,$target);
     1280            if ($metadata_out{nsState} eq 'CONFIRM_STACKING') {
     1281                if (defined($pretend)) {
     1282                    add_to_macro_list('check_confirm_stacks',$stackable_list{$target},$date,$target);
     1283                }
     1284                else {
     1285                    add_to_macro_list('confirm_stacks',$stackable_list{$target},$date,$target);
     1286                }
    12801287            }
    12811288        }

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/dist_bundle.pl

    r28963 r29124  
    154154#   2. magic is not required for this distRun
    155155#   3. the processing for the component produced no images (warp or diff with bad quality for example)
    156 my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack')) || $no_magic || $poor_quality);
     156my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack') || ($stage eq 'sky')) || $no_magic || $poor_quality);
    157157
    158158my ($image, $mask, $variance);

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/dist_defineruns.pl

    r28536 r29124  
    8383    push @stages, $stage;
    8484} else {
    85     @stages = qw( raw chip chip_bg camera fake warp warp_bg diff stack SSdiff);
     85    @stages = qw( raw chip chip_bg camera fake warp warp_bg diff stack SSdiff sky);
    8686}
    8787

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_destreak_cleanup.pl

    r27785 r29124  
    3838
    3939# Parse the command-line arguments
    40 my ($magic_ds_id, $camera);
     40my ($magic_ds_id, $camera, $stage);
    4141my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
    4242
    … …  
    4444           'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
    4545           'camera=s'       => \$camera,     # camera for evaluating file rules
     46           'stage=s'        => \$stage,     # camera for evaluating file rules
    4647           'save-temps'     => \$save_temps, # Save temporary files?
    4748           'dbname=s'       => \$dbname,     # Database name
    … …  
    5657           -exitval => 3) unless
    5758    defined $magic_ds_id and
    58     defined $camera;
    59 #    defined $stage and
    60 #    defined $stage_id;
     59    defined $camera and
     60    defined $stage;
    6161
    6262my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    … …  
    7979my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to mysql server\n";
    8080
    81 my $q1 = "SELECT magicDSRun.*, camera, camProcessedExp.path_base AS cam_path_base, camRun.reduction AS cam_reduction, magicRun.inverse"
    82          . " FROM magicDSRun JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN camRun USING(cam_id)"
    83          . " WHERE magic_ds_id = $magic_ds_id";
     81my $q1;
     82
     83if ($stage ne 'diff') {
     84    $q1 = "SELECT magicDSRun.*, camera, camProcessedExp.path_base AS cam_path_base, camRun.reduction AS cam_reduction"
     85     . " FROM magicDSRun JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN camRun USING(cam_id)";
     86} else {
     87    $q1 = "SELECT magicDSRun.*, diffRun.diff_mode FROM magicDSRun JOIN diffRun ON stage_id = diffRun.diff_id AND stage = 'diff'";
     88}
     89$q1 .= " WHERE magic_ds_id = $magic_ds_id";
     90
    8491my $q2 = "SELECT * from magicDSFile where magic_ds_id = $magic_ds_id";
    8592
    … …  
    8996&my_die ("Unable to find magicDSRun $magic_ds_id", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nrows;
    9097my $run = $stmt1->fetchrow_hashref();
     98$stmt1->finish();
    9199
    92100my $state = $run->{state};
    93 my $stage = $run->{stage};
    94101my $stage_id = $run->{stage_id};
    95102my $cam_path_base = $run->{cam_path_base};
    96103my $cam_reduction = $run->{cam_reduction};
    97104$cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL');
    98 my $inverse = $run->{inverse};
     105
     106my $warp_warp = ($stage eq 'diff' and $run->{diff_mode} eq 1);
    99107
    100108
    … …  
    105113&my_die("clean not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw";
    106114
    107 my $recipe_psastro = $ipprc->reduction($cam_reduction, 'PSASTRO'); # Recipe to use
    108 &my_die("Unrecognised PSASTRO recipe", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;
     115my $recipe_psastro;
     116if ($stage eq 'chip') {
     117    $recipe_psastro = $ipprc->reduction($cam_reduction, 'PSASTRO'); # Recipe to use
     118    &my_die("Unrecognised PSASTRO recipe", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;
     119}
    109120
    110121
    … …  
    217228        delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
    218229
    219         if ($stage eq "diff" and $inverse) {
     230        if ($stage eq "diff" and $warp_warp) {
    220231            my $name = "PPSUB.INVERSE";
    221232            if ($backup_path_base) {

  • branches/eam_branches/ipp-20100823/ippScripts/scripts/staticsky.pl

    r28962 r29124  
    7171$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $sky_id, $PS_EXIT_SYS_ERROR ) if $redirect;
    7272
     73my $traceDest      = $ipprc->filename("TRACE.EXP", $outroot);
    7374# my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_STATICSKY);
    7475
    … …  
    164165    $command .= " -recipe PSPHOT $recipe_psphot";
    165166    $command .= " -dumpconfig $configuration";
    166     # $command .= " -tracedest $traceDest -log $logDest";
     167    $command .= " -tracedest $traceDest -log $logDest";
    167168    # $command .= " -dbname $dbname" if defined $dbname;
    168169    # $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id;

  • branches/eam_branches/ipp-20100823/ippTasks/destreak.pro

    r27946 r29124  
    600600    book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID
    601601    book getword magicDSToCleanup $pageName camera -var CAMERA
     602    book getword magicDSToCleanup $pageName stage -var STAGE
    602603    book getword magicDSToCleanup $pageName outroot -var OUTROOT
    603604    book getword magicDSToCleanup $pageName dbname -var DBNAME
    … …  
    607608    host anyhost
    608609
    609     $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --logfile $logfile
     610    $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --stage $STAGE --camera $CAMERA --logfile $logfile
    610611
    611612    add_standard_args run

  • branches/eam_branches/ipp-20100823/ippTasks/dist.pro

    r28778 r29124  
    3030list DIST_STAGE -add "chip_bg"
    3131list DIST_STAGE -add "warp_bg"
     32list DIST_STAGE -add "sky"
    3233
    3334$currentStage = 0

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl

    r28954 r29124  
    1010use ippToPsps::IppToPspsDb;
    1111use ippToPsps::Datastore;
     12use ippToPsps::BatchManager;
    1213
    1314my $singleBatch = undef;
    … …  
    1718my $toTime = undef;
    1819my $product = undef;
     20my $filePath = undef;
    1921
    2022
    … …  
    2426        'to|t=s' => \$toTime,
    2527        'product|p=s' => \$product,
     28        'location|l=s' => \$filePath,
    2629        'verbose|v' => \$verbose,
    2730        'save_temps|s' => \$save_temps
    … …  
    3235}
    3336if (!defined $singleBatch) {
    34     print "* OPTIONAL: a single batch                  -b                   (default = none)\n";
     37    print "* OPTIONAL: a single batch                  -b <batchNum>        (default = none)\n";
    3538}
    3639if (!defined $fromTime) {
    3740    $fromTime = "2010-01-01";
    38     print "* OPTIONAL: from time                       -f                   (default = $fromTime)\n";
     41    print "* OPTIONAL: from time                       -f <dateTime>        (default = $fromTime)\n";
    3942}
    4043if (!defined $toTime) {
    4144    $toTime = "2099-12-31";
    42     print "* OPTIONAL: to time                         -t                   (default = $toTime)\n";
     45    print "* OPTIONAL: to time                         -t <dateTime>        (default = $toTime)\n";
     46}
     47if (!defined $filePath) {
     48    print "* OPTIONAL: location for files to be deleted    -l <path>\n";
    4349}
    4450if (!defined $verbose) {
    … …  
    5460my $datastore = new ippToPsps::Datastore($product, 0, 0);
    5561my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
     62my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $filePath, $verbose, $save_temps);
    5663my $odmUrl = "http://web01.psps.ifa.hawaii.edu/a01/OdmWebService/OdmWebService.asmx/GetBatchStatus";
    5764my $ua = LWP::UserAgent->new;
    … …  
    7279    my $numOfBatches;
    7380
    74     if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getSingleBatch($singleBatch, \$batches);}
    75     else { $numOfBatches = $ippToPspsDb->getBatchList(\$batches, $fromTime, $toTime);}
     81    if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getBatch($singleBatch, \$batches);}
     82    else { $numOfBatches = $ippToPspsDb->getBatches(\$batches, $fromTime, $toTime);}
    7683
    7784    if ($numOfBatches < 1) {return 0;}
    … …  
    8491    # loop round batches
    8592    my $batch;
     93    my $numBatchesToCheck = 0;
    8694    my $numChecked = 0;
     95    my $numRemovedFromDatastore = 0;
     96    my $numDeleted = 0;
    8797    foreach $batch ( @{$batches} ) {
    88         my ($timestamp, $expId, $batchId, $surveyType, $deleted) =  @{$batch};
    89 
    90         if (checkBatch($timestamp, $expId, $batchId, $surveyType, $deleted)) {$numChecked++;}
    91 
     98        my ($timestamp, $expId, $batchId, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) =  @{$batch};
     99
     100        if (!$processed) {next;}
     101
     102        $numBatchesToCheck++;
     103
     104        my $batchName = $batchManager->getBatchName($batchId);
     105
     106        # if not merged then update by polling ODM for status
     107        if (!$merged) {
     108
     109            if (checkODM($batchName, \$loadedToOdm, \$mergeWorthy, \$merged)) {$numChecked++;}
     110        }
     111
     112        # delete from datastore
     113        if (defined $product && !$deleted && $loadedToOdm && $mergeWorthy) {
     114
     115            $deleted = $datastore->remove($batchName);
     116            if ($deleted) {
     117                $ippToPspsDb->setBatchAsDeleted($batchId, $expId);
     118                $numRemovedFromDatastore++;
     119            }
     120        }
     121        # if merged and already removed from datastore then delete files
     122        if (defined $filePath && $merged && $deleted) {
     123
     124            if($batchManager->deleteBatch($batchId)) {$numDeleted++;}
     125        }
     126
     127        # update database
     128        $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $merged, $deleted);
     129        printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s     | %6s  |  %5s   |\n",
     130                $timestamp,
     131                $batchName,
     132                $expId,
     133                $loadedToOdm ? "yes" : "no",
     134                $mergeWorthy ? "yes" : "no",
     135                $merged ? "yes" : "no",
     136                $deleted ? "yes" : "no");
    92137    }
     138
    93139    printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
    94140
    95     printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numOfBatches);
    96 
    97 }
    98 
    99 ######################################################################################y
     141    printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numBatchesToCheck);
     142    printf( "* Successfully removed %d batch%s from the datastore\n", $numRemovedFromDatastore, ($numRemovedFromDatastore==1) ? "" : "es");
     143    printf( "* Successfully deleted %d batch%s from local file system\n", $numDeleted, ($numDeleted==1) ? "" : "es");
     144
     145}
     146
     147########################################################################################
    100148#
    101149# Check a single batch
    102150#
    103151########################################################################################
    104 sub checkBatch {
    105     my ($timestamp, $expId, $batchId, $surveyType, $deleted) = @_;
    106 
    107 
    108     my $batchFilter = sprintf("B%08d", $batchId);
     152sub checkODM {
     153    my ($batchName, $loadedToOdm, $mergeWorthy, $merged) = @_;
     154
    109155    my $statusFilter = "*";
    110156
    111157    my $response = $ua->post($odmUrl,
    112             [batchNameFilter => $batchFilter,
     158            [batchNameFilter => "%".$batchName,
    113159            statusFilter => $statusFilter,
    114160            fromFilter => "2010-01-01",
    … …  
    119165    if ($response->code != 200) {
    120166
    121         print "Problem connecting to web service for '$batchFilter', HTTP response status: ".$response->status_line."\n";
    122         return;
     167        print "Problem connecting to web service for '$batchName', HTTP response status: ".$response->status_line."\n";
     168        return 0;
    123169    }
    124170    #        print( "HTTP response status: ".$response->content."\n" );
    … …  
    128174    close($tempFile);
    129175
    130     my $loadedToOdm = 0;
    131     my $mergeWorthy = 0;
    132     my $mergeCompleted = 0;
    133 
    134     parseXml($tempName, \$loadedToOdm, \$mergeWorthy, \$mergeCompleted);
    135 
    136     # delete from datastore
    137     if(defined $product) {
    138    
    139         if (!$deleted && $loadedToOdm && $mergeWorthy) {
    140        
    141             $deleted = $datastore->remove($batchFilter);
    142             if ($deleted) {
    143                 $ippToPspsDb->setBatchAsDeleted($batchId, $expId);
    144             }
    145         }
    146     }
    147    
    148     printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s     | %6s  |  %5s   |\n",
    149             $timestamp,
    150             $batchFilter,
    151             $expId,
    152             $loadedToOdm ? "yes" : "no",
    153             $mergeWorthy ? "yes" : "no",
    154             $mergeCompleted ? "yes" : "no",
    155             $deleted ? "yes" : "no");
    156 
    157     $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $mergeCompleted, $deleted);
    158 
     176    ${$loadedToOdm} = 0;
     177    ${$mergeWorthy} = 0;
     178    ${$merged} = 0;
     179
     180    parseXml($tempName, $loadedToOdm, $mergeWorthy, $merged);
     181
     182    return 1;
    159183}
    160184
    … …  
    165189########################################################################################
    166190sub parseXml {
    167     my ($xmlFile, $loadedToOdm, $mergeWorthy, $mergeCompleted) = @_;
     191    my ($xmlFile, $loadedToOdm, $mergeWorthy, $merged) = @_;
    168192
    169193    my $parser = XML::LibXML->new;
    … …  
    174198    ${$loadedToOdm} = 0;
    175199    ${$mergeWorthy} = 0;
    176     ${$mergeCompleted} = 0;
     200    ${$merged} = 0;
    177201
    178202    if ($result =~ m/LoadStarted/) { ${$loadedToOdm} = 1;}
    179203    if ($result =~ m/MergeWorthy/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1;}
    180     if ($result =~ m/Merge[1-9]Completed/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1; ${$mergeCompleted} = 1;}
    181 }
    182 
     204    if ($result =~ m/Merge[1-9]Completed/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1; ${$merged} = 1;}
     205}
     206

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPspsDb.pm

    r28952 r29124  
    1515#
    1616###########################################################################
    17 sub getBatchList {
    18     my ($self, $exposures, $fromTime, $toTime) = @_;
    19 
    20     my $query = $self->{_db}->prepare(<<SQL);
    21     SELECT created, exp_id, batch_id, survey_id, deleted
     17sub getBatches {
     18    my ($self, $batches, $fromTime, $toTime) = @_;
     19
     20    my $query = $self->{_db}->prepare(<<SQL);
     21    SELECT created, exp_id, batch_id, survey_id, deleted, dvo_db, processed, on_datastore, loaded_to_ODM, merge_worthy, merged
    2222        FROM batches
    23         WHERE processed = 1
    24         AND on_datastore = 1
    25         AND created >= '$fromTime'
     23        WHERE created >= '$fromTime'
    2624        AND created <= '$toTime';
    2725SQL
    … …  
    2927    # TODO remove date restriction
    3028    $query->execute;
    31     ${$exposures} = $query->fetchall_arrayref();
    32     my $count = scalar @{${$exposures}};
     29    ${$batches} = $query->fetchall_arrayref();
     30    my $count = scalar @{${$batches}};
    3331
    3432   printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es");
    … …  
    4139#
    4240###########################################################################
    43 sub getSingleBatch{
    44     my ($self, $batch_id, $exposures) = @_;
    45 
    46     my $query = $self->{_db}->prepare(<<SQL);
    47     SELECT created, exp_id, batch_id, survey_id, deleted 
     41sub getBatch{
     42    my ($self, $batch_id, $batches) = @_;
     43
     44    my $query = $self->{_db}->prepare(<<SQL);
     45    SELECT created, exp_id, batch_id, survey_id, deleted, dvo_db, processed, on_datastore, loaded_to_ODM, merge_worthy, merged
    4846        FROM batches
    4947        WHERE batch_id = $batch_id
    50         AND processed = 1
    51         AND on_datastore = 1;
    5248SQL
    5349 
    5450    $query->execute;
    55     ${$exposures} = $query->fetchall_arrayref();
    56     my $count = scalar @{${$exposures}};
     51    ${$batches} = $query->fetchall_arrayref();
     52    my $count = scalar @{${$batches}};
    5753
    5854   printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es");
    … …  
    149145
    150146    my $processed = $query->fetchrow_array();
    151 
    152     if ($processed) {print "* Exposure ID '$expId' has already been processed\n";}
    153147
    154148    return $processed;
    … …  
    424418    return $row[0];
    425419}
     420
     421sub insetDetection {
     422    my ($self, $expId, $imageID, $objID,$ippObjID, $ippDetectID) = @_;
     423
     424   my $query = $self->{_db}->prepare(<<SQL);
     425    INSERT INTO detections
     426        (expId, imageID, objID, ippObjID, ippDetectID)
     427        VALUES
     428        ($expId, $imageID, $objID,$ippObjID, $ippDetectID);
     429SQL
     430
     431    $query->execute;
     432
     433}
     434
    4264351

  • branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps_run.pl

    r28907 r29124  
    1717
    1818# globals
    19 my $camera = 'GPC1';
     19my $camera = undef;
    2020my $batchType = undef;
    2121my $dvoLocation = undef;
    … …  
    3333# get user args
    3434GetOptions(
     35        'camera|c' => \$camera,
    3536        'output|o=s' => \$output,
    3637        'batch|b=s' => \$batchType,
    … …  
    6364    print "* REQUIRED: need to provide a DVO Db        -d <pathToDVO>\n";
    6465}
     66if (!defined $camera) {
     67    $camera = "GPC1";
     68    print "* OPTIONAL: select a camera                 -c                   (default = $camera)\n";
     69}
    6570if (!defined $singleExpId) {
    6671
    … …  
    175180        if (!$initBatch && $ippToPspsDb->isExposureAlreadyProcessed($expId)) {
    176181
    177             if ($force) {print "* Forcing....\n";}
    178             else {next};
     182            if ($force) {print "* Already processed '$expId', but forcing....\n";}
     183            else {
     184           
     185                print "* Exposure ID '$expId' has already been processed\n";
     186                next
     187            };
    179188        }
    180189

  • branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchDetection.c

    r28992 r29124  
    305305                        numInvalidFlux++;
    306306                    }
    307 
    308                     // store max/min objID
    309                     if (objID[s] > maxObjID) maxObjID = objID[s];
    310                     if (objID[s] < minObjID) minObjID = objID[s];
    311 
     307                    else {
     308
     309                        // store max/min objID
     310                        if (objID[s] > maxObjID) maxObjID = objID[s];
     311                        if (objID[s] < minObjID) minObjID = objID[s];
     312                    }
    312313                    totalDetections++;
    313314                }

  • branches/eam_branches/ipp-20100823/ippTools/configure.ac

    r28486 r29124  
    1919PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
    2020PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.63])
    21 PKG_CHECK_MODULES([PPSTAMP], [ippdb >= 1.1.63])
     21PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1])
    2222
    2323PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${PPSTAMP_CFLAGS=} ${IPPDB_CFLAGS=}"

  • branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_tocleanup.sql

    r25837 r29124  
    11SELECT
    22    magic_ds_id,
     3    stage,
    34    magicDSRun.state,
    45    magicDSRun.outroot,
    … …  
    910WHERE magicDSRun.state = 'goto_cleaned'
    1011-- XXX: need to add fault to magicDSRun
     12-- XXX: the database has been updated, but fault isn't yet used
    1113--    AND magicDSRun.fault = 0
    1214

  • branches/eam_branches/ipp-20100823/ippTools/src/disttool.c

    r28959 r29124  
    519519    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");;
    520520    PXOPT_COPY_STR(config->args, where, "-state", "distRun.state", "==");
    521     PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
     521    PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE");
    522522    PXOPT_COPY_STR(config->args, where, "-data_group", "distRun.data_group", "LIKE");
    523523    PXOPT_COPY_STR(config->args, where, "-dist_group", "distTarget.dist_group", "==");

  • branches/eam_branches/ipp-20100823/ippTools/src/disttoolConfig.c

    r28938 r29124  
    7676    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage",     0, "value for stage", NULL);
    7777    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state",     0, "value for state", NULL);
    78     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",     0, "limit updates to label", NULL);
    79     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group",     0, "limit updates to data_group", NULL);
     78    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",     0, "limit updates to label (LIKE comparison)", NULL);
     79    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group",     0, "limit updates to data_group (LIKE comparison)", NULL);
    8080    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group",     0, "limit updates to data_group", NULL);
    8181    psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-time_stamp_begin", 0, "limit updates by time_stamp (>=)", NULL);

  • branches/eam_branches/ipp-20100823/ippTools/src/staticskytool.c

    r28959 r29124  
    290290                                workdir,
    291291                                label,
    292                                 data_group,
     292                                data_group ? data_group : label,
    293293                                dist_group,
    294294                                reduction,

  • branches/eam_branches/ipp-20100823/ippconfig/recipes/fitstypes.mdc

    r27417 r29124  
    107107        BSCALE          F32     0.1     # Disk is stored as 1/10th sec
    108108        BZERO           F32     3276.8  # Store unsigned zero as zero
     109        # switch to NONE to work around ticket 1411
     110        #COMPRESSION    STR     NONE
    109111        COMPRESSION     STR     GZIP
    110112        TILE.X          S32     0

  • branches/eam_branches/ipp-20100823/ippconfig/recipes/nightly_science.config

    r28982 r29124  
    3939
    4040UNRECOVERABLE_QUALITY MULTI
    41 UNRECOVERABLE_QUALITY S16 0
    4241UNRECOVERABLE_QUALITY S16 4007
    4342

  • branches/eam_branches/ipp-20100823/ippconfig/recipes/ppSub.config

    r29005 r29124  
    5959INNER           S32     5               # Inner half-size for SPAM and FRIES kernels
    6060RINGS.ORDER     S32     2               # Polynomial order for RINGS kernels
    61 PENALTY         F32     1.0             # Penalty for wideness
     61PENALTY         F32     16.0            # Penalty for wideness
    6262BIN1            S32     4               # Binning factor for first level
    6363BIN2            S32     4               # Binning factor for second level

  • branches/eam_branches/ipp-20100823/ippconfig/recipes/psphot.config

    r29005 r29124  
    188188 #PGAUSS_PSF  EXTENDED_SOURCE_MODEL  PS_MODEL_PGAUSS   20.0    TRUE
    189189 #QGAUSS_PSF  EXTENDED_SOURCE_MODEL  PS_MODEL_QGAUSS   20.0    TRUE
    190  SERSIC_PSF  EXTENDED_SOURCE_MODEL  PS_MODEL_SERSIC   20.0    TRUE
     190 EXP_PCM  EXTENDED_SOURCE_MODEL  PS_MODEL_EXP      20.0    TRUE
     191 DEV_PCM  EXTENDED_SOURCE_MODEL  PS_MODEL_DEV      20.0    TRUE
     192 SER_PCM  EXTENDED_SOURCE_MODEL  PS_MODEL_SERSIC   20.0    TRUE
    191193END
    192194
    … …  
    325327# Extended source fit parameters
    326328STACKPHOT                             METADATA
     329  EXTENDED_SOURCE_FITS                BOOL  TRUE  # perform any of the aperture-like measurements?
    327330  EXTENDED_SOURCE_ANALYSIS            BOOL  TRUE  # perform any of the aperture-like measurements?
    328331  EXTENDED_SOURCE_SN_LIM              F32   20.0
    329332  EXTENDED_SOURCE_PETROSIAN           BOOL  TRUE
    330   EXTENDED_SOURCE_ISOPHOTAL           BOOL  FALSE
    331333  EXTENDED_SOURCE_ANNULI              BOOL  TRUE
    332   EXTENDED_SOURCE_KRON                BOOL  FALSE
    333334  PSPHOT.STACK.MATCH.PSF.SOURCE       STR   AUTO # which inputs to convolve? (RAW, CNV, AUTO)
    334335  PSPHOT.STACK.TARGET.PSF.AUTO        BOOL  F    # automatically determine target PSF size?
    … …  
    337338  RADIAL_APERTURES_SN_LIM             F32   0.0  # S/N limit for radial aperture calculation
    338339  OUTPUT.FORMAT                       STR   PS1_SV1
     340
     341  EXT_FIT_MAX_RADIUS                  F32   50.0
     342  PSF_MODEL                           STR   PS_MODEL_GAUSS
     343  EXT_MODEL                           STR   PS_MODEL_SERSIC
    339344END
    340345

  • branches/eam_branches/ipp-20100823/psLib/src/math/psUnaryOp.c

    r26892 r29124  
    6363
    6464// Conversion for degrees to radians
    65 #define D2R 0.01745329252111111  /* PI/180 */
    66 
     65// #define D2R 0.01745329252111111  /* PI/180 */
     66#define D2R    0.01745329251994329  /* PI/180 Corrected. Truncated digits: 576924 */
    6767// Conversion for radians to degrees
    68 #define R2D 57.29577950924861   /* 180.0/PI */
    69 
     68// #define R2D 57.29577950924861   /* 180.0/PI */
     69#define R2D    57.29577951308232   /* 180.0/PI Correcte.  Truncated digits: 087679 */
    7070
    7171// Unary SCALAR operations

  • branches/eam_branches/ipp-20100823/psModules/src/imcombine/pmSubtractionStamps.c

    r29004 r29124  
    830830    }
    831831
    832 #if 1
     832#if 0
    833833    {
    834834        psFits *fits = NULL;

  • branches/eam_branches/ipp-20100823/psModules/src/objects/models/pmModel_SERSIC.c

    r29004 r29124  
    252252    float Rminor = Rmajor * (axes.minor / axes.major);
    253253
    254     fprintf (stderr, "guess index: %f : %f, %f -> %f, %f\n", index, axes.major, axes.minor, Rmajor, Rminor);
     254    // fprintf (stderr, "guess index: %f : %f, %f -> %f, %f\n", index, axes.major, axes.minor, Rmajor, Rminor);
    255255
    256256    axes.major = Rmajor;
    … …  
    346346    psF64 radius = axes.major * sqrt (2.0) * pow(zn, 0.5 / PAR[PM_PAR_7]);
    347347
    348     fprintf (stderr, "sersic model %f %f, n %f, radius: %f, zn: %f, f/Io: %f, major: %f\n", PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], PAR[PM_PAR_7], radius, zn, flux/PAR[PM_PAR_I0], axes.major);
     348    // fprintf (stderr, "sersic model %f %f, n %f, radius: %f, zn: %f, f/Io: %f, major: %f\n", PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], PAR[PM_PAR_7], radius, zn, flux/PAR[PM_PAR_I0], axes.major);
    349349
    350350    psAssert (isfinite(radius), "fix this code: z should not be nan for %f", PAR[PM_PAR_7]);

  • branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_DV2.c

    r29004 r29124  
    186186        psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX_SIG",PS_DATA_F32, "Sigma of PSF instrumental magnitude",        source->psfFluxErr);
    187187        psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG",           PS_DATA_F32, "magnitude in standard aperture",             source->apMag);
     188        // XXX replace with AP_FLUX (& ERR)? psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG",           PS_DATA_F32, "magnitude in standard aperture",             source->apMag);
     189
    188190        psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS",    PS_DATA_F32, "radius used for aperture mags",              apRadius);
    189191        psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude",           peakMag);
    … …  
    214216        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XY",       PS_DATA_F32, "second moments (X*Y)",                      mxy);
    215217        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_YY",       PS_DATA_F32, "second moments (Y*Y)",                      myy);
     218
     219        float Mrf  = source->moments ? source->moments->Mrf : NAN;
     220        float Mrh  = source->moments ? source->moments->Mrh : NAN;
     221        float Krf  = source->moments ? source->moments->KronFlux : NAN;
     222        float dKrf = source->moments ? source->moments->KronFluxErr : NAN;
     223
     224        float Kinner = source->moments ? source->moments->KronFinner : NAN;
     225        float Kouter = source->moments ? source->moments->KronFouter : NAN;
     226
     227        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_R1",       PS_DATA_F32, "first radial moment",                       Mrf);
     228        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_RH",       PS_DATA_F32, "half radial moment",                        Mrh);
     229        psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX",        PS_DATA_F32, "Kron Flux (in 2.5 R1)",                     Krf);
     230        psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_ERR",    PS_DATA_F32, "Kron Flux Error",                          dKrf);
     231
     232        psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_INNER",  PS_DATA_F32, "Kron Flux (in 1.0 R1)",                     Kinner);
     233        psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_OUTER",  PS_DATA_F32, "Kron Flux (in 4.0 R1)",                     Kouter);
    216234
    217235        psMetadataAdd (row, PS_LIST_TAIL, "DIFF_NPOS",        PS_DATA_S32, "nPos (n pix > 3 sigma)",                    diffStats.nGood);

  • branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_SV1.c

    r29004 r29124  
    198198        psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX_SIG",PS_DATA_F32, "Sigma of PSF instrumental flux",             source->psfFluxErr);
    199199        psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG",           PS_DATA_F32, "magnitude in standard aperture",             source->apMag);
     200        psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RAW",       PS_DATA_F32, "magnitude in reported aperture",             source->apMagRaw);
    200201        psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS",    PS_DATA_F32, "radius used for aperture mags",              apRadius);
    201202        psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude",           peakMag);

  • branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_V3.c

    r29012 r29124  
    179179        psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RAW",       PS_DATA_F32, "magnitude in reported aperture",             source->apMagRaw);
    180180        psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS",    PS_DATA_F32, "radius used for aperture mags",              apRadius);
     181        // XXX need ap_mag error
    181182        psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude",           peakMag);
    182183        psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG",      PS_DATA_F32, "PSF Magnitude using supplied calibration",   calMag);
    … …  
    204205        float Myy = source->moments ? source->moments->Myy : NAN;
    205206
     207        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XX",       PS_DATA_F32, "second moments (X^2)",                      Mxx);
     208        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XY",       PS_DATA_F32, "second moments (X*Y)",                      Mxy);
     209        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_YY",       PS_DATA_F32, "second moments (Y*Y)",                      Myy);
     210
     211        float M_c3 = source->moments ? 1.0*source->moments->Mxxx - 3.0*source->moments->Mxyy : NAN;
     212        float M_s3 = source->moments ? 3.0*source->moments->Mxxy - 1.0*source->moments->Myyy : NAN;
     213        float M_c4 = source->moments ? 1.0*source->moments->Mxxxx - 6.0*source->moments->Mxxyy + 1.0*source->moments->Myyyy : NAN;
     214        float M_s4 = source->moments ? 4.0*source->moments->Mxxxy - 4.0*source->moments->Mxyyy : NAN;
     215
     216        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M3C",      PS_DATA_F32, "third momemt cos theta",                    M_c3);
     217        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M3S",      PS_DATA_F32, "third momemt sin theta",                    M_s3);
     218        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M4C",      PS_DATA_F32, "fourth momemt cos theta",                   M_c4);
     219        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M4S",      PS_DATA_F32, "fourth momemt sin theta",                   M_s4);
     220
    206221        float Mrf  = source->moments ? source->moments->Mrf : NAN;
    207222        float Mrh  = source->moments ? source->moments->Mrh : NAN;
    … …  
    211226        float Kinner = source->moments ? source->moments->KronFinner : NAN;
    212227        float Kouter = source->moments ? source->moments->KronFouter : NAN;
    213 
    214         float M_c3 = source->moments ? 1.0*source->moments->Mxxx - 3.0*source->moments->Mxyy : NAN;
    215         float M_s3 = source->moments ? 3.0*source->moments->Mxxy - 1.0*source->moments->Myyy : NAN;
    216         float M_c4 = source->moments ? 1.0*source->moments->Mxxxx - 6.0*source->moments->Mxxyy + 1.0*source->moments->Myyyy : NAN;
    217         float M_s4 = source->moments ? 4.0*source->moments->Mxxxy - 4.0*source->moments->Mxyyy : NAN;
    218 
    219         psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XX",       PS_DATA_F32, "second moments (X^2)",                      Mxx);
    220         psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XY",       PS_DATA_F32, "second moments (X*Y)",                      Mxy);
    221         psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_YY",       PS_DATA_F32, "second moments (Y*Y)",                      Myy);
    222 
    223         psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M3C",      PS_DATA_F32, "third momemt cos theta",                    M_c3);
    224         psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M3S",      PS_DATA_F32, "third momemt sin theta",                    M_s3);
    225         psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M4C",      PS_DATA_F32, "fourth momemt cos theta",                   M_c4);
    226         psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M4S",      PS_DATA_F32, "fourth momemt sin theta",                   M_s4);
    227228
    228229        psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_R1",       PS_DATA_F32, "first radial moment",                       Mrf);

  • branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceMoments.c

    r29004 r29124  
    7777    // well:
    7878    psF32 sky = 0.0;
     79    if (source->moments == NULL) {
     80      source->moments = pmMomentsAlloc();
     81    }
    7982    // XXX if (source->moments == NULL) {
    8083    // XXX     source->moments = pmMomentsAlloc();

  • branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourcePhotometry.c

    r29004 r29124  
    148148
    149149    // for PSFs, correct both apMag and psfMag to same system, consistent with infinite flux star in aperture RADIUS
     150    // XXX add a flag for "ap_mag is corrected?"
    150151    if ((mode & PM_SOURCE_PHOT_APCORR) && isPSF && psf && psf->ApTrend) {
    151152        // the source peak pixel is guaranteed to be on the image, and only minimally different from the source center
    … …  
    292293
    293294    // measure apMag
     295    // XXX save the apFlux and apFluxErr
     296    // XXX note that these fluxes/mags are uncorrected for masked pixels
     297    // XXX raise a bit if the aperture has a masked pixel (not marked)?
    294298    for (int iy = 0; iy < image->numRows; iy++) {
    295299        for (int ix = 0; ix < image->numCols; ix++) {

  • branches/eam_branches/ipp-20100823/psconfig/tagsets/ipp-2.9.perl

    r27951 r29124  
    9292  80    Abstract::Meta::Class          Abstract-Meta-Class-0.13.tar.gz          0              NONE      NONE
    9393  81    DBIx::Connection               DBIx-Connection-0.13.tar.gz              0              NONE      NONE
    94   82a   Test::Pod                      Test-Pod-1.40.tar.gz                     0              NONE      NONE
    95   82c   XML::NamespaceSupport          XML-NamespaceSupport-1.10.tar.gz         0              --skip    NONE
    96   82b   XML::SAX                       XML-SAX-0.96.tar.gz                      0              NONE      Y
    97   82d   Simple::SAX::Serializer        Simple-SAX-Serializer-0.05.tar.gz        0              NONE      NONE
     94  82a   Pod::Escapes                   Pod-Escapes-1.04.tar.gz                  0              NONE      NONE
     95  82b   Pod::Simple                    Pod-Simple-3.14.tar.gz                   0              NONE      NONE
     96  82c   Test::Pod                      Test-Pod-1.40.tar.gz                     0              NONE      NONE
     97  82d   XML::NamespaceSupport          XML-NamespaceSupport-1.10.tar.gz         0              --skip    NONE
     98  82e   XML::SAX                       XML-SAX-0.96.tar.gz                      0              NONE      Y
     99  82f   Simple::SAX::Serializer        Simple-SAX-Serializer-0.05.tar.gz        0              NONE      NONE
    98100  83    Test::Distribution             Test-Distribution-2.00.tar.gz            0              NONE      NONE
    99101  84    Test::DBUnit                   Test-DBUnit-0.20.tar.gz                  0.20           NONE      NONE

  • branches/eam_branches/ipp-20100823/psphot/doc/notes.20100715.txt

    r29004 r29124  
     1
     22010.08.24
     3
     4  Remaining work to be done:
     5
     6  * test and turn on CR masking
     7  * double-check source size analysis:
     8    * are we doing the right thing with sources flagged as bad in 'rough'?
     9    * are we doing the right thing for SAT, CR, EXT, PSF after SourceSize?
     10  * convert CR_SIGMA and EXT_SIGMA to probabilities
     11  * example results of model fits vs fake inputs
     12  * what is the right solution for the stack PSF photometry?
     13    * Nigel sees 4-6%
     14    * we see 1.9% at the worst
     15    * non-poisson errors are clearly better -- check on the chisq values
     16    * why is QGAUSS so good for the one example (better for poisson, if trendy)
    117
    2182010.08.12

  • branches/eam_branches/ipp-20100823/psphot/src/psphotBlendFit.c

    r29004 r29124  
    6666
    6767    float fitMinTol = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MIN_TOL"); // Fit tolerance
    68     assert (status && isfinite(fitMinTol) && fitMinTol > 0);
     68    if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) {
     69        fitMinTol = psMetadataLookupF32 (&status, recipe, "PSF_FIT_TOL"); // Fit tolerance
     70        if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) {
     71            psAbort("PSF_FIT_MIN_TOL (and PSF_FIT_TOL) not defined or positive");
     72        }
     73    }
    6974
    7075    float fitMaxTol = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MAX_TOL"); // Fit tolerance
    71     assert (status && isfinite(fitMaxTol) && fitMaxTol > 0);
     76    if (!status || !isfinite(fitMaxTol) || fitMaxTol <= 0) {
     77        fitMaxTol = 1.0;
     78    }
    7279
    7380    bool poisson = psMetadataLookupBool(&status, recipe, "POISSON.ERRORS.PHOT.LMM"); // Poisson errors?

  • branches/eam_branches/ipp-20100823/psphot/src/psphotExtendedSourceAnalysis.c

    r29004 r29124  
    121121        if (source->peak->y > AnalysisRegion.y1) continue;
    122122
     123        // fprintf (stderr, "xsrc: %f, %f : %f\n", source->peak->xf, source->peak->yf, source->peak->SN);
     124
    123125        // replace object in image
    124126        if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) {
    … …  
    178180    }
    179181
     182    // fprintf (stderr, "xsrc : tried %ld objects\n", sources->n);
     183
    180184    return true;
    181185}

  • branches/eam_branches/ipp-20100823/psphot/src/psphotExtendedSourceFits.c

    r29015 r29124  
    200200            }
    201201            psFree(job);
    202             }
     202        }
    203203    }
    204204    psFree (cellGroups);
    … …  
    270270        // set the radius based on the footprint (also sets the mask pixels)
    271271        if (!psphotSetRadiusFootprint(&radius, readout, source, markVal, 1.0)) {
     272            fprintf (stderr, "skipping (1) %f, %f\n", source->peak->xf, source->peak->yf);
    272273            psFree (fitOptions)
    273274            return false;
    … …  
    280281        // XXX save the psf-based moments for output
    281282        if (!pmSourceMoments (source, radius, 0.0, 0.0, maskVal)) {
     283            fprintf (stderr, "skipping (2) %f, %f\n", source->peak->xf, source->peak->yf);
    282284            // subtract the best fit from the object, leave local sky
    283285            pmSourceSub (source, PM_MODEL_OP_FULL, maskVal);
    … …  
    289291        psImage *modelFluxStart = psMemIncrRefCounter (source->modelFlux);
    290292        if (!modelFluxStart) {
    291           // XXX raise an error of some kind?
    292           continue;
     293            pmSourceCacheModel (source, maskVal);
     294            modelFluxStart = psMemIncrRefCounter (source->modelFlux);
     295            if (!modelFluxStart) {
     296                fprintf (stderr, "skipping (3) %f, %f\n", source->peak->xf, source->peak->yf);
     297                // XXX raise an error of some kind?
     298                continue;
     299            }
    293300        }
    294 
     301       
    295302        if (savePics) {
    296303          psphotSaveImage (NULL, readout->image, "image.xp.fits");
    … …  
    317324          float SNlim = psMetadataLookupF32 (&status, model, "SNLIM_VALUE");
    318325          assert (status);
     326
     327          // fprintf (stderr, "xfit: %f, %f : %f\n", source->peak->xf, source->peak->yf, source->peak->SN);
    319328
    320329          // limit selection to some SN limit
    … …  
    449458    psFree (fitOptions);
    450459
     460    // fprintf (stderr, "xfit : tried %ld objects\n", sources->n);
     461
    451462    // change the value of a scalar on the array (wrap this and put it in psArray.h)
    452463    scalar = job->args->data[7];

  • branches/eam_branches/ipp-20100823/psphot/src/psphotSourceFits.c

    r29015 r29124  
    506506    // psTraceSetLevel("psLib.math.psMinimizeLMChi2", 5);
    507507    pmSourceFitModel (source, model, &options, maskVal);
    508     fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix);
     508    // fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix);
    509509
    510510    // psTraceSetLevel("psLib.math.psMinimizeLMChi2", 0);
    … …  
    570570    // psTraceSetLevel("psLib.math.psMinimizeLMChi2", 5);
    571571    pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize);
    572     fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix);
     572    // fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix);
    573573
    574574    // psTraceSetLevel("psLib.math.psMinimizeLMChi2", 0);
    … …  
    613613       
    614614        pmSourceFitModel (source, model, &options, maskVal);
    615         fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix);
     615        // fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix);
    616616
    617617        chiSquare[i] = model->chisqNorm;
    … …  
    681681        pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize);
    682682# endif
    683         fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix);
     683        // fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix);
    684684
    685685        chiSquare[i] = model->chisq;

  • branches/eam_branches/ipp-20100823/psphot/src/psphotSourceSize.c

    r29004 r29124  
    366366        float nSigmaMYY = (Myy - psfClump->Y) / hypot(psfClump->dY, psfClump->Y*psfClump->Y*source->errMag);
    367367
    368         fprintf (stderr, "%f %f : Mxx: %f, Myy: %f, dx: %f, dy: %f, psfMag: %f, apMag: %f, dMag: %f, errMag: %f, nSigmaMag: %f, nSigmaMxx: %f, nSigmaMyy: %f\n",
    369                 source->peak->xf, source->peak->yf, Mxx, Myy, source->peak->xf - source->moments->Mx, source->peak->yf - source->moments->My,
    370                 source->psfMag, apMag, dMag, source->errMag, nSigmaMAG, nSigmaMXX, nSigmaMYY);
     368        // fprintf (stderr, "%f %f : Mxx: %f, Myy: %f, dx: %f, dy: %f, psfMag: %f, apMag: %f, dMag: %f, errMag: %f, nSigmaMag: %f, nSigmaMxx: %f, nSigmaMyy: %f\n",
     369        // source->peak->xf, source->peak->yf, Mxx, Myy, source->peak->xf - source->moments->Mx, source->peak->yf - source->moments->My,
     370        // source->psfMag, apMag, dMag, source->errMag, nSigmaMAG, nSigmaMXX, nSigmaMYY);
    371371
    372372        // XXX double check on ths stuff!! partially-masked sources are more likely to be mis-measured PSFs

  • branches/eam_branches/ipp-20100823/psphot/src/psphotStackMatchPSFs.c

    r28013 r29124  
    106106    rescaleData(readoutOut, config, options, index);
    107107
    108     dumpImage(readoutOut, readoutSrc, index, "convolved");
     108    // dumpImage(readoutOut, readoutSrc, index, "convolved");
    109109
    110110    return true;

  • branches/eam_branches/ipp-20100823/psphot/src/psphotStackMatchPSFsUtils.c

    r28013 r29124  
    346346    if (!fake) goto escape;
    347347
    348     dumpImage(fake, readoutSrc, index, "fake");
    349     dumpImage(readoutSrc,  readoutSrc, index, "real");
     348    // dumpImage(fake, readoutSrc, index, "fake");
     349    // dumpImage(readoutSrc,  readoutSrc, index, "real");
    350350
    351351    if (threads) pmSubtractionThreadsInit();
    … …  
    380380    }
    381381
    382     dumpImage(readoutOut, readoutSrc, index, "conv");
    383     dumpImageDiff(readoutOut, fake, readoutSrc, index, "diff");
     382    // dumpImage(readoutOut, readoutSrc, index, "conv");
     383    // dumpImageDiff(readoutOut, fake, readoutSrc, index, "diff");
    384384
    385385    psFree(fake);

  • branches/eam_branches/ipp-20100823/pstamp/scripts/Makefile.am

    r28038 r29124  
    1919        pstamp_webrequest.pl \
    2020        pstamp_get_image_job.pl \
     21        psmkreq \
    2122        pstamp_checkdependent.pl \
    2223        request_finish.pl \

  • branches/eam_branches/ipp-20100823/pstamp/scripts/detectability_respond.pl

    r28902 r29124  
    9898            my $key = shift(@key_values);
    9999            my $val = shift(@key_values);
     100            # if we have wisdom, then we should have updated already.  If not, we'll bomb out later in the code.
     101            if ($key eq 'FAULT') {
     102                $val = 0;
     103            }
    100104            $query{$fpa_id}{$key}[$i] = $val;
    101105        }
    … …  
    148152        # Determine the query style for this fpa_id
    149153        if ($fpa_id =~ /o.*g.*o/) {
    150         $query_style = 'byexp';
    151     }
     154            $query_style = 'byexp';
     155        }
    152156        elsif ($fpa_id =~ /\d+/) {
    153         $query_style = 'byid';
    154     }
     157            $query_style = 'byid';
     158        }
    155159        else {
    156         exit_with_failure(21,"Parse error in request file");
    157     }
     160            exit_with_failure(21,"Parse error in request file");
     161        }
    158162        # Confirm that we only have one stage/filter/mjd
    159163        for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
    160         $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
    161         $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
    162         $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
    163     }
     164            $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
     165            $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
     166            $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
     167        }
    164168        if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
    165169            $stage = (keys(%{ $temp_hash{STAGE} }))[0];
    166170        }
    167171        else {
    168         exit_with_failure(21,"Too many STAGEs specified");
    169     }
     172            exit_with_failure(21,"Too many STAGEs specified");
     173        }
    170174        if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
    171         $filter = (keys(%{ $temp_hash{FILTER} }))[0];
    172     }
     175            $filter = (keys(%{ $temp_hash{FILTER} }))[0];
     176        }
    173177        else {
    174         exit_with_failure(21,"Too many FILTERs specified");
    175     }
     178            exit_with_failure(21,"Too many FILTERs specified");
     179        }
    176180        if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
    177         $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
    178     }
     181            $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
     182        }
    179183        else {
    180         exit_with_failure(21,"Too many MJD-OBS specified");
    181     }
     184            exit_with_failure(21,"Too many MJD-OBS specified");
     185        }
    182186        # Set common request components
    183187        my $option_mask |= 1;
    … …  
    200204            $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i];
    201205            $rowList[$i]->{COORD_MASK} = 0;
     206            # Set default values
     207            $query{$fpa_id}{BAD_COMPONENT}[$i] = 1;
     208            $query{$fpa_id}{IMAGE}[$i] = 'no_image';
     209            $query{$fpa_id}{MASK}[$i] = 'no_mask';
     210            $query{$fpa_id}{WEIGHT}[$i] = 'no_weight';
     211            $query{$fpa_id}{PSF}[$i] = 'no_psf';
     212
     213            $query{$fpa_id}{STAGE_ID}[$i] = 'no_id';
     214            $query{$fpa_id}{IMAGE_DB}[$i] = 'no_imdb';
     215            $query{$fpa_id}{NEED_MAGIC}[$i] = 'no_magic';
     216            $query{$fpa_id}{MAGICKED}[$i] = 'no_magic';
     217            $query{$fpa_id}{CATALOG}[$i] = 'no_catalog';
     218            $query{$fpa_id}{COMPONENT_ID}[$i] = 'no_component';
     219            $query{$fpa_id}{CLASS_ID}[$i] = 'no_class';
     220
     221            $query{$fpa_id}{STATE}[$i] = 'no_state';
     222            $query{$fpa_id}{DATA_STATE}[$i] = 'no_dstate';
     223            $query{$fpa_id}{FAULT}[$i] = 'no_fault';
     224            $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate';
     225
    202226        }
    203227       
    … …  
    217241                    $value = join ' ', @{ $this_image_ref->{$key} };
    218242                }
     243
    219244#               print "$this_image_ref $key $value\n";
    220245                foreach my $valid_index (@{ $this_image_ref->{row_index} }) {
    … …  
    226251                    $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb};
    227252                    $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic;
     253                    $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0;
    228254                   
    229255                    if (exists($this_image_ref->{astrom})) {
    … …  
    231257                    }
    232258                    else {
    233                     $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf};
    234                 }
     259                        $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf};
     260                    }
    235261                    if (exists($this_image_ref->{class_id})) {
    236262                        $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id};
    … …  
    301327        }
    302328        print WISDOM "\n";
    303         @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } =
    304             ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
    305              $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
    306         push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
     329        if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
     330            @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } =
     331                ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
     332                 $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
     333            push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
     334        }
    307335    }
    308336}
    … …  
    356384        my $stage = $query{$fpa_id}{STAGE}[$index];
    357385        # if there's a fault, then we can't process this image.
    358         if ($fault != 0) {
     386        if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)) {
     387            $query{$fpa_id}{PROC_ERROR}[$index] = 23;
     388           
     389            $query{$fpa_id}{NPIX}[$index] = 0;
     390            $query{$fpa_id}{QFACTOR}[$index] = 0.0;
     391            $query{$fpa_id}{FLUX}[$index] = 0.0;
     392            $query{$fpa_id}{FLUX_SIG}[$index] = 0.0;
     393
    359394            next;
    360395        }
    … …  
    384419            my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line;
    385420            print $targetfile "$r_x $r_y\n";
    386             if (($r_ra == $query{$fpa_id}{RA1_DEG}[$processing_request{$fpa_id}{$image}[$i]])&&
    387                 ($r_dec == $query{$fpa_id}{DEC1_DEG}[$processing_request{$fpa_id}{$image}[$i]])) {
     421            if ((abs($r_ra - $query{$fpa_id}{RA1_DEG}[$processing_request{$fpa_id}{$image}[$i]]) < 1e-8)&&
     422                (abs($r_dec - $query{$fpa_id}{DEC1_DEG}[$processing_request{$fpa_id}{$image}[$i]]) < 1e-8)) {
    388423                $query{$fpa_id}{X_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_x;
    389424                $query{$fpa_id}{Y_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_y;
    … …  
    392427            else {
    393428                $error_code = $PS_EXIT_PROG_ERROR;
    394                 my_die("Unable to match input RA/DEC with output RA/DEC: ($query{$fpa_id}{RA1_DEG}[$i],$query{$fpa_id}{DEC1_DEG}[$i]) -> ($r_ra,$r_dec) i. $error_code",
     429                my_die("Unable to match input RA/DEC with output RA/DEC: ($query{$fpa_id}{RA1_DEG}[$i],$query{$fpa_id}{DEC1_DEG}[$i]) -> ($r_ra,$r_dec) $error_code",
    395430                   $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index],
    396431                   $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index],

  • branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_checkdependent.pl

    r28929 r29124  
    644644    # if the input file is already magicked no need to queue destreaking for this chipRun
    645645    if ($need_magic and !$input_magicked) {
    646         if ($dsRun_state eq 'cleaned') {
     646        if (!defined($dsRun_state) or ($dsRun_state eq 'NULL')) {
     647            print "No magicDSRun for chipRun $stage_id and magic is required\n";
     648            faultJobs('stop', undef, undef, $PSTAMP_NOT_DESTREAKED);
     649        } elsif ($dsRun_state eq 'cleaned') {
    647650            my $command = "$magicdstool -updaterun -set_state new -stage $stage -stage_id $stage_id";
    648651            $command .= " -set_label $rlabel" if $rlabel;
    … …  
    658661            }
    659662        } elsif ($dsRun_state eq 'failed_revert') {
    660             print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update";
     663            print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n";
    661664            faultJobs('stop', undef, undef, $PSTAMP_NOT_AVAILABLE);
    662665        } else {

  • branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_webrequest.pl

    r28106 r29124  
    44# pstampwebrequest.pl: take a postage stamp request command line and process it
    55#
    6 # The arguments are the command line parameters for the program pstamprequest
     6# The arguments are the command line parameters for the program psmkreq
    77#
    88# Unless the argument -list is provided the output is the request id for the resulting request
    … …  
    7070my $missing_tools;
    7171
    72 my $pstamprequest = can_run('pstamprequest')  or (warn "Can't find pstamprequest"  and $missing_tools = 1);
     72my $psmkreq = can_run('psmkreq')  or (warn "Can't find psmkreq"  and $missing_tools = 1);
    7373my $pstamptool = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
    7474my $pstampparse = can_run('pstampparse.pl')  or (warn "Can't find pstampparse.pl"  and $missing_tools = 1);
    75 my $pstampparser_run = can_run('pstamp_parser_run.pl')  or (warn "Can't find pstamp_parser_run.pl"  and $missing_tools = 1);
    7675
    7776if ($missing_tools) {
    … …  
    9695my $request_file = "$datedir/$request_name.fits";
    9796{
    98     my $command = "$pstamprequest -req_name $request_name -project $project $request_file @ARGV";
    99     $command .= " -$job_type" if $job_type;     # default job_type is pstamp
     97    my $command = "$psmkreq --req_name $request_name  --output $request_file @ARGV";
    10098
    10199    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    103101    unless ($success) {
    104102        print STDERR @$stderr_buf;
    105         die("Unable to perform pstamprequest: $error_code");
     103        die("Unable to perform psmkreq: $error_code");
    106104    }
    107105}

  • branches/eam_branches/ipp-20100823/pstamp/web/request.php

    r25322 r29124  
    1616// XXX This is just a prototype for testing purposes.
    1717
    18 
    19 // XXX: change to include pstampconfig.php
    20 
    21 // BEGIN Local configuration
    22 
    23 $WORKDIR = "/data/ipp049.0/pstamp/work";
    24 $dsroot = "/data/ipp049.0/datastore/dsroot";
    25 $dbname = "ippRequestServer";
    26 $dbserver = "ipp049";
    27 
    28 $PSCONFDIR = "/data/ipp053.0/home/bills/psconfig";
    29 $PSCONFIG  = "debug";
    30 $PSBINDIR  = "$PSCONFDIR/$PSCONFIG.lin64/bin";
    31 
    32 // END Local configuration
     18require "pstamp.php";
     19require "submitted.php";
    3320
    3421# this script sets up the environment to run IPP commands with current directory
    … …  
    5542$diff_checked = "";
    5643$list_checked = "";
     44$getstatus_checked = "";
    5745$pstamp_checked = "";
    5846$get_checked = "";
    … …  
    10290if ($rvar_project == "gpc1") {
    10391    $gpc1_selected = "selected";
    104     $require_class_id = 1;
     92//    $require_class_id = 1;
    10593} else if ($rvar_project == "megacam-mops") {
    10694    $mops_selected = "selected";
    … …  
    122110    $diff_checked = "CHECKED";
    123111} else {
    124     // nothing checked default to By Exposure
    125     $exp_checked = "CHECKED";
     112    // nothing checked default to By ID
     113    $file_checked = "CHECKED";
    126114}
    127115
    … …  
    140128
    141129// is the center is specified in Pixels or sky coordinates
    142 if ($rvar_center_type == "Sky") {
     130if ($rvar_center_type == "Pixels") {
     131    $sky_checked="";
     132    $pix_checked="checked";
     133} else {
    143134    $sky_checked="checked";
    144135    $pix_checked="";
    145 } else {
    146     $sky_checked="";
    147     $pix_checked="checked";
    148136}
    149137
    … …  
    168156}
    169157
     158$getstatus_checked = "";
     159$pstamp_checked = "";
     160$list_checked = "";
     161$get_checked = "";
    170162if ($rvar_cmd_mode == "Make Stamps") {
    171163    $pstamp_checked = "checked";
    172     $list_checked = "";
    173     $get_checked = "";
    174164} else if ($rvar_cmd_mode == "Get Images") {
    175165    $get_checked = "checked";
    176     $list_checked = "";
    177     $pstamp_checked = "";
     166} else if ($rvar_cmd_mode == "Get Status") {
     167    $getstatus_checked = "checked";
    178168} else {
    179     $list_checked = "checked";
    180     $get_checked = "";
    181     $pstamp_checked = "";
     169    // default
     170    $pstamp_checked = "checked";
    182171}
    183172
    … …  
    188177
    189178// How do we know whether or not this is the intial page load or not?
    190 // Well, in that case rvar_img_type is not set so key off of that
     179// Well, the first time rvar_img_type is not set. So we key off of that.
    191180// TODO: find a better way to decide whether or not to run commands
    192181
    193182if ($rvar_img_type) {
    194 
    195183    $jobFinished = 0;
    196     if ($request_id == 0) {
     184    if (! $getstatus_checked) {
    197185        try {
    198186            $command_line = build_request_cmd();
    … …  
    201189            if (! $list_checked) {
    202190                // The only output from a successful run is the request_id
     191                $request_name = trim(Array_pop($output_array));
    203192                $request_id = Array_pop($output_array);
    204193                $last_request_id = $request_id;
    205                 setcookie("our_request_id", $request_id);
    206                 // echo "The request id is $request_id\n";
    207                 if (count($output_array) != 0) {
    208                     throw new Exception("unexpected output returned by pstampwebrequest.");
     194                if ($request_id && $request_name) {
     195                    addRequest($request_id, $request_name);
     196                    // setcookie("our_request_id", $request_id);
     197                    // echo "The request id is $request_id\n";
     198                    $getstatus_checked = "checked";
     199                    $pstamp_checked = "";
     200                } else {
     201                    // XXX: TODO print out the error
     202                    if (count($output_array) != 0) {
     203                        throw new Exception("unexpected output returned by pstampwebrequest.");
     204                    }
    209205                }
    210 
    211                 $jobRunning = getRequestStatus();
    212                 if (!$jobRunning) {
    213                   //   echo "1  getRequestStatus reuturned 0\n";
    214                     $jobFinished = 1;
    215                     $request_id = 0;
    216                 }
    217 
    218206            } else {
    219207                $last_request_id = 0;
    … …  
    222210            $error_line = $e->getMessage();
    223211        }
    224     } else {
    225         try {
    226             // get the list of jobs for the request
    227             // echo "calling getRequestStatus\n";
    228             $jobRunning = getRequestStatus();
    229 
    230             if (!$jobRunning) {
    231                //  echo "2  getRequestStatus reuturned 0\n";
    232                 $jobFinished = 1;
    233                 $request_id = 0;
    234             }
    235 
    236         } catch (Exception $e) {
    237             echo "Got Exception $request_id $e\n";
    238             $error_line = $e->getMessage();
    239         }
    240     }
    241     if ($last_request_id) {
    242         // This doesn't work for get_image
    243         listJobs($last_request_id, $jobFinished);
    244     }
    245 }
     212    }
     213}
     214
     215// This is the end of the Logic
    246216
    247217function build_request_cmd()
    … …  
    296266                throw new Exception('RA and DEC must be specified.');
    297267            }
    298             $cmd .= " -skycenter $rvar_RA $rvar_DEC";
     268            $cmd .= " --ra $rvar_RA --dec $rvar_DEC";
    299269        } else {
    300270            if (! $rvar_X || ! $rvar_Y) {
    301271                throw new Exception('X and Y must be specified.');
    302272            }
    303             $cmd .= " -pixcenter $rvar_X $rvar_Y";
     273            $cmd .= " -pixcenter --x $rvar_X --y $rvar_Y";
    304274        }
    305275
    … …  
    308278                throw new Exception('dRA and dDEC must be specified.');
    309279            }
    310             $cmd .= " -arcrange $rvar_dRA $rvar_dDEC";
     280            $cmd .= " --arcseconds --width $rvar_dRA --height $rvar_dDEC";
    311281        } else {
    312282            if (! $rvar_W || ! $rvar_H) {
    313283                throw new Exception('width and height must be specified.');
    314284            }
    315             $cmd .= " -pixrange $rvar_W $rvar_H";
     285            $cmd .= " --width $rvar_W --height $rvar_H";
    316286        }
    317287    }
    … …  
    324294
    325295    if ($exp_checked) {
     296        if ($rvar_img_type == "stack") {
     297            throw new Exception('Lookup by exposure name not supported for stack images.');
     298        }
    326299        if (! $rvar_id ) {
    327300            throw new Exception('Must set ID to the Exposure ID.');
    328301        }
    329         $cmd .= " -byexp $rvar_img_type $rvar_id";
     302        $cmd .= " --req_type byexp --stage $rvar_img_type --id $rvar_id";
    330303    } else if ($file_checked) {
    331304        if (! $rvar_id ) {
    332305            throw new Exception('Must set ID to the exposure name.');
    333306        }
    334         $cmd .= " -byid $rvar_img_type $rvar_id";
     307        $cmd .= " --req_type byid --stage $rvar_img_type --id $rvar_id";
    335308    } else if ($coord_checked) {
    336         // $cmd .= " -bycoord $rvar_img_type";
    337         $coord_checked = "";
    338         $exp_checked = "checked";
    339         throw new Exception("Image selection by coordinate not implemented yet.");
     309        $cmd .= " --req_type bycoord --stage $rvar_img_type";
     310        $coord_checked = "checked";
     311//        throw new Exception("Image selection by coordinate not implemented yet.");
    340312    } else if ($diff_checked) {
    341313        if (! $rvar_id ) {
    342314            throw new Exception('Must set ID to Diff Image ID.');
    343315        }
    344         $cmd .= " -bydiff $rvar_img_type $rvar_id";
    345     }
    346 
    347     if (($rvar_img_type == "raw") || ($rvar_img_type == "chip")) {
    348         if ($require_class_id && ! $rvar_class_id ) {
    349             throw new Exception("must specify Class ID with Image Type $rvar_img_type.");
    350         }
    351         if ((!$rvar_class_id) || ($rvar_class_id == "all")) {
    352             $cmd .= " null";
    353         } else {
    354             $cmd .= " $rvar_class_id";
    355         }
    356     }
     316        $cmd .= " --req_type bydiff --stage $rvar_img_type --id $rvar_id";
     317    }
     318
     319// XXX: don't need to require class_id anymore
     320//    if (($rvar_img_type == "raw") || ($rvar_img_type == "chip")) {
     321//        if (!$sky_checked && ($require_class_id && ! $rvar_class_id )) {
     322//            throw new Exception("must specify Class ID with Image Type $rvar_img_type.");
     323//        }
     324        // leave off compoennt if we're looking up by coordinates. It breaks it
     325        if (!$coord_checked && (($rvar_class_id) && ($rvar_class_id != "all"))) {
     326            $cmd .= " --component $rvar_class_id";
     327        }
     328//    }
    357329
    358330    return escapeshellcmd($cmd);
    … …  
    390362}
    391363
     364// This is no longer used
    392365function printURL($line)
    393366{
    … …  
    442415    echo "</td></tr>";
    443416}
    444 
    445 function countRunningJobs()
    446 {
    447     global $output_array;
    448 
    449     $runningJobs = 0;
    450     $size = sizeof($output_array);
    451     for ($i = 0; $i < $size; $i++) {
    452         $elements = explode(" ", $output_array[$i]);
    453         if (count($elements) == 3) {
    454             $state    = $elements[1];
    455             if ($state != "stop") {
    456                 $runningJobs++;
    457             }
    458         } else {
    459             throw new Exception ("incorrect data in job status: $output_array[$i]");
    460         }
    461     }
    462     return $runningJobs;
    463 }
    464 
    465 function listJobs($request_id, $jobFinished)
    466 {
    467     global $WORKDIR;
    468     global $SCRIPT;
    469 
    470     $command_line = "$SCRIPT pstamp_listjobs.pl $request_id";
    471     global $dbname;
    472     global $dbserver;
    473     if ($dbname) {
    474         $command_line .= " --dbname $dbname --dbserver $dbserver";
    475     }
    476 
    477     run_command($command_line);
    478     if ($jobFinished) {
    479         global $outFileset;
    480         global $dsroot;
    481         $parse_error = "$WORKDIR/$request_id/parse_error.txt";
    482         #echo "reading $parse_error\n";
    483         // readfile( $parse_error );
    484         if (file_exists($parse_error)) {
    485             $fhandle = fopen($parse_error, "r");
    486             if ($fhandle) {
    487                 $contents = fread($fhandle, 1024);
    488                 if ($contents) {
    489                     global $last_request_id;
    490                     global $error_line;
    491                     $error_line = "Request $last_request_id: $contents\n";
    492                 }
    493             }
    494         }
    495     }
    496 }
    497 
    498 function getRequestStatus()
    499 {
    500     global $request_id;
    501     global $command_status;
    502     global $output_array;
    503     global $outFileset;
    504     global $dbname;
    505     global $dbserver;
    506     global $SCRIPT;
    507 
    508     $command_line = "$SCRIPT pstamptool -listreq -req_id $request_id -simple";
    509     if ($dbname) {
    510         $command_line .= " -dbname $dbname -dbserver $dbserver";
    511     }
    512     // echo "Running $command_line\n";
    513 
    514     run_command($command_line);
    515     if ($command_status == 0) {
    516         $size = sizeof($output_array);
    517         $runningReq = 0;
    518         for ($i = 0; $i < $size; $i++) {
    519             $elements = explode(" ", $output_array[$i]);
    520             if (count($elements) >= 4) {
    521                 $state = $elements[2];
    522                 $outFileset = $elements[4];
    523                 if ($state != "stop") {
    524                     $runningReq++;
    525                 }
    526             } else {
    527                 throw new Exception ("incorrect data in job status: $output_array[$i]");
    528             }
    529         }
    530         return $runningReq;
    531     } else {
    532         return 0;
    533     }
    534 }
    535 
    536417?>
    537418
    538 <!-- Beginning of the HTML --------------------------------------------- -->
     419<!----------------------Beginning of the HTML --------------------------------------------- -->
     420
    539421<html>
    540  <head>
    541   <title>Postage Stamp Request Form</title>
    542     <?php
    543         if ($request_id != 0) {
    544             // This doesn't do what I want. It does a get not a post
    545 
    546             // echo '<META HTTP-EQUIV="refresh" CONTENT="5">';
    547 
    548         }
    549     ?>
    550  </head>
     422<head>
     423  <title>
     424    Postage Stamp Request Form (prototype)
     425  </title>
     426</head>
     427<body>
     428
     429<H1 align=center>
     430Postage Stamp Request Form
     431</h1>
     432
     433<?php
     434    welcomeHeader($auth_user, "pstamp_links.php", "Postage Stamp Home");
     435?>
     436
    551437<form method="post">
    552 
    553 <body>
    554 
    555 <H1 align=center>
    556 Postage Stamp Request
    557 </h1>
    558 
    559438<!-- Whole page is a single column table -->
    560439
    … …  
    586465<td>
    587466&nbsp;<b>Select Images By:</b>&nbsp;&nbsp;&nbsp;
     467<input type=radio name="select_by" value="db_id" <?php echo $file_checked; ?> >Database ID
     468&nbsp;
    588469<input type=radio name="select_by" value="exposure_id" <?php echo $exp_checked; ?> >Exposure Name
    589470&nbsp;
    590 <input type=radio name="select_by" value="db_id" <?php echo $file_checked; ?> >Database ID
    591 &nbsp;
     471
    592472<input type=radio name="select_by" value="coord" <?php echo $coord_checked; ?> >Coordinates
    593473&nbsp;
    … …  
    625505        echo "Chip ID:";
    626506      } else {
    627         echo "Class ID:";
     507        echo "Component:";
    628508      }
    629509?>
    … …  
    685565            &nbsp;
    686566            <input type="text" name="dRA" size=10  value= <?php echo $rvar_dRA; ?> >
     567            &nbsp; "
    687568        </td>
    688569        <td>
    … …  
    691572            &nbsp;
    692573            <input type="text" name="dDEC" size=10 value="<?php echo $rvar_dDEC;?>" >
     574            &nbsp; "
    693575        </td>
    694576    </tr>
    … …  
    742624  <tr>
    743625
    744 <?php
    745   if ($request_id == 0): ?>
    746626    <td><input type=submit value="Submit"></td>
    747627    <td><b>Mode:</b>&nbsp;&nbsp;
     628    <input type=radio name="cmd_mode" value="Get Status"<?php echo $getstatus_checked; ?> >Get Status
    748629    <input type=radio name="cmd_mode" value="Make Stamps"<?php echo $pstamp_checked; ?> >Make Stamps
    749     <input type=radio name="cmd_mode" value="Get Images" <?php echo $get_checked; ?> >Get Images
     630    <input type=radio name="cmd_mode" value="Get Images" <?php echo $get_checked; ?> >Get Bundles
     631<!--
    750632    <input type=radio name="cmd_mode" value="List Images" <?php echo $list_checked; ?> >List Images
     633-->
    751634    </td>
    752 <?php
    753   else: ?>
    754     <td><input type=submit value="Check Status"></td>
    755 <!---    <td><input type=submit value="Cancel"></td> --->
    756635<?php
    757   echo "<td><b>Request Id: $request_id";
    758   endif; ?>
     636  // echo "<td><b>Request Id: $request_id";
     637?>
    759638
    760639  </tr>
    … …  
    766645
    767646<tr>
     647<!--- Don't show the command
    768648    <td>
    769649    <b>Command:</b>&nbsp;&nbsp; <?php echo "$command_line\n";?>
     650    </td>
     651--->
     652
     653</tr>
     654<tr>
     655    <td>
     656    <b>Last Command</b>
    770657    </td>
    771658</tr>
    … …  
    794681<td>
    795682<table align=center width=100% rules=none>
    796 <caption height=10 valign=center><b>Results</b></caption>
    797 
     683<caption height=10 valign=center><b>Request Results</b></caption>
    798684
    799685<?php
     686if (0) {
     687    // This is the old way of listing the status of the current request.
     688    // now we save the submitted requests in the session see listRequests() below
     689
    800690    $size = sizeof($output_array);
    801691    // echo "<pre>size of output array is $size\n</pre>";
    … …  
    818708        }
    819709    }
     710} // end if if(0)
    820711?>
    821712</table>
    … …  
    845736</table>
    846737
     738<?php
     739    listRequests("http://datastore.ipp.ifa.hawaii.edu/pstampresults", "pstamp_results_fileset");
     740?>
     741
    847742<!-- The end -->
    848743
    … …  
    861756?>
    862757
     758</form>
    863759</body>
    864 </form>
    865 
    866760</html>

  • branches/eam_branches/ipp-20100823/tools/czarclean.pl

    r29033 r29124  
    5959
    6060
    61 $czarDb->cleanupDateRange($begin, $end, "1 HOUR");
     61$czarDb->cleanupDateRange($begin, $end, $interval);
    6262

  • branches/eam_branches/ipp-20100823/tools/czarplot.pl

    r28922 r29124  
    88
    99use czartool::CzarDb;
    10 use czartool::Czarplot;
     10use czartool::Plotter;
    1111
    1212my $czarDbName = "czardb";
    … …  
    2323my $nebulous = undef;
    2424my $savingToFile = undef;
     25my $log = undef;
    2526
    2627GetOptions (
    … …  
    3637        "timeseries|t" => \$timeSeries,
    3738        "verbose|v" => \$verbose,
     39        "log|g" => \$log,
    3840        );
    3941
    40 print "\n";
     42print "\n*******************************************************************************\n";
     43print "* \n";
     44my $quit = 0;
     45if (@ARGV) {
     46    $quit=1;
     47    print "* UNKNKOWN: option                          @ARGV\n";
     48}
    4149if (!$histogram) {
    4250    print "* OPTIONAL: plot histogram                  -h                          (default=off)\n";}
    4351if (!$timeSeries) {
    4452    print "* OPTIONAL: plot timeseries                 -t                          (default=on)\n";}
     53if (!$log) {
     54    $log = 0;
     55    print "* OPTIONAL: use log plots                   -g                          (default=$log\n";}
    4556if (!$nebulous) {
    4657    print "* OPTIONAL: plot nebulous disk space        -n                          (default=off)\n";}
    4758if (!$label) {
    48     print "* OPTIONAL: choose a label                  -l <labellName>             (default='all_stdscience_labels')\n";}
     59    $label = "all_stdscience_labels";
     60    print "* OPTIONAL: choose a label                  -l <labellName>             (default=$label)\n";}
    4961if (!$stage) {
    5062    print "* OPTIONAL: choose a stage                  -s <chip|cam|warp|etc>      (default=none)\n";}
    5163if (!$interval) {
    52     print "* OPTIONAL: choose time interval in past    -i <'1 hour'|'1 day'|etc>   (default=interval since 7am this morning)\n";}
     64    print "* OPTIONAL: choose time interval in past    -i <'1 hour'|'1 day'|etc>   (default=none\n";}
    5365if (!$begin) {
    5466    print "* OPTIONAL: choose a begin time             -b <datetime>               (default=7am this morning)\n";}
    … …  
    5668    print "* OPTIONAL: choose an end time              -e <datetime>               (default=now)\n";}
    5769if (!$path) {
    58     print "* OPTIONAL: choose an output location       -o <path>                   (default=none)\n";}
     70    print "* OPTIONAL: choose an output location       -o <path>                   (default=none)\n";
     71}
     72print "*\n*******************************************************************************\n";
    5973
    60 print "\n";
     74if ($quit) { exit; }
    6175
    6276# default values
    63 if (!$label) {$label = "all_stdscience_labels";}
    6477if (!$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 0;}
    6578if (!$verbose) {$verbose = 0;}
    … …  
    7184$czarDb->setDateFormat("%Y%m%d-%H%i%s");
    7285
    73 my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11", $path, $save_temps);
    7486
    75 my @allStages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
     87# GENE PLOTS my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png size 1280,960 font \"/usr/share/fonts/corefonts/arial.ttf\" 12" : "X11", $path, $save_temps);
     88my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11", $path, $save_temps);
    7689
    7790# sort out times
    … …  
    7992if (!$begin) {
    8093    if ($interval) {$begin = $czarDb->subtractInterval($end, $interval);}
    81     else {$begin =  strftime('%Y-%m-%d 07:00',localtime);}
     94    else {$begin =  strftime('%Y-%m-%d 06:35',localtime);}
    8295}
    8396
    8497my $diskUsage = 1; # TODO
    8598
    86 if (!$nebulous && $timeSeries) {$czarplot->createTimeSeries($label, $stage, $begin, $end);}
    87 if ($histogram) {$czarplot->createHistogram($label, $begin, $end);}
    88 if ($nebulous && $timeSeries) {$czarplot->plotStorageTimeSeries($begin, $end);}
    89 elsif ($nebulous) {$czarplot->plotDiskUsageHistogram();}
     99if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log);}
     100if ($histogram) {$plotter->createHistogram($label, $begin, $end);}
     101if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);}
     102elsif ($nebulous) {$plotter->plotDiskUsageHistogram();}
    90103

  • branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm

    r28921 r29124  
    1 #!/usr/bin/perl i-w
     1#!/usr/bin/perl -w
    22
    33package czartool::CzarDb;
    … …  
    66use strict;
    77
    8 my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere
     8my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere
    99
    1010use base 'czartool::MySQLDb';
    … …  
    119119
    120120    $query->execute;
     121}
     122
     123###########################################################################
     124#
     125# Updates nightlyscience table
     126#
     127###########################################################################
     128sub updateNightlyScience {
     129    my ($self, $status) = @_;
     130
     131    my $query = $self->{_db}->prepare(<<SQL);
     132    DELETE FROM nightlyscience;
     133SQL
     134
     135    $query->execute;
     136
     137    $query = $self->{_db}->prepare(<<SQL);
     138    INSERT INTO nightlyscience
     139        (status)
     140        VALUES
     141        ('$status');
     142SQL
     143
     144       $query->execute;
    121145}
    122146
    … …  
    218242###########################################################################
    219243sub createTimeSeriesData {
    220     my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
    221 
    222     my $dataFile = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
    223     open (GNUDAT, ">$dataFile");
     244    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff, $dataFile, $isLog) = @_;
     245
     246    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
     247
     248    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
    224249
    225250    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    235260SQL
    236261
    237     if (!$query->execute) {return undef;}
     262    if (!$query->execute) {return 0;}
    238263
    239264    my $minProcessed = undef;
    240 
    241     ($minProcessed, ${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
    242 
    243     if (!defined $minProcessed) {return undef;}
     265    my ($_maxY, $_minY, $_timeDiff);
     266    ($minProcessed, $_maxY, $_minY, ${$maxX}, ${$minX}, $_timeDiff) = $query->fetchrow_array();
     267
     268    if ($_maxY > ${$maxY}) {${$maxY} = $_maxY;}
     269    if ($_minY < ${$minY}) {${$minY} = $_minY;}
     270    if ($_timeDiff > ${$timeDiff}) {${$timeDiff} = $_timeDiff;}
     271
     272    if (!defined $minProcessed) {return 0;}
    244273
    245274    $query = $self->{_db}->prepare(<<SQL);
    … …  
    256285    # loop round results
    257286    while (my @row = $query->fetchrow_array()) {
    258 
    259287        my ($timestamp, $pending, $faults, $processed) = @row;
    260     #    print  "'$timestamp' '$pending' '$faults' '$processed'\n";
     288
     289        if ($isLog) {
     290
     291            $processed = log($processed + 1);
     292            $pending = log($pending + 1);
     293            $faults = log($faults + 1);
     294        }
     295
    261296        print GNUDAT "$timestamp $pending $faults $processed\n";
    262297    }
    263     close(GNUDAT);
    264 
    265     return $dataFile;
     298
     299    close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n";
     300
     301    return 1;
    266302}
    267303
    … …  
    491527}
    492528
     529
     530###########################################################################
     531#
     532# Deletes all but one row per interval from all stage tables for all labels between the two dates
     533#
     534###########################################################################
     535sub cleanupDateRange {
     536    my ($self, $startDay, $endDay, $interval) = @_;
     537
     538    my $thisDay = $startDay;
     539    my $quit = 0;
     540    while(!$quit) {
     541
     542        if (!$self->isBefore($thisDay, $endDay)) {
     543       
     544            $quit = 1;
     545        }
     546
     547        print "* Running cleanup for $thisDay with an interval of $interval\n";
     548        $self->cleanupADay($thisDay, $interval);
     549        $thisDay = $self->addInterval($thisDay, "1 DAY");
     550
     551    }
     552}
     553###########################################################################
     554#
     555# Deletes all but one row per interval from all stage tables for all labels between the provided day
     556#
     557###########################################################################
     558sub cleanupADay {
     559    my ($self, $startDay, $interval) = @_;
     560
     561    my $endDay =  $self->addInterval($startDay, "1 DAY");
     562
     563    my $labels = undef;
     564    my $fromTime = $startDay;
     565    my $toTime = undef;
     566    my $totalDeleted = undef;
     567    my $quit = 0;
     568    while(!$quit) {
     569
     570        $toTime = $self->addInterval($fromTime, $interval);
     571        if (!$self->isBefore($toTime, $endDay)) {
     572       
     573            $toTime = $endDay;
     574            $quit = 1;
     575        }
     576   
     577        my $stage = undef;
     578        $totalDeleted = 0;
     579        foreach $stage (@stages) {
     580
     581            if (!$self->getLabelsInThisTimePeriod($stage, $fromTime, $toTime, \$labels)) {next;}
     582
     583            my $label = undef;
     584            my $row = undef;
     585            foreach $row ( @{$labels} ) {
     586                my ($label) = @{$row};
     587
     588                my $query = $self->{_db}->prepare(<<SQL);
     589                SELECT COUNT(*)
     590                    FROM $stage
     591                    WHERE timestamp > '$fromTime'
     592                    AND timestamp <= '$toTime'
     593                    AND label = '$label'
     594SQL
     595
     596                    $query->execute;
     597
     598                my $toDelete = scalar $query->fetchrow_array() - 1;
     599                if ($toDelete < 1) {next;}
     600
     601                $query = $self->{_db}->prepare(<<SQL);
     602                DELETE FROM $stage
     603                    WHERE timestamp > '$fromTime'
     604                    AND timestamp <= '$toTime'
     605                    AND label = '$label' ORDER BY timestamp DESC LIMIT $toDelete
     606SQL
     607                    $query->execute;
     608
     609                $totalDeleted += $toDelete;
     610            }
     611        }
     612        print "   * Deleted $totalDeleted between $fromTime and  $toTime\n";
     613        $fromTime = $toTime;
     614    }
     615}
     616
     617###########################################################################
     618#
     619# Returns an array of labels present during the provided time frame
     620#
     621###########################################################################
     622sub getLabelsInThisTimePeriod {
     623    my ($self, $stage, $fromTime, $toTime, $labels) = @_;
     624
     625    my $query = $self->{_db}->prepare(<<SQL);
     626    SELECT DISTINCT label
     627        FROM $stage
     628        WHERE timestamp > '$fromTime'
     629        AND timestamp <= '$toTime';
     630SQL
     631
     632        if (!$query->execute) {
     633
     634            return 0;
     635        }
     636
     637    ${$labels} = $query->fetchall_arrayref();
     638
     639    return 1;
     640}
     641
    493642###########################################################################
    494643#
    … …  
    500649
    501650    my $currentRevision = -1;
    502     my $latestRevision = 10;
    503 
    504     while ($currentRevision != $latestRevision) {
     651
     652    while (1) {
    505653
    506654        $currentRevision = $self->getRevision();
    … …  
    517665        elsif ($currentRevision == 8) {$self->createRevision_9();}
    518666        elsif ($currentRevision == 9) {$self->createRevision_10();}
     667        elsif ($currentRevision == 10) {$self->createRevision_11();}
     668        elsif ($currentRevision == 11) {$self->createRevision_12();}
     669        else {last;}
    519670    }
    520671}
    … …  
    841992}
    842993
     994#######################################################################################
     995#
     996# Create revision 11 of the database
     997#
     998#######################################################################################
     999sub createRevision_11 {
     1000    my ($self) = @_;
     1001
     1002    print "* Creating revision 11 of '$self->{_dbName}'\n";
     1003
     1004    # same shape as other stage tables to enable easy update
     1005    my $query = $self->{_db}->prepare(<<SQL);
     1006    CREATE TABLE burntool (
     1007            timestamp TIMESTAMP DEFAULT NOW(),
     1008            label VARCHAR(128) DEFAULT "NONE",
     1009            pending BIGINT NOT NULL,
     1010            processed BIGINT NOT NULL,
     1011            faults BIGINT NOT NULL);
     1012SQL
     1013
     1014      $query->execute;
     1015        $query = $self->{_db}->prepare(<<SQL);
     1016        CREATE INDEX burntoolIndex ON burntool (timestamp, label);
     1017SQL
     1018
     1019      $query->execute;
     1020
     1021    $self->setRevision(11);
     1022}
     1023
     1024#######################################################################################
     1025#
     1026# Create revision 12 of the database
     1027#
     1028#######################################################################################
     1029sub createRevision_12 {
     1030    my ($self) = @_;
     1031
     1032    print "* Creating revision 12 of '$self->{_dbName}'\n";
     1033
     1034    # same shape as other stage tables to enable easy update
     1035    my $query = $self->{_db}->prepare(<<SQL);
     1036    CREATE TABLE nightlyscience (
     1037            timestamp TIMESTAMP DEFAULT NOW(),
     1038            status VARCHAR(128) DEFAULT "NONE");
     1039SQL
     1040
     1041    $query->execute;
     1042
     1043    $self->setRevision(12);
     1044}
     1045
    84310461;
    8441047

  • branches/eam_branches/ipp-20100823/tools/czartool/MySQLDb.pm

    r28771 r29124  
    6464    return $self->{_dbHost};                                                 
    6565}                                                                               
     66
     67###########################################################################
     68#
     69# Adds the provided interval to the provided time
     70#
     71###########################################################################
     72sub addInterval {
     73    my ($self, $time, $interval) = @_;
     74
     75      my $query = $self->{_db}->prepare(<<SQL);
     76          SELECT '$time' + INTERVAL $interval;
     77SQL
     78    $query->execute;
     79
     80return scalar $query->fetchrow_array();
     81}
    6682
    6783###########################################################################

  • branches/eam_branches/ipp-20100823/tools/czartool/Pantasks.pm

    r28920 r29124  
    7373    }
    7474    return 1;
     75}
     76
     77###########################################################################
     78#
     79# Gets nightlyscience status for today
     80#
     81###########################################################################
     82sub getNightlyScienceStatus {
     83    my ($self, $status) = @_;
     84
     85    my @labels;
     86
     87    my ($day, $month, $year) = (localtime)[3,4,5];
     88    my $today = sprintf("%04d-%02d-%02d", $year+1900, $month+1, $day);
     89
     90    my @cmdOut = `echo "ns.show.dates;quit" | pantasks_client -c ~ipp/stdscience/ptolemy.rc 2>&1`;
     91    my $line;
     92    foreach $line (@cmdOut) {
     93
     94        chomp($line);
     95        if (!$self->outputOk($line)) {return 0;}
     96        if ($line =~ m/$today\s+(.+)/) {
     97            ${$status} = $1;
     98            return 1;
     99        }
     100    }
     101
     102    return 0;
    75103}
    76104

  • branches/eam_branches/ipp-20100823/tools/roboczar.pl

    r28926 r29124  
    1111use czartool::Pantasks;
    1212use czartool::Nebulous;
    13 use czartool::Czarplot;
     13use czartool::Plotter;
     14use czartool::Burntool;
    1415
    1516my $period = 60;
    … …  
    2627my $nebulous = new czartool::Nebulous($czarDb);
    2728my $pantasks = new czartool::Pantasks();
    28 my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
     29my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
     30my $burntool = new czartool::Burntool();
     31
    2932$czarDb->setDateFormat("%Y%m%d-%H%i%s");
    3033
    31 my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
     34my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
    3235
    3336
    … …  
    101104    my $priority = undef;
    102105    my $newState = undef;
     106    my $nsStatus = undef;
    103107
    104108    while (1) {
    105109
    106110        # sort out times
    107         $begin =  strftime('%Y-%m-%d 07:00',localtime);
     111        $begin =  strftime('%Y-%m-%d 06:35',localtime);
    108112        $end = $czarDb->getNowTimestamp();
    109113
    … …  
    112116            $begin = $czarDb->subtractInterval($begin, "1 DAY");
    113117        }
     118
     119        # check nightly science status
     120        print "* Checking nightly science status\n";
     121        if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";}
     122        $czarDb->updateNightlyScience($nsStatus);
    114123
    115124        # check nebulous
    116125        print "* Checking Nebulous\n";
    117126        $nebulous->updateClusterSpaceInfo();
    118         $czarplot->plotDiskUsageHistogram();
     127        $plotter->plotDiskUsageHistogram();
    119128        updateServerStatus();
    120129
    … …  
    177186
    178187            chomp($label);
    179             $czarplot->createTimeSeries($label,  $stage, $begin, $end);
     188            $plotter->createLogAndLinearTimeSeries($label,  $stage, $begin, $end);
    180189        }
    181190    }
    … …  
    185194        my ($label) = @{$row};
    186195
    187         $czarplot->createTimeSeries($label, undef, $begin, $end);
    188         $czarplot->createHistogram($label, $begin, $end);
     196        $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end);
     197        $plotter->createHistogram($label, $begin, $end);
    189198
    190199        #routineChecks($label, "1 HOUR");
    191200    }
    192     $czarplot->createTimeSeries("all_".$server."_labels", undef, $begin, $end);
    193     $czarplot->createHistogram("all_".$server."_labels", $begin, $end);
     201    $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", undef, $begin, $end);
     202    $plotter->createHistogram("all_".$server."_labels", $begin, $end);
    194203    foreach $stage (@stages) {
    195204
    196         $czarplot->createTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
     205        $plotter->createLogAndLinearTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
    197206    }
    198207}
    … …  
    233242            chomp($label);
    234243
    235             $new = $gpc1Db->countExposures($label, $stage, $newState);
    236             $full = $gpc1Db->countExposures($label, $stage, "full");
    237             $faults = $gpc1Db->countFaults($label, $stage, $newState);
    238 
     244            if ($stage eq "burntool") {
     245
     246                if ($labelServer eq "stdscience") {
     247
     248                    $burntool->getPendingAndProcessed($label, \$new, \$full);
     249                    $faults = 0;
     250                }
     251                else { $new = $full = $faults = 0;}
     252            }
     253            else {
     254
     255                $new = $gpc1Db->countExposures($label, $stage, $newState);
     256                $full = $gpc1Db->countExposures($label, $stage, "full");
     257                $faults = $gpc1Db->countFaults($label, $stage, $newState);
     258            }
    239259            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
    240260            $totalNew += $new;
Note: See TracChangeset for help on using the changeset viewer.