IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29906


Ignore:
Timestamp:
Dec 3, 2010, 2:35:32 AM (15 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20101103
Files:
1 deleted
46 edited
3 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20101103/Nebulous/Build.PL

    r28446 r29906  
    122122        bin/neb-touch
    123123        bin/neb-xattr
     124        bin/whichnode
    124125    )],
    125126)->create_build_script;

  • branches/eam_branches/ipp-20101103/Nebulous/lib/Nebulous/Client.pm

    r28525 r29906  
    379379        {
    380380            # volume
    381             type        => SCALAR,
     381            type        => SCALAR|UNDEF,
    382382            optional    => 1,
    383383        },
    … …  
    401401        }
    402402
     403        # set the number of user copies to 1 to prevent replication
     404        $self->setxattr($key, 'user.copies', 1, 'replace');
     405
    403406        if (defined $locations) {
    404             my $instances = $self->find_instances($key);
    405             if (scalar @$instances < 2) {
    406                 die "not enough instances";
     407            my $instances = $self->find_instances($key, undef, 'find them all');
     408            if (scalar @$instances == 1) {
     409                # only one instance nothing to do
     410                return 0;
     411            }
     412            if (scalar @$instances == 0) {
     413                die "no instances";
    407414            }
    408415            foreach my $victim (@$instances) {
    … …  
    416423            # start at one so cull() is called one less time then the # of
    417424            # instances
    418             if ($stats->[6] < 2) {
    419                 die "not enough instances";
     425            if ($stats->[6] ==  1) {
     426                # only one instance nothing to do
     427                return 0;
     428            }
     429            if ($stats->[6] == 0) {
     430                die "no instances";
    420431            }
    421432            for (my $i = 1; $i < $stats->[6]; $i++) {
    … …  
    439450}
    440451
     452sub storage_object_exists
     453{
     454    my $self = shift;
     455
     456    # HI!
     457
     458    my ($key) = validate_pos(@_,
     459        {
     460            type => SCALAR,
     461        },
     462    );
     463
     464    $log->debug( "entered - @_" );
     465
     466    my $found = 0;
     467    eval {
     468        my $objects = $self->find_objects($key);
     469        if  ($objects) {
     470            $found = (scalar @$objects) > 0;
     471        }
     472    };
     473    if ($@) {
     474        if ($@ =~ qr/invalid key/) {
     475            $self->set_err($@);
     476            return;
     477        }
     478        $log->logdie($@);
     479    }
     480
     481    $log->debug("leaving");
     482
     483    return $found;
     484}
    441485
    442486sub lock

  • branches/eam_branches/ipp-20101103/PS-IPP-Config/lib/PS/IPP/Config.pm

    r29844 r29906  
    10871087}
    10881088
     1089sub prepare_output
     1090{
     1091    my $self = shift;             # Configuration object
     1092    my $name = shift;             # Name of the filerule
     1093    my $outroot = shift;          # For replacing {OUTPUT}
     1094    my $component = shift;        # For replacing {CHIP.NAME} and {CELL.NAME}
     1095    my $delete_existing = shift;  # if file exists delete it
     1096    my $r_error = shift;          # reference to error code
     1097
     1098    die "prepare_output: invalid number of arguments" if !defined $r_error;
     1099
     1100    $$r_error = 0;
     1101
     1102    my $output = $self->filename($name, $outroot, $component);
     1103    if (!$output) {
     1104        # error message emitted by filename should be sufficient
     1105        $$r_error = $PS_EXIT_CONFIG_ERROR;
     1106        return undef;
     1107    }
     1108
     1109    if (file_scheme($output) ne 'neb') {
     1110        # non-nebulous file we're done
     1111        if ($delete_existing) {
     1112            if (!$self->file_delete($output)) {
     1113                carp "failed to delete $output";
     1114                $$r_error = $PS_EXIT_SYS_ERROR;
     1115                $output = undef;
     1116            }
     1117        }
     1118        return $output;
     1119    }
     1120
     1121    my $neb = $self->nebulous;
     1122    if ($neb->storage_object_exists($output)) {
     1123        if ($delete_existing) {
     1124            # avoid dead instances by moving the file before deleting it.
     1125            # append current time to form new name
     1126            my $todelete;
     1127            eval {
     1128                # parse the key so that we can compute the new name
     1129                require Nebulous::Key;
     1130            };
     1131            if ($@) {
     1132                carp "Can't find Nebulous::Key";
     1133                $$r_error = $PS_EXIT_CONFIG_ERROR;
     1134                return undef;
     1135            }
     1136            eval {
     1137                # parse the key so that we can compute the new name
     1138                my $neb_key = Nebulous::Key::parse_neb_key($output);
     1139                die "parse_neb_key failed" if !$neb_key;
     1140                my $path = $neb_key->path;
     1141                die "neb_key has no path" if !$path;
     1142                my $ticks = time();
     1143                $todelete = "ipp_trash/$path.$ticks";
     1144                $neb->move($output, $todelete);
     1145            };
     1146            if ($@) {
     1147                carp "nebulous move failed for $output";
     1148                $output = undef;
     1149                $$r_error = $PS_EXIT_SYS_ERROR;
     1150            }
     1151            if ($todelete) {
     1152                eval {
     1153                    $neb->delete($todelete);
     1154                };
     1155                if ($@) {
     1156                    carp "nebulous delete for $todelete failed. Ignoring.\n";
     1157                    $$r_error = $PS_EXIT_SYS_ERROR;
     1158                }
     1159            }
     1160        } else {
     1161            # Make sure that there is only 1 instance.
     1162
     1163            eval {
     1164                $neb->there_can_be_only_one($output);
     1165            };
     1166            if ($@) {
     1167                carp "nebulous there_can_be_only_one() failed for $output";
     1168                $output = undef;
     1169                $$r_error = $PS_EXIT_SYS_ERROR;
     1170            }
     1171        }
     1172    }
     1173    return $output;
     1174}
     1175
     1176# Cause a nebulous file to be replicated setting the user.copies value
     1177sub replicate_file
     1178{
     1179    my $self = shift;
     1180    my $file = shift;
     1181    my $copies = shift;
     1182
     1183    if (file_scheme($file) ne 'neb') {
     1184        carp "cannot replicate non-neulous file: $file";
     1185        return 0;
     1186    }
     1187
     1188    if (!$copies) {
     1189        $copies = 2;
     1190    }
     1191
     1192    my $neb = $self->nebulous;
     1193    eval {
     1194        $neb->setxattr($file, 'user.copies', $copies, 'replace');
     1195    };
     1196    if ($@) {
     1197        carp "failed to set user.copies for $file";
     1198        return 0;
     1199    }
     1200
     1201    # XXX: if copies > 2 should we make more replicants here ?
     1202    eval {
     1203        $neb->replicate($file);
     1204    };
     1205    if ($@) {
     1206        carp "failed to replicate $file";
     1207        return 0;
     1208    }
     1209    return 1;
     1210}
     1211
     1212
    10891213# Return catdir for tessellation, from TESSELLATIONS within the site configuration
    10901214sub tessellation_catdir

  • branches/eam_branches/ipp-20101103/dbconfig

  • branches/eam_branches/ipp-20101103/dbconfig/cam.md

    r28089 r29906  
    113113    maskfrac_max_magic  F32     0.0
    114114    maskfrac_max_advisory F32   0.0
     115    deteff         F32      0
     116    deteff_err     F32      0
     117    deteff_lq      F32      0
     118    deteff_uq      F32      0
    115119    quality        S16      0
    116120END

  • branches/eam_branches/ipp-20101103/dbconfig/changes.txt

    r29527 r29906  
    19811981    FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id)
    19821982) ENGINE=innodb DEFAULT CHARSET=latin1;
     1983
     1984ALTER TABLE chipProcessedImfile ADD column deteff_magref FLOAT;
     1985ALTER TABLE camProcessedExp ADD column deteff FLOAT AFTER maskfrac_max_advisory;
     1986ALTER TABLE camProcessedExp ADD column deteff_err FLOAT AFTER deteff;
     1987ALTER TABLE camProcessedExp ADD column deteff_lq FLOAT AFTER deteff_err;
     1988ALTER TABLE camProcessedExp ADD column deteff_uq FLOAT AFTER deteff_lq;
     1989UPDATE dbversion set schema_version = '1.1.66',  updated= CURRENT_TIMESTAMP();
     1990
     1991-- Version 1.1.67

  • branches/eam_branches/ipp-20101103/dbconfig/chip.md

    r28056 r29906  
    105105    maskfrac_magic  F32     0.0
    106106    maskfrac_advisory F32   0.0
     107    deteff_magref   F32     0.0
    107108END
    108109

  • branches/eam_branches/ipp-20101103/dbconfig/config.md

    r29499 r29906  
    22    pkg_name        STR     ippdb
    33    pkg_namespace   STR     ippdb
    4     pkg_version     STR     1.1.65
     4    pkg_version     STR     1.1.66
    55END

  • branches/eam_branches/ipp-20101103/ippScripts/scripts/camera_exp.pl

    r28043 r29906  
    7070my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $cam_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    7171
    72 my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    73 
    7472if (not defined $run_state) { $run_state = 'new'; }
    75 if ($run_state eq 'update') {
    76     $logDest .= '.update';
     73
     74my_die ("$run_state is an invalid value for run-state", $cam_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new' or $run_state eq 'update');
     75
     76
     77my $replicateOutputs = 1;
     78
     79my $logDest;
     80my $traceDest;
     81if ($run_state eq 'new') {
     82    $logDest = prepare_output("LOG.EXP", $outroot, undef, 0);
     83    $traceDest = prepare_output("TRACE.EXP", $outroot, undef, 0);
     84} else {
     85    $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, undef, 0);
     86    $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, undef, 0);
    7787}
    7888
    … …  
    188198    print $list4File ($chipMask . "\n");
    189199
    190     push @outMasks, $ipprc->filename("PSASTRO.OUTPUT.MASK", $outroot, $class_id) if $produceMasks;
     200    push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $produceMasks;
    191201}
    192202close $list1File;
    … …  
    199209
    200210# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs
    201 my $jpeg1      = $ipprc->filename("PPIMAGE.JPEG1",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    202 my $jpeg2      = $ipprc->filename("PPIMAGE.JPEG2",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    203 my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",     $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    204 my $fpaStats   = $ipprc->filename("PSASTRO.STATS",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    205 my $traceDest  = $ipprc->filename("TRACE.EXP",          $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    206 my $configuration = $ipprc->filename("PSASTRO.CONFIG",  $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    207 
    208 if ($run_state eq 'update') {
    209     $traceDest .= '.update';
    210     $fpaStats .= '.update';
    211 }
    212 
    213 # convert supplied DVO database name to UNIX filename
    214 my $dvodbReal;
    215 if (defined $dvodb) {
    216     $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO
    217     $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal );
    218 }
    219 
    220 #my $dtime_addstar = 0;
     211my $jpeg1      = prepare_output("PPIMAGE.JPEG1",      $outroot, undef, 1);
     212my $jpeg2      = prepare_output("PPIMAGE.JPEG2",      $outroot, undef, 1);
     213my $fpaObjects = prepare_output("PSASTRO.OUTPUT",     $outroot, undef, 1);
     214my $configuration = prepare_output("PSASTRO.CONFIG",  $outroot, undef, 1);
     215
     216my $do_stats;
     217my $fpaStats;
     218if ($run_state eq 'new') {
     219    $do_stats = 1;
     220    $fpaStats = prepare_output("PSASTRO.STATS",      $outroot, undef, 1);
     221} else {
     222    $do_stats = 0;
     223}
    221224
    222225unless ($no_op) {
    … …  
    236239            &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code);
    237240        }
    238         &my_die("Unable to find expected output file: $jpeg1", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1);
     241        check_output($jpeg1, $replicateOutputs);
    239242    }
    240243
    … …  
    251254            &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code);
    252255        }
    253         &my_die("Unable to find expected output file: $jpeg2", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2);
     256        check_output($jpeg2, $replicateOutputs);
    254257    }
    255258
    … …  
    265268        $command .= " -dbname $dbname" if defined $dbname;
    266269
    267         my $do_stats;
    268270        if ($run_state eq 'new') {
    269             $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS";
    270271            $command .= " -dumpconfig $configuration";
    271             $do_stats = 1;
    272272        } elsif ($run_state eq 'update') {
    273273            $command .= " -ipprc $configuration";
    … …  
    275275            &my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR);
    276276        }
     277        $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS" if $do_stats;
    277278
    278279        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    286287        my $quality;            # Quality flag
    287288        if ($do_stats) {
     289            check_output($fpaStats, $replicateOutputs);
     290
    288291            my $fpaStatsReal = $ipprc->file_resolve($fpaStats);
    289             &my_die("Couldn't find expected output file: $fpaStats", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $fpaStatsReal );
    290292
    291293            # parse stats from metadata
    … …  
    306308
    307309        if (!$quality) {
    308             &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($fpaObjects);
     310            check_output($fpaObjects, $replicateOutputs);
    309311
    310312            foreach my $outMask (@outMasks) {
    311                 &my_die("Unable to find expected output file: $outMask", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outMask);
     313                check_output($outMask, $replicateOutputs);
    312314            }
    313315
    314316            if ($run_state eq 'new') {
    315                 &my_die("Couldn't find expected output file: $configuration", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
     317                check_output($configuration, $replicateOutputs);
    316318            }
    317319        }
    318 
    319         # run addstar on the output fpaObjects (if a DVO database is defined)
    320 #         if (defined $dvodbReal and ($run_state eq 'new')) {
    321 
    322 #             ## XXX the camera analysis can either save the full set of
    323 #             ## detections, or just the image metadata, in the dvodb
    324 
    325 #             ## get the addstar recipe for this camera and CAMERA reduction
    326 #             $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -";
    327 #             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    328 #                 run(command => $command, verbose => $verbose);
    329 #             unless ($success) {
    330 #                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    331 #                 &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR);
    332 #             }
    333 #             my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
    334 #                 &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR);
    335 
    336 #             ## allow the dvodb to save only images, or the full detection set
    337 #             my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY');
    338 
    339 #             # XXX this construct requires the user to have a valid .ptolemyrc
    340 #             # XXX which in turn points at ippconfig/dvo.site
    341 #             # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
    342 #             # XXX this needs to be converted to addstar_client...
    343 
    344 #             my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
    345 #             my $command;
    346 #             $command  = "$addstar -D CAMERA $camdir -update";
    347 #             $command .= " -image" if $imagesOnly;
    348 #             $command .= " -D CATDIR $dvodbReal";
    349 
    350 #             my $realFile = $ipprc->file_resolve($fpaObjects);
    351 #             $command .= " $realFile";
    352 
    353 #             my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
    354 
    355 #             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    356 #                 run(command => $command, verbose => $verbose);
    357 #             unless ($success) {
    358 #                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    359 #                 &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code);
    360 #             }
    361 #             $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);   # MJD of starting script
    362 #         }
    363320    }
    364321}
    … …  
    374331    $fpaCommand .= " -hostname $host" if defined $host;
    375332    $fpaCommand .= " -dtime_script $dtime_script";
    376 #    $fpaCommand .= " -dtime_addstar $dtime_addstar";
    377333} else {
    378334    $fpaCommand .= " -updaterun -set_state full";
    … …  
    393349}
    394350
     351exit 0;
     352
     353sub prepare_output
     354{
     355    my $filerule = shift;
     356    my $outroot  = shift;
     357    my $class_id = shift;
     358    my $delete = shift;
     359    $delete = 0 if !defined $delete;
     360
     361    my $error;
     362    my $output = $ipprc->prepare_output($filerule, $outroot, $class_id, $delete, \$error)
     363                    or &my_die("failed to prepare output file for: $filerule", $cam_id, $error);
     364
     365    return $output;
     366}
     367
     368sub check_output
     369{
     370    my $file = shift;
     371    my $replicate = shift;
     372
     373    if (!defined $file) {
     374        return;
     375    }
     376
     377    &my_die("Couldn't find expected output file: $file",  $cam_id, $PS_EXIT_SYS_ERROR) unless
     378        $ipprc->file_exists($file);
     379
     380    if ($replicate and (file_scheme($file) eq 'neb')) {
     381        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $cam_id, $PS_EXIT_SYS_ERROR);
     382    }
     383}
    395384
    396385sub my_die

  • branches/eam_branches/ipp-20101103/ippScripts/scripts/chip_imfile.pl

    r28899 r29906  
    6666
    6767pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    68 pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state",
     68pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state --dbname",
    6969           -exitval => 3) unless
    7070    defined $exp_id and
    … …  
    7575    defined $camera and
    7676    defined $outroot and
     77    defined $dbname and
    7778    defined $run_state;
    7879
    7980my_die ("$run_state is an invalid value for run-state", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new' or $run_state eq 'update');
    8081
    81 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    82 
    83 my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    84 $logDest .= ".update" if $run_state eq "update";
     82my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR );
     83
     84my $neb;
     85my $scheme = file_scheme($outroot);
     86if ($scheme and $scheme eq 'neb') {
     87    $neb = $ipprc->nebulous();
     88}
     89
     90my ($logDest, $traceDest);
     91if ($run_state eq 'new') {
     92    $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, 0);
     93    $traceDest = prepare_output("TRACE.IMFILE",  $outroot, $class_id, 1);
     94} else {
     95    $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1);
     96    $traceDest = prepare_output("TRACE.IMFILE.UPDATE",  $outroot, $class_id, 1);
     97}
    8598
    8699if ($redirect) {
    … …  
    90103    print STDOUT "FULL COMMAND: $0 @ARGS\n\n";
    91104}
     105
    92106
    93107# Recipes to use based on reduction class
    … …  
    114128
    115129## these names are used in ppImage, and thus may be URIs
    116 my $outputImage   = $ipprc->filename("PPIMAGE.CHIP",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    117 my $outputMask    = $ipprc->filename("PPIMAGE.CHIP.MASK",   $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    118 my $outputWeight  = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    119 my $outputBin1    = $ipprc->filename("PPIMAGE.BIN1",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    120 my $outputBin2    = $ipprc->filename("PPIMAGE.BIN2",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    121 my $outputStats   = $ipprc->filename("PPIMAGE.STATS",       $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    122 my $traceDest     = $ipprc->filename("TRACE.IMFILE",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    123 my $configuration = $ipprc->filename("PPIMAGE.CONFIG",      $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    124 
    125 if ($run_state eq 'update') {
    126     $outputStats .= '.update';
    127     $traceDest .= '.update';
     130my $outputImage   = prepare_output("PPIMAGE.CHIP",          $outroot, $class_id, 1 );
     131my $outputMask    = prepare_output("PPIMAGE.CHIP.MASK",     $outroot, $class_id, 1);
     132my $outputWeight  = prepare_output("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id, 1);
     133my $pattern       = prepare_output("PPIMAGE.PATTERN",       $outroot, $class_id, 1);
     134my $backmdl       = prepare_output("PSPHOT.BACKMDL",        $outroot, $class_id, 1);
     135
     136my $configuration;
     137my $outputSources;
     138my $outputPsf;
     139my $outputStats;
     140my $outputBin1;
     141my $outputBin2;
     142my $dump_config = 1;
     143my $do_binned_images = 1;
     144if ($run_state eq 'new') {
     145    # prepare the files that are only created for a new run
     146    $configuration = prepare_output("PPIMAGE.CONFIG",        $outroot, $class_id, 1);
     147    $outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
     148} else {
     149    $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id)
     150            or &my_die("Missing entry from camera config: PPIMAGE.CONFIG", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     151    if ($ipprc->file_exists($configuration)) {
     152        $dump_config = 0;
     153    } else {
     154        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
     155
     156        # XXX: should we create a new config dump file?
     157        # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
     158        my $gone = 0;
     159        if (storage_object_exists($configuration, \$gone)) {
     160            if ($gone) {
     161                rename_gone_file($configuration);
     162                $configuration = prepare_output('PPIMAGE.CONFIG', $outroot, $class_id, 1);
     163                # if we dump the config we need to insure that the config dump represents
     164                # the full processing
     165            } else {
     166                # file is temporarily not available. Don't dump config.
     167                $dump_config = 0;
     168            }
     169        }
     170    }
     171
    128172    # make sure that any lingering destreak backup files are gone
    129173    $ipprc->delete_destreak_backup_file($outputImage)
    … …  
    134178        or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
    135179}
     180if ($do_binned_images) {
     181    $outputBin1    = prepare_output("PPIMAGE.BIN1",          $outroot, $class_id, 1);
     182    $outputBin2    = prepare_output("PPIMAGE.BIN2",          $outroot, $class_id, 1);
     183}
    136184
    137185my $cmdflags;
    … …  
    152200    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
    153201        &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     202
     203    my $do_photom = metadataLookupBool($recipeData, 'PHOTOM');
     204    if ($do_photom and ($run_state eq 'update')) {
     205        # If previous photometry outputs are ok skip running photometry
     206        if ($dump_config || rerun_photometry($outroot, $class_id)) {
     207            carp "Will rerun photometry\n";
     208        } else {
     209            $do_photom = 0;
     210        }
     211    }
     212    if ($do_photom) {
     213        $outputSources = prepare_output("PSPHOT.OUTPUT",   $outroot, $class_id, 1);
     214        $outputPsf     = prepare_output("PSPHOT.PSF.SAVE", $outroot, $class_id, 1);
     215    }
     216
    154217
    155218    ## XXX make the feature more general?
    … …  
    357420    }
    358421
     422    $command  = "$ppImage -file $uri $outroot";
     423    if ($dump_config) {
     424        $command .= " -recipe PPIMAGE $recipe_ppImage";
     425        $command .= " -dumpconfig $configuration";
     426    } else {
     427        $command .= " -ipprc $configuration";
     428    }
     429    if ($do_photom) {
     430        $command .= " -recipe PSPHOT $recipe_psphot";
     431    } else {
     432        $command .= " -Db PPIMAGE:PHOTOM FALSE";
     433    }
    359434    if ($run_state eq "new") {
    360         $command  = "$ppImage -file $uri $outroot";
    361         $command .= " -recipe PPIMAGE $recipe_ppImage";
    362         $command .= " -recipe PSPHOT $recipe_psphot";
    363         $command .= " -threads $threads" if defined $threads;
    364         $command .= " -dbname $dbname" if defined $dbname;
    365         $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
    366         $command .= " -source_id $source_id" if defined $source_id;
    367         $command .= " -dumpconfig $configuration";
    368         $command .= " -tracedest $traceDest -log $logDest";
    369         $do_stats = 1;
    370     } else {
    371         $command  = "$ppImage -file $uri $outroot";
    372         $command .= " -ipprc $configuration";
    373         $command .= " -threads $threads" if defined $threads;
    374         $command .= " -dbname $dbname" if defined $dbname;
    375         $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
    376         $command .= " -source_id $source_id" if defined $source_id;
    377         $command .= " -tracedest $traceDest -log $logDest";
    378         $command .= " -Db PPIMAGE:PHOTOM FALSE";
    379     }
    380     if ($do_stats) {
    381435        $command .= " -recipe PPSTATS CHIPSTATS";
    382436        $command .= " -stats $outputStats";
    383     }
     437        $do_stats = 1;
     438    }
     439    if (!$do_binned_images) {
     440        $command .= " -Db PPIMAGE:BIN1.FITS FALSE -Db PPIMAGE:BIN2.FITS FALSE";
     441    }
     442    $command .= " -threads $threads" if defined $threads;
     443    $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
     444    $command .= " -source_id $source_id" if defined $source_id;
     445    $command .= " -tracedest $traceDest -log $logDest";
     446    $command .= " -dbname $dbname" if defined $dbname;
    384447
    385448    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    394457    my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS');
    395458    my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.VARIANCE.FITS');
     459    my $outputBackmdlExpect = metadataLookupBool($recipeData, 'BACKGROUND');
     460    my $outputPatternExpect = (metadataLookupBool($recipeData, 'PATTERN.ROW') or metadataLookupBool($recipeData, 'PATTERN.CELL')) ;
    396461
    397462    my $quality;                # Quality flag
    … …  
    416481
    417482    if (!$quality) {
    418         &my_die("Couldn't find expected output file: $outputImage\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputImageExpect or $ipprc->file_exists($outputImage);
    419         &my_die("Couldn't find expected output file: $outputMask\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputMaskExpect or $ipprc->file_exists($outputMask);
    420         &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputWeightExpect or $ipprc->file_exists($outputWeight);
    421         &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
    422         &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
    423         if ($run_state eq 'new') {
    424             &my_die("Couldn't find expected output file: $configuration", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
    425         }
    426     }
    427 
     483        my $replicateImages = 0;
     484        check_output($outputImage, $replicateImages) if $outputImageExpect;
     485        check_output($outputMask, $replicateImages) if $outputMaskExpect;
     486        check_output($outputWeight, $replicateImages) if $outputWeightExpect;
     487        if ($do_binned_images) {
     488            check_output($outputBin1, 1);
     489            check_output($outputBin2, 1);
     490        }
     491        check_output($configuration, 1) if $dump_config;
     492        check_output($backmdl, 1) if $outputBackmdlExpect;
     493        check_output($pattern, 1) if $outputPatternExpect;
     494        if ($do_photom) {
     495            check_output($outputSources, 1);
     496            check_output($outputPsf, 1);
     497        }
     498        # XXX: Do we want to replicate the stats, logs, trace file?
     499    }
    428500}
    429501
    … …  
    462534}
    463535
     536exit 0;
     537
     538# check whether psphot outputs should be regenerated.
     539# Whether we need to or not is a somewhat complicated question.
     540sub rerun_photometry
     541{
     542    my $outroot = shift;
     543    my $class_id = shift;
     544    my $outputSources = $ipprc->filename('PSPHOT.OUTPUT', $outroot, $class_id)
     545                    or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     546
     547    my $make_sources = 0;
     548    my $sources_available = 0;
     549    if ($ipprc->file_exists($outputSources)) {
     550        $sources_available = 1;
     551    } else {
     552        carp "WARNING: photometry sources file $outputSources is not available";
     553        my $gone;
     554        if (storage_object_exists($outputSources, \$gone)) {
     555            # check whether the file is permanantely or temporarily gone
     556            if ($gone) {
     557                carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
     558                rename_gone_file($outputSources);
     559                $make_sources = 1;
     560            }
     561        } else {
     562            # storage object must have been deleted
     563            $make_sources = 1;
     564        }
     565    }
     566
     567    my $make_psf = 0;
     568    my $psf_available = 0;
     569    my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE",       $outroot, $class_id)
     570                    or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     571    if ($ipprc->file_exists($outputPsf)) {
     572        $psf_available = 1;
     573    } else {
     574        carp "PSF file $outputPsf is missing";
     575        my $gone = 0;
     576        if (storage_object_exists($outputPsf, \$gone)) {
     577            # object exists, but no instances are available. If they are permanently gone
     578            # rename the storage object
     579            if ($gone) {
     580                carp "WARNING: PSF storage object exists but all instances are permanently gone";
     581                rename_gone_file($outputPsf);
     582                $make_psf = 1;
     583            }
     584        } else {
     585            $make_psf = 1;
     586        }
     587    }
     588
     589    if ($sources_available && $psf_available) {
     590        return 0;
     591    }
     592
     593    # if either of the files are gone rerun photometry unless the other file is temporarily not available
     594
     595    if (!$sources_available && !$make_sources) {
     596        # destreak will die if the sources is not available
     597        &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     598    }
     599    if (!$psf_available && !$make_psf) {
     600        # warp updates need the psf file
     601        &my_die("PSPHOT.PSF.SAVE is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     602    }
     603
     604    return $make_psf || $make_sources;
     605}
     606
     607# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that
     608# does not exist and one that all of the instances have been lost.
     609# XXXX This should be implemented properly in Nebulous
     610# For now uses Bill's script 'whichnode' which queries the nebulous database directly
     611
     612my $whichnode;
     613sub storage_object_exists
     614{
     615    my $file = shift;
     616    my $ref_all_gone = shift;
     617
     618    my $exists = $neb->storage_object_exists($file);
     619    if (!$exists) {
     620        return 0;
     621    }
     622
     623    if (!$whichnode) {
     624        $whichnode = can_run('whichnode') or
     625            &my_die("Can't find whichnode",  $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     626    }
     627
     628    my $command = "$whichnode $file";
     629
     630    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     631        run(command => $command, verbose => $verbose);
     632    unless ($success) {
     633        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     634        &my_die("Unable to perform whichnode: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     635    }
     636
     637    my @lines = split "\n", (join "", @$stdout_buf);
     638
     639    if (scalar @lines == 0) {
     640        # no output the file is really and truely gone
     641        # XXX: this is now caught above
     642        print STDERR "storage object for $file does not exist\n";
     643        return 0;
     644    }
     645
     646    my $numGone = 0;
     647    my $numNotGone = 0;
     648    foreach my $line (@lines) {
     649        chomp $line;
     650
     651        # output lines are either
     652        #   "volume available"
     653        # or
     654        #   "volume not available"
     655
     656        my ($volume, $answer, undef) = split " ", $line;
     657        # our hack is if the volume has an X in the name it's gone
     658        if ($volume =~ /X/) {
     659            print STDERR "$file is on $volume which is gone\n";
     660            $numGone++;
     661        } elsif ($answer eq 'available') {
     662            $numNotGone++;
     663        } elsif ($answer eq 'not') {
     664            print STDERR "$file is on $volume which is not available\n";
     665            $numNotGone++;
     666        } else {
     667            print STDERR "unexpected output from whichnode: $line\n";
     668        }
     669    }
     670    # if there are any instances that are not on a gone volume set all_gone to 0
     671    if ($numNotGone == 0 and $numGone > 0) {
     672        $$ref_all_gone = 1;
     673    } else {
     674        $$ref_all_gone = 0;
     675    }
     676
     677    # storage object exists so return true
     678    return 1;
     679}
     680sub rename_gone_file
     681{
     682    # check whether the only instance of file is on a lost volume
     683    # XXX: we don't have a proper interface for this.
     684    # For now try to use Bill's hack: the script 'whichnode'
     685
     686    my $file = shift;
     687    $neb->move($file, "$file.gone") or
     688                    &my_die("failed to rename: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     689}
     690
     691# Prepare to write to an output file
     692#   Lookup the filename in the rules.
     693#   Make sure that if file exists and is a nebulous file that there is only one instance
     694#   Deal with files that have been lost.
     695sub prepare_output
     696{
     697    my $filerule = shift;
     698    my $outroot  = shift;
     699    my $class_id = shift;
     700    my $delete = shift;
     701    $delete = 0 if !defined $delete;
     702
     703    my $error;
     704    my $output = $ipprc->prepare_output($filerule, $outroot, $class_id, $delete, \$error)
     705                    or &my_die("failed to prepare output file for: $filerule", $exp_id, $chip_id, $class_id, $error);
     706    return $output;
     707}
     708
     709sub check_output
     710{
     711    my $file = shift;
     712    my $replicate = shift;
     713
     714    if (!defined $file) {
     715        return;
     716    }
     717
     718    &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
     719
     720    if ($replicate and (file_scheme($file) eq 'neb')) {
     721        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     722    }
     723}
    464724
    465725sub my_die

  • branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_cleanup.pl

    r29839 r29906  
    19841984sub file_gone
    19851985{
    1986     # if $check_for_gone check whether the only instance of file is on a lost volumen
     1986    # if $check_for_gone check whether the only instance of file is on a lost volume
    19871987    # XXX: we don't have a proper interface for this.
    19881988    # For now try to use Bill's hack the script 'whichnode'
    … …  
    19901990
    19911991    my $file = shift;
     1992
     1993    if (file_scheme($file) ne 'neb') {
     1994        return 0;
     1995    }
    19921996
    19931997    if (!$whichnode) {
    … …  
    20212025            print STDERR "$file is on $volume which is gone\n";
    20222026            $numGone++;
     2027        } elsif ($answer eq 'available') {
     2028            $numNotGone++;
    20232029        } elsif ($answer eq 'not') {
    20242030            print STDERR "$file is on $volume which is not available\n";

  • branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_inject_fileset.pl

    r24201 r29906  
    2424# Parse the command-line arguments
    2525my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help);
     26my $exp_name;
    2627GetOptions('camera|i=s'     => \$camera,    # user-supplied camera name
    2728           'telescope|t=s'  => \$telescope, # user-supplied telescope name
     29           'exp_name|=s'    => \$exp_name,  # user-supplied exp_name name
    2830           'workdir|w=s'    => \$workdir,   # working directory for output files
    2931           'reduction=s'    => \$reduction, # user-supplied camera name
    … …  
    7981}
    8082
    81 # use the first file name as the exp_name (strip off .fits)
     83# if not supplied, use the first file name as the exp_name (strip off .fits)
    8284my $num = 0;
    83 my $exp_name;
    8485foreach my $file ( @ARGV ) {
    8586    # check for file existence
    … …  
    8990        # strip off the extension
    9091        my ( $vol, $path, $name ) = File::Spec->splitpath( $file );
    91         ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/;
     92        ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts|flt)(|.gz)/;
    9293        print "exp_name : $exp_name.\n";
    9394    }

  • branches/eam_branches/ipp-20101103/ippTasks/destreak.pro

    r29561 r29906  
    305305task           destreak.revert.load
    306306  host         local
    307   periods      -poll 60.0
     307  periods      -poll 5
    308308  periods      -exec 120.
    309309  periods      -timeout 120

  • branches/eam_branches/ipp-20101103/ippTools/share/camtool_find_pendingimfile.sql

    r23617 r29906  
    22    camRun.cam_id,
    33    chipProcessedImfile.*,
     4    (IF(exp_time IS NOT NULL AND deteff_magref IS NOT NULL,
     5       (2.5 * LOG10(exp_time) + chipProcessedImfile.deteff_magref),  NULL))
     6        AS deteff_inst,
    47    rawExp.exp_name,
    58    rawExp.camera,

  • branches/eam_branches/ipp-20101103/ippTools/share/chiptool_setimfiletoupdate.sql

    r29561 r29906  
    1414    AND (chipRun.magicked = 0
    1515      OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update')
    16             AND (magicDSFile.data_state = 'cleaned`' OR magicDSFile.data_state = 'update'))
     16            AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state = 'update'))
    1717    )

  • branches/eam_branches/ipp-20101103/ippTools/share/pxadmin_create_tables.sql

    r29495 r29906  
    371371    maskfrac_magic FLOAT,
    372372    maskfrac_advisory FLOAT,
     373    deteff_magref FLOAT,
    373374    PRIMARY KEY(chip_id, exp_id, class_id),
    374375    KEY(data_state),
    … …  
    486487    n_astrom INT,
    487488    path_base VARCHAR(255),
    488     quality SMALLINT NOT NULL DEFAULT 0,
    489489    fault SMALLINT NOT NULL,
    490490    software_ver VARCHAR(16),
    … …  
    499499    maskfrac_max_magic FLOAT,
    500500    maskfrac_max_advisory FLOAT,
     501    deteff FLOAT,
     502    deteff_err FLOAT,
     503    deteff_lq FLOAT,
     504    deteff_uq FLOAT,
     505    quality SMALLINT NOT NULL DEFAULT 0,
    501506    PRIMARY KEY(cam_id),
    502507    KEY(fault),
    … …  
    10501055        hostname VARCHAR(64),
    10511056        good_frac FLOAT,
     1057        mjd_obs DOUBLE,
    10521058        quality SMALLINT NOT NULL DEFAULT 0,
    10531059        fault SMALLINT,

  • branches/eam_branches/ipp-20101103/ippTools/share/warptool_towarped_labels.sql

    r29835 r29906  
    1212        group by warpRun.label
    1313        ORDER BY priority DESC, warp_id
    14         limit 100
    15 

  • branches/eam_branches/ipp-20101103/ippTools/src

  • branches/eam_branches/ipp-20101103/ippTools/src/camtool.c

    r29553 r29906  
    519519    PXOPT_LOOKUP_F32(maskfrac_max_magic, config->args, "-maskfrac_max_magic", false, false);
    520520    PXOPT_LOOKUP_F32(maskfrac_max_advisory, config->args, "-maskfrac_max_advisory", false, false);
     521
     522    // we store actual detection efficiency by adding in zpt_obs
     523    PXOPT_LOOKUP_F32(deteff_inst, config->args, "-deteff_inst", false, false);
     524    PXOPT_LOOKUP_F32(deteff_inst_lq, config->args, "-deteff_inst_lq", false, false);
     525    PXOPT_LOOKUP_F32(deteff_inst_uq, config->args, "-deteff_inst_uq", false, false);
     526    // error is dd
     527    PXOPT_LOOKUP_F32(deteff_err, config->args, "-deteff_inst_err", false, false);
     528    psF32 deteff = NAN;
     529    psF32 deteff_uq = NAN;
     530    psF32 deteff_lq = NAN;
     531    if (isfinite(zpt_obs)) {
     532        if (isfinite(deteff_inst)) {
     533            deteff = deteff_inst + zpt_obs;
     534        }
     535        if (isfinite(deteff_inst_uq)) {
     536            deteff_uq = deteff_inst_uq + zpt_obs;
     537        }
     538        if (isfinite(deteff_inst_lq)) {
     539            deteff_lq = deteff_inst_lq + zpt_obs;
     540        }
     541    }
    521542
    522543/*     psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", */
    … …  
    666687        maskfrac_max_magic,
    667688        maskfrac_max_advisory,
     689        deteff,
     690        deteff_err,
     691        deteff_lq,
     692        deteff_uq,
    668693        quality
    669694        );

  • branches/eam_branches/ipp-20101103/ippTools/src/camtoolConfig.c

    r29553 r29906  
    198198    psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_max_advisory", 0, "define advisory mask fraction", NAN);
    199199
     200    psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst", 0, "define deteff", NAN);
     201    psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_err", 0, "define deteff_err", NAN);
     202    psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_lq", 0, "define deteff_lq", NAN);
     203    psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_uq", 0, "define deteff_uq", NAN);
     204
    200205
    201206    // -processedexp

  • branches/eam_branches/ipp-20101103/ippTools/src/chiptool.c

    r29416 r29906  
    601601    PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false);
    602602    PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false);
     603    PXOPT_LOOKUP_F32(deteff_magref, config->args, "-deteff_magref", false, false);
    603604
    604605    psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n",
    … …  
    706707                                   maskfrac_dynamic,
    707708                                   maskfrac_magic,
    708                                    maskfrac_advisory
     709                                   maskfrac_advisory,
     710                                   deteff_magref
    709711            )) {
    710712        // rollback
    … …  
    715717        return false;
    716718    }
    717 
    718 #if 0
    719     // XXX I've decided to make the transaction cover the Exp migration as
    720     // well.  Otherwise, if the last imfile in an exp is moved and the exp
    721     // migration fails then the data base is left in a situation where the exp
    722     // migration can't happen.
    723 
    724     if (!chipProcessedCompleteExp(config)) {
    725         // rollback
    726         if (!psDBRollback(config->dbh)) {
    727             psError(PS_ERR_UNKNOWN, false, "database error");
    728         }
    729         psError(PS_ERR_UNKNOWN, false, "database error");
    730         return false;
    731     }
    732 #endif
    733719
    734720    if (!psDBCommit(config->dbh)) {

  • branches/eam_branches/ipp-20101103/ippTools/src/chiptoolConfig.c

    r29416 r29906  
    196196    psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN);
    197197    psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN);
     198    psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-deteff_magref", 0, "define deteff_magref", NAN);
    198199
    199200    // -processedimfile

  • branches/eam_branches/ipp-20101103/ippTools/src/warptool.c

    r29835 r29906  
    994994        psString thisWhere = NULL;
    995995        if (whereStr) {
    996             psStringAppend(&thisWhere, whereStr);
     996            psStringAppend(&thisWhere, "\n%s", whereStr);
    997997        }
    998998        psStringAppend(&thisWhere, "\nAND warpRun.label = '%s'", label);

  • branches/eam_branches/ipp-20101103/ippconfig

  • branches/eam_branches/ipp-20101103/ippconfig/isp/camera.config

    r29662 r29906  
    7777PHOTCODE.RULE           STR     {DETECTOR}.{FILTER.ID}          # Rule for generating photcode
    7878# don't censor any masked pixels when making postage stamps
    79 MASK.NO.CENSOR               U32 0
     79MASK.NO.CENSOR               U32 4294967295

  • branches/eam_branches/ipp-20101103/ippconfig/megacam/camera.config

    r29394 r29906  
    155155
    156156# don't censor any masked pixels
    157 NO.CENSOR               U32 0xFFFFFFFF
     157MASK.NO.CENSOR          U32 4294967295

  • branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-mef.mdc

    r29849 r29906  
    319319LOG.IMFILE              OUTPUT {OUTPUT}.{CHIP.NAME}.log          TEXT      NONE       CHIP       TRUE      NONE
    320320LOG.EXP                 OUTPUT {OUTPUT}.log                      TEXT      NONE       FPA        TRUE      NONE
     321LOG.IMFILE.UPDATE       OUTPUT {OUTPUT}.{CHIP.NAME}.log.update   TEXT      NONE       CHIP       TRUE      NONE
     322LOG.EXP.UPDATE          OUTPUT {OUTPUT}.log.update               TEXT      NONE       FPA        TRUE      NONE
    321323                                                                                     
    322324TRACE.IMFILE            OUTPUT {OUTPUT}.{CHIP.NAME}.trace        TEXT      NONE       CHIP       TRUE      NONE
    323325TRACE.EXP               OUTPUT {OUTPUT}.trace                    TEXT      NONE       FPA        TRUE      NONE
     326TRACE.IMFILE.UPDATE     OUTPUT {OUTPUT}.{CHIP.NAME}.trace.update TEXT      NONE       CHIP       TRUE      NONE
     327TRACE.EXP.UPDATE        OUTPUT {OUTPUT}.trace.update             TEXT      NONE       FPA        TRUE      NONE
    324328
    325329PPNOISEMAP.OUTPUT       OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     NONE       CHIP       TRUE      NONE

  • branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-simple.mdc

    r29849 r29906  
    246246PPSUB.REF.CONV.VARIANCE      OUTPUT {OUTPUT}.refConv.wt.fits      VARIANCE        NONE       FPA        TRUE      NONE
    247247                                             
     248PPSUB.FORCED.SOURCES         OUTPUT {OUTPUT}.frc.cmf              CMF             NONE       FPA        TRUE      NONE
     249PPSUB.POS1.SOURCES           OUTPUT {OUTPUT}.pos1.cmf             CMF             NONE       FPA        TRUE      NONE
     250PPSUB.POS2.SOURCES           OUTPUT {OUTPUT}.pos2.cmf             CMF             NONE       FPA        TRUE      NONE
     251
    248252PPSTACK.OUTPUT.COMP          OUTPUT {OUTPUT}.fits                 IMAGE           COMP_IMG   FPA        TRUE      NONE
    249253PPSTACK.OUTPUT.RAW           OUTPUT {OUTPUT}.fits                 IMAGE           NONE       FPA        TRUE      NONE
    … …  
    290294LOG.IMFILE                   OUTPUT {OUTPUT}.imfile.log           TEXT            NONE       FPA        TRUE      NONE
    291295LOG.EXP                      OUTPUT {OUTPUT}.exp.log              TEXT            NONE       FPA        TRUE      NONE
     296LOG.IMFILE.UPDATE            OUTPUT {OUTPUT}.imfile.log.update    TEXT            NONE       FPA        TRUE      NONE
     297LOG.EXP.UPDATE               OUTPUT {OUTPUT}.exp.log.update       TEXT            NONE       FPA        TRUE      NONE
    292298
    293299TRACE.IMFILE                 OUTPUT {OUTPUT}.imfile.trace         TEXT            NONE       FPA        TRUE      NONE
    294300TRACE.EXP                    OUTPUT {OUTPUT}.exp.trace            TEXT            NONE       FPA        TRUE      NONE
     301TRACE.IMFILE.UPDATE          OUTPUT {OUTPUT}.imfile.trace.update  TEXT            NONE       FPA        TRUE      NONE
     302TRACE.EXP.UPDATE             OUTPUT {OUTPUT}.exp.trace.update     TEXT            NONE       FPA        TRUE      NONE
    295303
    296304PPNOISEMAP.OUTPUT            OUTPUT {OUTPUT}.{CHIP.NAME}.fits     IMAGE           NONE       CHIP       TRUE      NONE

  • branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-split.mdc

    r29849 r29906  
    323323LOG.IMFILE                   OUTPUT {OUTPUT}.{CHIP.NAME}.log          TEXT            NONE       CHIP       TRUE      NONE
    324324LOG.EXP                      OUTPUT {OUTPUT}.log                      TEXT            NONE       FPA        TRUE      NONE
     325LOG.IMFILE.UPDATE            OUTPUT {OUTPUT}.{CHIP.NAME}.log.update   TEXT            NONE       CHIP       TRUE      NONE
     326LOG.EXP.UPDATE               OUTPUT {OUTPUT}.log.update               TEXT            NONE       FPA        TRUE      NONE
    325327                                                                                                             
    326328TRACE.IMFILE                 OUTPUT {OUTPUT}.{CHIP.NAME}.trace        TEXT            NONE       CHIP       TRUE      NONE
    327329TRACE.EXP                    OUTPUT {OUTPUT}.trace                    TEXT            NONE       FPA        TRUE      NONE
     330TRACE.IMFILE.UPDATE          OUTPUT {OUTPUT}.{CHIP.NAME}.trace.update TEXT            NONE       CHIP       TRUE      NONE
     331TRACE.EXP.UPDATE             OUTPUT {OUTPUT}.trace.update             TEXT            NONE       FPA        TRUE      NONE
    328332
    329333PPNOISEMAP.OUTPUT            OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE           NONE       CHIP       TRUE      NONE

  • branches/eam_branches/ipp-20101103/ippconfig/recipes/ppStats.config

    r28089 r29906  
    8484  HEADER        STR     IQ_M4_LQ
    8585  HEADER        STR     IQ_M4_UQ
     86  HEADER        STR     DETEFF.MAGREF
     87
    8688
    8789  HEADER        STR     PSLIB_V

  • branches/eam_branches/ipp-20101103/ippconfig/recipes/ppStatsFromMetadata.config

    r29283 r29906  
    187187  ENTRY VAL MASKFRAC_MAGIC        F32  CONSTANT         -maskfrac_magic
    188188  ENTRY VAL MASKFRAC_ADVISORY     F32  CONSTANT         -maskfrac_advisory
     189
     190  # Detection Efficiency
     191  ENTRY  VAL  DETEFF.MAGREF       F32  CONSTANT         -deteff_magref           
    189192END
    190193
    … …  
    255258  ENTRY  VAL  IMAGE_V             STR  CONSTANT         -ver_ppimage
    256259  ENTRY  VAL  STREAK_V            STR  CONSTANT         -ver_streaks
     260
     261  # Detection Efficiency
     262  ENTRY  VAL  deteff_inst         F64  ROBUST_MEDIAN    -deteff_inst           
     263  ENTRY  VAL  deteff_inst         F64  ROBUST_STDEV     -deteff_inst_err           
     264  ENTRY  VAL  deteff_inst         F64  UQ               -deteff_inst_uq           
     265  ENTRY  VAL  deteff_inst         F64  LQ               -deteff_inst_lq           
    257266END
    258267

  • branches/eam_branches/ipp-20101103/ippconfig/simtest/camera.config

    r29661 r29906  
    8080
    8181# don't censor any masked pixels when making postage stamps
    82 # MASK.NO.CENSOR               U32 0xFFFFFFFF
    83 MASK.NO.CENSOR               U32 0
     82# MASK.NO.CENSOR               U32 0xFFFFFFFF -- need to fix the MDC parser(s) to handle hex values
     83MASK.NO.CENSOR               U32 4294967295

  • branches/eam_branches/ipp-20101103/ppImage/src/ppImageDetrendRecord.c

    r24485 r29906  
    7575    detrendRecord(options->doFringe,   detrend, config, view, "PPIMAGE.FRINGE",   "DETREND.FRINGE",   "Fringe filename");
    7676
     77    detrendRecord(options->doNonLin,   detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN",   "Non-linearity table filename");
    7778    psFree (detrend);
    7879    return true;

  • branches/eam_branches/ipp-20101103/pstamp/scripts/pstamp_job_run.pl

    r29380 r29906  
    119119    $command .= " -dbserver $dbserver" if $dbserver;
    120120    $command .= " -stage $params->{stage}" if $params->{stage};
     121    $command .= " -no_censor_masked" unless $nan_masked;
    121122    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    122123        run(command => $command, verbose => $verbose);

  • branches/eam_branches/ipp-20101103/pstamp/src/ppstampArguments.c

    r29379 r29906  
    2525    fprintf(stderr, "   [-mask   mk_image] :     mask image\n");
    2626    fprintf(stderr, "   [-variance var_image] :  variance image\n");
     27    fprintf(stderr, "   [-no_censor_masked] : do not set masked pixels to NAN\n");
    2728    fprintf(stderr, "\n");
    2829
    … …  
    9596    }
    9697   
     98    if ((argnum = psArgumentGet(argc, argv, "-no_censor_masked"))) {
     99        psArgumentRemove(argnum, &argc, argv);
     100        options->censorMasked = false;
     101    }
    97102
    98103    // finally the output file

  • branches/eam_branches/ipp-20101103/pstamp/src/ppstampMakeStamp.c

    r29388 r29906  
    327327            }
    328328
    329             if (!setMaskedToNAN(config, outReadout->image, outReadout->mask, outReadout->variance)) {
     329            if (options->censorMasked && !setMaskedToNAN(config, outReadout->image, outReadout->mask, outReadout->variance)) {
    330330                 psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp mask image\n");
    331331                 status = false;

  • branches/eam_branches/ipp-20101103/pstamp/src/ppstampOptions.c

    r16132 r29906  
    2828    options->roip.dDEC = 0;
    2929    options->chipName  = NULL;
     30    options->censorMasked = true;
    3031
    3132    return options;

  • branches/eam_branches/ipp-20101103/pstamp/src/ppstampOptions.h

    r29379 r29906  
    1515    //
    1616    psRegion    roi;            // roi in chip coordinates
     17    bool        censorMasked;
    1718
    1819} ppstampOptions;

  • branches/eam_branches/ipp-20101103/tools/czarmetrics.pl

    r29834 r29906  
    99
    1010use czartool::DayMetrics;
     11use czartool::MultiDayMetrics;
     12use czartool::MetricsIndex;
    1113use czartool::CzarDb;
    1214use czartool::Gpc1Db;
    … …  
    1618my $begin = undef;
    1719my $end = undef;
     20my $cumulative = undef;
     21my $index = undef;
    1822my $verbose = undef;
    1923my $save_temps = undef;
    … …  
    2327        "begin|b=s" => \$begin,
    2428        "end|e=s" => \$end,
    25         "day|y=s" => \$day,
     29        "day|d=s" => \$day,
     30        "index|i" => \$index,
     31        "cumulative|c" => \$cumulative,
    2632        "verbose|v" => \$verbose,
    2733        );
    … …  
    3541}
    3642if (!$day) {
    37     print "* OPTIONAL: choose a single day             -y <date>               (default=today)\n";
     43    print "* OPTIONAL: choose a single day                -d <date>               (default=today)\n";
    3844}
    39 if (!$begin) {
    40     print "* OPTIONAL: choose a begin date             -b <date>               (default=today)\n";}
    41 if (!$end)
    42 {
    43     print "* OPTIONAL: choose an end date              -e <date>               (default=today)\n";
     45if (!$begin) {
     46    print "* OPTIONAL: choose a begin date                -b <date>               (default=today)\n";
     47}
     48if (!$end) {
     49    print "* OPTIONAL: choose an end date                 -e <date>               (default=today)\n";
     50}
     51if (!$cumulative) {
     52    print "* OPTIONAL: cumulative metrics for date range  -c                      (default=off)\n";
     53}
     54if (!$index) {
     55    print "* OPTIONAL: create index file for all metrics  -i                      (default=off)\n";
    4456}
    4557
    … …  
    5264$czarDb->setDateFormat("%Y%m%d-%H%i%s");
    5365
     66if ($index) {
     67
     68    my $metricsIndex = new czartool::MetricsIndex($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0);
     69    $metricsIndex->writeHTML();
     70    exit;
     71}
    5472
    5573if (!$day && !$begin && !$end) {
    … …  
    6179
    6280my $thisDay = $begin;
    63 while (1) {
    6481
    65     my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $thisDay);
    66     $dayMetrics->writeHTML();
     82if ($cumulative) {
    6783
    68     $thisDay = $czarDb->addInterval($thisDay, "1 DAY");
    69     if ($czarDb->isBefore($end, $thisDay)) {last;}
     84        my $multiDayMetrics = new czartool::MultiDayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $begin, $end);
     85        $multiDayMetrics->writeHTML();
    7086}
     87else {
     88
     89    while (1) {
     90
     91        my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $thisDay);
     92        $dayMetrics->writeHTML();
     93
     94        $thisDay = $czarDb->addInterval($thisDay, "1 DAY");
     95        if ($czarDb->isBefore($end, $thisDay)) {last;}
     96    }
     97}
     98

  • branches/eam_branches/ipp-20101103/tools/czarplot.pl

    r29834 r29906  
    4545        "begin|b=s" => \$begin,
    4646        "end|e=s" => \$end,
    47         "day|y=s" => \$day,
     47        "day|d=s" => \$day,
    4848        "output|o=s" => \$path,
    4949        "histogram|h" => \$histogram,
    … …  
    5151        "cleanup|c" => \$showCleanup,
    5252        "rate|r" => \$rate,
    53         "deriv|d" => \$deriv,
     53        "deriv|f" => \$deriv,
    5454        "analysis|a" => \$analysis,
    5555        "timeseries|t" => \$timeSeries,
    … …  
    8080if (!$deriv) {
    8181    $deriv = 0;
    82     print "* OPTIONAL: plot first derivative           -d                          (default=$deriv)\n";}
     82    print "* OPTIONAL: plot first derivative           -f                          (default=$deriv)\n";}
    8383if (!$analysis) {
    8484    $analysis = 0;
    … …  
    106106    print "* OPTIONAL: choose an end time              -e <datetime>               (default=now)\n";}
    107107if (!$day) {
    108     print "* OPTIONAL: choose a single day to plot     -y <date>                   (default=today)\n";}
     108    print "* OPTIONAL: choose a single day to plot     -d <date>                   (default=today)\n";}
    109109if (!$path) {
    110110    print "* OPTIONAL: choose an output location       -o <path>                   (default=none)\n";
    … …  
    137137if($day) {
    138138
    139     # day plots should run from about 6:30am until midnight
    140     $begin =  "$day 06:35";
    141     $end = "$day 23:59";
    142         print "JKJKJK '$begin' '$end' \n";
     139    if ($magicMask) {
    143140
     141        $begin =  $day;
     142        $end = $day;
     143    }
     144    else {
     145
     146        # day plots should run from about 6:30am until midnight
     147        $begin = "$day 06:35";
     148        $end = "$day 23:59";
     149    }
    144150}
    145151else {
    … …  
    152158    }
    153159}
    154 my $diskUsage = 1; # TODO
     160if ($rate) {
    155161
    156 if ($rate) {
    157     $plotter->createRateTimeSeries($label, $stage, $begin, $end, $rateInterval, $log);
     162    $plotter->createRateTimeSeries($label, $stage, $begin, $end, $rateInterval);
    158163    exit;
    159164}
    … …  
    165170   
    166171    if ($exposureId) {$plotter->plotMagicMaskFractionForThisExposure($exposureId);}
    167     else {$plotter->plotMagicMaskFractionHistogram($begin, $end);}
     172    else {$plotter->plotMagicMaskFraction($begin, $end);}
    168173
    169174}

  • branches/eam_branches/ipp-20101103/tools/czarpoll.pl

    r29834 r29906  
    133133       
    134134                print "* Creating metrics for last 24 hours\n";
    135                 # TODO hardcopded path needs to be in config
    136                 my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $today);
     135                my $yesterday = $czarDb->subtractInterval($today, "1 DAY");
     136                # TODO hardcoded path needs to be in config
     137                my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $yesterday);
    137138                $dayMetrics->writeHTML();
    138 
    139139                $doneMetricsToday = 1;
    140140        }

  • branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm

    r29834 r29906  
    719719SQL
    720720
    721     $query->execute;
     721    if (!$query->execute) {return 0;}
     722
    722723    (${$pending}, ${$faults}) = $query->fetchrow_array();   
    723724
    724725    ${$processed} = $self->countProcessed($label, $stage, $fromTime, $toTime);
     726
     727    if (!${$pending}) {${$pending} = 0;}
     728    if (!${$faults}) {${$faults} = 0;}
     729    if (!${$processed}) {${$processed} = 0;}
    725730
    726731    return 1;
    … …  
    822827SQL
    823828
    824 
    825829    if (!$query->execute) {return undef;}
    826830
    827831    (${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
    828832
     833
     834    if (!${$maxY} || !${$minY} || !${$maxX} || !${$minX} || !${$timeDiff}) {return 0;}
    829835
    830836    $query = $self->{_db}->prepare(<<SQL);
    … …  
    846852
    847853    close(GNUDAT);
    848 }
    849 
    850 ###########################################################################
    851 #
    852 # TODO implement isLog
     854
     855    return 1;
     856}
     857
     858###########################################################################
     859#
     860# Creates data for processing rate plots
    853861#
    854862###########################################################################
    855863sub createProcessingRateData {
    856     my ($self, $stage, $label, $begin, $end, $interval, $dataFile, $isLog) = @_;
     864    my ($self, $stage, $label, $begin, $end, $interval, $dataFile) = @_;
    857865
    858866    my $startTime = $begin;
    … …  
    865873
    866874    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_r.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
    867     $tmpFile->unlink_on_destroy( 0 );
     875    $tmpFile->unlink_on_destroy(0);
    868876    ${$dataFile} = $tmpFile->filename;
    869877
    … …  
    873881    while(1) {
    874882
    875 
    876         #print "NNN $startTime, $end\n";
    877883        if (!$self->isBefore($startTime, $end)) {last;}
    878884        $endTime = $self->addInterval($startTime, $interval);
    879         $self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults);
     885        if (!$self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults)) {}
    880886        $timestamp = $self->getFormattedDate($endTime);
     887        if (!$processed) {$processed = 0;}
    881888        print GNUDAT "$timestamp $processed 0 0\n";
    882889
    … …  
    887894
    888895    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n";
    889 
    890     #return $someData;
    891896}
    892897

  • branches/eam_branches/ipp-20101103/tools/czartool/DayMetrics.pm

    r29834 r29906  
    4343
    4444    # create path, dir and html file
    45     $self->{dayDir} = "$self->{baseDir}/$self->{day}";
    46     rmdir($self->{dayDir});
    47     mkdir($self->{dayDir}, 0777);
    48     $self->{plotter}->setOutputPath($self->{dayDir});
     45    $self->{path} = "$self->{baseDir}/$self->{day}";
     46    rmdir($self->{path});
     47    mkdir($self->{path}, 0777);
     48    $self->{plotter}->setOutputPath($self->{path});
    4949
    5050    if ($self->{verbose}) {print "* Creating metrics for $self->{day}\n";}
    … …  
    6464
    6565    # create HTML file and write header stuff
    66     open (HTMLDOC, ">$self->{dayDir}/index.html");
    67     print HTMLDOC "<html>\n";
    68     print HTMLDOC "<head>\n";
    69     print HTMLDOC "<title>IPP Metrics for $self->{day}</title>\n";
    70     print HTMLDOC "<a name=\"top\"></a>\n";
    71     print HTMLDOC "<h1 align=\"middle\">IPP Metrics for $self->{day}</h1>\n";
     66
     67
     68    $self->createHtml("IPP Metrics for $self->{day}");
     69    my $htmlFile = $self->{htmlFile};
    7270
    7371    # summit and burntool exposures
    … …  
    7977    my $nextDay = $self->{czarDb}->addInterval($self->{day}, "1 DAY");
    8078
    81     print HTMLDOC "<h5  align=\"middle\">";
    82     print HTMLDOC "<a href=\"../$previousDay/index.html\"> \< previous day</a> | <a href=\"../$nextDay/index.html\">next day \></a><br>\n";
    83     print HTMLDOC "Measured from $self->{begin} to $self->{end} HST<br>\n";
    84     printf ( HTMLDOC "%d exposures taken on summit last night, %d through burntool today</h5>\n",
     79    print $htmlFile "<h5  align=\"middle\">";
     80    print $htmlFile "<a href=\"../$previousDay/index.html\"> \< previous day</a> | <a href=\"../index.html\">all</a> | <a href=\"../$nextDay/index.html\">next day \></a><br>\n";
     81    print $htmlFile "Measured from $self->{begin} to $self->{end} HST<br>\n";
     82    printf ( $htmlFile "%d exposures taken on summit last night, %d through burntool today</h5>\n",
    8583            $summitExposures, $burntoolMetrics->getProcessed() ? $burntoolMetrics->getProcessed() : 0 );
    86     print HTMLDOC "</head>\n";
    87     print HTMLDOC "<body>\n";
     84    print $htmlFile "</head>\n";
     85    print $htmlFile "<body>\n";
    8886
    8987
    … …  
    176174    }
    177175
    178     # create hyperlink list of list of active labels
    179     print HTMLDOC "<h2>Active labels for this day</h1>\n";
     176    # create contents
     177    print $htmlFile "<h2>Contents</h2>\n";
     178    print $htmlFile "<a href=\"#surveys\">Survey timings</a><br>\n";
     179    print $htmlFile "<a href=\"#summaryplot\">Summary plots</a><br>\n";
     180    print $htmlFile "<a href=\"#stages\">Stage totals</a><br>\n";
     181    print $htmlFile "<a href=\"#mask\">Magic mask fraction</a><br>\n";
    180182    foreach my $label (keys (%labelTables)) {
    181183
    182         print HTMLDOC "<a href=\"#$label\">$label</a><br>\n";
    183     }
    184 
    185     # some plots
    186     print HTMLDOC "<h1 align=\"middle\">Summary plots</h1>\n";
    187     $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 0, 0, 0);
    188     print HTMLDOC "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_t.png\" alt=\"All labels and all stages for $self->{day}\" />\n";
    189     $self->{plotter}->createHistogram("all_stdscience_labels", $self->{begin}, $self->{end}, 0, 0, 0);
    190     print HTMLDOC "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_h.png\" alt=\"All stages for all_stdscience_labels for $self->{day}\" />\n";
    191     $self->{plotter}->createRateTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, "1 HOUR", 0);
    192     print HTMLDOC "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_r.png\" alt=\"All stages for all_stdscience_labels for $self->{day}\" />\n";
    193     $self->{plotter}->plotMagicMaskFractionHistogram($self->{day}, $self->{day});
    194     print HTMLDOC "<img src=\"czarplot_magic_mask_fraction_h.png\" alt=\"\" />\n";
    195     $self->{plotter}->plotStorageTimeSeries($self->{begin}, $self->{end});
    196     print HTMLDOC "<img src=\"czarplot_cluster.png\" alt=\"All stages for all_stdscience_labels for $self->{day}\" />\n";
    197     print HTMLDOC "<img src=\"czarplot_magic_mask_fraction_d.png\" alt=\"\" />\n";
    198 
     184        print $htmlFile "<a href=\"#$label\">$label</a><br>\n";
     185    }
    199186
    200187    # survey table
    201188    $self->printSurveyTable();
    202189
     190    # some summary plots
     191    $self->createSummaryPlots();
     192
     193    # magic mask plots
     194    $self->createMaskPlots($self->{day}, $self->{day});
     195
    203196    # table for stage totals
    204     $table =  "<h2  align=\"middle\">Totals for all labels</h1>\n";
     197    $table = "<a name=\"stages\"></a>\n";
     198    $table .= "<h2  align=\"middle\">Stage totals <a href=\"#top\">(top)</a></h1>\n";
    205199    $table .= "<table border='1'>";
    206200    $table .= "  <tr>\n";
    … …  
    224218    $table .= "<br>\n";
    225219
    226     print HTMLDOC $table;
     220    print $htmlFile $table;
    227221
    228222    # print all label tables to page
    229223    foreach my $label (keys (%labelTables)) {
    230224
    231         print HTMLDOC "<a name=\"$label\"></a>\n";
    232         print HTMLDOC "<h2  align=\"middle\">$label <a href=\"#top\">(top)</a></h1>\n";
     225        print $htmlFile "<a name=\"$label\"></a>\n";
     226        print $htmlFile "<h2  align=\"middle\">$label <a href=\"#top\">(top)</a></h1>\n";
    233227        $self->{plotter}->createTimeSeries($label, undef, $self->{begin}, $self->{end}, 0, 0, 0);
    234         print HTMLDOC "<img src=\"czarplot_linear_".$label."_all_stages_t.png\" alt=\"All stages for $label for $self->{day}\" />\n";
     228        print $htmlFile "<img src=\"czarplot_linear_".$label."_all_stages_t.png\" alt=\"All stages for $label for $self->{day}\" />\n";
    235229        $self->{plotter}->createHistogram($label, $self->{begin}, $self->{end}, 0, 0, 0);
    236         print HTMLDOC "<img src=\"czarplot_linear_".$label."_all_stages_h.png\" alt=\"All stages for $label for $self->{day}\" />\n";
    237 
    238         print HTMLDOC $labelTables{$label};
    239     }
    240 
    241 
    242     print HTMLDOC "<br>\n";
    243     print HTMLDOC "</body>\n";
    244     print HTMLDOC "</html>\n";
    245     close(HTMLDOC);
     230        print $htmlFile "<img src=\"czarplot_linear_".$label."_all_stages_h.png\" alt=\"All stages for $label for $self->{day}\" />\n";
     231
     232        print $htmlFile $labelTables{$label};
     233    }
     234
     235
     236    $self->finishHtml();
    246237}
    247238
    … …  
    254245    my ($self) = @_;
    255246
    256     print HTMLDOC "<h2  align=\"middle\">Survey statistics</h1>\n";
    257     print HTMLDOC "<table border='1'>";
    258     print HTMLDOC "  <tr>\n";
    259     print HTMLDOC "    <th>Survey</th>\n";
    260     print HTMLDOC "    <th>Started burntool</th>\n";
    261     print HTMLDOC "    <th>Finished distribution</th>\n";
    262     print HTMLDOC "    <th>Time taken</th>\n";
    263     print HTMLDOC "  </tr>\n";
     247    my $htmlFile = $self->{htmlFile};
     248
     249    print $htmlFile "<a name=\"surveys\"></a>\n";
     250    print $htmlFile "<h2  align=\"middle\">Survey timings <a href=\"#top\">(top)</a></h1>\n";
     251    print $htmlFile "<table border='1'>";
     252    print $htmlFile "  <tr>\n";
     253    print $htmlFile "    <th>Survey</th>\n";
     254    print $htmlFile "    <th>Started burntool</th>\n";
     255    print $htmlFile "    <th>Finished distribution</th>\n";
     256    print $htmlFile "    <th>Time taken</th>\n";
     257    print $htmlFile "  </tr>\n";
    264258
    265259    # OSS survey
    … …  
    307301    $self->printSurveyDetails("All", $started, $finished, $timeTaken);
    308302
    309     print HTMLDOC "</table>\n";
     303    print $htmlFile "</table>\n";
    310304}
    311305
    … …  
    318312    my ($self, $survey, $started, $finished, $timeTaken, $processed, $pending) = @_;
    319313
    320     print HTMLDOC "  <tr>\n";
    321     print HTMLDOC "    <td>$survey</td>\n";
    322     printf (HTMLDOC "    <td>%s</td>\n", $started ? $started : "no");
    323     printf (HTMLDOC "    <td>%s</td>\n", $finished ? $finished : "no");
    324     printf (HTMLDOC "    <td>%s</td>\n", $timeTaken ? $timeTaken : "na");
    325     print HTMLDOC "  </tr>\n";
     314    my $htmlFile = $self->{htmlFile};
     315
     316    print $htmlFile "  <tr>\n";
     317    print $htmlFile "    <td>$survey</td>\n";
     318    printf ($htmlFile "    <td>%s</td>\n", $started ? $started : "no");
     319    printf ($htmlFile "    <td>%s</td>\n", $finished ? $finished : "no");
     320    printf ($htmlFile "    <td>%s</td>\n", $timeTaken ? $timeTaken : "na");
     321    print $htmlFile "  </tr>\n";
    326322}
    3273231;

  • branches/eam_branches/ipp-20101103/tools/czartool/Gpc1Db.pm

    r29834 r29906  
    246246###########################################################################
    247247#
    248 # Returns average magic mask fraction across all chips for a particular exposure
    249 #
    250 ###########################################################################
    251 sub getAverageMagicMaskFraction {
    252         my ($self, $exp_id) = @_;
     248# Returns average magic mask fraction, sum of mask fractions and chip count
     249# for a particular exposure
     250#
     251###########################################################################
     252sub getMagicMaskStats {
     253        my ($self, $exp_id, $mean, $sum, $chipCount) = @_;
    253254
    254255            my $query = $self->{_db}->prepare(<<SQL);
    255             SELECT AVG(streak_frac)
     256            SELECT AVG(streak_frac), SUM(streak_frac), COUNT(*)
    256257                FROM magicDSFile 
    257258                JOIN magicDSRun USING(magic_ds_id)
    … …  
    262263                AND component LIKE "XY%";
    263264SQL
    264     $query->execute;
    265     return scalar $query->fetchrow_array();
     265    if (!$query->execute) {return 0;} # TODO do this everywhere
     266
     267    (${$mean}, ${$sum}, ${$chipCount}) = $query->fetchrow_array();
     268
     269    return 1;
    266270}
    2672711;

  • branches/eam_branches/ipp-20101103/tools/czartool/Metrics.pm

    r29834 r29906  
    4747}
    4848
     49###########################################################################
     50#
     51# Writes HTML intro
     52#
     53###########################################################################
     54sub createHtml {
     55    my ($self, $title) = @_;
     56
     57    open (FILE, ">$self->{path}/index.html");
     58    $self->{htmlFile} = *FILE;
     59
     60    my $htmlFile = $self->{htmlFile};
     61    print $htmlFile "<html>\n";
     62    print $htmlFile "<head>\n";
     63    print $htmlFile "<title>$title</title>\n";
     64    print $htmlFile "<a name=\"top\"></a>\n";
     65    print $htmlFile "<h1 align=\"middle\">$title</h1>\n";
     66}
     67
     68###########################################################################
     69#
     70# Creates summary plots for period
     71#
     72###########################################################################
     73sub createSummaryPlots {
     74    my ($self) = @_;
     75
     76    my $htmlFile = $self->{htmlFile};
     77
     78    print $htmlFile "<a name=\"summaryplot\"></a>\n";
     79    print $htmlFile "<h2  align=\"middle\">Summary plots <a href=\"#top\">(top)</a></h1>\n";
     80    $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 0, 0, 0);
     81    print $htmlFile "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_t.png\" alt=\"timeseries\" />\n";
     82    $self->{plotter}->createHistogram("all_stdscience_labels", $self->{begin}, $self->{end}, 0, 0, 0);
     83    print $htmlFile "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_h.png\" alt=\"histogram\" />\n";
     84    $self->{plotter}->createRateTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, undef, 0);
     85    print $htmlFile "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_r.png\" alt=\"rate\" />\n";
     86    $self->{plotter}->plotStorageTimeSeries($self->{begin}, $self->{end});
     87    print $htmlFile "<img src=\"czarplot_cluster.png\" alt=\"storage plot not available\" />\n";
     88}
     89
     90###########################################################################
     91#
     92# Creates magic mask plots for provided period
     93#
     94###########################################################################
     95sub createMaskPlots {
     96    my ($self, $begin, $end) = @_;
     97
     98    my $htmlFile = $self->{htmlFile};
     99
     100    print $htmlFile "<a name=\"mask\"></a>\n";
     101    print $htmlFile "<h2  align=\"middle\">Magic mask fraction <a href=\"#top\">(top)</a></h1>\n";
     102
     103    $self->{plotter}->plotMagicMaskFraction($begin, $end);
     104    print $htmlFile "<img src=\"czarplot_magic_mask_fraction_h.png\" alt=\"\" />\n";
     105    print $htmlFile "<img src=\"czarplot_magic_mask_fraction_d.png\" alt=\"\" />\n";
     106}
     107
     108###########################################################################
     109#
     110# Writes HTML intro
     111#
     112###########################################################################
     113sub finishHtml {
     114    my ($self) = @_;
     115
     116    my $htmlFile = $self->{htmlFile};
     117
     118    print $htmlFile "<br>\n";
     119    print $htmlFile "</body>\n";
     120    print $htmlFile "</html>\n";
     121    close($htmlFile);
     122}
    491231;

  • branches/eam_branches/ipp-20101103/tools/czartool/Plotter.pm

    r29834 r29906  
    11#!/usr/bin/perl -w
     2
    23package czartool::Plotter;
    34
    … …  
    5657}
    5758
    58 
    5959###########################################################################
    6060#
    … …  
    6767
    6868    my $timeDiff = $self->{_czarDb}->diffTimes($end, $begin);
    69 
    7069
    7170    if ($self->{_czarDb}->isBefore($timeDiff, "00:00:01")) {return 0;}
    7271    elsif ($self->{_czarDb}->isBefore($timeDiff, "03:00:00")) {${$interval} = "15 MINUTE";}
    7372    elsif ($self->{_czarDb}->isBefore($timeDiff, "10:00:00")) {${$interval} = "30 MINUTE";}
    74     elsif ($self->{_czarDb}->isBefore($timeDiff, "24:00:00")) {${$interval} = "1 HOUR";} # under 1 day
     73    elsif ($self->{_czarDb}->isBefore($timeDiff, "26:00:00")) {${$interval} = "1 HOUR";} # under 1 day
    7574    elsif ($self->{_czarDb}->isBefore($timeDiff, "36:00:00")) {${$interval} = "2 HOUR";} # under 1.5 days
    7675    elsif ($self->{_czarDb}->isBefore($timeDiff, "48:00:00")) {${$interval} = "3 HOUR";} # under 2 days
    … …  
    8382    else {${$interval} = "1 MONTH";}
    8483
     84# ${$interval} = "2 HOUR";
     85
    8586    return 1;
    8687}
    … …  
    9293###########################################################################
    9394sub createRateTimeSeries {
    94     my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $isLog) = @_;
     95    my ($self, $label, $selectedStage, $beginTime, $endTime, $interval) = @_;
    9596
    9697    # stages
    … …  
    109110    my $stage = undef;
    110111    my $gnuplotFile = undef;
    111     my $outputFile = createImageFileName($self, $label, $selectedStage, "r", $isLog);
     112    my $outputFile = createImageFileName($self, $label, $selectedStage, "r", 0);
    112113    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
    113114    use FileHandle;
    … …  
    118119    print GP
    119120        "set term $self->{_outputFormat};" .
    120         "set title \"'$label', '$selectedStage' during '$beginTime' to '$endTime'\";" .
     121        "set title \"'$label', '$selectedStage'\\nFrom '$beginTime' to '$endTime' HST\";" .
    121122        "set boxwidth;" .
    122123        "set xtic rotate by -90 scale 0;" .
    … …  
    137138                    $endTime,
    138139                    $interval,
    139                     \$gnuplotFile,
    140                     $isLog)) {
     140                    \$gnuplotFile)) {
    141141
    142142            $gnuplotFiles{$stage} = $gnuplotFile;
    … …  
    251251                \$faults)) {next;}
    252252
    253         $pendingMinusFaults = $pending - $faults;
     253        $pendingMinusFaults = $pending - $faults; 
    254254        print GNUDAT "$stage $faults $processed $pendingMinusFaults\n";
    255255    }
    … …  
    264264    print GP
    265265        "set term $self->{_outputFormat};" .
    266         "set title \"'$label', '$beginTime' to '$endTime'\";" .
     266        "set title \"'$label'\\nFrom '$beginTime' to '$endTime' HST\";" .
    267267        "set grid;" .
    268268        "set boxwidth;" .
    … …  
    379379    my $yTitle = undef;
    380380    if ($isLog) {$yTitle = "Log( Exposures )";}
    381     elsif ($isDeriv) {$yTitle = "dExposures/dTime";}
     381    elsif ($isDeriv) {$yTitle = "dExposures/dTime(secs)";}
    382382    else {$yTitle = "Exposures";}
    383383
    … …  
    392392    else {$title .= "'all stages'"}
    393393
    394     $title .= " for '$label', '$fromTime' to '$toTime'";
     394    $title .= " for '$label'\\nFrom '$fromTime' to '$toTime' HST";
    395395
    396396    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
    … …  
    454454
    455455    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_storage_timeseries.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
    456     $self->{_czarDb}->createStorageTimeSeriesData($tmpFile, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
     456    if (!$self->{_czarDb}->createStorageTimeSeriesData($tmpFile, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff)) {return 0;}
    457457
    458458    my $timeFormat = undef;
    … …  
    468468    print GP
    469469        "set term $self->{_outputFormat};" .
    470         "set title \"Total available cluster space over time\";" .
     470        "set title \"Total available cluster space over time\\nFrom $fromTime to $toTime HST\";" .
    471471        "set key left top;" .
    472472        "set xdata time;" .
    … …  
    482482    print GP "\n";
    483483    close GP;
     484
     485    return 1;
    484486}
    485487
    … …  
    530532###########################################################################
    531533#
    532 # Generates 3D plot of magic mask fraction for provided exposure ID
     534# Generates 2D plot of magic mask fraction for provided exposure ID
    533535#
    534536###########################################################################
    … …  
    546548    open (GNUDAT, ">".$tmpFile->filename);
    547549
     550    # missing corner chip
    548551    print GNUDAT "0 0 0.0\n";
    549552    foreach my $row ( @{$fracs} ) {
    … …  
    551554        if (@{$row}[0] =~ m/XY([0-9])([0-9])/) {
    552555            print GNUDAT "$1 $2 @{$row}[1]\n";
     556            # missing corner chips
    553557            if($1 == 0 && $2 == 6) {print GNUDAT "0 7 0.0\n";}
    554558            if($1 == 6 && $2 == 7) {print GNUDAT "7 0 0.0\n";}
    555559        }
    556560    }
     561    # missing corner chip
    557562    print GNUDAT "7 7 0.0\n";
    558563
    … …  
    596601#
    597602###########################################################################
    598 sub plotMagicMaskFractionHistogram {
     603sub plotMagicMaskFraction {
    599604    my ($self, $begin, $end) = @_;
    600605
    … …  
    623628    }
    624629
     630    my $meanMask = undef;
     631    my $sumMask = undef;
     632    my $chipCount = undef;
     633    my $expCount = 0;
     634    my $totalChipCount = 0;
     635    my $totalMask = 0;
     636    my $exp_id = undef;
     637
    625638    # get mask for each exposure, and bin
    626     my $mask = undef;
    627     my $expCount = 0;
    628639    foreach my $row ( @{$exp_ids} ) {
    629640
    630         $mask = $self->{_gpc1Db}->getAverageMagicMaskFraction(@{$row}[0]);
    631 
    632         if (!$mask) {next;}
     641        $exp_id = @{$row}[0];
     642
     643        if (!$self->{_gpc1Db}->getMagicMaskStats($exp_id, \$meanMask, \$sumMask, \$chipCount )) {next;}
     644
     645
     646        if (!$meanMask) {next;}
     647        #print "expId=$exp_id meanMask=$meanMask sumMask=$sumMask nChips=$chipCount\n";
    633648        $expCount++;
     649        $totalMask += $sumMask;
     650        $totalChipCount += $chipCount;
    634651
    635652        foreach my $bin (@bins) {
    636             if ($mask <= ($bin+$interval)) {
     653            if ($meanMask <= ($bin+$interval)) {
    637654
    638655                $histogram{$bin}++;
    … …  
    640657            }
    641658        }
     659    }
     660
     661    my $totalMaskFrac;
     662   
     663    if ($totalChipCount > 0) {
     664
     665        $totalMaskFrac = sprintf( "%.1f", ($totalMask/$totalChipCount) * 100.0);
     666    }
     667    else {
     668
     669        $totalMaskFrac = "NA";
    642670    }
    643671
    … …  
    655683    close(GNUDAT);
    656684
    657     # now plot
     685
     686    $maxBin = $maxBin * 1.1;
     687    $accum = $accum * 1.1;
     688
     689    my $title = "Magic Mask Fraction for $expCount exposures\\nData taken between '$begin' and '$end'\\nTotal masked fraction is $totalMaskFrac%";
     690
     691    # make histogram
    658692    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
    659693    use FileHandle;
    660694    GP->autoflush(1);
    661 
    662     $maxBin = $maxBin * 1.1;
    663     $accum = $accum * 1.1;
    664 
    665695    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$histoOutputFile\";";}
    666696    print GP
    667697        "set term $self->{_outputFormat};" .
    668         "set title \"Magic mask fraction for $expCount exposures between '$begin' and '$end'\";" .
     698        "set title \"$title\";" .
    669699        "set grid;" .
    670700        "set boxwidth;" .
    … …  
    682712    close GP;
    683713
    684 
     714    # make cumulative distribution
    685715    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
    686716    use FileHandle;
    687717    GP->autoflush(1);
    688 
    689718    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$distOutputFile\";";}
    690719    print GP
    691720        "set term $self->{_outputFormat};" .
    692         "set title \"Cumulative distribution of magic streaks per image between '$begin' and '$end'\";" .
     721        "set title \"Cumulative distribution of $title\";" .
    693722        "set yrange [0:$accum];" .
    694723        "set key left top;" .
Note: See TracChangeset for help on using the changeset viewer.