IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 3, 2010, 8:50:52 AM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/simtest_nebulous_branches
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/simtest_nebulous_branches

  • branches/simtest_nebulous_branches/ippScripts/scripts/magic_destreak.pl

    r25029 r27840  
    88use Sys::Hostname;
    99my $host = hostname();
     10my $date = `date`;
    1011print "\n\n";
    11 print "Starting script $0 on $host\n\n";
     12print "Starting script $0 on $host at $date\n\n";
    1213
    1314use vars qw( $VERSION );
    … …  
    1516
    1617use IPC::Cmd 0.36 qw( can_run run );
    17 use File::Temp qw( tempfile );
     18use File::Temp qw( tempfile tempdir );
    1819use File::Basename qw( basename dirname );
    1920use PS::IPP::Metadata::Config;
    … …  
    3233my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
    3334my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1);
     35my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
     36my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    3437if ($missing_tools) {
    3538    warn("Can't find required tools.");
    … …  
    3841
    3942# Parse the command-line arguments
    40 my ($magic_ds_id, $camera, $streaks, $inv_streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
     43my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction);
    4144my ($outroot, $recoveryroot);
    4245my ($replace, $release);
    … …  
    4851           'streaks=s'      => \$streaks,    # file containing the list of streaks
    4952           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
     53           'exp_id=s'       => \$exp_id,     # exp_id, chip_id, warp_id, or diff_id
    5054           'stage=s'        => \$stage,      # raw, chip, warp, or diff
    5155           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
    … …  
    5458           'path_base=s'    => \$path_base,  # path_base of the input
    5559           'cam_path_base=s'=> \$cam_path_base,  # path_base from camera stage (for chip and raw)
     60           'cam_reduction=s'=> \$cam_reduction,  # reduction class from camera stage (for chip and raw)
    5661           'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
    5762           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
    … …  
    7479    defined $stage and
    7580    defined $stage_id and
     81    defined $exp_id and
    7682    defined $component and
    7783    defined $uri and
    … …  
    8187my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    8288$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile;
     89
     90$cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL');
     91
     92my $recipe_psastro = $ipprc->reduction($cam_reduction, 'PSASTRO'); # Recipe to use
     93&my_die("Unrecognised PSASTRO recipe", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;
    8394
    8495my ($skycell_args, $class_id, $skycell_id);
    … …  
    94105} elsif ($stage ne "camera") {
    95106    &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    96 }   
     107}
    97108$inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL");
    98109
    … …  
    111122    if ($stage eq 'camera') {
    112123        my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
    113         &my_die("cannot find NEB_SERVER in site configuration", 
    114                                         $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) 
     124        &my_die("cannot find NEB_SERVER in site configuration",
     125                                        $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR)
    115126            if !$ nebulousServer;
    116127
    … …  
    145156    }
    146157}
     158
     159my $statsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.stats";
    147160
    148161my $backup_path_base;
    … …  
    166179        if (! -e $recoveryroot ) {
    167180            my $code = system "mkdir -p $recoveryroot";
    168             &my_die("cannot create output directory $outroot", $magic_ds_id, $component,
     181            &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
    169182                    $code >> 8) if $code;
    170183        }
    … …  
    173186    $recovery_path_base = "$recoveryroot/$basename/";
    174187}
     188
     189my $temp_dir;
    175190
    176191if ($stage ne "camera") {
    … …  
    195210                    &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
    196211
     212            $temp_dir = tempdir( CLEANUP => !$save_temps);
    197213            ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
    198214
    199215            foreach my $skycell (@$skycells) {
    200                 my $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
     216                my $skycell_uri;
     217                if ($skycell->{data_state} eq "full") {
     218                    $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
     219                } else {
     220                    # diff run must have been cleaned up, need to create this skycell file on the fly
     221                    my $skycell_id = $skycell->{skycell_id};
     222                    $skycell_uri = "$temp_dir/$skycell_id";
     223                    $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
     224                        &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
     225                }
    201226                print $sfh "$skycell_uri\n";
    202227            }
    203             close $sfh;
     228            close $sfh
    204229        }
    205230    }
    … …  
    216241        $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id) if $release ;
    217242    } elsif ($stage eq "chip") {
     243
     244        # Check to see if we're using dynamic masks
     245        my $dynamicMasks;               # Use dynamic masks?
     246        {
     247            # Get the PSASTRO recipe
     248            my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
     249            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     250                run(command => $command, verbose => $verbose);
     251            unless ($success) {
     252                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     253                &my_die("Unable to perform ppConfigDump: $error_code", $magic_ds_id, $component,
     254                        $PS_EXIT_CONFIG_ERROR);
     255            }
     256            my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
     257                &my_die("Unable to parse metadata config doc", $magic_ds_id, $component,
     258                        $PS_EXIT_CONFIG_ERROR);
     259
     260            $dynamicMasks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
     261        }
     262
    218263        # we use the mask output from the camera stage for input and replace
    219264        # the output of the chip stage with that mask as well.
    220265        $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
    221         $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
    222         $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
    223266        $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
    224267        $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
    225         if (!$ipprc->file_exists($mask)) {
    226             carp("camera mask file $mask for $component not found. Continuing using mask from chip stage.");
    227             $mask = $ch_mask;
    228             $ch_mask = undef;
    229         }
     268
     269        if ($dynamicMasks) {
     270            $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
     271            $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
     272        } else {
     273            $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
     274        }
     275
    230276        # XXX: should we censor the sources as well? For now we're leaving them out of the distribution bundles
    231277        # because they confuse people
    … …  
    245291
    246292        if ($inv_streaks) {
    247             # create a temporary file containing the contents of the 
     293            # create a temporary file containing the contents of the
    248294            # two streaks files
    249295            ($allstreaks_fh, $allstreaks_name) = tempfile ("/tmp/all.streaks.XXXX",
    … …  
    260306        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
    261307
     308        $command .= " -stats $statsFile";
    262309        $command .= " -class_id $class_id" if defined $class_id;
    263310        $command .= " -recovery $recoveryroot" if defined $recoveryroot;
    … …  
    288335        $sources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $path_base);
    289336
     337        # Note: we create a stats file for the inverse procesing but we don't look at the results
     338        my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
     339
    290340        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
     341        $command .= " -stats $invStatsFile";
    291342
    292343        $command .= " -recovery $recoveryroot" if defined $recoveryroot;
    … …  
    335386        my $quality = $file->{quality};
    336387        if (!$quality) {
    337             my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $path_base, $class_id); 
     388            my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $path_base, $class_id);
    338389            if (! $ipprc->file_exists($mask)) {
    339390                # camera mask doesn't exist for this chip. Fall back to the chip mask
    … …  
    357408                &my_die("Unable to perform censorObjects: $error_code", $magic_ds_id, $component, $error_code);
    358409            }
    359             my $output = $ipprc->filename("CENSOR.OUTPUT", $backup_path_base); 
     410            my $output = $ipprc->filename("CENSOR.OUTPUT", $backup_path_base);
    360411            &my_die("expected output file $output not found ", $magic_ds_id, $component, $PS_EXIT_DATA_ERROR)
    361412                unless $ipprc->file_exists($output);
    … …  
    371422}
    372423
    373 # if recovery and/or backup files were expected make sure they exist
     424my $statsFlags;
     425if ($stage ne "camera") {
     426    file_check($statsFile);
     427    {
     428        my $resolvedStatsFile = $ipprc->file_resolve($statsFile);
     429        my $command = "$ppStatsFromMetadata $resolvedStatsFile - STREAKSREMOVE";
     430
     431        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     432            run(command => $command, verbose => $verbose);
     433        unless ($success) {
     434            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     435            &my_die("Unable to perform ppStatsFromMetadata: $error_code", $magic_ds_id, $component, $error_code);
     436        }
     437        foreach my $line (@$stdout_buf) {
     438            $statsFlags .= " $line";
     439        }
     440        chomp $statsFlags;
     441    }
     442}
     443
     444# XXX: if recovery and/or backup files were expected make sure they exist
    374445
    375446# Input result into database
    … …  
    381452    $command   .= " -backup_path_base $backup_path_base" if $backup_path_base;
    382453    $command   .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
     454    $command   .= " $statsFlags" if $statsFlags;
    383455    $command   .= " -dbname $dbname" if defined $dbname;
    384456
    … …  
    438510    }
    439511
    440     close $fout 
     512    close $fout
    441513        or my_die("failed to close combined streaks file", $magic_ds_id,
    442514                     $component, $PS_EXIT_UNKNOWN_ERROR);
Note: See TracChangeset for help on using the changeset viewer.