IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 24726


Ignore:
Timestamp:
Jul 8, 2009, 6:19:50 PM (17 years ago)
Author:
bills
Message:

updates to magic destreak processing

Location:
trunk
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/Build.PL

    r24512 r24726  
    5757        scripts/magic_process.pl
    5858        scripts/magic_destreak.pl
     59        scripts/magic_destreak_revert.pl
    5960        scripts/ippdb.pl
    6061        scripts/ipp_cleanup.pl

  • trunk/ippScripts/scripts/magic_destreak.pl

    r24702 r24726  
    2121
    2222use PS::IPP::Config 1.01 qw( :standard );
     23use Nebulous::Client;
    2324
    2425use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
    … …  
    3031my $streaksremove = can_run('streaksremove') or (warn "Can't find streaksremove" and $missing_tools = 1);
    3132my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
    32 # XXX: replace with real name for tool
    33 my $censordetections = "";
    34 # my $censordetections = can_run('censordetections') or (warn "Can't find censordetections" and $missing_tools = 1);
     33my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1);
    3534if ($missing_tools) {
    3635    warn("Can't find required tools.");
    … …  
    5453           'path_base=s'    => \$path_base,  # path_base of the input
    5554           'inverse'        => \$inverse,    # Inverse subtraction?
    56            'cam_path_base=s'=> \$cam_path_base,  # path_base of the associated camera run
     55           'cam_path_base=s'=> \$cam_path_base,  # path_base from camera stage (for chip and raw)
    5756           'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
    5857           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
    … …  
    102101my $basename = basename($path_base);
    103102my $nebulousInput = inNebulous($dirname);
     103my $nebulousOutput = inNebulous($outroot);
    104104
    105105# parse replace arguments check for errors and set up
    106106# the appropriate paths
     107my $nebulous;
    107108if (defined($replace) and ($replace eq "T")) {
     109    # for camera stage we need a handle to the nebulous server
     110    if ($stage eq 'camera') {
     111        my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
     112        &my_die("cannot find NEB_SERVER in site configuration",
     113                                        $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR)
     114            if !$ nebulousServer;
     115
     116        $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); };
     117        if ($@ or not defined $nebulous) {
     118            &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer",
     119                                    $magic_ds_id, $component , $PS_EXIT_UNKNOWN_ERROR);
     120        }
     121    }
    108122    $replace = 1;
    109123} else {
    … …  
    111125}
    112126
     127
    113128# default value is "NULL" do not use
    114129if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
    … …  
    122137
    123138# create the output directories if it is not a nebulous path and it doesn't exist
    124 if (!inNebulous($outroot)) {
     139if (! $nebulousOutput) {
    125140    if (! -e $outroot ) {
    126141        my $code = system "mkdir -p $outroot";
    … …  
    128143                $code >> 8) if $code;
    129144    }
    130 #} elsif ($replace) {
    131     # if input is in nebulous and we've been asked to replace the input with the destreaked
    132     # version the output must be in nebulous as well
    133     # XXX Is this necessary? if ($replace) the outputs are going to the nebulous "directory" of the
    134     # input anyways. Why require that other files (log) be in nebulous
    135145}
    136146
    … …  
    208218        $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
    209219        $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
     220        if (!$ipprc->file_exists($mask)) {
     221            carp("camera mask file $mask for $component not found. Continuing using mask from chip stage.");
     222            $mask = $ch_mask;
     223            $ch_mask = undef;
     224        }
     225        # XXX: should we censor the sources as well? For now we're leaving them out of the distribution bundles
     226        # because they confuse people
     227        # XXX: make sure that this is the right file rule
     228        # $sources = $ipprc->filename("PSPHOT.OUT.CMF.SPL",  $path_base);
     229
    210230    } elsif ($stage eq "warp") {
    211231        $image  = $ipprc->filename("PSWARP.OUTPUT", $path_base);
    … …  
    247267    }
    248268} else {
     269    # camera stage. The only work to do is to censor the detections file
     270
    249271    my $tempOutRoot = "/tmp/destreak";
    250272    my ($maskListFile,   $maskListName)   = tempfile( "$tempOutRoot.mask.list.XXXX",   UNLINK => !$save_temps);
    … …  
    272294        if (!$quality) {
    273295            my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $path_base, $class_id);
     296            if (! $ipprc->file_exists($mask)) {
     297                # camera mask doesn't exist for this chip. Fall back to the chip mask
     298                $mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $file->{path_base}, $class_id);
     299            }
    274300            print $maskListFile "$mask\n";
    275301        }
    … …  
    279305    my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $path_base);
    280306    {
    281         my $output = $tmproot . basename($astrom);
    282         my $command = "$censordetections -stage $stage -output $output";
    283         $command .= " -astrom $astrom";
    284         $command .= " -masklist $maskListName";
    285         $command .= " -replace" if $replace;
     307        my $command = "$censorObjects -file $astrom -masklist $maskListName $backup_path_base";
     308        # $command .= " -replace" if $replace;
    286309        $command .= " -dbname $dbname" if defined $dbname;
    287310        unless (defined $no_op) {
    … …  
    290313            unless ($success) {
    291314                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    292                 &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code);
     315                &my_die("Unable to perform censorObjects: $error_code", $magic_ds_id, $component, $error_code);
     316            }
     317            my $output = $ipprc->filename("CENSOR.OUTPUT", $backup_path_base);
     318            &my_die("expected output file $output not found ", $magic_ds_id, $component, $PS_EXIT_DATA_ERROR)
     319                unless $ipprc->file_exists($output);
     320
     321            if ($replace) {
     322                $nebulous->swap($astrom, $output) or
     323                    &my_die("nebulous swap failed $astrom $output", $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR);
    293324            }
    294325        } else {

  • trunk/ippScripts/scripts/magic_destreak_revert.pl

    r24686 r24726  
    7070    defined $outroot;
    7171
    72 # XXX: for testing
    73 $no_update = 1;
    74 
    7572&my_die("bytes and md5sum are is required for raw stage stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'raw' and (!$bytes or !$md5sum));
    7673
    … …  
    8784if ($@ or not defined $nebulous) {
    8885    &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    89     return undef;
    9086}
    9187
    … …  
    109105my $nebulousInput = inNebulous($dirname);
    110106
    111 # parse replace arguments check for errors and set up
    112 # the appropriate paths
    113 # this program doesn't make sense if $replace is false
    114 &my_die("Invalid value for replace: $replace", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if $replace ne "T";
    115 
    116107if (defined($replace) and ($replace eq "T")) {
    117108    $replace = 1;
    … …  
    120111}
    121112
    122 # default value is "NULL" do not use
     113&my_die("replace not allowed for non-nebulous files", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR)
     114    if ($replace eq "T") and !$nebulousInput;
     115
     116
     117# default value is "NULL" do not use it
    123118if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
    124119    $recoveryroot = undef;
    … …  
    135130    $backup_path_base = $tmproot . "$basename";
    136131} else {
    137     # note: trailing / is necessary
     132    # note: the trailing / here is necessary
    138133    $tmproot = "$outroot/";
    139134    $backup_path_base = $tmproot . $basename;
    … …  
    189184}
    190185
    191 do_swap($image, $bimage) or
     186revert_file($image, $bimage) or
    192187        &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     188
     189# don't commit next the next couple of lines and see xxx below
     190# change to if 0 to revert chip runs without masks
     191if (0) {
     192
    193193if ($mask) {
    194     my $swap_result = do_swap($mask, $bmask);
    195     if (! $swap_result) {
    196         # delete next 5 lines
    197         if ($stage eq "chip") {
    198             # XXX: this is a kludge to get around my test environment
    199             # Don't commit
    200             print STDERR "failed to resotre camera mask file, proceeding\n";
     194    if (!revert_file($mask, $bmask)) {
     195        if ($stage eq 'chip') {
     196            # XXX: don't fail if the mask file fails to revert. It probably doesn't exist
     197            # Handle this properly
     198            print STDERR "failure to revert mask file, ignored\n";
    201199        } else {
    202         &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    203         }  # delete this line
    204     }
    205 }
     200            &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     201        }
     202    }
     203}
     204
    206205if ($ch_mask) {
    207     do_swap($ch_mask, $bch_mask) or
    208         &my_die("failed to restore chip mask", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     206    # Handle this properly
     207    if (!revert_file($ch_mask, $bch_mask)) {
     208        # print STDERR "failure to revert chip mask file, ignored\n";
     209        &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     210    }
    209211}
     212
     213# xxx don't commmit next line
     214}
     215
    210216if ($weight) {
    211     do_swap($weight, $bweight) or
     217    revert_file($weight, $bweight) or
    212218        &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    213219}
    214220
    215 # now reverse the desreaked file
    216 # Change magicdstool to set the magicked value to zero
    217 # Actually we should probably do this first to insure that no data gets deleted
    218 # Input result into database
    219 if (0) {
    220     # XXX: this is the wrong mode
    221     my $command = "$magicdstool -adddestreakedfile";
     221# now revert the row in the database
     222{
     223    my $command = "$magicdstool -revertdestreakedfile";
    222224    $command   .= " -magic_ds_id $magic_ds_id";
    223225    $command   .= " -component $component";
    224     $command   .= " -setmagicked" if $replace;
    225     $command   .= " -backup_path_base $backup_path_base" if $backup_path_base;
    226     $command   .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
    227226    $command   .= " -dbname $dbname" if defined $dbname;
    228227
    … …  
    242241
    243242
    244 
    245243### Pau.
    246244
    … …  
    256254}
    257255
    258 sub do_swap
     256sub revert_file
    259257{
    260258    my $original = shift;
    … …  
    297295        if ($b_path and -e $b_path) {
    298296            print "ready to delete backup\n" if $verbose;
    299             if (! $nebulous->delete($backup)) {
     297            if (! $ipprc->file_delete($backup)) {
    300298                print "failed to delete $backup\n";
    301299                return 0;
    … …  
    307305        # delete the 'backup' (destreaked target) file if it exists
    308306        if ($b_path and -e $b_path) {
    309             if (! $nebulous->delete($backup)) {
     307            if (! $ipprc->file_delete($backup)) {
    310308                print "failed to delete $backup\n";
    311309                return 0;
    … …  
    348346    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
    349347
    350     my $command = "$magicdstool -adddestreakedfile";
    351     $command   .= " -magic_ds_id $magic_ds_id";
    352     $command   .= " -component $component";
    353     $command   .= " -fault $exit_code";
    354     $command   .= " -dbname $dbname" if defined $dbname;
    355 
    356     # Add the processed file to the database
    357     unless ($no_update) {
    358         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    359             run(command => $command, verbose => $verbose);
    360         unless ($success) {
    361             carp("failed to update database for $magic_ds_id $component");
    362         }
    363     } else {
    364         print "Skipping command: $command\n";
    365     }
     348    # we wouldn't be here unless an entry already exists in the database
     349#    my $command = "$magicdstool -adddestreakedfile";
     350#    $command   .= " -magic_ds_id $magic_ds_id";
     351#    $command   .= " -component $component";
     352#    $command   .= " -fault $exit_code";
     353#    $command   .= " -dbname $dbname" if defined $dbname;
     354
     355#    # Add the processed file to the database
     356#    unless ($no_update) {
     357#        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     358#            run(command => $command, verbose => $verbose);
     359#        unless ($success) {
     360#            carp("failed to update database for $magic_ds_id $component");
     361#        }
     362#    } else {
     363#        print "Skipping command: $command\n";
     364#    }
    366365
    367366    carp($msg);

  • trunk/ippTasks/magic.pro

    r24711 r24726  
    463463    sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
    464464
    465     host anyhost
     465    substr $COMPONENT 0 3 COMP_HEAD
     466    if ("$COMP_HEAD" == "sky")
     467        set.host.for.skycell $COMPONENT
     468    else
     469        # assume component is a class_id, if not we will default to anyhost
     470        set.host.for.camera $CAMERA $COMPONENT
     471    end
    466472
    467473    # TODO: do not add recoveryroot or replace if they are null or zero
Note: See TracChangeset for help on using the changeset viewer.