IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Nov 10, 2008, 4:52:52 PM (17 years ago)
Author:
bills
Message:

remove workaround for missing wcs in diff files.
List of input files to merge stage was incorrect.
Include a hack we use the other day will remove tomorrow

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/magic_process.pl

    r20502 r20675  
    3030
    3131# Parse the command-line arguments
    32 my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $redirect);
    33 my $skycellroot;
     32my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
     33my $hack_mask;
     34
    3435GetOptions(
    3536           'magic_id=s'      => \$magic_id,   # Magic identifier
    … …  
    4243           'no-update'       => \$no_update,  # Don't update the database?
    4344           'no-op'           => \$no_op,      # Don't do any operations?
    44            'redirect-output' => \$redirect,   # Redirect output?
    45            'skycellroot=s'   => \$skycellroot, # root of the warps to find skycells (temporary)
     45           'logfile=s'       => \$logfile,
     46           'hack-mask'       => \$hack_mask   # enable extra special mask substitution
    4647           ) or pod2usage( 2 );
    4748
    … …  
    5657$ipprc->define_camera($camera);
    5758
    58 # Remove streaks doesn't know about nebulous. It expects to be able to append strings to outroot
     59# RemoveStreaks doesn't know about nebulous. It expects to be able to append strings to outroot
    5960# to form valid file names.
    6061# So forbid nebulous path in outroot. We could relax this by change RemoveStreaks to take all
    … …  
    6869$outroot = $ipprc->file_resolve($outroot);
    6970   
    70 
    71 my $logDest = $ipprc->filename("LOG.EXP", $outroot, $magic_id) or
    72     &my_die("Missing entry from camera config", $magic_id, $PS_EXIT_CONFIG_ERROR);
    73 $ipprc->redirect_output($logDest) if $redirect;
     71$ipprc->redirect_output($logfile) if $logfile;
    7472
    7573# Look for programs we need
    7674my $missing_tools;
    77 my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
    78 my $streaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1);
     75my $magictool      = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
     76my $removestreaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1);
    7977if ($missing_tools) {
    8078    warn("Can't find required tools.");
    … …  
    104102
    105103
    106 my @outputs = ("${outroot}.clusters", "${outroot}_hough.fits", "${outroot}.streaks");
     104my @outputs;
    107105### Do the heavy lifting
    108106{
    109     my $mode;
    110107    my $command;                # Command to execute
    111     $command = "$streaks --outroot $outroot";
     108    $command = "$removestreaks --outroot $outroot";
     109    $command .= " --verbose" if $verbose;
     110
     111#    $command .= " -u 6";
     112#    $command .= " --test";
     113
    112114    if (scalar @$inputs == 1 and $node ne "root") {
    113115        #
    … …  
    115117        #
    116118        # Leaf node: 'detect' stage
    117         $mode = 'detect';
    118119        my $innode = $$inputs[0];     # Input node
    119120
    120         my ($image, $mask, $weight) = resolve_inputs($innode);
     121        # expected outputs for detect stage
     122        @outputs = ("${outroot}.clusters", "${outroot}_hough.fits", "${outroot}.streaks");
     123
     124        my $diff_id;
     125        $diff_id = $innode->{diff_id} if $hack_mask;
     126        my ($image, $mask, $weight) = resolve_inputs($innode, $diff_id);
    121127
    122128        $command .= " --detect --image $image --mask $mask --weight $weight";
    123129
     130        # set threshold to 4 sigma
     131        $command .= ' -t 4';
     132
    124133        # create the list of inputs used at this stage. At higher levels the
    125         # thes files will get catenated together to create the file for the subsquent stage
     134        # these files will get catenated together to create the file for the subsquent stage
     135        # this causes major file pollution, but avoids multi-level queries
     136        # at higher level nodes.
    126137       
     138        my ($in_fh, $input_list)  = open_list_file($outroot, "input.list");
     139        print $in_fh "$outroot\n";
     140        close $in_fh;
    127141        my ($ifh, $image_list)  = open_list_file($outroot, "image.list");
    128142        print $ifh "$image\n";
    … …  
    134148        print $wfh "$weight\n";
    135149        close $wfh;
    136 
    137         # work around missing wcs in difference images. Use the skycells from warp stage
    138         my ($sfh, $skycell_list);
    139         if ($skycellroot) {
    140            ($sfh, $skycell_list) = open_list_file($outroot, "wcs.list");
    141             # at this level the skycell_id is equal to the node
    142             my $skycell_id = $node;
    143             my $skycell = $ipprc->file_resolve("${skycellroot}.${skycell_id}.skycell");
    144             print $sfh "$skycell\n";
    145             close $sfh;
    146         }
    147150    } else {
    148151        #
    … …  
    152155
    153156        # Branch node: 'merge' stage
    154         $mode = 'merge';
    155 
    156         my ($infh, $input_list) = open_list_file($outroot, "input.list");
    157         my ($ifh, $image_list)  = open_list_file($outroot, "image.list");
    158         my ($mfh, $mask_list)   = open_list_file($outroot, "mask.list");
    159         my ($wfh, $weight_list) = open_list_file($outroot, "weight.list");
    160         my ($sfh, $skycell_list);
    161         if ($skycellroot) {
    162            ($sfh, $skycell_list) = open_list_file($outroot, "wcs.list");
    163         }
     157
     158        # expected outputs from merge stage
     159        @outputs = ("${outroot}.streaks");
     160
     161        my ($in_fh, $input_list) = open_list_file($outroot, "input.list");
     162        my ($ifh, $image_list)   = open_list_file($outroot, "image.list");
     163        my ($mfh, $mask_list)    = open_list_file($outroot, "mask.list");
     164        my ($wfh, $weight_list)  = open_list_file($outroot, "weight.list");
     165        my ($sfh, $streaks_list) = open_list_file($outroot, "streaks.list");
    164166
    165167        # do this in eval so we can fault the exposure without
    166168        # passing the $magic_id and $node everywhere
     169        # XXX  (they're global dummy fix this)
    167170        eval {
    168171            foreach my $innode (@$inputs) {
    169172                # root for inputs from previous stage
    170173                my $in_uri = $innode->{uri};
    171                 print $infh "$in_uri\n";
    172 
    173                 # build image lists by combining the lists from
     174                print $sfh "$in_uri\n";
     175
     176                cat_list_to_list($in_fh, $in_uri, "input.list");
     177                # build input lists by combining the lists from
    174178                # previous stages
    175179                cat_list_to_list($ifh, $in_uri, "image.list");
    176180                cat_list_to_list($mfh, $in_uri, "mask.list");
    177181                cat_list_to_list($wfh, $in_uri, "weight.list");
    178                 cat_list_to_list($sfh, $in_uri, "wcs.list") if $skycell_list;
    179182            }
    180             close $infh;
     183            close $in_fh;
    181184            close $ifh;
    182185            close $mfh;
    183186            close $wfh;
    184             close $sfh if $skycell_list;
     187            close $sfh;
    185188
    186189            $command .= " --merge --inputs $input_list";
    187190            $command .= " --images $image_list --masks $mask_list --weights $weight_list" ;
    188             $command .= " --wcsList $skycell_list" if $skycell_list;
     191            $command .= " --inputstreaks $streaks_list";
    189192        };
    190193        if ($@) {
    191             &my_die("failed to create file lists: $@", $PS_EXIT_UNKNOWN_ERROR, $magic_id, $node,
    192                 $PS_EXIT_UNKNOWN_ERROR);
     194            &my_die("failed to create file lists: $@", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
    193195        }
    194196    }
    … …  
    214216    }
    215217}
    216 
    217218
    218219
    … …  
    254255}
    255256
    256 sub cat_list_to_list   { # ($infh, $in_uri, "input.list");
     257sub cat_list_to_list   {
    257258    my $out = shift;        # output file handle
    258259    my $uri = shift;        # uri to append ...
    259     my $extension = shift;  # extension to
     260    my $extension = shift;  # ... the extension to
     261
    260262    my $filename = "$uri.$extension";
    261263
    … …  
    267269
    268270sub resolve_inputs {
     271    my %substitute_mask = (
     272        30 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_30_mask_extra.fits",
     273        31 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_31_mask_extra.fits",
     274        34 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_34_mask_extra.fits",
     275        35 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_35_mask_extra.fits",
     276        36 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_36_mask_extra.fits",
     277        39 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_39_mask_extra.fits",
     278        40 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_40_mask_extra.fits",
     279        41 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_41_mask_extra.fits",
     280        44 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_44_mask_extra.fits",
     281        45 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_45_mask_extra.fits",
     282        49 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_49_mask_extra.fits",
     283        77 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_77_mask_extra.fits",
     284        81 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_81_mask_extra.fits",
     285        86 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_86_mask_extra.fits",
     286        89 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_89_mask_extra.fits",
     287        90 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_90_mask_extra.fits",
     288        94 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_94_mask_extra.fits",
     289        95 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_95_mask_extra.fits"
     290    );
     291
    269292    my $node = shift;
    270293    my $input_base = $node->{path_base};
     294
     295    my $diff_id = shift;
     296
    271297    my $image = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT", $input_base));
    272     my $mask  = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
     298    my $mask;
     299
     300    # if a diff_id is provided look up the extra special difference image mask from the table
     301    # XXX: perhaps if the mask doesn't exist, we should fall through and use the original
     302    if ($diff_id) {
     303        $mask = $substitute_mask{$diff_id};
     304        &my_die("substitute mask not found for $diff_id", $magic_id, $node->{node}, $PS_EXIT_SYS_ERROR) unless $mask;
     305    } else {
     306        $mask = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
     307    }
    273308    my $weight= $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.WEIGHT", $input_base));
    274309
Note: See TracChangeset for help on using the changeset viewer.