IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 27537


Ignore:
Timestamp:
Mar 30, 2010, 3:45:08 PM (16 years ago)
Author:
Paul Price
Message:

Branching to branches/pap_delete for development of deletion of convolved diffs.

  • ppSub will now only write the convolved images if explicitly requested on the command-line.
  • diff_skyfile.pl only requests convolved images for warps (because we don't need to run magic on stacks).
  • magic_process.pl deletes the convolved images when done.
  • If magic_process.pl can't find the convolved image to use (e.g., it's already been deleted because we're re-running magic), then it is created on the fly using ppSubConvolve and the previously calculated convolution kernel.
Location:
branches/pap_delete
Files:
7 edited
3 copied

Legend:

Unmodified
Added
Removed

  • branches/pap_delete/ippScripts/scripts/diff_skycell.pl

    r27215 r27537  
    102102my $camera;                     # Camera
    103103my ($inputMagic, $templateMagic); # Are the inputs been magicked?
     104my ($saveInConv, $saveRefConv);   # Save the input or reference convolved images?
    104105# Prescan to decide if this is or is not a stack stack diff. The check above confirms we only have two entries.
    105106if ((${ $files }[0]->{warp_id} == 0)&&
    … …  
    140141            $templateSources = "PSWARP.OUTPUT.SOURCES";
    141142            $templateMagic = $file->{magicked};
     143            $saveRefConv = 1;
    142144        }
    143145    } else {
    … …  
    162164            $inputVariance = "PSWARP.OUTPUT.VARIANCE";
    163165            $inputSources = "PSWARP.OUTPUT.SOURCES";
     166            $saveInConv = 1;
    164167        }
    165168    }
    … …  
    284287        $command .= " -ipprc $configurationReal";
    285288    }
     289    $command .= " -save-inconv" if defined $saveInConv;
     290    $command .= " -save-refconv" if defined $saveRefConv;
    286291    $command .= " -recipe PPSUB $recipe_ppSub";
    287292    $command .= " -recipe PSPHOT $recipe_psphot";

  • branches/pap_delete/ippScripts/scripts/magic_process.pl

    r27240 r27537  
    3030my $missing_tools;
    3131my $magictool      = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
     32my $difftool       = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
     33my $ppSubConvolve = can_run('ppSubConvolve') or (warn "Can't find ppSubConvolve" and $missing_tools = 1);
    3234my $detectstreaks = can_run('DetectStreaks') or (warn "Can't find DetectStreaks" and $missing_tools = 1);
    3335my $VerifyStreaks = can_run('VerifyStreaks') or (warn "Can't find VerifyStreaks, will not produce png images");
    … …  
    116118    $command .= " --threshold 2.35";
    117119
     120    my @deletions;          # Files to delete
    118121    if (scalar @$inputs == 1 and $node ne "root") {
    119122        #
    … …  
    135138        }
    136139
    137         my $template = resolve_template($innode);
    138         &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR)
    139             unless defined $template;
     140        my $diff_base = $innode->{diff_path_base}; # Base name for diff
     141        my $tempName = $innode->{inverse} ? "PPSUB.INPUT.CONV" : "PPSUB.REF.CONV"; # File rule of interest
     142        my $template = $ipprc->file_resolve($ipprc->filename($tempName, $diff_base));
     143        &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless defined $template;
     144
     145        # Delete the convolved products when done (we can recreate them as we need)
     146        push @deletions, $template;
     147        push @deletions, $ipprc->filename($tempName . ".MASK", $diff_base);
     148        push @deletions, $ipprc->filename($tempName . ".VARIANCE", $diff_base);
     149
     150        unless ($ipprc->file_exists($template)) {
     151            # Template doesn't exist (or can't be found); try to recreate it
     152            my $tempPath = "/tmp/magic.$magic_id.$node.template";
     153
     154            my $kernel = $ipprc->filename("PPSUB.OUTPUT.KERNELS", $diff_base); # Name of kernel file
     155            &my_die("Unable to find kernel file", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless $ipprc->file_exists($kernel);
     156
     157            my ($image, $mask);   # Image and mask
     158            {
     159                my $command = "$difftool -inputskyfile -diff_id $diff_id -skycell_id $node"; # Command to run
     160                $command .= " -dbname $dbname" if defined $dbname;
     161                if ($node->{inverse}) {
     162                    # Want the input because we're magicking the reference
     163                    $command .= " -input";
     164                } else {
     165                    # Want the reference because we're magicking the input
     166                    $command .= " -template";
     167                }
     168
     169                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     170                    run(command => $command, verbose => $verbose);
     171                unless ($success) {
     172                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     173                    &my_die("Unable to determine convolution inputs: $error_code", $magic_id, $node, $error_code);
     174                }
     175
     176                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     177                    &my_die("Unable to parse metadata config doc", $magic_id, $node, $PS_EXIT_PROG_ERROR);
     178
     179                my $inputs = parse_md_list($metadata) or
     180                    &my_die("Unable to parse metadata list", $magic_id, $node, $PS_EXIT_PROG_ERROR);
     181                &my_die("Unexpected number of outputs", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless scalar @$inputs == 1;
     182                my $input = $$inputs[0];
     183                my $path = $input->{path_base}; # Path of interest
     184                if (defined $input->{warp_id} and $input->{warp_id} > 0) {
     185                    $image = $ipprc->filename("PSWARP.OUTPUT", $path);
     186                    $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path);
     187                } elsif (defined $input->{stack_id} and $input->{stack_id} > 0) {
     188                    $image = $ipprc->filename("PPSTACK.UNCONV", $path);
     189                    $mask = $ipprc->filename("PPSTACK.UNCONV.MASK", $path);
     190                }
     191                &my_die("Unable to determine image and mask name", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless defined $image and defined $mask;
     192            }
     193
     194            {
     195                &my_die("Unable to find image and mask", $magic_id, $node, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image) and $ipprc->file_exists($mask);
     196
     197                my $command = "$ppSubConvolve $tempPath -image $image -mask $mask -kernel $kernel";
     198                $command .= " -reference" unless $node->{inverse};
     199                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     200                    run(command => $command, verbose => $verbose);
     201                unless ($success) {
     202                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     203                    &my_die("Unable to create template image: $error_code", $magic_id, $node, $error_code);
     204                }
     205
     206                $template = $ipprc->filename("PPSUB.INPUT.CONV", $tempPath) or &my_die("Unable to determine filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
     207                $template = $ipprc->file_resolve($template) or &my_die("Unable to resolve filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
     208                &my_die("Unable to find created template", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($template);
     209                push @deletions, $template;
     210            }
     211        }
    140212
    141213        $command .= " --detect --image $image --mask $mask --weight $weight -k $template";
    … …  
    226298        print "Skipping command: $command\n";
    227299    }
     300
     301    foreach my $file (@deletions) {
     302        $ipprc->file_delete($file);
     303    }
    228304}
    229305
    … …  
    387463}
    388464
    389 sub resolve_template
    390 {
    391     my $node = shift;
    392 
    393     my $path_base = $node->{diff_path_base}; # Base name for name
    394     my $image = $node->{inverse} ? "PPSUB.INPUT.CONV" : "PPSUB.REF.CONV"; # File rule of interest
    395 
    396     $image = $ipprc->file_resolve($ipprc->filename($image, $path_base));
    397 
    398     return $image
    399 }
    400 
    401465sub file_check
    402466{

  • branches/pap_delete/ippconfig/recipes/ppSub.config

    r26902 r27537  
    7676INVERSE         BOOL    FALSE           # Generate inverse subtraction?
    7777PHOTOMETRY      BOOL    FALSE           # Perform photometry?
    78 SAVE.CONVOLVED  BOOL    TRUE            # Save convolved images?
    7978
    8079

  • branches/pap_delete/ppSub/src/ppSub.h

    r27109 r27537  
    4545    bool photometry;                    // Perform photometry?
    4646    bool inverse;                       // Output inverse subtraction as well?
     47    bool saveInConv;                    // Save convolved input?
     48    bool saveRefConv;                   // Save convolved reference?
    4749    psString stamps;                    // Stamps file
    4850    pmPSF *psf;                         // Point Spread Function

  • branches/pap_delete/ppSub/src/ppSubArguments.c

    r27143 r27537  
    8787    psMetadataAddBool(arguments, PS_LIST_TAIL, "-photometry", 0, "Perform photometry?", NULL);
    8888    psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp", 0, "Zero point for photometry", NAN);
    89     psMetadataAddBool(arguments, PS_LIST_TAIL, "-inverse", 0, "Generate inverse subtractions?", NULL);
     89    psMetadataAddBool(arguments, PS_LIST_TAIL, "-inverse", 0, "Generate inverse subtractions?", false);
     90    psMetadataAddBool(arguments, PS_LIST_TAIL, "-save-inconv", 0, "Save input convolved images?", false);
     91    psMetadataAddBool(arguments, PS_LIST_TAIL, "-save-refconv", 0, "Save reference convolved images?", false);
    9092    psMetadataAddBool(arguments, PS_LIST_TAIL, "-visual", 0, "Show diagnostic plots", NULL);
    9193
    … …  
    154156    }
    155157
     158    data->saveInConv = psMetadataLookupBool(NULL, arguments, "-save-inconv");
     159    data->saveRefConv = psMetadataLookupBool(NULL, arguments, "-save-refconv");
     160
    156161    if (psMetadataLookupBool(NULL, arguments, "-visual")) {
    157162        pmVisualSetVisual(true);

  • branches/pap_delete/ppSub/src/ppSubCamera.c

    r27040 r27537  
    219219    data->photometry = psMetadataLookupBool(NULL, recipe, "PHOTOMETRY");
    220220
    221     bool mdok;                          // Status of MD lookup
    222     bool saveConv = psMetadataLookupBool(&mdok, recipe, "SAVE.CONVOLVED"); // Save convolved images?
    223 
    224221    // Convolved input image
    225222    pmFPAfile *inConvImage = defineOutputFile(config, input, true, "PPSUB.INPUT.CONV", PM_FPA_FILE_IMAGE);
    … …  
    230227        return false;
    231228    }
    232     if (saveConv) {
    233         inConvImage->save = true;
    234         inConvMask->save = true;
    235     }
     229    inConvImage->save = data->saveInConv;
     230    inConvMask->save = data->saveInConv;
    236231    if (inVar) {
    237232        pmFPAfile *inConvVar = defineOutputFile(config, inConvImage, false, "PPSUB.INPUT.CONV.VARIANCE",
    … …  
    241236            return false;
    242237        }
    243         if (saveConv) {
    244             inConvVar->save = true;
    245         }
     238        inConvVar->save = data->saveInConv;
    246239    }
    247240
    … …  
    254247        return false;
    255248    }
    256     if (saveConv) {
    257         refConvImage->save = true;
    258         refConvMask->save = true;
    259     }
     249    refConvImage->save = data->saveRefConv;
     250    refConvMask->save = data->saveRefConv;
    260251    if (refVar) {
    261252        pmFPAfile *refConvVar = defineOutputFile(config, refConvImage, false, "PPSUB.REF.CONV.VARIANCE",
    … …  
    265256            return false;
    266257        }
    267         if (saveConv) {
    268             refConvVar->save = true;
    269         }
     258        refConvVar->save = data->saveRefConv;
    270259    }
    271260

  • branches/pap_delete/ppSub/src/ppSubData.c

    r27061 r27537  
    3333    data->photometry = false;
    3434    data->inverse = false;
     35    data->saveInConv = false;
     36    data->saveRefConv = false;
    3537    data->stamps = NULL;
    3638    data->psf = NULL;
Note: See TracChangeset for help on using the changeset viewer.