Changeset 30321

Timestamp:

Jan 20, 2011, 11:11:01 AM (15 years ago)

Author:

bills

Message:

Merge in changes to support saving pixels excised by magic and making uncensored postage stamps
for authorized users

Location:

tags/ipp-20101215

Files:

• PS-IPP-Config (modified) (1 prop)
• PS-IPP-Config/lib/PS/IPP/Config.pm (modified) (3 diffs)
• PS-IPP-PStamp (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (1 diff)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (5 diffs)
• dbconfig/pstamp.md (modified) (1 diff, 1 prop)
• ippScripts/scripts/ipp_cleanup.pl (modified) (4 diffs, 1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (3 diffs, 1 prop)
• ippScripts/scripts/magic_destreak_defineruns.pl (modified) (3 diffs, 1 prop)
• ippScripts/scripts/magic_destreak_revert.pl (modified) (9 diffs, 1 prop)
• ippTasks/pstamp.pro (modified) (2 diffs, 1 prop)
• ippTasks/survey.pro (modified) (2 diffs, 1 prop)
• ippTools/share (modified) (1 prop)
• ippTools/share/disttool_pending_camera.sql (modified) (1 prop)
• ippTools/share/disttool_revertrun.sql (modified) (1 diff)
• ippTools/share/magicdstool_todestreak_camera.sql (modified) (1 prop)
• ippTools/share/magicdstool_torevert_diff.sql (modified) (1 diff)
• ippTools/share/pstamptool_pendingreq.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff)
• ippTools/src/magicdstool.c (modified) (2 diffs, 1 prop)
• ippTools/src/magicdstoolConfig.c (modified) (1 diff, 1 prop)
• ippTools/src/pstamptool.c (modified) (2 diffs, 1 prop)
• ippTools/src/pstamptoolConfig.c (modified) (1 diff, 1 prop)
• magic/remove (modified) (1 prop)
• magic/remove/src/streaksVersion.c (modified) (1 diff)
• magic/remove/src/streakscompare.c (modified) (1 diff)
• magic/remove/src/streaksio.c (modified) (18 diffs)
• magic/remove/src/streaksio.h (modified) (3 diffs)
• magic/remove/src/streaksrelease.c (modified) (1 diff)
• magic/remove/src/streaksremove.c (modified) (7 diffs)
• magic/remove/src/streaksremove.h (modified) (1 diff)
• magic/remove/src/streaksreplace.c (modified) (4 diffs)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (1 diff)
• pstamp/scripts/pstamp_job_run.pl (modified) (8 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_server_status (modified) (1 diff)
• pstamp/scripts/pstampparse.pl (modified) (5 diffs)

tags/ipp-20101215/PS-IPP-Config/lib/PS/IPP/Config.pm

@@ -1048 +1048 @@
 }
 
+sub magic_filename
+{
+    my $file = shift;
+    my $prefix = shift;
+    my $dirname = dirname($file);
+    my $base = basename($file);
+
+    return "$dirname/$prefix$base";
+}
+
+sub destreaked_filename
+{
+    my $self  = shift;
+    my $file  = shift;
+
+    return magic_filename($file, "SR_");
+}
+sub recovery_filename
+{
+    my $self  = shift;
+    my $file  = shift;
+
+    return magic_filename($file, "REC_");
+}
+
 # Return an EXTNAME From the EXTNAME.RULE table in the camera configuration
 sub extname_rule
@@ -1107 +1132 @@
     }
 
-    if (file_scheme($output) ne 'neb') {
+    my $scheme = file_scheme($output);
+    if (!$scheme or ($scheme ne 'neb')) {
         # non-nebulous file we're done
-        if ($delete_existing) {
+        if ($delete_existing and $self->file_exists($output)) {
             if (!$self->file_delete($output)) {
                 carp "failed to delete $output";
@@ -1181 +1207 @@
     my $copies = shift;
 
-    if (file_scheme($file) ne 'neb') {
+    my $scheme = file_scheme($file);
+    if (!$scheme or ($scheme ne 'neb')) {
         carp "cannot replicate non-neulous file: $file";
         return 0;

tags/ipp-20101215/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -17 +17 @@
                     resolve_project
                     getCamRunByCamID
+                    runToolAndParse
                     parse_md_fast
                     );

tags/ipp-20101215/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -29 +29 @@
                     $PSTAMP_SELECT_INVERSE
                     $PSTAMP_SELECT_UNCONV
+                    $PSTAMP_REQUEST_UNCENSORED
+                    $PSTAMP_REQUIRE_UNCENSORED
                     $PSTAMP_USE_IMFILE_ID
                     $PSTAMP_NO_WAIT_FOR_UPDATE
@@ -44 +46 @@
                     $PSTAMP_NO_JOBS_QUEUED
                     $PSTAMP_NO_OVERLAP
+                    $PSTAMP_NOT_AUTHORIZED
                     );
 our %EXPORT_TAGS = (standard => [@EXPORT_OK]);
@@ -64 +67 @@
 our $PSTAMP_USE_IMFILE_ID    = 16384;
 
-our $PSTAMP_NO_WAIT_FOR_UPDATE  = 32768;
+our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768;
+our $PSTAMP_REQUEST_UNCENSORED = 1 << 16;
+our $PSTAMP_REQUIRE_UNCENSORED = 1 << 17;
 
 # job and result codes
@@ -82 +87 @@
 our $PSTAMP_NO_JOBS_QUEUED   = 27;
 our $PSTAMP_NO_OVERLAP       = 28;
+our $PSTAMP_NOT_AUTHORIZED   = 29;
 
 
@@ -119 +125 @@
 PSTAMP_NO_JOBS_QUEUED
 PSTAMP_NO_OVERLAP
+PSTAMP_NOT_AUTHORIZED
 );
 

tags/ipp-20101215/dbconfig/pstamp.md

@@ -8 +8 @@
     uri         STR         255
     pollInterval S32        0
+    need_magic  S32         0
 END
 

tags/ipp-20101215/ippScripts/scripts/ipp_cleanup.pl

@@ -167 +167 @@
 #            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
 #            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
-            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
-            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
-            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1);
+            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1);
+            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1);
             if ($mode eq "goto_purged") {
                 # additional files to remove for 'purge' mode
@@ -429 +429 @@
 
                 # delete the temporary image datafiles
-                addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id );
-                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
-                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
+                addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id, 1);
+                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1);
+                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
 #            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
                 if ($mode eq "goto_purged") {
@@ -706 +706 @@
             my @files = ();
             # delete the temporary image datafiles
-            addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id);
-
-            addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id);
+            addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
+
+            addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id, 1);
 
             addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id);
@@ -2048 +2048 @@
     my $path_base  = shift; # base filename
     my $class_id   = shift; # class_id, if needed
+    my $recovery   = shift; # is there is a recovery file to clean?
 
     my $file = $ipprc->filename($filerule, $path_base, $class_id);
 
     push @$files, $file;
+    
+    if ($recovery) {
+        # need to clean up the recovery file (the pixels censored by streaksremove)
+        $file = $ipprc->recovery_filename($file);
+        push @$files, $file;
+    }
     return 1;
 }

tags/ipp-20101215/ippScripts/scripts/magic_destreak.pl

@@ -65 +65 @@
            'cam_reduction=s'=> \$cam_reduction,  # reduction class from camera stage (for chip and raw)
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
-           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
+           'recoveryroot=s' => \$recoveryroot,# "prefix" for saving the images of excised pixels
            'replace=s'      => \$replace,    # replace the input images with the results.
            'magicked=s'     => \$magicked,   # magicked state of the run
@@ -190 +190 @@
 my $recovery_path_base;
 if ($recoveryroot) {
-    if (!inNebulous($recoveryroot)) {
-        if (! -e $recoveryroot ) {
-            my $code = system "mkdir -p $recoveryroot";
-            &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
-                    $code >> 8) if $code;
-        }
-    }
-    # note: trailing / is necessary
-    $recovery_path_base = "$recoveryroot/$basename/";
+    # recoveryroot is a path to prepend to the basenames of the input files
+    if (inNebulous($recoveryroot)) {
+        # if recoveryroot is a nebulous path we ignore the actual path and put the files in the
+        # the same "directory" as the input files
+        $recoveryroot = $dirname;
+    } else {
+        # otherwise we put the files in recoveryroot. Regardless, we prefix the basename with 'REC_'
+        $ipprc->outroot_prepare($recoveryroot);
+    }
+    $recovery_path_base = "$recoveryroot/REC_$basename";
 }
 
@@ -601 +602 @@
     $command   .= " -magic_ds_id $magic_ds_id";
     $command   .= " -component $component";
+    $command .= " -backup_path_base $backup_path_base" if $backup_path_base;
+    $command .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
     $command   .= " -fault $exit_code";
     $command   .= " -dbname $dbname" if defined $dbname;

tags/ipp-20101215/ippScripts/scripts/magic_destreak_defineruns.pl

@@ -42 +42 @@
 
 # Parse the command-line arguments
-my ($stage, $stage_limit, $workdir);
+my ($stage, $stage_limit, $workdir, $recoveryroot);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 my @labels;
@@ -51 +51 @@
            'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
            'workdir=s'      => \$workdir,    # output destination
+           'recoveryroot=s' => \$recoveryroot, # recovery pixels destination
            'dbname=s'       => \$dbname,     # Database name
            'verbose'        => \$verbose,    # Print stuff?
@@ -78 +79 @@
     foreach my $label (@labels) {
         my $command = "$magicdstool -definebyquery -stage $stage -workdir $workdir -label $label";
+        $command .= " -recoveryroot $recoveryroot" if $recoveryroot;
         $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;

tags/ipp-20101215/ippScripts/scripts/magic_destreak_revert.pl

@@ -39 +39 @@
 # Parse the command-line arguments
 my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction, $magicked, $run_state);
-my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum);
+my ($outroot, $recovery_path_base, $replace, $release, $bytes, $md5sum);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -53 +53 @@
            'cam_reduction=s'=> \$cam_reduction,  # reduction class of the associated camera run
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
-           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
+           'recovery_path_base=s' => \$recovery_path_base,# "directory" for saving the images of excised pixels
            'replace=s'      => \$replace,    # replace the input images with the results.
            'bothways=s'     => \$bothways,   # run has inverse files (bothways diff)
@@ -130 +130 @@
 
 # default value is "NULL" do not use it
-if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
-    $recoveryroot = undef;
+if (defined($recovery_path_base) and ($recovery_path_base eq "NULL")) {
+    $recovery_path_base = undef;
 }
 
@@ -149 +149 @@
 }
 
-my $recovery_path_base;
-if ($recoveryroot) {
-    # note: trailing / is necessary
-    $recovery_path_base = "$recoveryroot/$basename/";
-}
-
 
 my ($image, $mask, $ch_mask, $weight, $sources, $astrom);
 my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
+my ($rimage, $rmask, $rch_mask, $rweight);
 
 if ($stage eq "raw") {
@@ -214 +209 @@
     $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id);
     $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id);
+
+    $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $class_id);
+    # This is somewhat kludgey but it works whether the mask is camera mask or chip mask
+    $rmask   = dirname($recovery_path_base) . "/REC_" . basename($mask);
+    $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $class_id);
+    $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $class_id);
 } elsif ($stage eq "camera") {
     $astrom =  $ipprc->filename("PSASTRO.OUTPUT", $path_base);
@@ -226 +227 @@
     $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base);
     $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
+    $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
+    $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
+    $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
 } elsif ($stage eq "diff") {
     my $name = "PPSUB.OUTPUT";
@@ -236 +240 @@
     $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
     $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+    if ($recovery_path_base) {
+        $rimage  = $ipprc->filename($name, $recovery_path_base);
+        $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
+        $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
+    }
 }
 
 revert_files($replace, $image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
+if ($recovery_path_base) {
+    delete_recovery_files($rimage, $rmask, $rweight, $rch_mask);
+}
 
 if ($stage eq "diff" and $bothways) {
@@ -251 +263 @@
     $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
     revert_files($replace, $image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef);
+    if ($recovery_path_base) {
+        $rimage  = $ipprc->filename($name, $recovery_path_base);
+        $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
+        $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
+        delete_recovery_files($rimage, $rmask, $rweight);
+    }
 }
 
@@ -426 +444 @@
 
 
-sub file_check
-{
-    my $file = shift;           # Name of file
-    &my_die("Unable to find output file: $file", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless
-        $ipprc->file_exists($file);
+sub delete_recovery_files
+{
+    foreach my $file (@_) {
+        if ($ipprc->file_exists($file)) {
+            if (!$ipprc->file_delete($file)) {
+                &my_die("failed to delete recovery file $file", $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+            }
+        }
+    }
 }
 

tags/ipp-20101215/ippTasks/pstamp.pro

@@ -258 +258 @@
         book getword pstampRequest $pageName ds_outProduct -var PRODUCT
         book getword pstampRequest $pageName outdir -var OUTDIR
+        book getword pstampRequest $pageName need_magic -var NEED_MAGIC
         book getword pstampRequest $pageName label -var LABEL
 
@@ -263 +264 @@
 
         $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL --redirect-output
+
+        if ($NEED_MAGIC != 0)
+            $run = $run --need_magic
+        end
 
         add_standard_args run

tags/ipp-20101215/ippTasks/survey.pro

@@ -259 +259 @@
 # user functions to manipulate destreak labels
 macro survey.add.destreak
-  if ($0 != 3)
-    echo "USAGE: survey.add.destreak (label) (workdir base)"
+  if ($0 != 4)
+    echo "USAGE: survey.add.destreak (label) (workdir base) (recovery root)"
     break
   end
   book newpage SURVEY_DESTREAK $1
   book setword SURVEY_DESTREAK $1 WORKDIR $2
+  book setword SURVEY_DESTREAK $1 RECOVERYROOT $3
   book setword SURVEY_DESTREAK $1 STATE PENDING
 end
@@ -821 +822 @@
     book setword SURVEY_DESTREAK $label STATE DONE
     book getword SURVEY_DESTREAK $label WORKDIR -var workdir
+    book getword SURVEY_DESTREAK $label RECOVERYROOT -var recoveryroot
   
     $run = magic_destreak_defineruns.pl --label $label --workdir $workdir/$label
+    if ("$recoveryroot" != "NULL") 
+        $run = $run --recoveryroot $recoveryroot/$label
+    end
 
     if ($DB:n == 0)

tags/ipp-20101215/ippTools/share/disttool_revertrun.sql

@@ -1 +1 @@
 UPDATE distRun
 SET distRun.fault = 0
-WHERE distRun.state = 'new'
+WHERE (distRun.state = 'new' OR distRun.state = 'goto_cleaned')
     AND distRun.fault != 0

tags/ipp-20101215/ippTools/share/magicdstool_torevert_diff.sql

@@ -10 +10 @@
     outroot,
     diffSkyfile.path_base,
+    magicDSFile.backup_path_base,
+    magicDSFile.recovery_path_base,
     "NULL" AS cam_path_base,
     CAST(diffRun.bothways AS SIGNED) AS bothways,

tags/ipp-20101215/ippTools/share/pstamptool_pendingreq.sql

@@ -3 +3 @@
     pstampRequest.*,
     pstampDataStore.outProduct AS ds_outProduct,
+    IFNULL(pstampDataStore.need_magic, 1) AS need_magic,
     IFNULL(Label.priority, 10000) AS priority
 FROM pstampRequest

tags/ipp-20101215/ippTools/share/pxadmin_create_tables.sql

@@ -1391 +1391 @@
         uri VARCHAR(255),
         pollInterval INTEGER DEFAULT 60,
+        need_magic TINYINT,
         PRIMARY KEY(ds_id),
         KEY(ds_id)

tags/ipp-20101215/ippTools/src/magicdstool.c

@@ -1724 +1724 @@
     PXOPT_LOOKUP_STR(component, config->args, "-component", false, false);
     PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_STR(recoveryroot, config->args, "-set_recoveryroot", false, false);
 
     psString query = pxDataGet("magicdstool_setfiletoupdate.sql");
@@ -1734 +1735 @@
     if (label) {
         psStringAppend(&setHook, "\n , magicDSRun.label = '%s'", label);
+    }
+    if (recoveryroot) {
+        psStringAppend(&setHook, "\n , magicDSRun.recoveryroot = '%s'", recoveryroot);
     }
 

tags/ipp-20101215/ippTools/src/magicdstoolConfig.c

@@ -230 +230 @@
     psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL);
     psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "set new label", NULL);
+    psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-set_recoveryroot", 0, "define new recovery directory", NULL);
 
     // -destreakedfile

tags/ipp-20101215/ippTools/src/pstamptool.c

@@ -134 +134 @@
     PXOPT_LOOKUP_STR(label,       config->args, "-set_label",         false, false);
     PXOPT_LOOKUP_S32(pollInterval, config->args, "-set_poll_interval",false, false);
+    PXOPT_LOOKUP_BOOL(no_magic, config->args, "-set_no_magic", false);
+
+    bool need_magic = ! no_magic;
 
     if (!pstampDataStoreInsert(config->dbh,
@@ -143 +146 @@
             outProduct,
             uri,
-            pollInterval
+            pollInterval,
+            need_magic
         )) {
         psError(PS_ERR_UNKNOWN, false, "database error");

tags/ipp-20101215/ippTools/src/pstamptoolConfig.c

@@ -51 +51 @@
     psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_label",        0, "define datastore label", NULL);
     psMetadataAddS32(adddatastoreArgs, PS_LIST_TAIL, "-set_poll_interval", 0, "define datastore poll interval (seconds)", 60);
+    psMetadataAddBool(adddatastoreArgs,PS_LIST_TAIL, "-set_no_magic",   0, "set magic not required", true);
 
     // -datastore

tags/ipp-20101215/magic/remove/src/streaksVersion.c

@@ -58 +58 @@
     psString source = streaksSource();   // Software source
 
-    psMetadataAddStr(header, PS_LIST_TAIL, "STREAK_V", 0, NULL, source);
+    psMetadataAddStr(header, PS_LIST_TAIL, "STREAK_V", PS_META_REPLACE, NULL, source);
     
     psStringPrepend(&version, "%s version: ", streaksProgram);

tags/ipp-20101215/magic/remove/src/streakscompare.c

@@ -18 +18 @@
     ippStage stage = psMetadataLookupS32(&status, config->arguments, "STAGE");
 
-    sFile *file1 = sFileOpen(config, stage, "INPUT1", NULL, true);
-    sFile *file2 = sFileOpen(config, stage, "INPUT2", NULL, true);
+    sFile *file1 = sFileOpen(config, stage, "INPUT1", NULL, true, false);
+    sFile *file2 = sFileOpen(config, stage, "INPUT2", NULL, true, false);
 
     int ncomponents;

tags/ipp-20101215/magic/remove/src/streaksio.c

@@ -42 +42 @@
     sf->class_id = psMetadataLookupStr(&status, config->arguments, "CLASS_ID");
 
-    sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true);
+    sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true, false);
     sf->nHDU = sf->inImage->nHDU;
 
@@ -48 +48 @@
     // The names of the temporary and recovery files are taken from the input
     char *inputBasename = basename(sf->inImage->name);
-    sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+    sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
 
     if (remove) {
         // XXX: the recovery file should be required if stage is raw and the replace flag is set
         // that is if the input raw image is to be destroyed
-        sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
-    } else {
-        sf->recImage = sFileOpen(config, stage, "RECOVERY.IMAGE", NULL, true);
-    }
-
-    sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false);
+        sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false);
+    } else {
+        sf->recImage = sFileOpen(config, stage, "RECOVERY.IMAGE", NULL, true, false);
+    }
+
+    sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false, false);
     if (sf->inMask && (sf->stage != IPP_STAGE_RAW)) {
         inputBasename = basename(sf->inMask->name);
-        sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+        sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
         if (remove) {
-            sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
+            sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false);
         } else {
-            sf->recMask = sFileOpen(config, stage, "RECOVERY.MASK", NULL, true);
+            sf->recMask = sFileOpen(config, stage, "RECOVERY.MASK", NULL, true, false);
         }
     }
@@ -73 +73 @@
     // If it doesn't exist, we didn't have a camera mask
     if (remove && sf->inMask && (stage == IPP_STAGE_CHIP)) {
-        sf->inChMask = sFileOpen(config, stage, "INPUT.CHMASK", NULL, false);
+        sf->inChMask = sFileOpen(config, stage, "INPUT.CHMASK", NULL, false, false);
         if (sf->inChMask) {
             inputBasename = basename(sf->inChMask->name);
-            sf->outChMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
-            sf->recChMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
-        }
-    }
-
-    sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false);
+            sf->outChMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
+            sf->recChMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false);
+        }
+    }
+
+    sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false, false);
     if (sf->inWeight) {
         inputBasename = basename(sf->inWeight->name);
-        sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+        sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
         if (remove) {
-            sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
+            sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false);
         } else {
-            sf->recWeight = sFileOpen(config, stage, "RECOVERY.WEIGHT", NULL, true);
+            sf->recWeight = sFileOpen(config, stage, "RECOVERY.WEIGHT", NULL, true, false);
         }
     }
     if (remove) {
-        sf->inSources = sFileOpen(config, stage, "SOURCES", NULL, false);
+        sf->inSources = sFileOpen(config, stage, "SOURCES", NULL, false, false);
         if (sf->inSources) {
             inputBasename = basename(sf->inSources->name);
-            sf->outSources = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+            sf->outSources = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
         }
     }
@@ -121 +121 @@
     sf->transparentStreaks = psMetadataLookupF64(&status, config->arguments, "TRANSPARENT_STREAKS");
 
-    sf->stats = psMetadataAlloc();
-    psString statsFileName= psMetadataLookupStr(&status, config->arguments, "STATS");
-    if (statsFileName) {
-        sf->statsFile = fopen(statsFileName, "w");
-        if (!sf->statsFile) {
-            psError(PS_ERR_IO, true, "failed to open stats file %s", statsFileName);
-            streaksExit("", PS_EXIT_CONFIG_ERROR);
-        }
-    }
-
     return sf;
 }
@@ -138 +128 @@
 {
     freeImages(sf);
+    psFree(sf->stats);
     psFree(sf->diffedPixels);
     psFree(sf->tiles);
@@ -221 +212 @@
 
 static psString
-resolveFilename(pmConfig *config, sFile *sfile, bool create)
+resolveFilename(pmConfig *config, sFile *sfile, bool create, bool checkIfDestreaked)
 {
     sfile->inNebulous = IN_NEBULOUS(sfile->name);
@@ -232 +223 @@
             // instance. It will get created below in pmConfigConvertFilename
             if ((sfile->resolved_name = nebFind(server, sfile->name)) != NULL) {
-                if (!nebFileIsDestreaked(sfile)) {
+                if (checkIfDestreaked && !nebFileIsDestreaked(sfile)) {
                     psError(PS_ERR_IO, false, "attempting to delete file that has not been destreaked %s", sfile->name);
                     return NULL;
@@ -247 +238 @@
 
 sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect,
-    psString outputFilename, bool required)
+    psString outputFilename, bool required, bool checkIfDestreaked)
 {
     bool status;
@@ -322 +313 @@
     // and the file is to be opened for writing
     if (outputFilename) {
+        sfile->write = true;
         psStringAppend(&sfile->name, "%s%s", name, outputFilename);
-        sfile->resolved_name = resolveFilename(config, sfile, true);
+        sfile->resolved_name = resolveFilename(config, sfile, true, checkIfDestreaked);
         if (!sfile->resolved_name) {
             psError(PS_ERR_IO, false, "Failed to resolve filename for %s", sfile->name);
@@ -335 +327 @@
     } else {
         sfile->name = psStringCopy(name);
-        sfile->resolved_name = resolveFilename(config, sfile, false);
+        sfile->resolved_name = resolveFilename(config, sfile, false, false);
         if (!sfile->resolved_name) {
             psError(PS_ERR_IO, false, "Failed to resolve name for %s", sfile->name);
@@ -402 +394 @@
 
 void
-addDestreakKeyword(psMetadata *header)
+addDestreakKeyword(psMetadata *header, bool value)
 {
     psMetadataAddBool(header, PS_LIST_TAIL, "PSDESTRK", PS_META_REPLACE,
-        "Have streaks been removed from image?", true);
+        "Have streaks been removed from image?", value);
 }
 
@@ -416 +408 @@
 
 void
-copyPHU(streakFiles *sfiles, bool remove)
+copyPHU(streakFiles *sfiles, bool remove, bool destreak)
 {
     psAssert(sfiles->stage == IPP_STAGE_RAW, "copyPHU should only be used for raw stage");
@@ -433 +425 @@
 
     // add keyword indicating that streaks have been removed
-    addDestreakKeyword(imageHeader);
+    addDestreakKeyword(imageHeader, destreak);
 
     if (!psFitsWriteBlank(sfiles->outImage->fits, imageHeader, NULL)) {
@@ -463 +455 @@
         }
         // add keyword indicating that streaks have been removed
-        addDestreakKeyword(maskHeader);
+        addDestreakKeyword(maskHeader, destreak);
         if (!psFitsWriteBlank(sfiles->outMask->fits, maskHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
@@ -493 +485 @@
 
         // add keyword indicating that streaks have been removed
-        addDestreakKeyword(weightHeader);
+        addDestreakKeyword(weightHeader, destreak);
         if (!psFitsWriteBlank(sfiles->outWeight->fits, weightHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
@@ -683 +675 @@
     psVector *tiles)
 {
-    if (!sfile) {
+    if (!sfile || !sfile->write) {
         return;
     }
@@ -861 +853 @@
         streaksExit("", PS_EXIT_DATA_ERROR);
     }
+    sfile->fits = NULL;
     psFree(sfile->header);
     sfile->header = NULL;
@@ -1250 +1243 @@
     } else if (in->image->type.type == PS_TYPE_F32) {
         in->exciseValue = NAN;
+    } else if (in->image->type.type == PS_TYPE_F64) {
+        in->exciseValue = NAN;
     } else {
         psError(PS_ERR_PROGRAMMING, true, "unexpected image type found: %d\n", in->image->type.type);

tags/ipp-20101215/magic/remove/src/streaksio.h

@@ -5 +5 @@
 
 sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect,
-    psString outputFilename, bool required);
+    psString outputFilename, bool required, bool checkIfDestreaked);
 
 void closeImages(streakFiles *sfiles);
@@ -12 +12 @@
 
 void readImage(sFile *sfile, int extnum, ippStage stage, bool isMask);
-void copyPHU(streakFiles *sfiles, bool remove);
+void copyPHU(streakFiles *sfiles, bool remove, bool destreaked);
 void copyTable(sFile *out, sFile *in, int extnum);
 void copyFitsOptions(sFile *out, sFile *rec, sFile *in, psVector *tiles);
@@ -24 +24 @@
 void readImageFrom_pmFile(streakFiles *sf);
 
-void addDestreakKeyword(psMetadata *);
+void addDestreakKeyword(psMetadata *, bool value);
 void addRecoveryKeyword(psMetadata *);
 

tags/ipp-20101215/magic/remove/src/streaksrelease.c

@@ -24 +24 @@
     if (sfiles->stage == IPP_STAGE_RAW) {
         // copy PHU to output files
-        copyPHU(sfiles, true);
+        copyPHU(sfiles, false, true);
 
         // advance to the first image extension

tags/ipp-20101215/magic/remove/src/streaksremove.c

@@ -93 +93 @@
         // Raw files have a phu and multiple extensions, one per chip
         // Since this is a raw file, copy it's PHU to output files
-        copyPHU(sfiles, true);
+        copyPHU(sfiles, true, true);
 
         // advance to the first image extension
@@ -275 +275 @@
     psLogMsg("streaksremove", PS_LOG_INFO, "time to run streaksremove: %f\n", total_time);
 
-    if (sfiles->statsFile) {
-        const char *statsMDC = psMetadataConfigFormat(sfiles->stats);
-        if (!statsMDC || strlen(statsMDC) == 0) {
-            psError(PS_ERR_IO, false, "Unable to get statistics MDC file.\n");
-        } else {
-            fprintf(sfiles->statsFile, "%s", statsMDC);
-            psFree(statsMDC);
-            fclose(sfiles->statsFile);
-            sfiles->statsFile = NULL;
-            psFree(sfiles->stats);
-            sfiles->stats = NULL;
-        }
-    }
+    psString statsFileName= psMetadataLookupStr(&status, config->arguments, "STATS");
+
+    if (statsFileName) {
+    // Write out
+        psString resolved = pmConfigConvertFilename(statsFileName, config, true, true); // Resolved filename
+        if (!resolved) {
+            psError(psErrorCodeLast(), false, "Unable to resolve statistics file name");
+            return false;
+        }
+        if (!psMetadataConfigWrite(sfiles->stats, resolved, NULL)) {
+            psError(psErrorCodeLast(), false, "Unable to serialize stats metadata.\n");
+            psFree(resolved);
+            return false;
+        }
+        psFree(resolved);
+    }
+
     // all done. Clean up to look for memory leaks.
 
@@ -613 +617 @@
         addRecoveryKeyword(sf->recImage->header);
     }
-    addDestreakKeyword(sf->outImage->header);
+    addDestreakKeyword(sf->outImage->header, true);
 
     if (!SFILE_IS_IMAGE(sf->inImage)) {
@@ -639 +643 @@
                 addRecoveryKeyword(sf->recMask->header);
             }
-            addDestreakKeyword(sf->outMask->header);
+            addDestreakKeyword(sf->outMask->header, true);
             // Note: we don't excise the mask pixels even if exciseAll is true.
             setupImageRefs(sf->outMask, sf->recMask, sf->inMask, sf->extnum, false);
@@ -671 +675 @@
             addRecoveryKeyword(sf->recWeight->header);
         }
-        addDestreakKeyword(sf->outWeight->header);
+        addDestreakKeyword(sf->outWeight->header, true);
         setupImageRefs(sf->outWeight, sf->recWeight, sf->inWeight, sf->extnum, exciseAll);
 
@@ -912 +916 @@
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
-        addDestreakKeyword(header);
+        addDestreakKeyword(header, true);
 
         if (!psFitsWriteBlank(out->fits, header, extname)) {
@@ -973 +977 @@
         psArrayRealloc(outTable, j);
 
-        addDestreakKeyword(header);
+        addDestreakKeyword(header, true);
         if (psArrayLength(outTable) > 0) {
             printf("Censored %d sources\n", numCensored);

tags/ipp-20101215/magic/remove/src/streaksremove.h

@@ -18 +18 @@
     psString    name;
     bool        inNebulous;
+    bool        write;
     psFits      *fits;
     int         nHDU;

tags/ipp-20101215/magic/remove/src/streaksreplace.c

@@ -36 +36 @@
     if (sfiles->stage == IPP_STAGE_RAW) {
         // copy PHU to output files
-        copyPHU(sfiles, false);
+        copyPHU(sfiles, false, false);
 
         // advance to the first image extension
@@ -296 +296 @@
 
     sf->outImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header);
+    addDestreakKeyword(sf->outImage->header, false);
 
     if (!SFILE_IS_IMAGE(sf->inImage)) {
@@ -317 +318 @@
         readImage(sf->recMask, sf->extnum, sf->stage, true);
         sf->outMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
+        
+        addDestreakKeyword(sf->outMask->header, false);
 
         // XXX: TODO
@@ -329 +332 @@
 
         sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header);
+        addDestreakKeyword(sf->outWeight->header, false);
         setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false);
 

tags/ipp-20101215/pstamp/scripts/pstamp_checkdependent.pl

@@ -680 +680 @@
             foreach my $c (@$components) {
                 my $command = "$magicdstool -destreakedfile -magic_ds_id $magic_ds_id -component $c";
+                # XXX: get this from a config file
+                $command .= " -set_recoveryroot neb://any/gpc1/destreak/recover";
                 my $dsfile = runToolAndParseExpectOne($command, $verbose);
                 if (!$dsfile) {

tags/ipp-20101215/pstamp/scripts/pstamp_job_run.pl

@@ -14 +14 @@
 use File::Basename;
 use File::Copy;
+use File::Temp qw(tempfile);
 use Digest::MD5::File qw( file_md5_hex );
 use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 use IPC::Cmd 0.36 qw( can_run run );
 use POSIX;
@@ -26 +28 @@
 
 my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); 
-my ($verbose, $dbname, $dbserver, $no_update);
+my ($verbose, $dbname, $dbserver, $no_update, $save_temps);
 
 GetOptions(
@@ -39 +41 @@
     'verbose'           =>  \$verbose,
     'no-update'         =>  \$no_update,
+    'save-temps'        =>  \$save_temps,
 );
 
@@ -75 +78 @@
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
 my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1);
+my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
+my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -88 +93 @@
     my $argString;
     $argString = $params->{job_args};
+    my $stage = $params->{stage};
 
     # XXX: should we do any other sanity checking?
     my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
 
-    $argString .= " -file $params->{image}";
+    my $nan_masked = 1;
+    my $muggle = 0;
+    if (!$params->{magicked}) {
+        $nan_masked = 0;
+    } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) {
+        # Attempt to find or create a muggle image
+        $nan_masked = 0;
+        $muggle = 1;
+    }
+
+    my $image = $params->{image};
+    my $mask;
+    my $variance;
+    my $fileArgs = " -file $params->{image}";
     my @file_list = ($params->{image});
     
-    my $nan_masked = 1;
-    if (!$params->{magicked}) {
-        $nan_masked = 0;
-    }
     if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
-        $argString .= " -mask $params->{mask}";
-        push @file_list, $params->{mask};
+        $mask = $params->{mask};
+        $fileArgs .= " -mask $mask";
+        push @file_list, $mask;
     }
     if ($options & $PSTAMP_SELECT_VARIANCE) {
-        $argString .= " -variance $params->{weight}";
-        push @file_list, $params->{weight};
+        $variance = $params->{weight};
+        $fileArgs .= " -variance $variance";
+        push @file_list, $variance;
     }
 
@@ -113 +130 @@
     }
 
-    check_files(@file_list);
-
-    my $command = "$ppstamp $outputBase $argString";
+    # check that actual input files exist
+    check_files($PSTAMP_GONE, @file_list);
+
+    # find our output directory
+    my $outdir = dirname($outputBase);
+    my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
+    
+    if ($muggle) {
+        # first see if the original uncensored images are around
+        if (check_for_backups($params, \$fileArgs)) {
+            # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images
+            print "Making stamps from backup images\n";
+        } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
+            # user required uncensored but since stage isn't chip we can't rebuild them
+            my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE);
+        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($stage ne 'chip')) {
+            # make stamps from uncensored images
+            $muggle = 0;
+        } else {
+            # Try and replace the streaks from the recovery images
+
+            @file_list = ();
+            $tmproot = "$outdir/$job_id";
+            mkdir $tmproot or
+                my_die( "failed to create temporary directory $tmproot", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+            my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
+            # find the "directory" of the input path_base
+            my $inputdir = dirname($image);
+            my $base = basename($image);
+            $tmpImage = "$tmproot/$base";
+
+            # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
+            # stage work. Import the rule here.
+            my $recImage = "$inputdir/REC_$base";
+            my $newFileArgs = " -file $tmpImage";
+            $muggle_command .= " -image $image -recimage $recImage";
+            push @file_list, $recImage;
+
+            if ($mask) {
+                $base = basename($mask);
+                $tmpMask = "$tmproot/$base";
+                $newFileArgs .= " -mask $tmpMask";
+                my $recMask = "$inputdir/REC_$base";
+                $muggle_command .= " -mask $mask -recmask $recMask";
+                push @file_list, $recMask;
+            }
+
+            if ($variance) {
+                $base = basename($variance);
+                $tmpVariance = "$tmproot/$base";
+                $newFileArgs .= " -weight $tmpVariance";
+                my $recVariance = "$inputdir/REC_$base";
+                $muggle_command .= " -weight $variance -recweight $recVariance";
+                push @file_list, $recVariance;
+            }
+            if (check_files(0, @file_list)) {
+                # recovery files exist and are accessible restore the excised pixels
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $muggle_command, verbose => $verbose);
+                unless ($success) {
+                    my $exitStatus = WEXITSTATUS($error_code);
+                    my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus);
+                }
+
+                # set the ppstamp file arguments
+                $fileArgs = $newFileArgs;
+            } else {
+                if ($options & $PSTAMP_REQUIRE_UNCENSORED) {
+                    my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE);
+                }
+                # just make stamps from the censored images
+                print "Unable to restore uncensored images, will extract stamps from censored images\n";
+                # these files won't be used so zap them
+                ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef);
+            }
+        }
+    }
+
+    my $command = "$ppstamp $outputBase $argString $fileArgs";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
-    $command .= " -stage $params->{stage}" if $params->{stage};
+    $command .= " -stage $stage";
     $command .= " -no_censor_masked" unless $nan_masked;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -131 +225 @@
     }
 
+    if (!$save_temps) {
+        unlink $tmpImage if $tmpImage;
+        unlink $tmpMask if $tmpMask;
+        unlink $tmpVariance if $tmpVariance;
+        rmdir $tmproot if $tmproot;
+    }
+
     if ($exitStatus == 0) {
-        my $dir = dirname($outputBase);
-
-        my $reglist = "$dir/reglist$job_id";
+        my $reglist = "$outdir/reglist$job_id";
 
         my $F;
@@ -363 +462 @@
 
 sub check_files {
+    my $error_code = shift;
+    my $return_code = 1;
     foreach my $f (@_) {
         if (!$ipprc->file_exists($f)) {
-            my_die( "file $f does not exist:", $job_id, $PSTAMP_GONE, 'stop');
-        }
-    }
+            if ($error_code) {
+                my_die( "file $f does not exist:", $job_id, $error_code, 'stop');
+            } else {
+                print STDERR "file $f does not exist\n";
+                $return_code = 0;
+            }
+        }
+    }
+    return $return_code;
+}
+
+sub check_for_backups {
+    my $params = shift;
+    my $r_fileArgs = shift;
+
+    my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}";
+    my $results = runToolAndParse($command, $verbose);
+    my $dsComponent  = $results->[0];
+
+    if ($dsComponent and ($dsComponent->{data_state} eq 'full')) {
+
+        print "magicDSFile state is full. Backup images should exist\n";
+
+        # replace the file names with the backup paths
+        # fileArgs has the form:  -file imagename [-mask maskname] [-weight weightname]
+        my @args = split " ", $$r_fileArgs;
+
+        my $newFileArgs;
+        while (@args) {
+            my $a = shift @args;
+            my $f = shift @args;
+            my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f;
+
+            my $backup_image_name = $ipprc->destreaked_filename($f);
+            my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name;
+            $newFileArgs .= " $a $backup_image_name";
+        }
+        $$r_fileArgs = $newFileArgs;
+
+        return 1;
+    }
+
+    return 0;
 }
 

tags/ipp-20101215/pstamp/scripts/pstamp_parser_run.pl

@@ -31 +31 @@
 my $outdir;
 my $label;
+my $need_magic;
 my $verbose;
 my $dbname;
@@ -41 +42 @@
     'outdir=s'          =>  \$outdir,
     'label=s'           =>  \$label,
+    'need_magic'        =>  \$need_magic,
     'redirect-output'   =>  \$redirect_output,
     'verbose'           =>  \$verbose,
@@ -160 +162 @@
             $parse_cmd = "$pstampparse";
             $parse_cmd .= " --label $label" if $label;
+            $parse_cmd .= " --need_magic" if $need_magic;
             $request_fault = 0;
         } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {

tags/ipp-20101215/pstamp/scripts/pstamp_server_status

@@ -44 +44 @@
 chdir $rundir or die "failed to cd to $rundir";
 
+
+my $down = 0;
+if ($down) {
+    print "Postage Stamp Server will be down for maintenance it will back at approximately 00:00 2010-12-11 UTC\n";
+    exit 0;
+}
 
 my ($pts, $pantasks_script) = tempfile ('/tmp/pts.XXXX', UNLINK => !$save_temps);

tags/ipp-20101215/pstamp/scripts/pstampparse.pl

@@ -30 +30 @@
 my $save_temps;
 my $no_update;
+my $dest_requires_magic;
 
 GetOptions(
@@ -38 +39 @@
     'label=s'   =>  \$label,
     'mode=s'    =>  \$mode,
+    'need_magic'=>  \$dest_requires_magic,
     'dbname=s'  =>  \$dbname,
     'dbserver=s'=>  \$dbserver,
@@ -126 +128 @@
 }
 
-# Adjust the label for requests coming in over the web interaface
+
+# Adjust the label for requests coming in over the web interface
 
 my $label_changed = 0;
@@ -384 +387 @@
     my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
     if (!$proj_hash) {
-        foreach $row (@$rowList) {
-            insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
+        foreach my $r (@$rowList) {
+            insertFakeJobForRow($r, 1, $PSTAMP_UNKNOWN_PRODUCT);
             $num_jobs++;
         }
@@ -409 +412 @@
     $need_magic    = $proj_hash->{need_magic};
 
-    # Temporary hack so that IFA can get at non-magicked data
-    my $allow_mops_unmagicked = 1;
-    if ($allow_mops_unmagicked) {
-        if ($product and (($product eq "mops-pstamp-results") or
-                          ($product eq "mops-pstamp-results2") or
-                          ($product eq "ifa-pstamp-results"))) {
-            $need_magic = 0;
-        }
-    }
-
     $need_magic = 0 if $stage eq 'stack';
+
+    if ($need_magic) {
+
+        # this project requires that postage stamps be extracted from destreaked images
+
+        if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
+            # The user has requested uncensored stamps
+
+            if (!$dest_requires_magic) {
+                # and this user's data store destination is allowed uncensored stamps, so accept the request
+                $need_magic = 0;
+            } else {
+                print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
+                if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
+                    # user required uncensored stamps. Can't do it so fail.
+                    foreach my $r (@$rowList) {
+                        insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
+                        $num_jobs++;
+                    }
+                    return $num_jobs;
+                }
+
+                # user will accept censored stamps. alter OPTION_MASK and continue
+
+                print STDERR "    Will attempt to make destreaked stamps\n";
+                # zap the offending bit in the option mask
+                $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
+                foreach my $r (@$rowList) {
+                    $r->{OPTION_MASK} = $option_mask;
+                }
+            }
+        }
+    }
     
     my $numRows = scalar @$rowList;