Changeset 30317

Timestamp:

Jan 20, 2011, 10:28:02 AM (15 years ago)

Author:

bills

Message:

Implement extraction of postage stamps from uncensored images either by using the backup images
or by replacing the censored pixels from destreak recovery images.

Location:

trunk

Files:

• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (1 diff)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (5 diffs)
• ippTasks/pstamp.pro (modified) (2 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (1 diff)
• pstamp/scripts/pstamp_job_run.pl (modified) (8 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_server_status (modified) (1 diff)
• pstamp/scripts/pstampparse.pl (modified) (5 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -17 +17 @@
                     resolve_project
                     getCamRunByCamID
+                    runToolAndParse
                     parse_md_fast
                     );

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -29 +29 @@
                     $PSTAMP_SELECT_INVERSE
                     $PSTAMP_SELECT_UNCONV
+                    $PSTAMP_REQUEST_UNCENSORED
+                    $PSTAMP_REQUIRE_UNCENSORED
                     $PSTAMP_USE_IMFILE_ID
                     $PSTAMP_NO_WAIT_FOR_UPDATE
@@ -44 +46 @@
                     $PSTAMP_NO_JOBS_QUEUED
                     $PSTAMP_NO_OVERLAP
+                    $PSTAMP_NOT_AUTHORIZED
                     );
 our %EXPORT_TAGS = (standard => [@EXPORT_OK]);
@@ -64 +67 @@
 our $PSTAMP_USE_IMFILE_ID    = 16384;
 
-our $PSTAMP_NO_WAIT_FOR_UPDATE  = 32768;
+our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768;
+our $PSTAMP_REQUEST_UNCENSORED = 1 << 16;
+our $PSTAMP_REQUIRE_UNCENSORED = 1 << 17;
 
 # job and result codes
@@ -82 +87 @@
 our $PSTAMP_NO_JOBS_QUEUED   = 27;
 our $PSTAMP_NO_OVERLAP       = 28;
+our $PSTAMP_NOT_AUTHORIZED   = 29;
 
 
@@ -119 +125 @@
 PSTAMP_NO_JOBS_QUEUED
 PSTAMP_NO_OVERLAP
+PSTAMP_NOT_AUTHORIZED
 );
 

trunk/ippTasks/pstamp.pro

@@ -258 +258 @@
         book getword pstampRequest $pageName ds_outProduct -var PRODUCT
         book getword pstampRequest $pageName outdir -var OUTDIR
+        book getword pstampRequest $pageName need_magic -var NEED_MAGIC
         book getword pstampRequest $pageName label -var LABEL
 
@@ -263 +264 @@
 
         $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL --redirect-output
+
+        if ($NEED_MAGIC != 0)
+            $run = $run --need_magic
+        end
 
         add_standard_args run

trunk/pstamp/scripts/pstamp_checkdependent.pl

@@ -680 +680 @@
             foreach my $c (@$components) {
                 my $command = "$magicdstool -destreakedfile -magic_ds_id $magic_ds_id -component $c";
+                # XXX: get this from a config file
+                $command .= " -set_recoveryroot neb://any/gpc1/destreak/recover";
                 my $dsfile = runToolAndParseExpectOne($command, $verbose);
                 if (!$dsfile) {

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -14 +14 @@
 use File::Basename;
 use File::Copy;
+use File::Temp qw(tempfile);
 use Digest::MD5::File qw( file_md5_hex );
 use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 use IPC::Cmd 0.36 qw( can_run run );
 use POSIX;
@@ -26 +28 @@
 
 my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); 
-my ($verbose, $dbname, $dbserver, $no_update);
+my ($verbose, $dbname, $dbserver, $no_update, $save_temps);
 
 GetOptions(
@@ -39 +41 @@
     'verbose'           =>  \$verbose,
     'no-update'         =>  \$no_update,
+    'save-temps'        =>  \$save_temps,
 );
 
@@ -75 +78 @@
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
 my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1);
+my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
+my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -88 +93 @@
     my $argString;
     $argString = $params->{job_args};
+    my $stage = $params->{stage};
 
     # XXX: should we do any other sanity checking?
     my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
 
-    $argString .= " -file $params->{image}";
+    my $nan_masked = 1;
+    my $muggle = 0;
+    if (!$params->{magicked}) {
+        $nan_masked = 0;
+    } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) {
+        # Attempt to find or create a muggle image
+        $nan_masked = 0;
+        $muggle = 1;
+    }
+
+    my $image = $params->{image};
+    my $mask;
+    my $variance;
+    my $fileArgs = " -file $params->{image}";
     my @file_list = ($params->{image});
     
-    my $nan_masked = 1;
-    if (!$params->{magicked}) {
-        $nan_masked = 0;
-    }
     if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
-        $argString .= " -mask $params->{mask}";
-        push @file_list, $params->{mask};
+        $mask = $params->{mask};
+        $fileArgs .= " -mask $mask";
+        push @file_list, $mask;
     }
     if ($options & $PSTAMP_SELECT_VARIANCE) {
-        $argString .= " -variance $params->{weight}";
-        push @file_list, $params->{weight};
+        $variance = $params->{weight};
+        $fileArgs .= " -variance $variance";
+        push @file_list, $variance;
     }
 
@@ -113 +130 @@
     }
 
-    check_files(@file_list);
-
-    my $command = "$ppstamp $outputBase $argString";
+    # check that actual input files exist
+    check_files($PSTAMP_GONE, @file_list);
+
+    # find our output directory
+    my $outdir = dirname($outputBase);
+    my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
+    
+    if ($muggle) {
+        # first see if the original uncensored images are around
+        if (check_for_backups($params, \$fileArgs)) {
+            # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images
+            print "Making stamps from backup images\n";
+        } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
+            # user required uncensored but since stage isn't chip we can't rebuild them
+            my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE);
+        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($stage ne 'chip')) {
+            # make stamps from uncensored images
+            $muggle = 0;
+        } else {
+            # Try and replace the streaks from the recovery images
+
+            @file_list = ();
+            $tmproot = "$outdir/$job_id";
+            mkdir $tmproot or
+                my_die( "failed to create temporary directory $tmproot", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+            my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
+            # find the "directory" of the input path_base
+            my $inputdir = dirname($image);
+            my $base = basename($image);
+            $tmpImage = "$tmproot/$base";
+
+            # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
+            # stage work. Import the rule here.
+            my $recImage = "$inputdir/REC_$base";
+            my $newFileArgs = " -file $tmpImage";
+            $muggle_command .= " -image $image -recimage $recImage";
+            push @file_list, $recImage;
+
+            if ($mask) {
+                $base = basename($mask);
+                $tmpMask = "$tmproot/$base";
+                $newFileArgs .= " -mask $tmpMask";
+                my $recMask = "$inputdir/REC_$base";
+                $muggle_command .= " -mask $mask -recmask $recMask";
+                push @file_list, $recMask;
+            }
+
+            if ($variance) {
+                $base = basename($variance);
+                $tmpVariance = "$tmproot/$base";
+                $newFileArgs .= " -weight $tmpVariance";
+                my $recVariance = "$inputdir/REC_$base";
+                $muggle_command .= " -weight $variance -recweight $recVariance";
+                push @file_list, $recVariance;
+            }
+            if (check_files(0, @file_list)) {
+                # recovery files exist and are accessible restore the excised pixels
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $muggle_command, verbose => $verbose);
+                unless ($success) {
+                    my $exitStatus = WEXITSTATUS($error_code);
+                    my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus);
+                }
+
+                # set the ppstamp file arguments
+                $fileArgs = $newFileArgs;
+            } else {
+                if ($options & $PSTAMP_REQUIRE_UNCENSORED) {
+                    my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE);
+                }
+                # just make stamps from the censored images
+                print "Unable to restore uncensored images, will extract stamps from censored images\n";
+                # these files won't be used so zap them
+                ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef);
+            }
+        }
+    }
+
+    my $command = "$ppstamp $outputBase $argString $fileArgs";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
-    $command .= " -stage $params->{stage}" if $params->{stage};
+    $command .= " -stage $stage";
     $command .= " -no_censor_masked" unless $nan_masked;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -131 +225 @@
     }
 
+    if (!$save_temps) {
+        unlink $tmpImage if $tmpImage;
+        unlink $tmpMask if $tmpMask;
+        unlink $tmpVariance if $tmpVariance;
+        rmdir $tmproot if $tmproot;
+    }
+
     if ($exitStatus == 0) {
-        my $dir = dirname($outputBase);
-
-        my $reglist = "$dir/reglist$job_id";
+        my $reglist = "$outdir/reglist$job_id";
 
         my $F;
@@ -363 +462 @@
 
 sub check_files {
+    my $error_code = shift;
+    my $return_code = 1;
     foreach my $f (@_) {
         if (!$ipprc->file_exists($f)) {
-            my_die( "file $f does not exist:", $job_id, $PSTAMP_GONE, 'stop');
-        }
-    }
+            if ($error_code) {
+                my_die( "file $f does not exist:", $job_id, $error_code, 'stop');
+            } else {
+                print STDERR "file $f does not exist\n";
+                $return_code = 0;
+            }
+        }
+    }
+    return $return_code;
+}
+
+sub check_for_backups {
+    my $params = shift;
+    my $r_fileArgs = shift;
+
+    my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}";
+    my $results = runToolAndParse($command, $verbose);
+    my $dsComponent  = $results->[0];
+
+    if ($dsComponent and ($dsComponent->{data_state} eq 'full')) {
+
+        print "magicDSFile state is full. Backup images should exist\n";
+
+        # replace the file names with the backup paths
+        # fileArgs has the form:  -file imagename [-mask maskname] [-weight weightname]
+        my @args = split " ", $$r_fileArgs;
+
+        my $newFileArgs;
+        while (@args) {
+            my $a = shift @args;
+            my $f = shift @args;
+            my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f;
+
+            my $backup_image_name = $ipprc->destreaked_filename($f);
+            my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name;
+            $newFileArgs .= " $a $backup_image_name";
+        }
+        $$r_fileArgs = $newFileArgs;
+
+        return 1;
+    }
+
+    return 0;
 }
 

trunk/pstamp/scripts/pstamp_parser_run.pl

@@ -31 +31 @@
 my $outdir;
 my $label;
+my $need_magic;
 my $verbose;
 my $dbname;
@@ -41 +42 @@
     'outdir=s'          =>  \$outdir,
     'label=s'           =>  \$label,
+    'need_magic'        =>  \$need_magic,
     'redirect-output'   =>  \$redirect_output,
     'verbose'           =>  \$verbose,
@@ -160 +162 @@
             $parse_cmd = "$pstampparse";
             $parse_cmd .= " --label $label" if $label;
+            $parse_cmd .= " --need_magic" if $need_magic;
             $request_fault = 0;
         } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {

trunk/pstamp/scripts/pstamp_server_status

@@ -44 +44 @@
 chdir $rundir or die "failed to cd to $rundir";
 
+
+my $down = 0;
+if ($down) {
+    print "Postage Stamp Server will be down for maintenance it will back at approximately 00:00 2010-12-11 UTC\n";
+    exit 0;
+}
 
 my ($pts, $pantasks_script) = tempfile ('/tmp/pts.XXXX', UNLINK => !$save_temps);

trunk/pstamp/scripts/pstampparse.pl

@@ -30 +30 @@
 my $save_temps;
 my $no_update;
+my $dest_requires_magic;
 
 GetOptions(
@@ -38 +39 @@
     'label=s'   =>  \$label,
     'mode=s'    =>  \$mode,
+    'need_magic'=>  \$dest_requires_magic,
     'dbname=s'  =>  \$dbname,
     'dbserver=s'=>  \$dbserver,
@@ -126 +128 @@
 }
 
-# Adjust the label for requests coming in over the web interaface
+
+# Adjust the label for requests coming in over the web interface
 
 my $label_changed = 0;
@@ -384 +387 @@
     my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
     if (!$proj_hash) {
-        foreach $row (@$rowList) {
-            insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
+        foreach my $r (@$rowList) {
+            insertFakeJobForRow($r, 1, $PSTAMP_UNKNOWN_PRODUCT);
             $num_jobs++;
         }
@@ -409 +412 @@
     $need_magic    = $proj_hash->{need_magic};
 
-    # Temporary hack so that IFA can get at non-magicked data
-    my $allow_mops_unmagicked = 1;
-    if ($allow_mops_unmagicked) {
-        if ($product and (($product eq "mops-pstamp-results") or
-                          ($product eq "mops-pstamp-results2") or
-                          ($product eq "ifa-pstamp-results"))) {
-            $need_magic = 0;
-        }
-    }
-
     $need_magic = 0 if $stage eq 'stack';
+
+    if ($need_magic) {
+
+        # this project requires that postage stamps be extracted from destreaked images
+
+        if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
+            # The user has requested uncensored stamps
+
+            if (!$dest_requires_magic) {
+                # and this user's data store destination is allowed uncensored stamps, so accept the request
+                $need_magic = 0;
+            } else {
+                print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
+                if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
+                    # user required uncensored stamps. Can't do it so fail.
+                    foreach my $r (@$rowList) {
+                        insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
+                        $num_jobs++;
+                    }
+                    return $num_jobs;
+                }
+
+                # user will accept censored stamps. alter OPTION_MASK and continue
+
+                print STDERR "    Will attempt to make destreaked stamps\n";
+                # zap the offending bit in the option mask
+                $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
+                foreach my $r (@$rowList) {
+                    $r->{OPTION_MASK} = $option_mask;
+                }
+            }
+        }
+    }
     
     my $numRows = scalar @$rowList;