Changeset 25256 for trunk/ippScripts/scripts/publish_file.pl

Timestamp:

Sep 2, 2009, 2:36:52 PM (17 years ago)

Author:

Paul Price

Message:

Merging branches/pap_mops into trunk. ppMops now merges multiple skycells, and these get published for MOPS as a single file per exposure. Tested and works.

Location:

trunk

Files:

• . (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (8 diffs)

trunk/ippScripts/scripts/publish_file.pl

@@ -39 +39 @@
 
 # Parse the command-line arguments
-my ( $pub_id, $camera, $stage, $stage_id, $format, $product, $workdir );
-my ( $dbname, $verbose, $no_update, $save_temps, $redirect );
+my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
+my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect );
 
 GetOptions(
@@ -48 +48 @@
     'stage_id=s'        => \$stage_id,    # Stage identifier
     'product=s'         => \$product,     # Datastore product name
+    'fileset=s'         => \$fileset,     # Fileset name
     'workdir=s'         => \$workdir,     # Working directory
     'dbname=s'          => \$dbname,    # Database name
     'verbose'           => \$verbose,   # Print to stdout
     'no-update'         => \$no_update, # Don't update the database?
+    'no-op'             => \$no_op, # Don't do any operations
     'save-temps'        => \$save_temps, # Save temporary files?
     'redirect-output'   => \$redirect,   # Redirect output to log file?
@@ -77 +79 @@
 my $mdcParser = PS::IPP::Metadata::Config->new;
 
-
-# Retrieve file name of interest
-my %files;                      # Input filenames
-my %zp;                         # Zero points
-my %exp_id;                     # Exposure identifiers
-my %exp_name;                   # Exposure names
-my %direction;                  # Direction of subtraction
-{
-    my $command;                # Command to run
-
-    if ($stage eq 'diff') {
-        $command =  "difftool -diffskyfile -diff_id $stage_id";
-    } elsif ($stage eq 'camera') {
-        $command =  "camtool -processedexp -cam_id $stage_id";
-    } else {
-        &my_die( "Unrecognised stage: $stage", $pub_id, $PS_EXIT_CONFIG_ERROR );
-    }
+my ($dsFile, $dsFileName) = tempfile("/tmp/publish.$pub_id.ds.XXXX", UNLINK => !$save_temps );
+
+if ($stage eq 'camera') {
+    my $command =  "camtool -processedexp -cam_id $stage_id";
     $command .= " -dbname $dbname" if defined $dbname;
-
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -106 +94 @@
         &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
 
+    &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$components > 1;
+
+    my $comp = $$components[0]; # Component of interest
+    my $path_base = $comp->{path_base}; # Base name for file
+    my $file = $ipprc->filename( "PSASTRO.OUTPUT", $path_base );
+    $file = $ipprc->file_resolve($file);
+    my $cam_id = $comp->{cam_id};
+    my $name = "cam_$cam_id";
+    print $dsFile "$file|||$product|$name|\n";
+} elsif ($stage eq 'diff') {
+    my $command =  "difftool -diffskyfile -diff_id $stage_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
+
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
+
+    my $components = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
+
+    my ($mopsPositiveFile, $mopsPositiveFileName) = tempfile("/tmp/publish.$pub_id.mops.pos.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
+    my ($mopsNegativeFile, $mopsNegativeFileName) = tempfile("/tmp/publish.$pub_id.mops.neg.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
+
+    my %positive;               # Data for positive diff detections
+    my %negative;               # Data for negative diff detections
     foreach my $comp ( @$components ) {
         my $path_base = $comp->{path_base}; # Base name for file
@@ -112 +127 @@
         (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time};
         my $zp = $comp->{zpt_obs} + 2.5 * log($comp->{exp_time}) / log(10);
-
-        if ($stage eq 'diff') {
-            my $skycell_id = $comp->{skycell_id};
-
-            # Positive direction
-            $files{"$skycell_id.pos"} = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base );
-            $zp{"$skycell_id.pos"} = $zp;
-            $exp_id{"$skycell_id.pos"} = $comp->{exp_id_1};
-            $exp_name{"$skycell_id.pos"} = $comp->{exp_name_1};
-            $direction{"$skycell_id.pos"} = 1;
-
-            # Negative direction
-            if (defined $comp->{bothways} and $comp->{bothways}) {
-                $files{"$skycell_id.neg"} = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base );
-                $zp{"$skycell_id.neg"} = $zp;
-                $exp_id{"$skycell_id.neg"} = $comp->{exp_id_2};
-                $direction{"$skycell_id.neg"} = 0;
-                $exp_name{"$skycell_id.neg"} = $comp->{exp_name_2};
+        my $astrom = sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2);
+
+        my $skycell_id = $comp->{skycell_id};
+        my $filename = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base );
+        $filename = $ipprc->file_resolve($filename);
+
+        my $data = { zp => $zp,
+                     zp_err => $comp->{zpt_stdev},
+                     astrom => sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2),
+                     exp_name => $comp->{exp_name_1},
+                     exp_id => $comp->{exp_id_1},
+                     chip_id => $comp->{chip_id_1},
+                     cam_id => $comp->{cam_id_1},
+                     fake_id => $comp->{fake_id_1},
+                     warp_id => $comp->{warp1},
+                     diff_id => $comp->{diff_id},
+                     direction => 1,
+        };
+
+        diff_check(\%positive, $data, "positive");
+
+        if ($product eq 'IPP-MOPS') {
+            print $mopsPositiveFile "$filename\n";
+        } else {
+            print $dsFile "$filename|||$product|${skycell_id}.pos|\n";
+        }
+
+        # Negative direction
+        if (defined $comp->{bothways} and $comp->{bothways}) {
+            my $filename = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base );
+            $filename = $ipprc->file_resolve($filename);
+
+            my $data = { zp => $zp,
+                         zp_err => $comp->{zpt_stdev},
+                         astrom => sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2),
+                         exp_name => $comp->{exp_name_2},
+                         exp_id => $comp->{exp_id_2},
+                         chip_id => $comp->{chip_id_2},
+                         cam_id => $comp->{cam_id_2},
+                         fake_id => $comp->{fake_id_2},
+                         warp_id => $comp->{warp2},
+                         diff_id => $comp->{diff_id},
+                         direction => 0,
+            };
+
+            diff_check(\%negative, $data, "negative");
+
+            if ($product eq 'IPP-MOPS') {
+                print $mopsNegativeFile "$filename\n";
+            } else {
+                print $dsFile "$filename|||$product|${skycell_id}.neg|\n";
             }
-
-        } elsif ($stage eq 'camera') {
-            my $cam_id = $comp->{cam_id};
-            $files{$cam_id} = $ipprc->filename( "PSASTRO.OUTPUT", $path_base );
-            $zp{$cam_id} = $zp;
-            $exp_id{$cam_id} = $comp->{exp_id};
-            $exp_name{$cam_id} = $comp->{exp_name};
-            $direction{$cam_id} = 1;
         }
     }
-}
-
-# Prepare for data store input
-my ($listFile, $listFileName) = tempfile("/tmp/publish.$pub_id.list.XXXX", UNLINK => !$save_temps );
-
-# Process each file
-foreach my $comp ( keys %files ) {
-    my $file = $ipprc->file_resolve( $files{$comp} ) or
-        &my_die("Unable to resolve file $files{$comp}", $pub_id, $PS_EXIT_SYS_ERROR);
-    my_die("Unable to find file $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $file );
-
-    my $zp = $zp{$comp};
-    my $exp_id = $exp_id{$comp};
-    my $exp_name = $exp_name{$comp};
-    my $direction = $direction{$comp};
-    if ($product eq "IPP-MOPS") {
-        my $outuri = "$outroot.$comp.fits";
-        my $out = $ipprc->file_resolve( $outuri, 'create' ) or
-            &my_die( "Unable to resolve output file $outuri", $pub_id, $PS_EXIT_SYS_ERROR);
-
-        my $command = "$ppMops $file $zp $exp_id $exp_name $direction $out";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        &my_die( "Unable to translate $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
-        &my_die( "Unable to find translated file $out", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $out );
-        $file = $out;
+
+    close $mopsPositiveFile;
+    close $mopsNegativeFile;
+
+    if ($product eq 'IPP-MOPS') {
+        if (scalar keys %positive > 0) {
+            my $output = mops_combine(\%positive, "$outroot.pos.mops", $mopsPositiveFileName);
+            print $dsFile "$output|||$product|positive|\n";
+        }
+        if (scalar keys %negative > 0) {
+            my $output = mops_combine(\%negative, "$outroot.neg.mops", $mopsNegativeFileName);
+            print $dsFile "$output|||$product|negative|\n";
+        }
     }
-
-    # format: filename|filesize|md5sum|filetype|
-    # note: since we omit filesize and md5sum, dsreg will calculate them
-    print $listFile "$file|||$product|$comp|\n";
-}
-
+}
+
+close $dsFile;
 
 unless ($no_update) {
-    my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $listFileName";
+    my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $dsFileName";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -185 +209 @@
 unless ($no_update) {
     my $command = "$pubtool -add -pub_id $pub_id -path_base $outroot";
+    $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
+    $command .= " -hostname $host" if defined $host;
+
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -195 +222 @@
 
 ### Pau.
+
+# Check inputs for a diff
+sub diff_check
+{
+    my $data = shift;           # Data hash
+    my $comp = shift;           # Component data
+    my $name = shift;           # Name of component
+
+    $data->{zp} = $comp->{zp} unless defined $data->{zp};
+    $data->{zp_err} = $comp->{zp_err} unless defined $data->{zp_err};
+    $data->{astrom} = $comp->{astrom} unless defined $data->{astrom};
+    $data->{exp_name} = $comp->{exp_name} unless defined $data->{exp_name};
+    $data->{exp_id} = $comp->{exp_id} unless defined $data->{exp_id};
+    $data->{chip_id} = $comp->{chip_id} unless defined $data->{chip_id};
+    $data->{cam_id} = $comp->{cam_id} unless defined $data->{cam_id};
+    $data->{fake_id} = $comp->{fake_id} unless defined $data->{fake_id};
+    $data->{warp_id} = $comp->{warp_id} unless defined $data->{warp_id};
+    $data->{diff_id} = $comp->{diff_id} unless defined $data->{diff_id};
+    $data->{direction} = $comp->{direction} unless defined $data->{direction};
+
+    &my_die("zp value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp} and $comp->{zp} != $data->{zp};
+    &my_die("zp_err value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp_err} and $comp->{zp_err} != $data->{zp_err};
+    &my_die("astrom value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{astrom} and $comp->{astrom} != $data->{astrom};
+    &my_die("exp_name value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_name} and $comp->{exp_name} ne $data->{exp_name};
+    &my_die("exp_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_id} and $comp->{exp_id} != $data->{exp_id};
+    &my_die("chip_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{chip_id} and $comp->{chip_id} != $data->{chip_id};
+    &my_die("cam_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{cam_id} and $comp->{cam_id} != $data->{cam_id};
+    &my_die("fake_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{fake_id} and $comp->{fake_id} != $data->{fake_id};
+    &my_die("warp_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{warp_id} and $comp->{warp_id} != $data->{warp_id};
+    &my_die("diff_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{diff_id} and $comp->{diff_id} != $data->{diff_id};
+    &my_die("direction value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{direction} and $comp->{direction} != $data->{direction};
+
+    return 1;
+}
+
+# Combine multiple files for MOPS
+sub mops_combine
+{
+    my $data = shift;           # Data
+    my $output = shift;         # Output name
+    my $input = shift;          # Input name
+
+    $output = $ipprc->file_resolve( $output, 'create' ) or
+        &my_die( "Unable to resolve output file $output", $pub_id, $PS_EXIT_SYS_ERROR);
+
+    my $command = "$ppMops $input $output";
+    $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name};
+    $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id};
+    $command .= " -chip_id " . $data->{chip_id} if defined $data->{chip_id};
+    $command .= " -cam_id " . $data->{cam_id} if defined $data->{cam_id};
+    $command .= " -fake_id " . $data->{fake_id} if defined $data->{fake_id};
+    $command .= " -warp_id " . $data->{warp_id} if defined $data->{warp_id};
+    $command .= " -diff_id " . $data->{diff_id} if defined $data->{diff_id};
+    $command .= " -inverse" if defined $data->{direction} and $data->{direction} == 0;
+    $command .= " -zp " . $data->{zp} if defined $data->{zp};
+    $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err};
+    $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom};
+
+    unless ($no_op) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        &my_die( "Unable to translate", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
+        &my_die( "Unable to find translated file $output", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $output );
+    } else {
+        print "Not running: $command\n";
+    }
+
+    return $output;
+}
 
 
@@ -210 +306 @@
         $command .= " -pub_id $pub_id";
         $command .= " -path_base $outroot";
+        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
+        $command .= " -hostname $host" if defined $host;
         $command .= " -fault $fault";
         $command .= " -dbname $dbname" if defined $dbname;