IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 25225


Ignore:
Timestamp:
Aug 31, 2009, 6:45:15 PM (17 years ago)
Author:
Paul Price
Message:

Reworking to support merging of skycells for MOPS.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/pap_mops/ippScripts/scripts/publish_file.pl

    r25202 r25225  
    3939
    4040# Parse the command-line arguments
    41 my ( $pub_id, $camera, $stage, $stage_id, $format, $product, $workdir );
     41my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
    4242my ( $dbname, $verbose, $no_update, $save_temps, $redirect );
    4343
    … …  
    4848    'stage_id=s'        => \$stage_id,    # Stage identifier
    4949    'product=s'         => \$product,     # Datastore product name
     50    'fileset=s'         => \$fileset,     # Fileset name
    5051    'workdir=s'         => \$workdir,     # Working directory
    5152    'dbname=s'          => \$dbname,    # Database name
    … …  
    7778my $mdcParser = PS::IPP::Metadata::Config->new;
    7879
    79 
    80 # Retrieve file name of interest
    81 my %data;                       # Data to publish
    82 {
    83     my $command;                # Command to run
    84 
    85     if ($stage eq 'diff') {
    86         $command =  "difftool -diffskyfile -diff_id $stage_id";
    87     } elsif ($stage eq 'camera') {
    88         $command =  "camtool -processedexp -cam_id $stage_id";
    89     } else {
    90         &my_die( "Unrecognised stage: $stage", $pub_id, $PS_EXIT_CONFIG_ERROR );
    91     }
    92     $command .= " -dbname $dbname" if defined $dbname;
    93 
     80my ($dsFile, $dsFileName) = tempfile("/tmp/publish.$pub_id.ds.XXXX", UNLINK => !$save_temps );
     81
     82if ($stage eq 'camera') {
     83    my $command =  "camtool -processedexp -cam_id $stage_id";
    9484    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    9585        run(command => $command, verbose => $verbose);
    … …  
    10292        &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
    10393
     94    &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$components > 1;
     95
     96    my $comp = $$components[0]; # Component of interest
     97    my $path_base = $comp->{path_base}; # Base name for file
     98    my $file = $ipprc->filename( "PSASTRO.OUTPUT", $path_base );
     99    my $cam_id = $comp->{cam_id};
     100    my $name = "cam_$cam_id";
     101    print $dsFile "$file|||$product|$name|\n";
     102} elsif ($stage eq 'diff') {
     103    my $command =  "difftool -diffskyfile -diff_id $stage_id";
     104    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     105        run(command => $command, verbose => $verbose);
     106    &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
     107
     108    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     109        &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
     110
     111    my $components = parse_md_list($metadata) or
     112        &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
     113
     114    my ($mopsPositiveFile, $mopsPositiveFileName) = tempfile("/tmp/publish.$pub_id.mops.pos.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
     115    my ($mopsNegativeFile, $mopsNegativeFileName) = tempfile("/tmp/publish.$pub_id.mops.neg.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
     116
     117    my %positive;               # Data for positive diff detections
     118    my %negative;               # Data for negative diff detections
    104119    foreach my $comp ( @$components ) {
    105120        my $path_base = $comp->{path_base}; # Base name for file
    … …  
    108123        (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time};
    109124        my $zp = $comp->{zpt_obs} + 2.5 * log($comp->{exp_time}) / log(10);
    110 
    111         if ($stage eq 'diff') {
    112             my $skycell_id = $comp->{skycell_id};
    113 
    114             my $data = { filename => $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base ),
    115                          zp => $zp,
    116                          zp_err => $comp->{zpt_stdev},
    117                          astrom => sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2),
    118                          exp_name => $comp->{exp_name_1},
    119                          exp_id => $comp->{exp_id_1},
    120                          chip_id => $comp->{chip_id_1},
    121                          cam_id => $comp->{cam_id_1},
    122                          fake_id => $comp->{fake_id_1},
    123                          warp_id => $comp->{warp1},
    124                          direction => 1,
    125                      };
    126 
    127             $data{"$skycell_id.pos"} = $data;
    128 
    129             # Negative direction
    130             if (defined $comp->{bothways} and $comp->{bothways}) {
    131                 my $data = { filename => $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base ),
    132                              zp => $zp,
    133                              zp_err => $comp->{zpt_stdev},
    134                              astrom => sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2),
    135                              exp_name => $comp->{exp_name_2},
    136                              exp_id => $comp->{exp_id_2},
    137                              chip_id => $comp->{chip_id_2},
    138                              cam_id => $comp->{cam_id_2},
    139                              fake_id => $comp->{fake_id_2},
    140                              warp_id => $comp->{warp2},
    141                              direction => 0,
    142                          };
    143 
    144                 $data{"$skycell_id.neg"} = $data;
    145             }
    146         } elsif ($stage eq 'camera') {
    147             my $data = { filename => $ipprc->filename( "PSASTRO.OUTPUT", $path_base ),
    148                          zp => $zp,
     125        my $astrom = sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2);
     126
     127        my $skycell_id = $comp->{skycell_id};
     128        my $filename = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base );
     129
     130        my $data = { zp => $zp,
     131                     zp_err => $comp->{zpt_stdev},
     132                     astrom => sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2),
     133                     exp_name => $comp->{exp_name_1},
     134                     exp_id => $comp->{exp_id_1},
     135                     chip_id => $comp->{chip_id_1},
     136                     cam_id => $comp->{cam_id_1},
     137                     fake_id => $comp->{fake_id_1},
     138                     warp_id => $comp->{warp1},
     139                     direction => 1,
     140        };
     141
     142        diff_check(\%positive, $data);
     143
     144        if ($product eq 'IPP-MOPS') {
     145            print $mopsPositiveFile "$filename\n";
     146        } else {
     147            print $dsFile "$filename|||$product|${skycell_id}.pos|\n";
     148        }
     149
     150        # Negative direction
     151        if (defined $comp->{bothways} and $comp->{bothways}) {
     152            my $filename = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base );
     153            my $data = { zp => $zp,
    149154                         zp_err => $comp->{zpt_stdev},
    150155                         astrom => sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2),
    151                          exp_name => $comp->{exp_name},
    152                          exp_id => $comp->{exp_id},
    153                          chip_id => $comp->{chip_id},
    154                          cam_id => $comp->{cam_id},
    155                      };
    156 
    157             my $cam_id = $comp->{cam_id};
    158             $data{"cam.$cam_id"} = $data;
     156                         exp_name => $comp->{exp_name_2},
     157                         exp_id => $comp->{exp_id_2},
     158                         chip_id => $comp->{chip_id_2},
     159                         cam_id => $comp->{cam_id_2},
     160                         fake_id => $comp->{fake_id_2},
     161                         warp_id => $comp->{warp2},
     162                         direction => 0,
     163            };
     164
     165            diff_check(\%negative, $data);
     166
     167            if ($product eq 'IPP-MOPS') {
     168                print $mopsNegativeFile "$filename\n";
     169            } else {
     170                print $dsFile "$filename|||$product|${skycell_id}.neg|\n";
     171            }
    159172        }
    160173    }
    161 }
    162 
    163 # Prepare for data store input
    164 my ($listFile, $listFileName) = tempfile("/tmp/publish.$pub_id.list.XXXX", UNLINK => !$save_temps );
    165 
    166 # Process each file
    167 my ($zp, $zp_err, $astrom, $exp_name, $exp_id, $chip_id, $cam_id, $fake_id, $warp_id, $direction);
    168 foreach my $comp ( keys %data ) {
    169     my $data = $data{$comp};
    170     my $file = $ipprc->file_resolve( $data->{filename} ) or
    171         &my_die("Unable to resolve file $files{$comp}", $pub_id, $PS_EXIT_SYS_ERROR);
    172     my_die("Unable to find file $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $file );
    173 
    174     $zp = $data->{zp} unless defined $zp;
    175     $zp_err = $data->{zp_err} unless defined $zp_err;
    176     $astrom = $data->{astrom} unless defined $astrom;
    177     $exp_name = $data->{exp_name} unless defined $exp_name;
    178     $exp_id = $data->{exp_id} unless defined $exp_id;
    179     $chip_id = $data->{chip_id} unless defined $chip_id;
    180     $cam_id = $data->{cam_id} unless defined $cam_id;
    181     $fake_id = $data->{fake_id} unless defined $fake_id;
    182     $warp_id = $data->{warp_id} unless defined $warp_id;
    183     $direction = $data->{direction} unless defined $direction;
    184 
    185     &my_die("zp value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $zp and $zp != $data->{zp};
    186     &my_die("zp_err value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $zp_err and $zp_err != $data->{zp_err};
    187     &my_die("astrom value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $astrom and $astrom != $data->{astrom};
    188     &my_die("exp_name value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $exp_name and $exp_name ne $data->{exp_name};
    189     &my_die("exp_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $exp_id and $zp != $data->{exp_id};
    190     &my_die("chip_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $chip_id and $chip_id != $data->{chip_id};
    191     &my_die("cam_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $cam_id and $cam_id != $data->{cam_id};
    192     &my_die("fake_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $fake_id and $fake_id != $data->{fake_id};
    193     &my_die("warp_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $warp_id and $warp_id != $data->{warp_id};
    194     &my_die("direction value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $direction and $direction != $data->{direction};
    195 
    196     if ($product eq "IPP-MOPS") {
    197         my $outuri = "$outroot.$comp.fits";
    198         my $out = $ipprc->file_resolve( $outuri, 'create' ) or
    199             &my_die( "Unable to resolve output file $outuri", $pub_id, $PS_EXIT_SYS_ERROR);
    200 
    201         my $command = "$ppMops $file $zp $exp_id $exp_name $direction $out";
    202         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    203             run(command => $command, verbose => $verbose);
    204         &my_die( "Unable to translate $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
    205         &my_die( "Unable to find translated file $out", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $out );
    206         $file = $out;
     174
     175    close $mopsPositiveFile;
     176    close $mopsNegativeFile;
     177
     178    if ($product eq 'IPP-MOPS') {
     179        if (scalar keys %positive > 0) {
     180            my $output = mops_combine(\%positive, "$outroot.pos.mops", $mopsPositiveFileName);
     181            print $dsFile "$output|||$product|positive|\n";
     182        }
     183        if (scalar keys %negative > 0) {
     184            my $output = mops_combine(\%negative, "$outroot.neg.mops", $mopsNegativeFileName);
     185            print $dsFile "$output|||$product|negative|\n";
     186        }
    207187    }
    208 
    209     # format: filename|filesize|md5sum|filetype|
    210     # note: since we omit filesize and md5sum, dsreg will calculate them
    211     print $listFile "$file|||$product|$comp|\n";
    212 }
    213 
     188}
     189
     190close $dsFile;
    214191
    215192unless ($no_update) {
    216     my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $listFileName";
     193    my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $dsFileName";
    217194
    218195    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    233210
    234211### Pau.
     212
     213# Check inputs for a diff
     214sub diff_check
     215{
     216    my $data = shift;           # Data hash
     217    my $comp = shift;           # Component data
     218    my $name = shift;           # Name of component
     219
     220    $data->{zp} = $comp->{zp} unless defined $data->{zp};
     221    $data->{zp_err} = $comp->{zp_err} unless defined $data->{zp_err};
     222    $data->{astrom} = $comp->{astrom} unless defined $data->{astrom};
     223    $data->{exp_name} = $comp->{exp_name} unless defined $data->{exp_name};
     224    $data->{exp_id} = $comp->{exp_id} unless defined $data->{exp_id};
     225    $data->{chip_id} = $comp->{chip_id} unless defined $data->{chip_id};
     226    $data->{cam_id} = $comp->{cam_id} unless defined $data->{cam_id};
     227    $data->{fake_id} = $comp->{fake_id} unless defined $data->{fake_id};
     228    $data->{warp_id} = $comp->{warp_id} unless defined $data->{warp_id};
     229    $data->{direction} = $comp->{direction} unless defined $data->{direction};
     230
     231    &my_die("zp value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp} and $comp->{zp} != $data->{zp};
     232    &my_die("zp_err value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp_err} and $comp->{zp_err} != $data->{zp_err};
     233    &my_die("astrom value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{astrom} and $comp->{astrom} != $data->{astrom};
     234    &my_die("exp_name value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_name} and $comp->{exp_name} ne $data->{exp_name};
     235    &my_die("exp_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_id} and $comp->{zp} != $data->{exp_id};
     236    &my_die("chip_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{chip_id} and $comp->{chip_id} != $data->{chip_id};
     237    &my_die("cam_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{cam_id} and $comp->{cam_id} != $data->{cam_id};
     238    &my_die("fake_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{fake_id} and $comp->{fake_id} != $data->{fake_id};
     239    &my_die("warp_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{warp_id} and $comp->{warp_id} != $data->{warp_id};
     240    &my_die("direction value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{direction} and $comp->{direction} != $data->{direction};
     241
     242    return 1;
     243}
     244
     245# Combine multiple files for MOPS
     246sub mops_combine
     247{
     248    my $data = shift;           # Data
     249    my $output = shift;         # Output name
     250    my $input = shift;          # Input name
     251
     252    $output = $ipprc->file_resolve( $output, 'create' ) or
     253        &my_die( "Unable to resolve output file $output", $pub_id, $PS_EXIT_SYS_ERROR);
     254
     255    my $command = "$ppMops $input $output";
     256    $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name};
     257    $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id};
     258    $command .= " -chip_id " . $data->{chip_id} if defined $data->{chip_id};
     259    $command .= " -cam_id " . $data->{cam_id} if defined $data->{cam_id};
     260    $command .= " -fake_id " . $data->{fake_id} if defined $data->{fake_id};
     261    $command .= " -warp_id " . $data->{warp_id} if defined $data->{warp_id};
     262    $command .= " -diff_id " . $data->{diff_id} if defined $data->{diff_id};
     263    $command .= " -inverse" if defined $data->{direction} and $data->{direction} == 0;
     264    $command .= " -zp " . $data->{zp} if defined $data->{zp};
     265    $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err};
     266    $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom};
     267
     268    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     269        run(command => $command, verbose => $verbose);
     270    &my_die( "Unable to translate", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
     271    &my_die( "Unable to find translated file $output", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $output );
     272
     273    return $output;
     274}
    235275
    236276
Note: See TracChangeset for help on using the changeset viewer.