IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Sep 15, 2009, 4:02:42 PM (17 years ago)
Author:
eugene
Message:

updates from trunk

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/20090715/ippScripts/scripts/publish_file.pl

    r25022 r25406  
    3939
    4040# Parse the command-line arguments
    41 my ( $pub_id, $exp_id, $camera, $stage, $stage_id, $format, $product, $workdir );
    42 my ( $dbname, $verbose, $no_update, $save_temps, $redirect );
     41my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
     42my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect );
    4343
    4444GetOptions(
    4545    'pub_id=s'          => \$pub_id, # Publish identifier
    46     'exp_id=s'          => \$exp_id, # Exposure identifier
    4746    'camera=s'          => \$camera, # Camera name
    4847    'stage=s'           => \$stage,       # Stage of interest
    4948    'stage_id=s'        => \$stage_id,    # Stage identifier
    5049    'product=s'         => \$product,     # Datastore product name
     50    'fileset=s'         => \$fileset,     # Fileset name
    5151    'workdir=s'         => \$workdir,     # Working directory
    5252    'dbname=s'          => \$dbname,    # Database name
    5353    'verbose'           => \$verbose,   # Print to stdout
    5454    'no-update'         => \$no_update, # Don't update the database?
     55    'no-op'             => \$no_op, # Don't do any operations
    5556    'save-temps'        => \$save_temps, # Save temporary files?
    5657    'redirect-output'   => \$redirect,   # Redirect output to log file?
    … …  
    5859
    5960pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    60 pod2usage( -msg => "Required options: --pub_id --exp_id --camera --stage --stage_id --product --workdir",
     61pod2usage( -msg => "Required options: --pub_id --camera --stage --stage_id --product --workdir",
    6162           -exitval => $PS_EXIT_CONFIG_ERROR) unless
    6263    defined $pub_id and
    63     defined $exp_id and
    6464    defined $camera and
    6565    defined $product and
    … …  
    6868    defined $workdir;
    6969
    70 my $outroot = "$workdir/$product.$pub_id.$exp_id"; # Output root name
     70my $outroot = "$workdir/$product.$pub_id"; # Output root name
    7171
    7272my $ipprc = PS::IPP::Config->new( $camera ) or
    … …  
    7979my $mdcParser = PS::IPP::Metadata::Config->new;
    8080
    81 
    82 # Retrieve file name of interest
    83 my %files;                      # Input filenames
    84 my %zp;                         # Zero points
    85 {
    86     my $command;                # Command to run
    87 
    88     if ($stage eq 'diff') {
    89         $command =  "difftool -diffskyfile -diff_id $stage_id";
    90     } elsif ($stage eq 'camera') {
    91         $command =  "camtool -processedexp -cam_id $stage_id";
    92     } else {
    93         &my_die( "Unrecognised stage: $stage", $pub_id, $PS_EXIT_CONFIG_ERROR );
    94     }
     81my ($dsFile, $dsFileName) = tempfile("/tmp/publish.$pub_id.ds.XXXX", UNLINK => !$save_temps );
     82
     83if ($stage eq 'camera') {
     84    my $command =  "camtool -processedexp -cam_id $stage_id";
    9585    $command .= " -dbname $dbname" if defined $dbname;
    96 
    9786    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    9887        run(command => $command, verbose => $verbose);
    … …  
    10594        &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
    10695
     96    &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$components > 1;
     97
     98    my $comp = $$components[0]; # Component of interest
     99    my $path_base = $comp->{path_base}; # Base name for file
     100    my $file = $ipprc->filename( "PSASTRO.OUTPUT", $path_base );
     101    $file = $ipprc->file_resolve($file);
     102    my $cam_id = $comp->{cam_id};
     103    my $name = "cam_$cam_id";
     104    print $dsFile "$file|||$product|$name|\n";
     105} elsif ($stage eq 'diff') {
     106    my $command =  "difftool -diffskyfile -diff_id $stage_id";
     107    $command .= " -dbname $dbname" if defined $dbname;
     108    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     109        run(command => $command, verbose => $verbose);
     110    &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
     111
     112    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     113        &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
     114
     115    my $components = parse_md_list($metadata) or
     116        &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
     117
     118    my ($mopsPositiveFile, $mopsPositiveFileName) = tempfile("/tmp/publish.$pub_id.mops.pos.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
     119    my ($mopsNegativeFile, $mopsNegativeFileName) = tempfile("/tmp/publish.$pub_id.mops.neg.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
     120
     121    my %positive;               # Data for positive diff detections
     122    my %negative;               # Data for negative diff detections
    107123    foreach my $comp ( @$components ) {
    108124        my $path_base = $comp->{path_base}; # Base name for file
    … …  
    111127        (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time};
    112128        my $zp = $comp->{zpt_obs} + 2.5 * log($comp->{exp_time}) / log(10);
    113 
    114         if ($stage eq 'diff') {
    115             my $skycell_id = $comp->{skycell_id};
    116             $files{"$skycell_id.pos"} = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base );
    117             $files{"$skycell_id.neg"} = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base ) if
    118                 defined $comp->{bothways} and $comp->{bothways};
    119             $zp{"$skycell_id.pos"} = $zp;
    120             $zp{"$skycell_id.neg"} = $zp if defined $comp->{bothways} and $comp->{bothways};
    121         } elsif ($stage eq 'camera') {
    122             my $cam_id = $comp->{cam_id};
    123             $files{$cam_id} = $ipprc->filename( "PSASTRO.OUTPUT", $path_base );
    124             $zp{$cam_id} = $zp;
     129        my $astrom = sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2);
     130
     131        my $skycell_id = $comp->{skycell_id};
     132        my $filename = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base );
     133        $filename = $ipprc->file_resolve($filename);
     134
     135        my $data = { zp => $zp,
     136                     zp_err => $comp->{zpt_stdev},
     137                     astrom => sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2),
     138                     exp_name => $comp->{exp_name_1},
     139                     exp_id => $comp->{exp_id_1},
     140                     chip_id => $comp->{chip_id_1},
     141                     cam_id => $comp->{cam_id_1},
     142                     fake_id => $comp->{fake_id_1},
     143                     warp_id => $comp->{warp1},
     144                     diff_id => $comp->{diff_id},
     145                     direction => 1,
     146        };
     147
     148        diff_check(\%positive, $data, "positive");
     149
     150        if ($product eq 'IPP-MOPS') {
     151            print $mopsPositiveFile "$filename\n";
     152        } else {
     153            print $dsFile "$filename|||$product|${skycell_id}.pos|\n";
     154        }
     155
     156        # Negative direction
     157        if (defined $comp->{bothways} and $comp->{bothways}) {
     158            my $filename = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base );
     159            $filename = $ipprc->file_resolve($filename);
     160
     161            my $data = { zp => $zp,
     162                         zp_err => $comp->{zpt_stdev},
     163                         astrom => sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2),
     164                         exp_name => $comp->{exp_name_2},
     165                         exp_id => $comp->{exp_id_2},
     166                         chip_id => $comp->{chip_id_2},
     167                         cam_id => $comp->{cam_id_2},
     168                         fake_id => $comp->{fake_id_2},
     169                         warp_id => $comp->{warp2},
     170                         diff_id => $comp->{diff_id},
     171                         direction => 0,
     172            };
     173
     174            diff_check(\%negative, $data, "negative");
     175
     176            if ($product eq 'IPP-MOPS') {
     177                print $mopsNegativeFile "$filename\n";
     178            } else {
     179                print $dsFile "$filename|||$product|${skycell_id}.neg|\n";
     180            }
    125181        }
    126182    }
    127 }
    128 
    129 # Prepare for data store input
    130 my ($listFile, $listFileName) = tempfile("/tmp/publish.$pub_id.list.XXXX", UNLINK => !$save_temps );
    131 
    132 # Process each file
    133 foreach my $comp ( keys %files ) {
    134     my $file = $ipprc->file_resolve( $files{$comp} ) or
    135         &my_die("Unable to resolve file $files{$comp}", $pub_id, $PS_EXIT_SYS_ERROR);
    136     my_die("Unable to find file $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $file );
    137 
    138     my $zp = $zp{$comp};
    139     if ($product eq "IPP-MOPS") {
    140         my $outuri = "$outroot.$comp.fits";
    141         my $out = $ipprc->file_resolve( $outuri, 'create' ) or
    142             &my_die( "Unable to resolve output file $outuri", $pub_id, $PS_EXIT_SYS_ERROR);
    143 
    144         my $command = "$ppMops $file $zp $out";
    145         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    146             run(command => $command, verbose => $verbose);
    147         &my_die( "Unable to translate $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
    148         &my_die( "Unable to find translated file $out", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $out );
    149         $file = $out;
     183
     184    close $mopsPositiveFile;
     185    close $mopsNegativeFile;
     186
     187    if ($product eq 'IPP-MOPS') {
     188        if (scalar keys %positive > 0) {
     189            my $output = mops_combine(\%positive, "$outroot.pos.mops", $mopsPositiveFileName);
     190            print $dsFile "$output|||$product|positive|\n";
     191        }
     192        if (scalar keys %negative > 0) {
     193            my $output = mops_combine(\%negative, "$outroot.neg.mops", $mopsNegativeFileName);
     194            print $dsFile "$output|||$product|negative|\n";
     195        }
    150196    }
    151 
    152     # format: filename|filesize|md5sum|filetype|
    153     # note: since we omit filesize and md5sum, dsreg will calculate them
    154     print $listFile "$file|||$product|$comp|\n";
    155 }
    156 
     197}
     198
     199close $dsFile;
    157200
    158201unless ($no_update) {
    159     my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $listFileName";
     202    my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $dsFileName";
    160203
    161204    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    166209unless ($no_update) {
    167210    my $command = "$pubtool -add -pub_id $pub_id -path_base $outroot";
     211    $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     212    $command .= " -hostname $host" if defined $host;
     213
    168214    $command .= " -dbname $dbname" if defined $dbname;
    169215
    … …  
    176222
    177223### Pau.
     224
     225# Check inputs for a diff
     226sub diff_check
     227{
     228    my $data = shift;           # Data hash
     229    my $comp = shift;           # Component data
     230    my $name = shift;           # Name of component
     231
     232    $data->{zp} = $comp->{zp} unless defined $data->{zp};
     233    $data->{zp_err} = $comp->{zp_err} unless defined $data->{zp_err};
     234    $data->{astrom} = $comp->{astrom} unless defined $data->{astrom};
     235    $data->{exp_name} = $comp->{exp_name} unless defined $data->{exp_name};
     236    $data->{exp_id} = $comp->{exp_id} unless defined $data->{exp_id};
     237    $data->{chip_id} = $comp->{chip_id} unless defined $data->{chip_id};
     238    $data->{cam_id} = $comp->{cam_id} unless defined $data->{cam_id};
     239    $data->{fake_id} = $comp->{fake_id} unless defined $data->{fake_id};
     240    $data->{warp_id} = $comp->{warp_id} unless defined $data->{warp_id};
     241    $data->{diff_id} = $comp->{diff_id} unless defined $data->{diff_id};
     242    $data->{direction} = $comp->{direction} unless defined $data->{direction};
     243
     244    &my_die("zp value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp} and $comp->{zp} != $data->{zp};
     245    &my_die("zp_err value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp_err} and $comp->{zp_err} != $data->{zp_err};
     246    &my_die("astrom value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{astrom} and $comp->{astrom} != $data->{astrom};
     247    &my_die("exp_name value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_name} and $comp->{exp_name} ne $data->{exp_name};
     248    &my_die("exp_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_id} and $comp->{exp_id} != $data->{exp_id};
     249    &my_die("chip_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{chip_id} and $comp->{chip_id} != $data->{chip_id};
     250    &my_die("cam_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{cam_id} and $comp->{cam_id} != $data->{cam_id};
     251    &my_die("fake_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{fake_id} and $comp->{fake_id} != $data->{fake_id};
     252    &my_die("warp_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{warp_id} and $comp->{warp_id} != $data->{warp_id};
     253    &my_die("diff_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{diff_id} and $comp->{diff_id} != $data->{diff_id};
     254    &my_die("direction value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{direction} and $comp->{direction} != $data->{direction};
     255
     256    return 1;
     257}
     258
     259# Combine multiple files for MOPS
     260sub mops_combine
     261{
     262    my $data = shift;           # Data
     263    my $output = shift;         # Output name
     264    my $input = shift;          # Input name
     265
     266    $output = $ipprc->file_resolve( $output, 'create' ) or
     267        &my_die( "Unable to resolve output file $output", $pub_id, $PS_EXIT_SYS_ERROR);
     268
     269    my $command = "$ppMops $input $output";
     270    $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name};
     271    $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id};
     272    $command .= " -chip_id " . $data->{chip_id} if defined $data->{chip_id};
     273    $command .= " -cam_id " . $data->{cam_id} if defined $data->{cam_id};
     274    $command .= " -fake_id " . $data->{fake_id} if defined $data->{fake_id};
     275    $command .= " -warp_id " . $data->{warp_id} if defined $data->{warp_id};
     276    $command .= " -diff_id " . $data->{diff_id} if defined $data->{diff_id};
     277    $command .= " -inverse" if defined $data->{direction} and $data->{direction} == 0;
     278    $command .= " -zp " . $data->{zp} if defined $data->{zp};
     279    $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err};
     280    $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom};
     281
     282    unless ($no_op) {
     283        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     284            run(command => $command, verbose => $verbose);
     285        &my_die( "Unable to translate", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
     286        &my_die( "Unable to find translated file $output", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $output );
     287    } else {
     288        print "Not running: $command\n";
     289    }
     290
     291    return $output;
     292}
    178293
    179294
    … …  
    191306        $command .= " -pub_id $pub_id";
    192307        $command .= " -path_base $outroot";
     308        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     309        $command .= " -hostname $host" if defined $host;
    193310        $command .= " -fault $fault";
    194311        $command .= " -dbname $dbname" if defined $dbname;
Note: See TracChangeset for help on using the changeset viewer.