IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 3, 2010, 8:50:52 AM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/simtest_nebulous_branches
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/simtest_nebulous_branches

  • branches/simtest_nebulous_branches/ippScripts/scripts/publish_file.pl

    r24957 r27840  
    77use Sys::Hostname;
    88my $host = hostname();
    9 print "Starting script $0 on $host\n\n";
     9my $date = `date`;
     10print "\n\n";
     11print "Starting script $0 on $host at $date\n\n";
    1012
    1113use DateTime;
    … …  
    3234my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
    3335my $ppMops = can_run('ppMops') or (warn "Can't find ppMops" and $missing_tools = 1);
     36my $ppMonet = can_run('ppMonet') or (warn "Can't find ppMonet" and $missing_tools = 1);
    3437my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1);
    3538if ($missing_tools) {
    … …  
    3942
    4043# Parse the command-line arguments
    41 my ( $pub_id, $exp_id, $camera, $stage, $stage_id, $format, $product, $workdir );
    42 my ( $dbname, $verbose, $no_update, $save_temps, $redirect );
     44my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
     45my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect );
    4346
    4447GetOptions(
    4548    'pub_id=s'          => \$pub_id, # Publish identifier
    46     'exp_id=s'          => \$exp_id, # Exposure identifier
    4749    'camera=s'          => \$camera, # Camera name
    4850    'stage=s'           => \$stage,       # Stage of interest
    4951    'stage_id=s'        => \$stage_id,    # Stage identifier
    5052    'product=s'         => \$product,     # Datastore product name
     53    'fileset=s'         => \$fileset,     # Fileset name
    5154    'workdir=s'         => \$workdir,     # Working directory
    5255    'dbname=s'          => \$dbname,    # Database name
    5356    'verbose'           => \$verbose,   # Print to stdout
    5457    'no-update'         => \$no_update, # Don't update the database?
     58    'no-op'             => \$no_op, # Don't do any operations
    5559    'save-temps'        => \$save_temps, # Save temporary files?
    5660    'redirect-output'   => \$redirect,   # Redirect output to log file?
    … …  
    5862
    5963pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    60 pod2usage( -msg => "Required options: --pub_id --exp_id --camera --stage --stage_id --product --workdir",
     64pod2usage( -msg => "Required options: --pub_id --camera --stage --stage_id --product --workdir",
    6165           -exitval => $PS_EXIT_CONFIG_ERROR) unless
    6266    defined $pub_id and
    63     defined $exp_id and
    6467    defined $camera and
    6568    defined $product and
    … …  
    6871    defined $workdir;
    6972
    70 my $outroot = "$workdir/$product.$pub_id.$exp_id"; # Output root name
     73my $outroot = "$workdir/$product.$pub_id"; # Output root name
    7174
    7275my $ipprc = PS::IPP::Config->new( $camera ) or
    … …  
    7982my $mdcParser = PS::IPP::Metadata::Config->new;
    8083
    81 
    82 # Retrieve file name of interest
    83 my %files;                      # Input filenames
    84 my %zp;                         # Zero points
    85 {
    86     my $command;                # Command to run
    87 
    88     if ($stage eq 'diff') {
    89         $command =  "difftool -diffskyfile -diff_id $stage_id";
    90     } elsif ($stage eq 'camera') {
    91         $command =  "camtool -processedexp -cam_id $stage_id";
    92     } else {
    93         &my_die( "Unrecognised stage: $stage", $pub_id, $PS_EXIT_CONFIG_ERROR );
    94     }
     84my ($dsFile, $dsFileName) = tempfile("/tmp/publish.$pub_id.ds.XXXX", UNLINK => !$save_temps );
     85
     86my $comment;                    # Comment for exposure
     87if ($stage eq 'camera') {
     88    my $command =  "camtool -processedexp -cam_id $stage_id";
    9589    $command .= " -dbname $dbname" if defined $dbname;
    96 
    9790    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    9891        run(command => $command, verbose => $verbose);
    … …  
    10598        &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
    10699
     100    &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$components > 1;
     101
     102    my $comp = $$components[0]; # Component of interest
     103    my $path_base = $comp->{path_base}; # Base name for file
     104    my $file = $ipprc->filename( "PSASTRO.OUTPUT", $path_base );
     105    $file = $ipprc->file_resolve($file);
     106    my $exp_name = $comp->{exp_name};
     107    my $exp_id = $comp->{exp_id};
     108    my $chip_id = $comp->{chip_id};
     109    my $cam_id = $comp->{cam_id};
     110    my $zp = $comp->{zpt_obs};
     111    my $zp_err = $comp->{zpt_stdev};
     112    my $astrom = sqrt($comp->{sigma_ra}**2 + $comp->{sigma_dec}**2);
     113    my $name = "cam_$cam_id";
     114    $comment = $comp->{comment};
     115
     116    if ($product eq "MONET") {
     117        my $output = $ipprc->file_resolve( "$outroot.csv.gz", 'create' ) or
     118            &my_die( "Unable to resolve output file", $pub_id, $PS_EXIT_SYS_ERROR);
     119
     120        my $command = "$ppMonet $file $output";
     121        $command .= " -exp_name " . $exp_name if defined $exp_name;
     122        $command .= " -exp_id " . $exp_id if defined $exp_id;
     123        $command .= " -chip_id " . $chip_id if defined $chip_id;
     124        $command .= " -cam_id " . $cam_id if defined $cam_id;
     125        $command .= " -zp " . $zp if defined $zp;
     126        $command .= " -zp_error " . $zp_err if defined $zp_err;
     127        $command .= " -astrom_rms " . $astrom if defined $astrom;
     128
     129        unless ($no_op) {
     130            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     131                run(command => $command, verbose => $verbose);
     132            &my_die( "Unable to translate", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
     133            &my_die( "Unable to find translated file $output", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $output );
     134        } else {
     135            print "Not running: $command\n";
     136        }
     137
     138        $file = $output;
     139    }
     140    print $dsFile "$file|||$product|$name|\n";
     141
     142} elsif ($stage eq 'diff') {
     143    my $command =  "difftool -diffskyfile -diff_id $stage_id";
     144    $command .= " -dbname $dbname" if defined $dbname;
     145    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     146        run(command => $command, verbose => $verbose);
     147    &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
     148
     149    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     150        &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
     151
     152    my $components = parse_md_list($metadata) or
     153        &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
     154
     155    my ($mopsPositiveFile, $mopsPositiveFileName) = tempfile("/tmp/publish.$pub_id.mops.pos.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
     156    my ($mopsNegativeFile, $mopsNegativeFileName) = tempfile("/tmp/publish.$pub_id.mops.neg.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
     157
     158    my %positive;               # Data for positive diff detections
     159    my %negative;               # Data for negative diff detections
    107160    foreach my $comp ( @$components ) {
    108161        my $path_base = $comp->{path_base}; # Base name for file
    109162        next if defined $comp->{quality} and $comp->{quality} > 0;
     163        print "Warning: mis-matched comments\n" if defined $comment and $comment ne $comp->{comment};
     164        $comment = $comp->{comment} unless defined $comment;
    110165
    111166        (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time};
    112167        my $zp = $comp->{zpt_obs} + 2.5 * log($comp->{exp_time}) / log(10);
    113 
    114         if ($stage eq 'diff') {
    115             my $skycell_id = $comp->{skycell_id};
    116             $files{"$skycell_id.pos"} = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base );
    117             $files{"$skycell_id.neg"} = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base ) if
    118                 defined $comp->{bothways} and $comp->{bothways};
    119             $zp{"$skycell_id.pos"} = $zp;
    120             $zp{"$skycell_id.neg"} = $zp if defined $comp->{bothways} and $comp->{bothways};
    121         } elsif ($stage eq 'camera') {
    122             my $cam_id = $comp->{cam_id};
    123             $files{$cam_id} = $ipprc->filename( "PSASTRO.OUTPUT", $path_base );
    124             $zp{$cam_id} = $zp;
    125         }
    126     }
    127 }
    128 
    129 # Prepare for data store input
    130 my ($listFile, $listFileName) = tempfile("/tmp/publish.$pub_id.list.XXXX", UNLINK => !$save_temps );
    131 
    132 # Process each file
    133 foreach my $comp ( keys %files ) {
    134     my $file = $ipprc->file_resolve( $files{$comp} ) or
    135         &my_die("Unable to resolve file $files{$comp}", $pub_id, $PS_EXIT_SYS_ERROR);
    136     my_die("Unable to find file $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $file );
    137 
    138     my $zp = $zp{$comp};
    139     if ($product eq "IPP-MOPS") {
    140         my $outuri = "$outroot.$comp.fits";
    141         my $out = $ipprc->file_resolve( $outuri, 'create' ) or
    142             &my_die( "Unable to resolve output file $outuri", $pub_id, $PS_EXIT_SYS_ERROR);
    143 
    144         my $command = "$ppMops $file $zp $out";
    145         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    146             run(command => $command, verbose => $verbose);
    147         &my_die( "Unable to translate $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
    148         &my_die( "Unable to find translated file $out", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $out );
    149         $file = $out;
    150     }
    151 
    152     # format: filename|filesize|md5sum|filetype|
    153     # note: since we omit filesize and md5sum, dsreg will calculate them
    154     print $listFile "$file|||$product|$comp|\n";
    155 }
    156 
     168        my $astrom = sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2);
     169
     170        my $skycell_id = $comp->{skycell_id};
     171        my $filename = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base );
     172        $filename = $ipprc->file_resolve($filename);
     173
     174        my $data = { zp => $zp,
     175                     zp_err => $comp->{zpt_stdev},
     176                     astrom => sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2),
     177                     exp_name => $comp->{exp_name_1},
     178                     exp_id => $comp->{exp_id_1},
     179                     chip_id => $comp->{chip_id_1},
     180                     cam_id => $comp->{cam_id_1},
     181                     fake_id => $comp->{fake_id_1},
     182                     warp_id => $comp->{warp1},
     183                     diff_id => $comp->{diff_id},
     184                     direction => 1,
     185        };
     186
     187        diff_check(\%positive, $data, "positive");
     188
     189        if ($product eq 'IPP-MOPS') {
     190            print $mopsPositiveFile "$filename\n";
     191        } else {
     192            print $dsFile "$filename|||$product|${skycell_id}.pos|\n";
     193        }
     194
     195        # Negative direction
     196        if (defined $comp->{bothways} and $comp->{bothways}) {
     197            my $filename = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base );
     198            $filename = $ipprc->file_resolve($filename);
     199
     200            my $data = { zp => $zp,
     201                         zp_err => $comp->{zpt_stdev},
     202                         astrom => sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2),
     203                         exp_name => $comp->{exp_name_2},
     204                         exp_id => $comp->{exp_id_2},
     205                         chip_id => $comp->{chip_id_2},
     206                         cam_id => $comp->{cam_id_2},
     207                         fake_id => $comp->{fake_id_2},
     208                         warp_id => $comp->{warp2},
     209                         diff_id => $comp->{diff_id},
     210                         direction => 0,
     211            };
     212
     213            diff_check(\%negative, $data, "negative");
     214
     215            if ($product eq 'IPP-MOPS') {
     216                print $mopsNegativeFile "$filename\n";
     217            } else {
     218                print $dsFile "$filename|||$product|${skycell_id}.neg|\n";
     219            }
     220        }
     221    }
     222
     223    close $mopsPositiveFile if $product eq 'IPP-MOPS';
     224    close $mopsNegativeFile if $product eq 'IPP-MOPS';
     225
     226    if ($product eq 'IPP-MOPS') {
     227        if (scalar keys %positive > 0) {
     228            my $output = mops_combine(\%positive, "$outroot.pos.mops", $mopsPositiveFileName);
     229            print $dsFile "$output|||$product|positive|\n";
     230        }
     231        if (scalar keys %negative > 0) {
     232            my $output = mops_combine(\%negative, "$outroot.neg.mops", $mopsNegativeFileName);
     233            print $dsFile "$output|||$product|negative|\n";
     234        }
     235    }
     236}
     237
     238close $dsFile;
    157239
    158240unless ($no_update) {
    159     my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $listFileName";
     241    my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $dsFileName";
     242    $command .= " --ps0 \"$comment\"" if defined $comment;
    160243
    161244    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    166249unless ($no_update) {
    167250    my $command = "$pubtool -add -pub_id $pub_id -path_base $outroot";
     251    $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     252    $command .= " -hostname $host" if defined $host;
     253
    168254    $command .= " -dbname $dbname" if defined $dbname;
    169255
    … …  
    176262
    177263### Pau.
     264
     265# Check inputs for a diff
     266sub diff_check
     267{
     268    my $data = shift;           # Data hash
     269    my $comp = shift;           # Component data
     270    my $name = shift;           # Name of component
     271
     272    $data->{zp} = $comp->{zp} unless defined $data->{zp};
     273    $data->{zp_err} = $comp->{zp_err} unless defined $data->{zp_err};
     274    $data->{astrom} = $comp->{astrom} unless defined $data->{astrom};
     275    $data->{exp_name} = $comp->{exp_name} unless defined $data->{exp_name};
     276    $data->{exp_id} = $comp->{exp_id} unless defined $data->{exp_id};
     277    $data->{chip_id} = $comp->{chip_id} unless defined $data->{chip_id};
     278    $data->{cam_id} = $comp->{cam_id} unless defined $data->{cam_id};
     279    $data->{fake_id} = $comp->{fake_id} unless defined $data->{fake_id};
     280    $data->{warp_id} = $comp->{warp_id} unless defined $data->{warp_id};
     281    $data->{diff_id} = $comp->{diff_id} unless defined $data->{diff_id};
     282    $data->{direction} = $comp->{direction} unless defined $data->{direction};
     283
     284    &my_die("zp value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp} and $comp->{zp} != $data->{zp};
     285    &my_die("zp_err value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp_err} and $comp->{zp_err} != $data->{zp_err};
     286    &my_die("astrom value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{astrom} and $comp->{astrom} != $data->{astrom};
     287    &my_die("exp_name value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_name} and $comp->{exp_name} ne $data->{exp_name};
     288    &my_die("exp_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_id} and $comp->{exp_id} != $data->{exp_id};
     289    &my_die("chip_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{chip_id} and $comp->{chip_id} != $data->{chip_id};
     290    &my_die("cam_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{cam_id} and $comp->{cam_id} != $data->{cam_id};
     291    &my_die("fake_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{fake_id} and $comp->{fake_id} != $data->{fake_id};
     292    &my_die("warp_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{warp_id} and $comp->{warp_id} != $data->{warp_id};
     293    &my_die("diff_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{diff_id} and $comp->{diff_id} != $data->{diff_id};
     294    &my_die("direction value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{direction} and $comp->{direction} != $data->{direction};
     295
     296    return 1;
     297}
     298
     299# Combine multiple files for MOPS
     300sub mops_combine
     301{
     302    my $data = shift;           # Data
     303    my $output = shift;         # Output name
     304    my $input = shift;          # Input name
     305
     306    $output = $ipprc->file_resolve( $output, 'create' ) or
     307        &my_die( "Unable to resolve output file $output", $pub_id, $PS_EXIT_SYS_ERROR);
     308
     309    my $command = "$ppMops $input $output";
     310    $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name};
     311    $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id};
     312    $command .= " -chip_id " . $data->{chip_id} if defined $data->{chip_id};
     313    $command .= " -cam_id " . $data->{cam_id} if defined $data->{cam_id};
     314    $command .= " -fake_id " . $data->{fake_id} if defined $data->{fake_id};
     315    $command .= " -warp_id " . $data->{warp_id} if defined $data->{warp_id};
     316    $command .= " -diff_id " . $data->{diff_id} if defined $data->{diff_id};
     317    $command .= " -inverse" if defined $data->{direction} and $data->{direction} == 0;
     318    $command .= " -zp " . $data->{zp} if defined $data->{zp};
     319    $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err};
     320    $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom};
     321
     322    unless ($no_op) {
     323        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     324            run(command => $command, verbose => $verbose);
     325        &my_die( "Unable to translate", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
     326        &my_die( "Unable to find translated file $output", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $output );
     327    } else {
     328        print "Not running: $command\n";
     329    }
     330
     331    return $output;
     332}
    178333
    179334
    … …  
    191346        $command .= " -pub_id $pub_id";
    192347        $command .= " -path_base $outroot";
     348        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     349        $command .= " -hostname $host" if defined $host;
    193350        $command .= " -fault $fault";
    194351        $command .= " -dbname $dbname" if defined $dbname;
Note: See TracChangeset for help on using the changeset viewer.