IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 8309


Ignore:
Timestamp:
Aug 11, 2006, 6:24:04 PM (20 years ago)
Author:
Paul Price
Message:

Updating following the big workflow discussion.

Location:
trunk/ippScripts/scripts
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/phase0exp.pl

    r8239 r8309  
    66use IPC::Cmd qw( can_run run );
    77use PS::IPP::Metadata::Config;
     8use PS::IPP::Metadata::List qw( parse_md_list );
     9use Statistics::Descriptive;
    810use Data::Dumper;
    9 
    10 use constant RECIPE => "PPSTATS_PHASE0_EXP"; # Recipe to use for ppStats
    1111
    1212use constant TYPE => "OBSTYPE"; # Observation type keyword
    … …  
    1414use constant DETREND_FLAG => "-detrend"; # Flag to use to mark detrend exposure
    1515
    16 use constant IMFILE_URI => 'uri'; # Key for the URI out of p0search -pendingimfile
    17 use constant IMFILE_BG => 'background'; # Key for the background out of p0search -pendingimfile
    18 use constant IMFILE_BGSD => 'background_stdev'; # Key for the background standard deviation
     16use constant PHASE0_URI => 'uri'; # Key for the URI
     17use constant PHASE0_CLASSID => 'class_id'; # Key for the class id
     18use constant PHASE0_BG => 'bg'; # Key for the background
     19use constant PHASE0_BG_STDEV => 'bg_stdev'; # Key for the background standard deviation
     20use constant PHASE0_BG_MEAN_STDEV => 'bg_mean_stdev'; # Key for the mean of the background standard deviation
    1921
    2022use constant EXP_BG => '-background'; # Switch to add background for exposure
    … …  
    2325
    2426# These values should be constant for all components
    25 # The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp"
    26 use constant CONSTANTS => {
    27                            "FPA.FILTER"   => "-filter", # Filter used
    28                            "FPA.AIRMASS"  => "-airmass", # Airmass
    29                            "FPA.RA"       => "-ra", # Right ascension
    30                            "FPA.DEC"      => "-decl", # Declination
    31                            "FPA.POSANGLE" => "-posang", # Position angle
    32                            "TELALT"       => "-alt", # Altitude
    33                            "TELAZ"        => "-az", # Azimuth
    34                            "DETTEM"       => "-ccd_temp" # CCD temperature
    35     };
     27use constant CONSTANTS => [
     28                           "filter", # Filter used
     29                           "airmass", # Airmass
     30                           "ra", # Right ascension
     31                           "decl", # Declination
     32                           "posang", # Position angle
     33                           "alt", # Altitude
     34                           "az", # Azimuth
     35                           "ccd_temp" # CCD temperature
     36                       ];
    3637
    3738# These values may vary across components; we will take the average
    38 # The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp"
    39 use constant VARIABLES => {
    40                            "CELL.EXPOSURE" => "-exp_time", # Exposure time
    41 ###                        "CELL.TIME" # Time of exposure --- not yet implemented
    42                            };
     39use constant VARIABLES => [
     40                           "exp_time", # Exposure time
     41###                        "time" # Time of exposure --- not yet implemented
     42                           ];
    4343
    4444if (scalar @ARGV == 0 || scalar @ARGV >= 2) {
    … …  
    4646        "Usage: $0 EXP_ID\n\n";
    4747}
     48my $expid = shift @ARGV;        # Exposure id
    4849
    4950
    5051# Look for commands we need
    5152my $missing_tools;
    52 my $p0search = can_run('p0search')
    53     or (warn "can't find p0search" and $missing_tools = 1);
     53my $p0tool = can_run('p0tool')
     54    or (warn "can't find p0tool" and $missing_tools = 1);
    5455my $ppStats = can_run('ppStats')
    5556    or (warn "can't find ppStats" and $missing_tools = 1);
    5657die "Can't find required tools.\n" if $missing_tools;
    5758
    58 my $expid = shift @ARGV;        # Exposure id
    59 
    6059my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    6160
    62 # Get the list of filenames
    63 my $filenames;                  # Hash of input files, with their background and stdev
     61# Get the list of imfiles
     62my $imfiles;
    6463{
    65     my $command = "$p0search -pendingimfile -exp_id $expid";
     64    my $command = "$p0tool -rawimfile -exp_id $expid";
    6665    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    67         run(command => $command, verbose => 0);
    68     die "Unable to perform p0search on exposure id $expid: $error_code\n" if not $success;
    69     my $parsed = $mdcParser->parse(join "", @$stdout_buf); # Parsed metadata
    70     $filenames = mdFileInfo($parsed); # Get filenames from the output
     66        run(command => $command, verbose => 1);
     67    die "Unable to perform p0tool on exposure id $expid: $error_code\n" if not $success;
     68    my $metadata = $mdcParser->parse(join "", @$stdout_buf); # Parsed metadata
     69    $imfiles = parse_md_list($metadata); # Data for imfiles
    7170}
    7271
    73 my %values;                     # Values to return
    74 foreach my $fileName (keys %$filenames) {
    75     my $command = "$ppStats $fileName -recipe PPSTATS " . RECIPE; # Command to run
    76     print "Executing: $command\n";
    77     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    78         run(command => $command, verbose => 0);
    79     die "Unable to perform ppStats on $fileName: $error_code\n" if not $success;
    80     my $parsed = $mdcParser->parse(join "", @$stdout_buf); # Parsed metadata
    81 
    82     mdParse($parsed, \%values);
     72# Process the values
     73my %values;                     # Values to update
     74foreach my $variable (@{VARIABLES()}) {
     75    $values{$variable} = [];
     76}
     77my @backgrounds;                # Array of backgrounds
     78my @stdevs;                     # Array of background standard deviations\
     79my $obsType;                    # Observation type
     80 
     81foreach my $imfile (@$imfiles) {
     82    foreach my $constant (@{CONSTANTS()}) {
     83        my $value = get_value($imfile, $constant); # Value for imfile
     84        if (not defined $values{$constant}) {
     85            $values{$constant} = $value;
     86        } elsif ($values{$constant} != $value) {
     87            die "Value of $constant for " . $imfile->{PHASE0_CLASSID} .
     88                " doesn't match previous value.\n";
     89        }
     90    }
     91   
     92    foreach my $variable (@{VARIABLES()}) {
     93        my $value = get_value($imfile, $variable); # Value for imfile
     94        my $array = $values{$variable}; # Array of data
     95        push @$array, $value;
     96    }
     97   
     98    my $bg = get_value($imfile, PHASE0_BG());
     99    push @backgrounds, $bg;
     100   
     101    my $stdev = get_value($imfile, PHASE0_BG_MEAN_STDEV());
     102    push @stdevs, $stdev;
     103   
     104    my $type = get_value($imfile, TYPE());
     105    if (not defined $obsType) {
     106        $obsType = $type;
     107    } elsif ($obsType ne $type) {
     108        die "Observation types differ.\n";
     109    }
    83110}
    84111
    85 my $command = "$p0search -updateexp"; # Command to execute to update exposure parameters
     112# Output results to the database
     113{
     114    my $command = "$p0tool -updateexp"; # Command to execute to update exposure parameters
     115   
     116    # Add the values of interest
     117    foreach my $constant (@{CONSTANTS()}) {
     118        $command .= " -" . $constant . " " . $values{$constant};
     119    }
     120    foreach my $variable (@{VARIABLES()}) {
     121        my $array = $values{$variable}; # Array of values
     122        my $stats = Statistics::Descriptive->new; # Statistics calculator
     123        $stats->add_data(@$array);
     124        $command .= " -" . $variable . " " . $stats->mean();
     125    }
    86126
    87 foreach my $constant (keys %{CONSTANTS()}) {
    88     if (not defined $values{$constant}) {
    89         die "Couldn't find value for $constant.\n";
     127    # Add the statistics
     128    {
     129        my $stats = Statistics::Descriptive->new; # Statistics calculator
     130        $stats->add_data(@backgrounds);
     131        $command .= " -" . PHASE0_BG() . " " . $stats->mean();
     132        $command .= " -" . PHASE0_BG_STDEV() . " " . $stats->standard_deviation();
    90133    }
    91     my $switch = CONSTANTS->{$constant}; # The switch to use on the value
    92     my $value = $values{$constant}; # The value of interest
    93     $command .= " $switch $value";
     134    {
     135        my $stats = Statistics::Descriptive->new; # Statistics calculator
     136        $stats->add_data(@stdevs);
     137        $command .= " -" . PHASE0_BG_MEAN_STDEV() . " " . $stats->mean();
     138    }
     139   
     140    # Add the detrend flag
     141    foreach my $detrendType (@{DETRENDS()}) {
     142        if (lc($obsType) eq lc($detrendType)) {
     143            $command .= " " . DETREND_FLAG;
     144            last;
     145        }
     146    }
     147 
     148    print "$command\n";
     149   
     150#    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     151#       run(command => $command, verbose => 1);
     152#    die "Unable to run phase0 update for $expid: $error_code\n" if not $success;
    94153}
    95 
    96 foreach my $variable (keys %{VARIABLES()}) {
    97     if (not defined $values{$variable}) {
    98         die "Couldn't find value for $variable.\n";
    99     }
    100     my $switch = VARIABLES->{$variable}; # The switch to use on the value
    101     # Take the mean
    102     my $value = $values{$variable}->{value} / $values{$variable}->{num}; # Value of interest
    103     $command .= " $switch $value";
    104 }
    105 
    106 # Check if it's a detrend
    107 if ($values{TYPE}) {
    108     $command .= " " . DETREND_FLAG;
    109 }
    110 
    111 ### Now to do the background: mean background, SD of backgrounds, mean of background SDs
    112 my $bgMean = 0;                 # Mean background
    113 my $bgsdMean = 0;               # Mean of background SDs
    114 foreach my $filename (keys %$filenames) {
    115     my $stats = $filenames->{$filename}; # Statistics for that file
    116     $bgMean += $stats->{bg};
    117     $bgsdMean += $stats->{bgsd};
    118 }
    119 $bgMean /= scalar(keys %$filenames);
    120 $bgsdMean /= scalar(keys %$filenames);
    121 my $bgSD = 0;                   # SD of backgrounds
    122 foreach my $filename (keys %$filenames) {
    123     my $stats = $filenames->{$filename}; # Statistics for that file
    124     $bgSD += ($stats->{bg} - $bgMean)**2;
    125 }
    126 $bgSD = sqrt($bgSD / (scalar(keys %$filenames) - 1));
    127 
    128 $command .= " " . EXP_BGMEAN() . " $bgMean " . EXP_BGSD() . " $bgSD " . EXP_BGSDMEAN() . " $bgsdMean";
    129 
    130 print "$command";
    131 
    132 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    133     run(command => $command, verbose => 0);
    134 die "Unable to run phase0 update for $expid: $error_code\n" if not $success;
    135 
    136154
    137155### Pau.
    138156
    139157
    140 # Parse a metadata containing a list of files; return a hash keyed by the URI, containing bg and sd
    141 sub mdFileInfo
    142 {
    143     my $mdItemArray = shift;    # Array of metadata items from "p0search -pendingimfile"
     158# Get the value for a particular imfile, along with a strong check for its existence
     159sub get_value {
     160    my $imfile = shift;         # The hash of values
     161    my $name = shift;           # The name of the value to check
    144162
    145     my %uriHash;                # Hash of URIs to return
    146   IMFILE: foreach my $mdItem (@$mdItemArray) {
    147       die "Unexpected format for filenames: does not consist solely of METADATAs.\n"
    148           if $mdItem->{class} ne "metadata";
    149       my %imfileInfo = values2hash($mdItem->{value});           # File information
    150       die "Unexpected format for filenames: METADATAs do not contain " . IMFILE_URI . " element.\n"
    151           if not defined $imfileInfo{IMFILE_URI};
    152       die "Unexpected format for filenames: METADATAs do not contain " . IMFILE_BG . " element.\n"
    153           if not defined $imfileInfo{IMFILE_BG};
    154       die "Unexpected format for filenames: METADATAs do not contain " . IMFILE_BGSD . " element.\n"
    155           if not defined $imfileInfo{IMFILE_BGSD};
    156       $uriHash{$imfileInfo{IMFILE_URI}} = { 'bg' => $imfileInfo{IMFILE_BG},
    157                                             'bgsd' => $imfileInfo{IMFILE_BGSD}
    158                                         };
    159   }
    160    
    161     return \%uriHash;
     163    my $source = $imfile->{PHASE0_CLASSID()}; # Where it comes from
     164    die "Couldn't find value of $name for class_id=$source\n" if not defined $imfile->{$name};
     165    return $imfile->{$name};
    162166}
    163 
    164 
    165 # Parse a metadata, looking for the appropriate values
    166 sub mdParse
    167 {
    168     my $mdItemArray = shift;    # Array of metadata items
    169     my $values = shift;         # The values
    170 
    171   MDITEM:    foreach my $mdItem (@$mdItemArray) {
    172       # Recurse if required
    173       if ($mdItem->{class} eq "metadata") {
    174           mdParse($mdItem->{value}, $values);
    175           return;
    176       }
    177      
    178       return if not defined $mdItem->{name};
    179 
    180       if ($mdItem->{name} eq TYPE) {
    181           foreach my $type (DETRENDS) {
    182               if (lc($mdItem->{value}) eq lc($type)) {
    183                   $values->{TYPE} = 1;
    184                   next MDITEM;
    185               }
    186           }
    187           # If we got here, it's not a detrend
    188           $values->{TYPE} = 0;
    189           next MDITEM;
    190       }
    191      
    192       foreach my $constant (keys %{CONSTANTS()}) {
    193           if ($mdItem->{name} eq $constant) {
    194               if (defined $values->{$constant} and $mdItem->{value} ne $values->{$constant}) {
    195                   die "Differing values found for $constant\n";
    196               }
    197               $values->{$constant} = $mdItem->{value};
    198               next MDITEM;
    199           }
    200       }
    201        
    202       foreach my $variable (keys %{VARIABLES()}) {
    203           if ($mdItem->{name} eq $variable) {
    204               $values->{$variable}->{value} += $mdItem->{value};
    205               $values->{$variable}->{num}++;
    206               next MDITEM;
    207           }
    208       }
    209   }
    210 }
    211 
    212 
    213 # Convert values to a hash
    214 sub values2hash
    215 {
    216     my $values = shift;
    217  
    218     my %hash;
    219     foreach my $data (@$values) {
    220         $hash{$data->{name}} = $data->{value};
    221     }
    222  
    223     return \%hash;
    224 }

  • trunk/ippScripts/scripts/phase0imfile.pl

    r8256 r8309  
    66use IPC::Cmd qw( can_run run );
    77use PS::IPP::Metadata::Config;
     8use PS::IPP::Metadata::Stats;
    89use Data::Dumper;
    910
    10 use constant RECIPE => "PPSTATS_PHASE0_IMFILE"; # Recipe to use for ppStats
     11use constant RECIPE => "PPSTATS_PHASE0"; # Recipe to use for ppStats
    1112
    12 # Switches for p0search
    13 use constant P0SEARCH_MODE => '-updateimfile'; # Mode for p0search
    14 use constant P0SEARCH_EXPID => '-exp_id'; # Switch to specify the exposure id
    15 use constant P0SEARCH_CLASSID => '-class_id'; # Switch to specify the class id
    16 use constant P0SEARCH_BG => '-bg'; # Switch to add the background
    17 use constant P0SEARCH_BGSD => '-background_stdev'; # Switch to add the bg stdev
    18 use constant P0SEARCH_EXPTIME => '-exp_time'; # Switch to add the exposure time
     13# These values should be constant for all components
     14# The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp"
     15use constant CONSTANTS => {
     16    "FPA.FILTER"   => "-filter", # Filter used
     17    "FPA.AIRMASS"  => "-airmass", # Airmass
     18    "FPA.RA"       => "-ra",    # Right ascension
     19    "FPA.DEC"      => "-decl",  # Declination
     20    "TELALT"       => "-alt",   # Altitude
     21    "TELAZ"        => "-az",    # Azimuth
     22    "DETTEM"       => "-ccd_temp", # CCD temperature
     23    "FPA.POSANGLE" => "-posang", # Position angle
     24    };
    1925
     26# These values may vary across components; we will take the average
     27# The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp"
     28use constant VARIABLES => {
     29    "CELL.EXPOSURE" => "-exp_time" # Exposure time
     30    };
     31
     32
     33# Switches for p0tool
     34use constant P0TOOL_MODE => '-updateimfile'; # Mode for p0tool
     35use constant P0TOOL_EXPID => '-exp_id'; # Switch to specify the exposure id
     36use constant P0TOOL_CLASSID => '-class_id'; # Switch to specify the class id
     37use constant P0TOOL_BG_MEAN => '-bg'; # Switch to add the background
     38use constant P0TOOL_BG_STDEV => '-bg_stdev'; # Switch to add the bg stdev
     39use constant P0TOOL_BG_MEAN_STDEV => '-bg_mean_stdev'; # Switch to add the bg mean stdev
     40
     41# Parse command-line arguments
    2042if (scalar @ARGV == 0 || scalar @ARGV > 3) {
    2143    die "Perform phase 0 processing at the imfile level.\n\n" .
    … …  
    2749my $file = shift @ARGV;         # Input filename
    2850
    29 # Look for commands we need
     51
     52# Look for programs we need
    3053my $missing_tools;
    31 my $p0search = can_run('p0search')
    32     or (warn "can't find p0search" and $missing_tools = 1);
    33 my $ppStats = can_run('ppStats')
    34     or (warn "can't find ppStats" and $missing_tools = 1);
     54my $p0tool = can_run('p0tool') or (warn "Can't find p0tool" and $missing_tools = 1);
     55my $ppStats = can_run('ppStats') or (warn "Can't find ppStats" and $missing_tools = 1);
    3556die "Can't find required tools.\n" if $missing_tools;
    3657
    37 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    38 
    3958# Run ppStats on the input file
    40 my $parsed;                     # Parsed metadata
     59my $stats;
    4160{
    4261    my $command = "$ppStats $file -recipe PPSTATS " . RECIPE; # Command to run ppStats
    … …  
    4463        run(command => $command, verbose => 1);
    4564    die "Unable to perform ppStats on exposure id $expid: $error_code\n" if not $success;
    46     $parsed = $mdcParser->parse(join "", @$stdout_buf);
     65   
     66    # Parse the output
     67    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     68    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
     69    my @constants = keys %{CONSTANTS()}; # List of constants to parse out
     70    my @variables = keys %{VARIABLES()}; # List of variables to parse out
     71    $stats = PS::IPP::Metadata::Stats->new(\@constants, \@variables); # Stats parser
     72    $stats->parse($metadata) or die "Unable to find all values.\n";
    4773}
    48 
    49 my $bg = 0;                     # Mean background level
    50 my $sd = 0;                     # Mean standard deviation
    51 my $expTime = 0;                # Mean exposure time
    52 my $bgNum = 0;                  # Number of background entries
    53 my $sdNum = 0;                  # Number of standard deviation entries
    54 my $expTimeNum = 0;             # Number of exposure time entries
    55 foreach my $fpaItem (@$parsed) {
    56     if ($fpaItem->{class} eq "metadata") {
    57         my $chipName = $fpaItem->{name}; # Name of chip
    58         my $chipData = $fpaItem->{value}; # Chip-level data
    59         foreach my $chipItem (@$chipData) {
    60             if ($chipItem->{class} eq "metadata") {
    61                 my $cellName = $chipItem->{name}; # Name of cell
    62                 my $cellData = $chipItem->{value}; # Cell-level data
    63                 foreach my $cellItem (@$cellData) {
    64                     if ($cellItem->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) {
    65                         # It's a statistic of some sort
    66                         if ($cellItem->{name} =~ /STDEV$/) {
    67                             $sd += $cellItem->{value};
    68                             $sdNum++;
    69                         } else {
    70                             $bg += $cellItem->{value};
    71                             $bgNum++;
    72                         }
    73                     } elsif ($cellItem->{name} =~ /CELL\.EXPOSURE/) {
    74                         $expTime += $cellItem->{value};
    75                         $expTimeNum++;
    76                     }
    77                 }
    78             }
    79         }
    80     }
    81 }
    82 die "Unable to find statistics.\n" if ($bgNum == 0 or $sdNum == 0);
    83 $bg /= $bgNum;
    84 $sd /= $sdNum;
    85 die "Unable to find exposure times.\n" if ($expTimeNum == 0);
    86 $expTime /= $expTimeNum;
    8774
    8875# Push the results into the database
    8976{
    90     my $command = "$p0search " . P0SEARCH_MODE . " " . P0SEARCH_EXPID() . " $expid " .
    91         P0SEARCH_CLASSID() . " $classid " . P0SEARCH_BG() ." $bg " . P0SEARCH_BGSD() ." $sd " .
    92         P0SEARCH_EXPTIME() . " $expTime"; # Command to run p0search
     77    my $command = $p0tool . " " . P0TOOL_MODE() . " " . P0TOOL_EXPID() . " " . $expid . " " .
     78        P0TOOL_CLASSID() . " " . $classid; # Command to run p0tool
     79   
     80    foreach my $constant (keys %{CONSTANTS()}) {
     81        $command .= " " . CONSTANTS->{$constant} . " " . ($stats->data($constant))->{value};
     82    }
     83    foreach my $variable (keys %{VARIABLES()}) {
     84        # Just use the mean value
     85        $command .= " " . VARIABLES->{$variable} . " " . ($stats->data($variable))->{mean};
     86    }
     87   
     88    $command .= " " . P0TOOL_BG_MEAN() . " " . $stats->bg_mean();
     89    $command .= " " . P0TOOL_BG_STDEV() . " " . $stats->bg_stdev();
     90    $command .= " " . P0TOOL_BG_MEAN_STDEV() . " " . $stats->bg_mean_stdev();
     91 
    9392    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    9493        run(command => $command, verbose => 1);
    95     die "Unable to perform p0search -updateimfile: $error_code\n" if not $success;
     94    die "Unable to perform p0tool -updateimfile: $error_code\n" if not $success;
    9695}
     96
     97# Pau.
     98
     99__END__
Note: See TracChangeset for help on using the changeset viewer.