Index: /branches/eam_branch_20080706/ippScripts/scripts/detrend_reject_exp.pl =================================================================== --- /branches/eam_branch_20080706/ippScripts/scripts/detrend_reject_exp.pl (revision 18467) +++ /branches/eam_branch_20080706/ippScripts/scripts/detrend_reject_exp.pl (revision 18467) @@ -0,0 +1,371 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use Statistics::Descriptive; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my $ITER_LIMIT = 20; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $det_type, $camera, $outroot, $filter, $dbname, $verbose, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'det_type|t=s' => \$det_type, + 'camera=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'filter=s' => \$filter, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $det_type and + defined $camera and + defined $outroot; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# these stats are used it the rejections but not passed to the database +# there is some duplication with the above, but the calculation time is minimal +my $REJSTATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "clipmean", flag => "ensMeanMean", dtype => "float" }, + { name => "bg", type => "clipstdev", flag => "ensMeanStdev", dtype => "float" }, + { name => "bg_mean_stdev", type => "clipmean", flag => "ensMeanStdevMean", dtype => "float" }, + { name => "bg_mean_stdev", type => "clipstdev", flag => "ensMeanStdevStdev", dtype => "float" }, + { name => "bg_stdev", type => "clipmean", flag => "ensStdevMean", dtype => "float" }, + { name => "bg_stdev", type => "clipstdev", flag => "ensStdevStdev", dtype => "float" }, + { name => "bg_skewness", type => "clipmean", flag => "ensSkewness", dtype => "float" }, + { name => "bg_kurtosis", type => "clipmean", flag => "ensKurtosis", dtype => "float" }, + + ]; +my $rejstats = PS::IPP::Metadata::Stats->new($REJSTATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($exposures, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + # dettool command to select exp data for this det_run + $command = "$dettool -residexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -residexp: $error_code", $det_id, $iter, $error_code); + } + + # Parse the stdout buffer into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $exposures = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics for rejections + $rejstats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); +} + +# each image has a mean mean (average of means over all chips) +# a standard deviation mean (average of standard deviations over all chips) +# and a + + + +# we use the statistics of the ensemble to accept/reject exposurs +my $ensMeanMean = $rejstats->value_for_flag ("ensMeanMean"); # average of all exposure means (in turn averaged over all chips) +my $ensMeanStdev = $rejstats->value_for_flag ("ensMeanStdev"); # standard deviation of all exposure means +my $ensMeanStdevMean = $rejstats->value_for_flag ("ensMeanStdevMean"); # average over all exposures of the stdev of the means for each chip +my $ensMeanStdevStdev = $rejstats->value_for_flag ("ensMeanStdevStdev"); # standard deviation over all exposures of the stdev of the means for each chip +my $ensStdevMean = $rejstats->value_for_flag ("ensStdevMean"); # average over all exposures of the sum of the squares of the stdevs for each chip +my $ensStdevStdev = $rejstats->value_for_flag ("ensStdevStdev"); # standard deviation over all exposures of the sum of the squares of the stdevs for each chip + +$ipprc->define_camera($camera); +# Rejection thresholds +my $reject_mean = rejection_limit( 'ENSEMBLE.MEAN', $det_type, $filter ); +my $reject_stdev = rejection_limit( 'ENSEMBLE.STDEV', $det_type, $filter ); +my $reject_meanstdev = rejection_limit( 'ENSEMBLE.MEANSTDEV', $det_type, $filter ); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $logName = "$outroot.log"; # Name for log + +my $logFile; +unless ($no_op) { + $logFile = $ipprc->file_create_append( $logName ); + print $logFile "Ensemble mean $ensMeanMean +/- $ensMeanStdev\n"; + print $logFile "Ensemble stdev $ensStdevMean +/- $ensStdevStdev\n"; + print $logFile "Ensemble mean rms (over imfiles) $ensMeanStdevMean +/- $ensMeanStdevStdev\n\n"; +} + +# Go through again to do rejection, and update the database for each exposure +my $numChanges = 0; # Number of exposures with changed status +my $numReject = 0; # Number of exposures rejected +my $firstElement = 1; + +foreach my $exposure (@$exposures) { + my $mean = $exposure->{bg}; # Mean for this exposure + my $stdev = $exposure->{bg_stdev}; # Stdev for this exposure + my $meanStdev = $exposure->{bg_mean_stdev}; # Stdev of Means for this exposure + my $expID = $exposure->{exp_id}; + my $accept = $exposure->{accept}; + my $include = $exposure->{include}; + + &my_die("Unable to find exposure id.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $expID; + &my_die("Unable to find accept.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $accept; + &my_die("Unable to find include.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $include; + + my $reject = 0; # Reject this exposure? + + $command = "$dettool -updateresidexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $expID"; + $command .= " -dbname $dbname" if defined $dbname; + + if (not $accept) { + # Rejected this at an earlier stage + unless ($no_op) { + print $logFile "Rejecting $expID based on earlier determination.\n"; + } + $reject = 1; + goto UPDATE; + } + + # Cop-out if we're not doing any operations + if ($no_op) { + # Make sure something gets rejected (just once!), just so that + # we can trace the full range of the workflow + if ($firstElement and $iter == 0) { + $reject = 1; + } + goto UPDATE; + } + + if ($reject_mean > 0 and $ensMeanStdev > 0) { + my $delta = abs($mean - $ensMeanMean); + if ($delta > ($reject_mean * $ensMeanStdev)) { + print $logFile "Rejecting $expID based on ensemble mean value: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble mean: "; + } + print $logFile "$mean --> $delta vs " . $reject_mean * $ensMeanStdev . "\n"; + } else { + print $logFile "No rejection of $expID for ensemble mean\n"; + } + + if ($reject_stdev > 0 and $ensStdevStdev > 0) { + my $delta = abs($stdev - $ensStdevMean); + if ($delta > ($reject_stdev * $ensStdevStdev)) { + print $logFile "Rejecting $expID based on ensemble stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble stdev: "; + } + print $logFile "$stdev --> $delta sigma vs " . $reject_stdev * $ensStdevStdev . "\n"; + } else { + print $logFile "No rejection of $expID for ensemble stdev\n"; + } + + if ($reject_meanstdev > 0 and $ensMeanStdevStdev > 0) { + my $delta = abs($meanStdev - $ensMeanStdevMean); + if ($delta > ($reject_meanstdev * $ensMeanStdevStdev)) { + print $logFile "Rejecting $expID based on ensemble mean stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble mean stdev: "; + } + print $logFile "$meanStdev --> $delta sigma vs " . $reject_meanstdev * $ensMeanStdevStdev. "\n"; + } else { + print $logFile "No rejection of $expID for ensemble mean stdev\n"; + } + + UPDATE: + if ($reject) { + $command .= ' -reject'; + $numReject++; + } + + # Check for status changes + if ((not $include and not $reject) or ($include and $reject)) { + unless ($no_op) { + print $logFile "Status of $expID has changed.\n"; + } + $numChanges++; + } + + unless ($no_update) { + # Update + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -updateresidexp: $error_code", $det_id, $iter, $error_code); + } + } +} + +# Decide if the current is sufficient to use as a master, and if we can stop iterating +my $master = 1; # This is good enough for a master +my $stop = 1; # Stop iterating + +if ($numChanges > 0) { + $master = 0; + $stop = 0; +} + +# Rejecting everything --- stop before something bad happens! +if ($numReject == scalar @$exposures) { + $master = 0; + $stop = 1; + carp "All inputs rejected!\n"; +} + +unless ($no_op) { + print $logFile "Master: $master\n"; + print $logFile "Stop: $stop\n"; + close $logFile; +} + +# Allow iteration to be turned off +my $allow_iter = metadataLookupBool($ipprc->{rejection}, "ITERATION"); # Allow iteration? +my $force_master = metadataLookupBool($ipprc->{rejection}, "MASTER"); # Force the stack to be accepted +$stop = 1 unless $allow_iter; +$master = 1 if $force_master and $stop; + +# attempt to prevent endless, pathological iterations +if ($iter >= $ITER_LIMIT) { + warn("iteration limit reached -- bailing out"); + exit($PS_EXIT_PROG_ERROR); +} + +## add the summary statistics, and request a new iteration if needed +$command = "$dettool -adddetrunsummary"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -accept" if $master; +$command .= " -dbname $dbname" if defined $dbname; +### XXX WE NEED to make this a recipe-driven option +$command .= " -again" unless $stop; +$command .= $stats->cmdflags(); + +# Put results into the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -adddetrunsummary: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -adddetrunsummary"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Retrieve the requested rejection limit, dying unless extant +sub rejection_limit +{ + my $name = shift; # Rejection limit to + my $type = shift; # Type of exposure + my $filter = shift; # Filter + + my $value = $ipprc->rejection( $name, $det_type, $filter ); + if (not defined $value) { + $filter = "(no filter)" unless defined $filter; + die "Unable to determine $name rejection limit for $det_type with $filter.\n"; + } + + return $value; +} + +__END__ Index: /branches/eam_branch_20080706/ippScripts/scripts/ipp_image_path.pl =================================================================== --- /branches/eam_branch_20080706/ippScripts/scripts/ipp_image_path.pl (revision 18467) +++ /branches/eam_branch_20080706/ippScripts/scripts/ipp_image_path.pl (revision 18467) @@ -0,0 +1,162 @@ +#!/usr/bin/env perl + +# ipp_image_path.pl print out the unix path name(s) for rawImage files for an exposure +# + +use warnings; +use strict; + +# get images that are on ipp008 from alternative location until the +# dataase is updated +my $use_008_workaround = 1; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use DBI; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($exp_name, $class_id, $from_registered, $dbname, $verbose); + +GetOptions( + 'exp_name|e=s' => \$exp_name, + 'class_id|c=s' => \$class_id, + 'registered|r' => \$from_registered, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_name", + -exitval => 3) unless + defined $exp_name; + +my $ota_num; +if ($class_id) { + $class_id = lc($class_id); + my $extra; + (undef, $ota_num, $extra) = $class_id =~ m/(\D*)(\d\d)(.*)/; + if ($extra || !$ota_num) { + print STDERR "$class_id is not a valid class_id\n"; + exit 1; + } +} + +# Look for commands we need +my $missing_tools; +my $neb_locate = can_run('neb-locate') + or (warn "can't find neb-locate" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $ipprc = PS::IPP::Config->new(); +my $dbh = getDBHandle(); + +# Get the list of imfiles +{ + # XXX: If there are multiple exposures with the same exposure name + # this query will return them all + my $query; + if ($from_registered) { + $query = "SELECT exp_id, class_id, uri FROM rawImfile" + . " WHERE exp_name = \'$exp_name\'"; + $query .= " AND class_id = \'xy$ota_num\'" if ($ota_num); + } else { + $query = "SELECT exp_id FROM newExp WHERE tmp_exp_name = ?"; + my $stmt = $dbh->prepare($query); + $stmt->execute($exp_name); + my $exp_ref = $stmt->fetchrow_hashref(); + if (!$exp_ref) { + print STDERR "exposure $exp_name not found\n"; + exit 1; + } + $stmt->finish(); + $query = "SELECT exp_id, tmp_class_id, uri FROM newImfile WHERE exp_id = $exp_ref->{exp_id}"; + $query .= " AND tmp_class_id = \'ota$ota_num\'" if ($ota_num); + } + + my $stmt = $dbh->prepare($query); + $stmt->execute(); + my $nfiles = 0; + while (my $ref = $stmt->fetchrow_hashref()) { + my $uri = $ref->{uri}; + my $path; + my ($scheme) = $uri =~/^(path|neb|file):/; + + if (!$scheme) { + $path = $uri; + } else { + if ($use_008_workaround && ($uri =~ /ipp008/)) { + $path = resolve_ipp008_file($uri); + } else { + $path = $ipprc->file_resolve($uri); + } + } + if ($path) { + # remove the leading "file://" if present + $path =~ s/^file\:\/\///; + } + $nfiles++; + print "$path\n"; + } + if (!$nfiles) { + my $cstr = defined($class_id) ? $class_id . " " : ""; + print STDERR "exposure $exp_name " . $cstr . "not found\n"; + exit 2; + } +} + +sub getDBHandle { + my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER"); + my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER"); + my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD"); + if (!$dbname) { + $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME"); + } + + die "database configuration set up" unless defined($dbserver) and defined($dbuser) + and defined($dbpassword) and defined($dbname); + + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database.\n"; + + return $dbh; +} + +sub resolve_ipp008_file { + my %locations = ( "ota24" => "ipp006", + "ota25" => "ipp009", + "ota26" => "ipp011", + "ota27" => "ipp019", + "ota30" => "ipp020", + "ota31" => "ipp021" + ); + + my $uri = shift; + + # uri's look like neb://ipp008.0/gpc1/20080625/o4642g0400o/o4642g0400o.ota24.fits + + my ($left, $middle, $ota, $fits) = split '\.', $uri; + + my $node = $locations{$ota}; + + die "can't find node for $ota: $uri" if (!$node); + + my $path = $uri; + + $path =~ s%neb://ipp008.0%/data/${node}.0/recover08%; + + die "sorry backup image for $uri not found\n" if (!-e $path); + + return $path; +} Index: /branches/eam_branch_20080706/ippScripts/scripts/ipp_serial_chip.pl =================================================================== --- /branches/eam_branch_20080706/ippScripts/scripts/ipp_serial_chip.pl (revision 18467) +++ /branches/eam_branch_20080706/ippScripts/scripts/ipp_serial_chip.pl (revision 18467) @@ -0,0 +1,105 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir_default, # Default working directory + $verbose, # Verbose operations? + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_default, + 'verbose' => \$verbose, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Required options: --dbname --workdir", + -exitval => 3, + ) unless + defined $dbname; + +$workdir_default = `pwd` unless defined $workdir_default; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $chip = can_run('chip_imfile.pl') or (warn "Can't find chip_imfile.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Imfile processing +my @whole; # The whole list for processing +{ + my $command = "$chiptool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 2 imfile list: $error_code\n" if not $success; + @whole = split /\n/, join( '', @$stdout_buf ); +} + +my @single = (); + +while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from chiptool.\n"; + + foreach my $item (@$list) { + my $chip_id = $item->{chip_id}; + my $exp_id = $item->{exp_id}; + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + my $reduction = $item->{reduction}; + my $workdir = $item->{workdir}; + $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL"); + + my $outroot = caturi( $workdir, $exp_tag, "$exp_tag.ch.$chip_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$chip --chip_id $chip_id --exp_id $exp_id --class_id $class_id --uri $uri --dbname $dbname --camera $camera --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 2 processing on $chip_id $class_id: $error_code\n" if not $success; + } + + @single = (); + + } +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branch_20080706/ippTasks/chiphosts.mhpcc.config =================================================================== --- /branches/eam_branch_20080706/ippTasks/chiphosts.mhpcc.config (revision 18467) +++ /branches/eam_branch_20080706/ippTasks/chiphosts.mhpcc.config (revision 18467) @@ -0,0 +1,170 @@ +chiphosts MULTI + +chiphosts METADATA + camera STR MEGACAM + ccd00 STR po00 + ccd01 STR po01 + ccd02 STR po02 + ccd03 STR po03 + ccd04 STR po04 + ccd05 STR po05 + ccd06 STR po06 + ccd07 STR po07 + ccd08 STR po08 + ccd09 STR po09 +END + +chiphosts METADATA + camera STR CFH12K + ccd00 STR po10 + ccd01 STR po11 + ccd02 STR po12 +END + +chiphosts METADATA + camera STR GPC1 + + XY01 STR ipp005 + XY02 STR ipp005 + XY03 STR ipp005 + XY04 STR ipp005 + XY05 STR ipp005 + XY06 STR ipp005 + + XY10 STR ipp006 + XY11 STR ipp006 + XY12 STR ipp006 + XY13 STR ipp006 + XY14 STR ipp006 + XY15 STR ipp006 + + XY16 STR ipp007 + XY17 STR ipp007 + XY20 STR ipp007 + XY21 STR ipp007 + XY22 STR ipp007 + XY23 STR ipp007 + + XY24 STR ipp006 + XY25 STR ipp009 + XY26 STR ipp010 + XY27 STR ipp011 + XY30 STR ipp016 + XY31 STR ipp020 + + XY32 STR ipp009 + XY33 STR ipp009 + XY34 STR ipp009 + XY35 STR ipp009 + XY36 STR ipp009 + XY37 STR ipp009 + + XY40 STR ipp010 + XY41 STR ipp010 + XY42 STR ipp010 + XY43 STR ipp010 + XY44 STR ipp010 + XY45 STR ipp010 + + XY46 STR ipp011 + XY47 STR ipp011 + XY50 STR ipp011 + XY51 STR ipp011 + XY52 STR ipp011 + XY53 STR ipp011 + + XY54 STR ipp016 + XY55 STR ipp016 + XY56 STR ipp016 + XY57 STR ipp016 + XY60 STR ipp016 + XY61 STR ipp016 + + XY62 STR ipp020 + XY63 STR ipp020 + XY64 STR ipp020 + XY65 STR ipp020 + XY66 STR ipp020 + XY67 STR ipp020 + + XY71 STR ipp021 + XY72 STR ipp021 + XY73 STR ipp021 + XY74 STR ipp021 + XY75 STR ipp021 + XY76 STR ipp021 +END + +chiphosts METADATA + camera STR gpc1 + + ota01 STR ipp005 + ota02 STR ipp005 + ota03 STR ipp005 + ota04 STR ipp005 + ota05 STR ipp005 + ota06 STR ipp005 + + ota10 STR ipp006 + ota11 STR ipp006 + ota12 STR ipp006 + ota13 STR ipp006 + ota14 STR ipp006 + ota15 STR ipp006 + + ota16 STR ipp007 + ota17 STR ipp007 + ota20 STR ipp007 + ota21 STR ipp007 + ota22 STR ipp007 + ota23 STR ipp007 + + ota24 STR ipp006 + ota25 STR ipp009 + ota26 STR ipp010 + ota27 STR ipp011 + ota30 STR ipp016 + ota31 STR ipp020 + + ota32 STR ipp009 + ota33 STR ipp009 + ota34 STR ipp009 + ota35 STR ipp009 + ota36 STR ipp009 + ota37 STR ipp009 + + ota40 STR ipp010 + ota41 STR ipp010 + ota42 STR ipp010 + ota43 STR ipp010 + ota44 STR ipp010 + ota45 STR ipp010 + + ota46 STR ipp011 + ota47 STR ipp011 + ota50 STR ipp011 + ota51 STR ipp011 + ota52 STR ipp011 + ota53 STR ipp011 + + ota54 STR ipp016 + ota55 STR ipp016 + ota56 STR ipp016 + ota57 STR ipp016 + ota60 STR ipp016 + ota61 STR ipp016 + + ota62 STR ipp020 + ota63 STR ipp020 + ota64 STR ipp020 + ota65 STR ipp020 + ota66 STR ipp020 + ota67 STR ipp020 + + ota71 STR ipp021 + ota72 STR ipp021 + ota73 STR ipp021 + ota74 STR ipp021 + ota75 STR ipp021 + ota76 STR ipp021 +END Index: /branches/eam_branch_20080706/ippTasks/replicate.pro =================================================================== --- /branches/eam_branch_20080706/ippTasks/replicate.pro (revision 18467) +++ /branches/eam_branch_20080706/ippTasks/replicate.pro (revision 18467) @@ -0,0 +1,168 @@ +## replicate.pro : tasks for data replication : -*- sh -*- +## this file contains the tasks for maintaining duplicates as needed in Nebulous +## these tasks use the books replicatePending + +# test for required global variables +check.globals + +$NEB_DB = nebulous +$NEB_HOST = alala +$NEB_USER = jhipp +$NEB_PASS = jhipp + +$LOGSUBDIR = $LOGDIR/replicate +exec mkdir -p $LOGSUBDIR + +book init replicatePending + +macro replicate.reset + book init replicatePending +end + +macro replicate.status + book listbook replicatePending +end + +macro replicate.on + task replicate.load + active true + end + task replicate.run + active true + end +end + +macro replicate.off + task replicate.load + active false + end + task replicate.run + active false + end +end + +macro set.host.for.replicate + if ($0 != 2) + echo "USAGE: set.host.for.replicate (hostname)" + break + end + + if (not($PARALLEL)) + host local + return + end + +# parse volume name + + if ("$1" == "NULL") + host anyhost + else + host $1 + end +end + +# the replicate process interacts with only the single Nebulous server + +# select Nebulous objects which desire additional copies +task replicate.load + host local + + # modify these after the tasks are tested + periods -poll 10 + periods -exec 1 + periods -timeout 300 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/replicate.log + + task.exec + # command does not need to be dynamic, but having it so allows us to adjust the periods + # so that we dont have to wait 10 minutes for things to start up + command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 2000 + periods -exec 600 + + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout replicatePending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook replicatePending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup replicatePending + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create the desired replicas +task replicate.run + periods -poll 5 + periods -exec 30 + periods -timeout 30 + + task.exec + book npages replicatePending -var N + if ($NETWORK == 0) break + if ($N == 0) + periods -exec 30 + break + end + periods -exec 0.1 + + # look for new objects in replicatePending + book getpage replicatePending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword replicatePending $pageName pantaskState RUN + + # XXX what values do I need to get back? + book getword replicatePending $pageName key -var KEY + book getword replicatePending $pageName need_copies -var NEED_COPIES + book getword replicatePending $pageName volume_name -var VOLUME_NAME + book getword replicatePending $pageName volume_host -var VOLUME_HOST + + set.host.for.replicate $VOLUME_HOST + + stdout NULL + stderr $LOGSUBDIR/replicate.log + + # these operations do not require a database to be specified + $run = neb-replicate --copies $NEED_COPIES $KEY + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit replicatePending $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword replicatePending $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branch_20080706/ippTasks/summit.copy.pro =================================================================== --- /branches/eam_branch_20080706/ippTasks/summit.copy.pro (revision 18467) +++ /branches/eam_branch_20080706/ippTasks/summit.copy.pro (revision 18467) @@ -0,0 +1,434 @@ +## summit.copy.pro : tasks for the summit to IPP download : -*- sh -*- +## PanTasks scripts for Summit Copy + +## XXX note that this currently works with a single database as defined in .ipprc +## XXX tie the database for output to the database from which the datastore was determined + +# pztool -adddatastore -inst isp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/skyprobe/index.txt +# pztool -adddatastore -inst gpc1 -telescope ps1 -uri http://conductor/ds/gpc1/index.txt +# pztool -adddatastore -inst allskycam -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/allskycam/index.txt + +# NOTE: workdir / volume mangling and nebulous. these tasks copy the +# imfiles from the summit, placing the output files in a directory +# which is based on the chip/host relationship. If nebulous is being +# used, the volume name is set based on the host; if nebulous is not +# being used, the workdir is set to include the host name. The copy +# operation is targetted to the same host by pcontrol. The value of +# workdir is set to be a template into which the appropriate value of +# @HOST@ may be substituted in later scripts. + +# test for required global variables +check.globals + +# list of DataStores to pull data from +book init pzDataStore +# list of summit exps that need to be queried +book init pzPendingExp +# list of summit imfiles that need to be downloaded +book init pzPendingImfile + +macro copy.on + task pztool.datastore + active true + end + task pzgetexp + active true + end + task pztool.pendingexp + active true + end + task pzgetimfile + active true + end + task pztool.pendingimfile + active true + end + task summit_copy + active true + end +end + +macro copy.off + task pztool.datastore + active false + end + task pzgetexp + active false + end + task pztool.pendingexp + active false + end + task pzgetimfile + active false + end + task pztool.pendingimfile + active false + end + task summit_copy + active false + end +end + +# these variables will cycle through the known database names +$pztoolDatastore_DB = 0 +$pztoolPendingExp_DB = 0 +$pztoolPendingImfile_DB = 0 + +# build a book of datastores to poll for data +task pztool.datastore + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 10 + periods -poll 1 + periods -timeout 20 + npending 1 + # trange 16:00 23:59 + # trange 00:00 04:00 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -datastore + else + # save the DB name for the exit tasks + option $DB:$pztoolDatastore_DB + command pztool -datastore -dbname $DB:$pztoolDatastore_DB + $pztoolDatastore_DB ++ + if ($pztoolDatastore_DB >= $DB:n) set pztoolDatastore_DB = 0 + end + end + + # success + task.exit 0 + # flush pzDataStore book + book init pzDataStore + # convert 'stdout' to book format + ipptool2book stdout pzDataStore -key camera:telescope -uniq -setword dbname $options:0 + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +$datastore_index = 0 + +# run pzgetexp periodically to populate pzPendingExp in the database (no I/O) +# this task is querying the data store for a list of exposures ("filesets") +# and inserting these into a db table on the local cluster (pzPendingExp) +task pzgetexp + periods -exec 10 + periods -poll 1 + periods -timeout 20 + # trage 16:00 23:59 + # trage 00:00 04:00 + npending 1 + host local + + task.exec + # find an exp that needs imfiles fetched + book getpage pzDataStore $datastore_index -var pageName + if ("$pageName" == "NULL") break + + # increment our pzDataStore index and loop back to 0 and the end of the + # book + $datastore_index ++ + book npages pzDataStore -var npages + if ($datastore_index == $npages ) + $datastore_index = 0 + end + + book getword pzDataStore $pageName camera -var CAMERA + book getword pzDataStore $pageName telescope -var TELESCOPE + book getword pzDataStore $pageName uri -var URI + book getword pzDataStore $pageName dbname -var DBNAME + + # store the current page + options $pageName + + $run = pzgetexp -uri $URI -inst $CAMERA -telescope $TELESCOPE -dbname $DBNAME + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + + command $run + end + + task.exit 0 + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# build a book of exps/filesetids that need to be queried +task pztool.pendingexp + host local + + periods -exec 10 + periods -poll 1 + periods -timeout 20 + # trange 16:00 23:59 + # trange 00:00 04:00 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -pendingexp -limit 5 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingExp_DB + command pztool -pendingexp -limit 5 -dbname $DB:$pztoolPendingExp_DB + $pztoolPendingExp_DB ++ + if ($pztoolPendingExp_DB >= $DB:n) set pztoolPendingExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingExp + end + + task.exit default + showcommand failure + if ($VERBOSE) + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + end + task.exit timeout + showcommand timeout + end +end + +# run pzgetimfiles on pending exps. analogous to pzgetexp, this task +# is downloading a list of imfiles reported by the datastore for a +# given exposure ("fileset"), and placing the result in the local +# database table of imfiles +task pzgetimfile + periods -exec 0.05 + periods -poll 0.025 + periods -timeout 700 + # trage 16:00 23:59 + # trage 00:00 04:00 + host local + npending 10 + + task.exec + if ($NETWORK == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingExp -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingExp $pageName pantaskState RUN + + book getword pzPendingExp $pageName exp_name -var EXP_NAME + book getword pzPendingExp $pageName camera -var CAMERA + book getword pzPendingExp $pageName telescope -var TELESCOPE + book getword pzPendingExp $pageName dateobs -var DATEOBS + book getword pzPendingExp $pageName exp_type -var EXP_TYPE + book getword pzPendingExp $pageName uri -var URI + book getword pzPendingExp $pageName imfiles -var IMFILES + book getword pzPendingExp $pageName dbname -var DBNAME + + # store the current page + options $pageName + + $batman = $EXP_NAME + # Sidik says we should use a longer timeout + $run = pzgetimfiles -uri $URI -filesetid $batman -inst $CAMERA -telescope $TELESCOPE -dbname $DBNAME -timeout 650 + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit 0 + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pzPendingExp $options:0 pantaskState TIMEOUT + end +end + + +# build a book of imfiles/files that need to be downloaded +task pztool.pendingimfile + host local + + periods -exec 30 + periods -poll 1 + periods -timeout 120 + # trage 16:00 23:59 + # trage 00:00 04:00 + npending 1 + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -pendingimfile -limit 40 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingImfile_DB + command pztool -pendingimfile -limit 60 -dbname $DB:$pztoolPendingImfile_DB + $pztoolPendingImfile_DB ++ + if ($pztoolPendingImfile_DB >= $DB:n) set pztoolPendingImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope:class:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle pzPendingImfile + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingImfile + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# retreive an imfile with dsget and then call pztool -copydone +task summit_copy + periods -exec 5 + periods -poll 0.05 + periods -timeout 650 + # trage 16:00 23:59 + # trage 00:00 04:00 + + task.exec + if ($NETWORK == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingImfile -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingImfile $pageName pantaskState RUN + + book getword pzPendingImfile $pageName uri -var URI + book getword pzPendingImfile $pageName bytes -var BYTES + book getword pzPendingImfile $pageName md5sum -var MD5SUM + book getword pzPendingImfile $pageName dateobs -var DATEOBS + book getword pzPendingImfile $pageName exp_name -var EXP_NAME + book getword pzPendingImfile $pageName camera -var CAMERA + book getword pzPendingImfile $pageName telescope -var TELESCOPE + book getword pzPendingImfile $pageName class -var CLASS + book getword pzPendingImfile $pageName class_id -var CLASS_ID + book getword pzPendingImfile $pageName dbname -var DBNAME + + set.host.for.camera $CAMERA $CLASS_ID + + # 2007-08-30T05:09:59Z + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 5 2 MONTH + substr $DATEOBS 8 2 DAY + + # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship + # this function uses workdir_template, default_host, volume_template, volume_default, + # it sets workdir and volume + set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + + # figure out filename + # XXX may need to use sprintf here + $FILENAME = $workdir_base/$CAMERA/$YEAR\$MONTH\$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits + $workdir = $workdir_template/$CAMERA/$YEAR\$MONTH\$DAY + + # workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb://@HOST@.0/gpc1/20080130 + + # filename examples: + # file://data/ipp005.0/gpc1/20080130/o4437g0025d/o4437g0025d.XY05.fits + # neb://@HOST@.0/gpc1/20080130/o4437g0025d/o4437g0025d.XY05.fits + + book setword pzPendingImfile $pageName filename $FILENAME + + $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --end_stage reg --workdir $workdir --dbname $DBNAME --timeout 120 --verbose + if ($COMPRESS) + $run = $run --compress + else + $run = $run --bytes $BYTES + end + if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS))) + $run = $run --md5 $MD5SUM + end + if ($NEBULOUS) + $run = $run --nebulous + end + # add_standard_args run + + # store the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pzPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pzPendingImfile $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080706/ippTools/notes.txt =================================================================== --- /branches/eam_branch_20080706/ippTools/notes.txt (revision 18467) +++ /branches/eam_branch_20080706/ippTools/notes.txt (revision 18467) @@ -0,0 +1,27 @@ + +2008.07.09 EAM + + In order to implement the 'cleanup' and 'update' strategies, we have + changed the state names and defined addtional states as follows: + + * new : entry is unprocessed (was 'run') + * full : entry is processed and has all output data products (was 'done') + * goto_cleaned : entry should be processed by the 'cleanup' system + * cleaned : entry has been processed by the 'cleanup' system -- + some output data products are now missing. + + * goto_full : entry should be re-processed by the 'update' system + * full : entry has been re-processed by the 'update' system + + * goto_purged : entry should be processed by the 'cleanup' ssytem + to purge all output data + * purged : entry has been purged + + + +2008.05.16 EAM + +caltool + * add the active field, -active flag to -dbs + * add the -region + * change 'catdir' to 'dvo_id' Index: /branches/eam_branch_20080706/ippTools/share/chiptool_completely_processed_exp.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/chiptool_completely_processed_exp.sql (revision 18467) +++ /branches/eam_branch_20080706/ippTools/share/chiptool_completely_processed_exp.sql (revision 18467) @@ -0,0 +1,34 @@ +-- the output of this query must match the format of chipRun row +SELECT DISTINCT + chip_id, + exp_id, + state, + workdir, + workdir_state, + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage +FROM + (SELECT + chipRun.*, + rawImfile.class_id as rawimfile_class_id, + chipProcessedImfile.class_id + FROM chipRun + JOIN rawImfile + USING(exp_id) + LEFT JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND rawImfile.exp_id = chipProcessedImfile.exp_id + AND rawImfile.class_id = chipProcessedImfile.class_id + WHERE + chipRun.state = 'new' + GROUP BY + chipRun.chip_id, + chipRun.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id) + AND SUM(chipProcessedImfile.fault) = 0 + ) as Foo Index: /branches/eam_branch_20080706/ippTools/share/difftool_inputskyfile.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/difftool_inputskyfile.sql (revision 18467) +++ /branches/eam_branch_20080706/ippTools/share/difftool_inputskyfile.sql (revision 18467) @@ -0,0 +1,73 @@ +SELECT * FROM + (SELECT + diffRun.diff_id, + diffRun.skycell_id, + diffRun.tess_id, + 0 as stack_id, + warpSkyfile.warp_id, + warpSkyfile.uri, + warpSkyfile.path_base, + diffInputSkyfile.template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN warpSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + JOIN warpRun + ON diffInputSkyfile.warp_id = warpRun.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + WHERE + diffRun.state = 'run' + AND warpRun.state = 'stop' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'full' + UNION + SELECT + diffRun.diff_id, + diffRun.skycell_id, + diffRun.tess_id, + stackSumSkyfile.stack_id, + 0 as warp_id, + stackSumSkyfile.uri, + stackSumSkyfile.path_base, + diffInputSkyfile.template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN stackSumSkyfile + ON diffInputSkyfile.stack_id = stackSumSkyfile.stack_id + JOIN stackInputSkyfile + ON diffInputSkyfile.stack_id = stackInputSkyfile.stack_id + JOIN warpRun + ON stackInputSkyfile.warp_id = warpRun.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + WHERE + diffRun.state = 'run' + AND warpRun.state = 'stop' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'full' + ) as Foo Index: /branches/eam_branch_20080706/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/difftool_todiffskyfile.sql (revision 18467) +++ /branches/eam_branch_20080706/ippTools/share/difftool_todiffskyfile.sql (revision 18467) @@ -0,0 +1,32 @@ +SELECT DISTINCT + diffRun.diff_id, + diffRun.workdir, + diffRun.skycell_id, + diffRun.tess_id, + rawExp.camera +FROM diffRun +JOIN diffInputSkyfile + USING(diff_id) +JOIN warpSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN diffSkyfile + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id +WHERE + diffRun.state = 'run' + AND warpRun.state = 'stop' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'full' + AND diffSkyfile.diff_id IS NULL Index: /branches/eam_branch_20080706/ippTools/share/faketool_find_camrun.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/faketool_find_camrun.sql (revision 18467) +++ /branches/eam_branch_20080706/ippTools/share/faketool_find_camrun.sql (revision 18467) @@ -0,0 +1,36 @@ +SELECT + * +FROM + (SELECT DISTINCT + camRun.cam_id, + chipRun.*, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_tag, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.solang + FROM camRun + JOIN chipRun + using(chip_id) + JOIN rawExp + using(exp_id) + WHERE + camRun.state = 'stop' + AND chipRun.state = 'full' +) as Foo Index: /branches/eam_branch_20080706/ippTools/share/faketool_find_pendingexp.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/faketool_find_pendingexp.sql (revision 18467) +++ /branches/eam_branch_20080706/ippTools/share/faketool_find_pendingexp.sql (revision 18467) @@ -0,0 +1,34 @@ +-- this query is used by both camtool -pendingexp & camtool -addprocessedexp it +-- does a little more work then is necessary for -addprocessed but it seems +-- "cleaner" to use the same query for both cases +SELECT * FROM + (SELECT + fakeRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + LEFT JOIN fakeProcessedImfile + USING(fake_id) + LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label + WHERE + fakeRun.state = 'run' + AND camRun.state = 'stop' + AND chipRun.state = 'full' + AND fakeMask.label IS NULL + AND fakeProcessedImfile.fake_id IS NULL + GROUP BY + fakeRun.fake_id + ) as Foo Index: /branches/eam_branch_20080706/ippTools/share/warptool_imfile.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/warptool_imfile.sql (revision 18467) +++ /branches/eam_branch_20080706/ippTools/share/warptool_imfile.sql (revision 18467) @@ -0,0 +1,27 @@ +SELECT DISTINCT + rawImfile.*, + warpRun.fake_id, + camRun.cam_id as cam_id, + chipProcessedImfile.uri as chip_uri, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base +FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id +JOIN rawImfile -- is there any reason not to refer back to rawimfiles? + ON chipProcessedImfile.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id +WHERE + warpRun.state = 'run' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'full' + Index: /branches/eam_branch_20080706/ippTools/share/warptool_scmap.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/warptool_scmap.sql (revision 18467) +++ /branches/eam_branch_20080706/ippTools/share/warptool_scmap.sql (revision 18467) @@ -0,0 +1,24 @@ +SELECT + warpSkyCellMap.*, + chipProcessedImfile.uri, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base +FROM warpRun +JOIN warpSkyCellMap + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND warpSkyCellMap.class_id = chipProcessedImfile.class_id +WHERE +-- warpRun.state = 'run' + fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'full' Index: /branches/eam_branch_20080706/ippTools/share/warptool_tooverlap.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/warptool_tooverlap.sql (revision 18467) +++ /branches/eam_branch_20080706/ippTools/share/warptool_tooverlap.sql (revision 18467) @@ -0,0 +1,29 @@ +SELECT + warpRun.warp_id, + warpRun.fake_id, + warpRun.workdir, + warpRun.tess_id, + warpRun.label, + rawExp.camera, + exp_id, + warpRun.magiced +FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN warpSkyCellMap + USING(warp_id) +LEFT JOIN warpMask + ON warpRun.label = warpMask.label +WHERE + warpRun.state = 'run' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'full' + AND warpSkyCellMap.warp_id IS NULL + AND warpMask.label IS NULL Index: /branches/eam_branch_20080706/ippTools/share/warptool_towarped.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/warptool_towarped.sql (revision 18467) +++ /branches/eam_branch_20080706/ippTools/share/warptool_towarped.sql (revision 18467) @@ -0,0 +1,37 @@ +SELECT DISTINCT + warpSkyCellMap.warp_id, + warpSkyCellMap.skycell_id, + warpSkyCellMap.tess_id, + warpRun.fake_id, + camRun.cam_id, + rawExp.camera, + warpRun.workdir +FROM warpRun +JOIN warpSkyCellMap + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN warpSkyfile + ON warpRun.warp_id = warpSkyfile.warp_id + AND warpSkyCellMap.skycell_id = warpSkyfile.skycell_id + AND warpSkyCellMap.tess_id = warpSkyfile.tess_id +LEFT JOIN warpMask + ON warpRun.label = warpMask.label +WHERE + warpRun.state = 'run' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'full' + AND warpSkyfile.warp_id IS NULL + AND warpSkyfile.skycell_id IS NULL + AND warpSkyfile.tess_id IS NULL + AND warpSkyCellMap.fault = 0 + AND warpMask.label IS NULL Index: /branches/eam_branch_20080706/ippTools/src/camtool.c =================================================================== --- /branches/eam_branch_20080706/ippTools/src/camtool.c (revision 18467) +++ /branches/eam_branch_20080706/ippTools/src/camtool.c (revision 18467) @@ -0,0 +1,911 @@ +/* + * camtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" +#include "camtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = camtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(CAMTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(CAMTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(CAMTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(CAMTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(CAMTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(CAMTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(CAMTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(CAMTOOL_MODE_BLOCK, blockMode); + MODECASE(CAMTOOL_MODE_MASKED, maskedMode); + MODECASE(CAMTOOL_MODE_UNBLOCK, unblockMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("camtool_find_chip_id.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + // loop over our list of chipRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + chipRunRow *row = chipRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxcamQueueByChipID(config, + row->chip_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + reduction ? reduction : row->reduction, + expgroup ? expgroup : row->expgroup, + dvodb ? dvodb : row->dvodb, + tess_id ? tess_id : row->tess_id, + end_stage ? end_stage : row->end_stage + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->chip_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set chipRun.state to state + if (!pxcamRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set chipRun.label to label + if (!pxcamRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("camtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("camtool_find_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + // optional + PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); + PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); + PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); + PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); + PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); + PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); + PXOPT_LOOKUP_F32(zp_mean, config->args, "-zp_mean", false, false); + PXOPT_LOOKUP_F32(zp_stdev, config->args, "-zp_stdev", false, false); + PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); + PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); + PXOPT_LOOKUP_F32(dtime_detrend, config->args, "-dtime_detrend", false, false); + PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); + PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(n_stars, config->args, "-n_stars", false, false); + PXOPT_LOOKUP_F32(n_extended, config->args, "-n_extended", false, false); + if (n_extended < 0) { + psError(PS_ERR_UNKNOWN, true, "-n_extended is required"); + return false; + } + PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); + if (n_cr < 0) { + psError(PS_ERR_UNKNOWN, true, "-n_cr is required"); + return false; + } + PXOPT_LOOKUP_F32(n_astrom, config->args, "-n_astrom", false, false); + if (n_astrom < 0) { + psError(PS_ERR_UNKNOWN, true, "-n_astrom is required"); + return false; + } + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // default + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + psString query = pxDataGet("camtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + { + // build a query to search by cam_id + psMetadata *where = psMetadataAlloc(); + if (cam_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "cam_id", 0, "==", cam_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item cam_id"); + psFree(where); + psFree(query); + return false; + } + } + + psString whereClaus = psDBGenerateWhereSQL(where, NULL); + psFree(where); + if (whereClaus) { + psStringAppend(&query, " %s", whereClaus); + psFree(whereClaus); + } + + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]); + psFree(output); + camProcessedExpRow *row = camProcessedExpRowAlloc( + pendingRow->cam_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + sigma_ra, + sigma_dec, + ap_resid, + ap_resid_stdev, + zp_mean, + zp_stdev, + fwhm_major, + fwhm_minor, + dtime_detrend, + dtime_photom, + dtime_astrom, + hostname, + n_stars, + n_extended, + n_cr, + n_astrom, + path_base, + code + ); + + if (!camProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + // since there is only one exp per 'run' set camRun.state = 'stop' + if (!pxcamRunSetState(config, row->cam_id, "stop")) { + psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); + psFree(row); + psFree(pendingRow); + return false; + } + + // NULL for end_stage means go as far as possible + // EAM : skip here if code != 0 + // Also, we can run fake even if tess_id is not defined + if (code || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam"))) { + psFree(row); + psFree(pendingRow); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + psFree(row); + // else continue on... + + if (!pxfakeQueueByCamID(config, + pendingRow->cam_id, + pendingRow->workdir, + pendingRow->label, + pendingRow->reduction, + pendingRow->expgroup, + pendingRow->dvodb, + pendingRow->tess_id, + pendingRow->end_stage + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); + psFree(pendingRow); + return false; + } + psFree(pendingRow); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = pxDataGet("camtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND camProcessedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND camProcessedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + +{ + psString query = pxDataGet("camtool_reset_faulted_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); +} + +{ + psString query = pxDataGet("camtool_revertprocessedexp.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); +} + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + if (!pxSetFaultCode(config->dbh, "camProcessedExp", config->where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!camMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM camMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM camMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} Index: /branches/eam_branch_20080706/ippTools/src/camtoolConfig.c =================================================================== --- /branches/eam_branch_20080706/ippTools/src/camtoolConfig.c (revision 18467) +++ /branches/eam_branch_20080706/ippTools/src/camtoolConfig.c (revision 18467) @@ -0,0 +1,584 @@ +/* + * camtoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "camtool.h" + +pxConfig *camtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebydefinebyqueryArgs = psMetadataAlloc(); + // XXX need to allow multiple chip_ids + psMetadataAddS64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chip_id", 0); + // XXX need to allow multiple exp_ids + psMetadataAddS64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exp_id", 0); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_name", 0, + "search by exp_name", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-inst", 0, + "search for camera", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, + "search for telescope", NULL); + psMetadataAddTime(definebydefinebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, + "search for exposures by time (>=)", NULL); + psMetadataAddTime(definebydefinebyqueryArgs, PS_LIST_TAIL, "-dateobs_end", 0, + "search for exposures by time (<)", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exp_tag", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exp_type", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, + "search by filelevel", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, + "search by reduction class", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-filter", 0, + "search for filter", NULL); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, + "define min", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, + "define min", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, + "define max", NAN); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min", NAN); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max", NAN); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, + "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, + "define min fraction of saturated pixels", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, + "define min", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, + "define min", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, + "define max", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(definebydefinebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-object", 0, + "search by exposure object", NULL); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF32(definebydefinebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, + "define workdir", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, + "define label", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, + "define reduction class", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_expgroup", 0, + "define exposure group", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, + "define DVO db", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, + "define tess ID", NULL); + psMetadataAddStr(definebydefinebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, + "define end stage", NULL); + psMetadataAddBool(definebydefinebyqueryArgs, PS_LIST_TAIL, "-all", 0, + "allow everything to be queued without search terms", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + // XXX need to allow multiple cam_ids + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by cam_id", 0); + // XXX need to allow multiple chip_ids + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chip_id", 0); + // XXX need to allow multiple exp_ids + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exp_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_name", 0, + "search by exp_name", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-inst", 0, + "search for camera", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-telescope", 0, + "search for telescope", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, + "search for exposures by time (>=)", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_end", 0, + "search for exposures by time (<)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exp_tag", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exp_type", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filelevel", 0, + "search by filelevel", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, + "search by reduction class", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filter", 0, + "search for filter", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_min", 0, + "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_min", 0, + "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_max", 0, + "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, + "define max fraction of saturated pixels", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, + "define min fraction of saturated pixels", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_min", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_min", 0, + "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_min", 0, + "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-object", 0, + "search by exposure object", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, + "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, + "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, + "set label", NULL); + + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", 0); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", 0); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, + "search by class", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "define camtool ID (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-uri", 0, + "define URI (required)", NULL); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-bias", 0, + "define bias", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-bias_stdev", 0, + "define bias stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe term 0", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe term 1", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe term 2", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sigma_ra", 0, + "define exposure E ra", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sigma_dec", 0, + "define exposure E dec", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ap_resid", 0, + "define aperture residual", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ap_resid_stdev", 0, + "define aperture residual stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zp_mean", 0, + "define zero point mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zp_stdev", 0, + "define zero point stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_major", 0, + "define FWHM (major axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_minor", 0, + "define FWHM (minor axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_detrend", 0, + "define elapsed detrend processing time (seconds)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_photom", 0, + "define elapsed photometry processing time (seconds)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_astrom", 0, + "define elapsed astrometry processing time (seconds)", NAN); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-hostname", 0, + "define hostname", NULL); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, + "define number of stars", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_extended", 0, + "define number of extended objects", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_cr", 0, + "define number of cosmic rays", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_astrom", 0, + "define number of astrometry reference objects", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + psMetadataAddBool(addprocessedexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", 0); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", 0); + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + + // -revertprocessedexp + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + // XXX need to allow multiple cam_ids + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by cam_id", 0); + // XXX need to allow multiple chip_ids + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chip_id", 0); + // XXX need to allow multiple exp_ids + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exp_id", 0); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_name", 0, + "search by exp_name", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-inst", 0, + "search for camera", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-telescope", 0, + "search for telescope", NULL); + psMetadataAddTime(revertprocessedexpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, + "search for exposures by time (>=)", NULL); + psMetadataAddTime(revertprocessedexpArgs, PS_LIST_TAIL, "-dateobs_end", 0, + "search for exposures by time (<)", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exp_tag", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exp_type", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-filelevel", 0, + "search by filelevel", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction", 0, + "search by reduction class", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-filter", 0, + "search for filter", NULL); + psMetadataAddF32(revertprocessedexpArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(revertprocessedexpArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-ra_min", 0, + "define min", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-ra_max", 0, + "define max", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-decl_min", 0, + "define min", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-decl_max", 0, + "define max", NAN); + psMetadataAddF32(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min", NAN); + psMetadataAddF32(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max", NAN); + psMetadataAddF32(revertprocessedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, + "define max fraction of saturated pixels", NAN); + psMetadataAddF32(revertprocessedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, + "define min fraction of saturated pixels", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-bg_min", 0, + "define max", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-bg_max", 0, + "define max", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-alt_min", 0, + "define min", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-alt_max", 0, + "define max", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-az_min", 0, + "define min", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-az_max", 0, + "define max", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(revertprocessedexpArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-object", 0, + "search by exposure object", NULL); + psMetadataAddF32(revertprocessedexpArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF32(revertprocessedexpArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all", 0, + "allow everything to be queued without search terms", false); + psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + + // -updateprocessedexp + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", 0); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class", 0, + "search by class", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code (required)", INT16_MAX); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, + "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, + "name of a label to unmask (required)", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "", CAMTOOL_MODE_DEFINEBYQUERY, definebydefinebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "", CAMTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "", CAMTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-pendingimfile", "", CAMTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-addprocessedexp", "", CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "", CAMTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "", CAMTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "", CAMTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-block", "", CAMTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "", CAMTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "", CAMTOOL_MODE_UNBLOCK, unblockArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + // generate SQL where clause + config->where = psMetadataAlloc(); + +{ + psS64 cam_id = -1; + bool status = false; + if ((cam_id = psMetadataLookupS64(&status, config->args, "-cam_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "cam_id", 0, "==", cam_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item cam_id"); + psFree(config); + return NULL; + } + } +} + +{ + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-chip_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "chip_id", 0, "==", (psS64)atoll(str))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); + psFree(config); + return NULL; + } + } +} + + // convert '-inst' to 'camera' + { + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(config); + return NULL; + } + } + } + addWhereStr(telescope); + addWhereStr(exp_type); + { + int imfiles = 0; + bool status = false; + if ((imfiles = psMetadataLookupS32(&status, config->args, "-imfiles"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "imfiles", 0, "==", imfiles)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item imfiles"); + psFree(config); + return NULL; + } + } + } + addWhereStr(class_id); + addWhereStr(filter); + + // convert '-code' to 'fault' + { + psS16 fault = 0; + bool status = false; + if ((fault = psMetadataLookupS16(&status, config->args, "-code"))) { + if (!psMetadataAddS16(config->where, PS_LIST_TAIL, "fault", 0, "==", fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fault"); + psFree(config); + return NULL; + } + } + } + + if (psListLength(config->where->list) < 1) { + psFree(config->where); + config->where = NULL; + } + + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branch_20080706/ippTools/src/warptoolConfig.c =================================================================== --- /branches/eam_branch_20080706/ippTools/src/warptoolConfig.c (revision 18467) +++ /branches/eam_branch_20080706/ippTools/src/warptoolConfig.c (revision 18467) @@ -0,0 +1,506 @@ +/* + * warptoolConfig.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "warptool.h" + +pxConfig *warptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-fake_id", 0, + "define camtool ID (required)", 0); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-mode", 0, + "define mode (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, + "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, + "define dvodb", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, + "define tess_id", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-end_stage", 0, + "define end stage", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-magiced", 0, + "has this exposure been magiced", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + // XXX need to allow multiple fake_ids + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", 0); + // XXX need to allow multiple fake_ids + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by fake_id", 0); + // XXX need to allow multiple chip_ids + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chip_id", 0); + // XXX need to allow multiple exp_ids + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exp_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_name", 0, + "search by exp_name", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-inst", 0, + "search for camera", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-telescope", 0, + "search for telescope", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, + "search for exposures by time (>=)", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_end", 0, + "search for exposures by time (<)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exp_tag", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exp_type", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filelevel", 0, + "search by filelevel", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, + "search by reduction class", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filter", 0, + "search for filter", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_min", 0, + "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_min", 0, + "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_max", 0, + "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, + "define max fraction of saturated pixels", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, + "define min fraction of saturated pixels", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_min", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_min", 0, + "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_min", 0, + "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_max", 0, + "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-object", 0, + "search by exposure object", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, + "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, + "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, + "set label", NULL); +#if 0 + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); +#endif + + // -exp + psMetadata *expArgs = psMetadataAlloc(); + psMetadataAddS64(expArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", 0); + psMetadataAddS64(expArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by camtool ID", 0); + psMetadataAddU64(expArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(expArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -imfile + psMetadata *imfileArgs = psMetadataAlloc(); + psMetadataAddS64(imfileArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", 0); + psMetadataAddS64(imfileArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by camtool ID", 0); + psMetadataAddU64(imfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(imfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -tooverlap + psMetadata *tooverlapArgs = psMetadataAlloc(); + psMetadataAddS64(tooverlapArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warp ID", 0); + psMetadataAddU64(tooverlapArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tooverlapArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addoverlap + psMetadata *addoverlapArgs = psMetadataAlloc(); + psMetadataAddStr(addoverlapArgs, PS_LIST_TAIL, "-mapfile", 0, + "path to skycell <-> imfile mapping file", NULL); + psMetadataAddS64(addoverlapArgs, PS_LIST_TAIL, "-warp_id", 0, + "set warp ID", 0); + psMetadataAddS16(addoverlapArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -scmap + psMetadata *scmapArgs = psMetadataAlloc(); + psMetadataAddS64(scmapArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", 0); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-skycell_id", 0, + "search by skycell ID", NULL); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-tess_id", 0, + "search by tess ID", NULL); + psMetadataAddU64(scmapArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(scmapArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -towarped + psMetadata *towarpedArgs = psMetadataAlloc(); + psMetadataAddS64(towarpedArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", 0); + psMetadataAddU64(towarpedArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(towarpedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addwarped + psMetadata *addwarpedArgs = psMetadataAlloc(); + psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, + "define warptool ID (required)", 0); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-skycell_id", 0, + "define skycell ID (required)", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, + "define tessellation ID (required)", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-uri", 0, + "define URI of file", 0); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", 0); + psMetadataAddF64(addwarpedArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(addwarpedArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-dtime_warp", 0, + "define elapsed processing time", NAN); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-xmin", 0, + "define minimum x value", INT_MAX); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-xmax", 0, + "define maximum x value", -INT_MAX); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-ymin", 0, + "define minimum y value", INT_MAX); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-ymax", 0, + "define maximum y value", -INT_MAX); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0, + "define hostname", 0); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac", 0, + "define %% of good pixels", NAN); + psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-ignore", 0, + "define if this skycell should be ignored", false); + psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -warped + psMetadata *warpedArgs = psMetadataAlloc(); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id", 0, + "define skycell ID", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id", 0, + "define tessellation ID", NULL); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, + "define exposure tag", 0); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, + "define phase 3 version of exposure tag", 0); + psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -revertwarped + psMetadata *revertwarpedArgs = psMetadataAlloc(); + // XXX need to allow multiple fake_ids + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", 0); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-skycell_id", 0, + "search by skycell ID", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, + "searcy by tessellation ID", NULL); + // XXX need to allow multiple fake_ids + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by fake_id", 0); + // XXX need to allow multiple chip_ids + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chip_id", 0); + // XXX need to allow multiple exp_ids + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exp_id", 0); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_name", 0, + "search by exp_name", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-inst", 0, + "search for camera", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-telescope", 0, + "search for telescope", NULL); + psMetadataAddTime(revertwarpedArgs, PS_LIST_TAIL, "-dateobs_begin", 0, + "search for exposures by time (>=)", NULL); + psMetadataAddTime(revertwarpedArgs, PS_LIST_TAIL, "-dateobs_end", 0, + "search for exposures by time (<)", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exp_tag", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exp_type", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-filelevel", 0, + "search by filelevel", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-reduction", 0, + "search by reduction class", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-filter", 0, + "search for filter", NULL); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ra_min", 0, + "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ra_max", 0, + "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-decl_min", 0, + "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-decl_max", 0, + "define max", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, + "define max fraction of saturated pixels", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, + "define min fraction of saturated pixels", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_min", 0, + "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_max", 0, + "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, + "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, + "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-alt_min", 0, + "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-alt_max", 0, + "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-az_min", 0, + "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-az_max", 0, + "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-object", 0, + "search by exposure object", NULL); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + + psMetadataAddS16(revertwarpedArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + psMetadataAddBool(revertwarpedArgs, PS_LIST_TAIL, "-all", 0, + "allow everything to be queued without search terms", false); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, + "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, + "name of a label to unmask (required)", NULL); + + psFree(now); + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definerun", "", WARPTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-updaterun", "", WARPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-exp", "", WARPTOOL_MODE_EXP, expArgs); + PXOPT_ADD_MODE("-imfile", "", WARPTOOL_MODE_IMFILE, imfileArgs); + PXOPT_ADD_MODE("-tooverlap", "", WARPTOOL_MODE_TOOVERLAP, tooverlapArgs); + PXOPT_ADD_MODE("-addoverlap", "", WARPTOOL_MODE_ADDOVERLAP, addoverlapArgs); + PXOPT_ADD_MODE("-scmap", "", WARPTOOL_MODE_SCMAP, scmapArgs); + PXOPT_ADD_MODE("-towarped", "", WARPTOOL_MODE_TOWARPED, towarpedArgs); + PXOPT_ADD_MODE("-addwarped", "", WARPTOOL_MODE_ADDWARPED, addwarpedArgs); + PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); + PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs); + PXOPT_ADD_MODE("-block", "set a label block", WARPTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked lables", WARPTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", WARPTOOL_MODE_UNBLOCK, unblockArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define addWhereS32(name) \ +{ \ + psS32 s32 = 0; \ + bool status = false; \ + if ((s32= psMetadataLookupS32(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, #name, 0, "==", s32)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + + // generate SQL where clause + config->where = psMetadataAlloc(); + +{ + psS64 warp_id = -1; + bool status = false; + if ((warp_id = psMetadataLookupS64(&status, config->args, "-warp_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "warp_id", 0, "==", warp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item warp_id"); + psFree(config); + return NULL; + } + } +} + addWhereStr(skycell_id); + addWhereStr(tess_id); + addWhereStr(exp_id); + + // convert '-code' to 'fault' + { + psS16 fault = 0; + bool status = false; + if ((fault = psMetadataLookupS16(&status, config->args, "-code"))) { + if (!psMetadataAddS16(config->where, PS_LIST_TAIL, "fault", 0, "==", fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fault"); + psFree(config); + return NULL; + } + } + } + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branch_20080706/ippconfig/Makefile.am =================================================================== --- /branches/eam_branch_20080706/ippconfig/Makefile.am (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/Makefile.am (revision 18467) @@ -0,0 +1,56 @@ +SUBDIRS = \ + recipes \ + isp \ + gpc1 \ + cfh12k \ + megacam \ + mosaic2 \ + simple \ + simmosaic \ + simtest \ + tc3 \ + skycell \ + sdssmosaic \ + sdss \ + esowfi \ + lbc_red \ + lulin \ + uh8k + +install_files = \ + system.config \ + GSCregions.tbl \ + dvo.photcodes + +built_sources = \ + ipprc.config.in \ + dvo.site.in + +built_files = \ + ipprc.config \ + dvo.site + +installdir = $(datadir)/ippconfig + +install_DATA = $(install_files) $(built_files) + +install-data-hook: + chmod 0755 $(installdir) + +EXTRA_DIST = $(install_files) $(built_sources) + +BUILT_SOURCES = $(built_files) + +#ACLOCAL_AMFLAGS = -I m4 + +# From the autoconf manual, section 4.7.2 "Installation Directory Variables" +edit = sed \ + -e 's|@pkgdatadir[@]|$(pkgdatadir)|g' \ + -e 's|@prefix[@]|$(prefix)|g' + +ipprc.config dvo.site: Makefile + rm -f $@ + $(edit) '$(srcdir)/$@.in' > $@ + +ipprc.config: $(srcdir)/ipprc.config.in +dvo.site: $(srcdir)/dvo.site.in Index: /branches/eam_branch_20080706/ippconfig/configure.ac =================================================================== --- /branches/eam_branch_20080706/ippconfig/configure.ac (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/configure.ac (revision 18467) @@ -0,0 +1,31 @@ +AC_PREREQ(2.59) + +AC_INIT([ippConfig], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([ipprc.config.in]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_MAINTAINER_MODE + +AC_PROG_INSTALL + +AC_CONFIG_FILES([ + Makefile + recipes/Makefile + isp/Makefile + gpc1/Makefile + cfh12k/Makefile + megacam/Makefile + mosaic2/Makefile + simple/Makefile + simmosaic/Makefile + simtest/Makefile + tc3/Makefile + skycell/Makefile + sdssmosaic/Makefile + sdss/Makefile + esowfi/Makefile + lbc_red/Makefile + lulin/Makefile + uh8k/Makefile +]) +AC_OUTPUT Index: /branches/eam_branch_20080706/ippconfig/gpc1/format_raw.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/gpc1/format_raw.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/gpc1/format_raw.config (revision 18467) @@ -0,0 +1,588 @@ +# The raw GPC data comes off the telescope with each of the chips stored in separate files + +# How to identify this type +RULE METADATA +# ORIGIN STR PS1 +# TELESCOP STR PS1 +### SOMETHING STR GPC1 + CONTROLR STR STARGRASP + NAMPS S32 64 +END + +# How to read this data +FILE METADATA + PHU STR CHIP # The FITS file represents a single chip + EXTENSIONS STR CELL # The extensions represent cells + FPA.OBS STR CONTROLR # A PHU keyword for unique identifier within the hierarchy level + CONTENT STR DETECTOR # How to determine content of FITS file + CONTENT.RULE STR {CHIP.ID} # How to derive the CONTENT when writing +END + +# What's in the FITS file? +CONTENTS METADATA + # CONTENT = chip name : type + CCID58-2-15a1 STR XY01:GPCChip # chip 00 + CCID58-2-18a1 STR XY02:GPCChip # chip 01 + CCID58-1-13a1 STR XY03:GPCChip # chip 02 + CCID58-1-09a1 STR XY04:GPCChip # chip 03 + CCID58-2-16b1 STR XY05:GPCChip # chip 04 + CCID58-2-09b1 STR XY06:GPCChip # chip 05 + CCID58-2-22a1 STR XY10:GPCChip # chip 06 + CCID58-2-13a2 STR XY11:GPCChip # chip 07 + CCID58-1-17a2 STR XY12:GPCChip # chip 08 + CCID58-1-14a2 STR XY13:GPCChip # chip 09 + CCID58-1-09a2 STR XY14:GPCChip # chip 10 + CCID58-2-13b1 STR XY15:GPCChip # chip 11 + CCID58-1-09b1 STR XY16:GPCChip # chip 12 + CCID58-2-10b2 STR XY17:GPCChip # chip 13 + CCID58-1-19a1 STR XY20:GPCChip # chip 14 + CCID58-1-04a1 STR XY21:GPCChip # chip 15 + CCID58-2-09a2 STR XY22:GPCChip # chip 16 + CCID58-1-02a1 STR XY23:GPCChip # chip 17 + CCID58-1-04a2 STR XY24:GPCChip # chip 18 + CCID58-1-01b1 STR XY25:GPCChip # chip 19 + CCID58-2-07b2 STR XY26:GPCChip # chip 20 + CCID58-1-18b1 STR XY27:GPCChip # chip 21 + CCID58-1-12a1 STR XY30:GPCChip # chip 22 + CCID58-2-01a1 STR XY31:GPCChip # chip 23 + CCID58-2-01a2 STR XY32:GPCChip # chip 24 + CCID58-1-02a2 STR XY33:GPCChip # chip 25 + CCID58-2-04a1 STR XY34:GPCChip # chip 26 + CCID58-1-01b2 STR XY35:GPCChip # chip 27 + CCID58-1-02b1 STR XY36:GPCChip # chip 28 + CCID58-2-16b2 STR XY37:GPCChip # chip 29 + CCID58-2-18b2 STR XY40:GPCChip # chip 30 + CCID58-1-03b2 STR XY41:GPCChip # chip 31 + CCID58-1-07b1 STR XY42:GPCChip # chip 32 + CCID58-2-11b2 STR XY43:GPCChip # chip 33 + CCID58-1-05a2 STR XY44:GPCChip # chip 34 + CCID58-1-21a2 STR XY45:GPCChip # chip 35 + CCID58-2-07a2 STR XY46:GPCChip # chip 36 + CCID58-1-21a1 STR XY47:GPCChip # chip 37 + CCID58-1-17b1 STR XY50:GPCChip # chip 38 + CCID58-2-09b2 STR XY51:GPCChip # chip 39 + CCID58-1-04b1 STR XY52:GPCChip # chip 40 + CCID58-1-05b2 STR XY53:GPCChip # chip 41 + CCID58-2-09a1 STR XY54:GPCChip # chip 42 + CCID58-2-12a2 STR XY55:GPCChip # chip 43 + CCID58-1-10a2 STR XY56:GPCChip # chip 44 + CCID58-2-12a1 STR XY57:GPCChip # chip 45 + CCID58-2-22b2 STR XY60:GPCChip # chip 46 + CCID58-1-03b1 STR XY61:GPCChip # chip 47 + CCID58-2-23b2 STR XY62:GPCChip # chip 48 + CCID58-1-02b2 STR XY63:GPCChip # chip 49 + CCID58-1-07a1 STR XY64:GPCChip # chip 50 + CCID58-2-17a2 STR XY65:GPCChip # chip 51 + CCID58-2-14a1 STR XY66:GPCChip # chip 52 + CCID58-2-16a2 STR XY67:GPCChip # chip 53 + CCID58-2-04b2 STR XY71:GPCChip # chip 54 + CCID58-1-14b1 STR XY72:GPCChip # chip 55 + CCID58-1-25b1 STR XY73:GPCChip # chip 56 + CCID58-1-18a1 STR XY74:GPCChip # chip 57 + CCID58-2-16a1 STR XY75:GPCChip # chip 58 + CCID58-2-05a1 STR XY76:GPCChip # chip 59 +END + +CHIPS METADATA + GPCChip METADATA + # Extension name, cellName:cellType + xy00 STR xy00:GPCCell + xy10 STR xy10:GPCCell + xy20 STR xy20:GPCCell + xy30 STR xy30:GPCCell + xy40 STR xy40:GPCCell + xy50 STR xy50:GPCCell + xy60 STR xy60:GPCCell + xy70 STR xy70:GPCCell + xy01 STR xy01:GPCCell + xy11 STR xy11:GPCCell + xy21 STR xy21:GPCCell + xy31 STR xy31:GPCCell + xy41 STR xy41:GPCCell + xy51 STR xy51:GPCCell + xy61 STR xy61:GPCCell + xy71 STR xy71:GPCCell + xy02 STR xy02:GPCCell + xy12 STR xy12:GPCCell + xy22 STR xy22:GPCCell + xy32 STR xy32:GPCCell + xy42 STR xy42:GPCCell + xy52 STR xy52:GPCCell + xy62 STR xy62:GPCCell + xy72 STR xy72:GPCCell + xy03 STR xy03:GPCCell + xy13 STR xy13:GPCCell + xy23 STR xy23:GPCCell + xy33 STR xy33:GPCCell + xy43 STR xy43:GPCCell + xy53 STR xy53:GPCCell + xy63 STR xy63:GPCCell + xy73 STR xy73:GPCCell + xy04 STR xy04:GPCCell + xy14 STR xy14:GPCCell + xy24 STR xy24:GPCCell + xy34 STR xy34:GPCCell + xy44 STR xy44:GPCCell + xy54 STR xy54:GPCCell + xy64 STR xy64:GPCCell + xy74 STR xy74:GPCCell + xy05 STR xy05:GPCCell + xy15 STR xy15:GPCCell + xy25 STR xy25:GPCCell + xy35 STR xy35:GPCCell + xy45 STR xy45:GPCCell + xy55 STR xy55:GPCCell + xy65 STR xy65:GPCCell + xy75 STR xy75:GPCCell + xy06 STR xy06:GPCCell + xy16 STR xy16:GPCCell + xy26 STR xy26:GPCCell + xy36 STR xy36:GPCCell + xy46 STR xy46:GPCCell + xy56 STR xy56:GPCCell + xy66 STR xy66:GPCCell + xy76 STR xy76:GPCCell + xy07 STR xy07:GPCCell + xy17 STR xy17:GPCCell + xy27 STR xy27:GPCCell + xy37 STR xy37:GPCCell + xy47 STR xy47:GPCCell + xy57 STR xy57:GPCCell + xy67 STR xy67:GPCCell + xy77 STR xy77:GPCCell + END +END + +# Specify the cell data +CELLS METADATA + GPCCell METADATA + CELL.TRIMSEC.SOURCE STR HEADER + CELL.TRIMSEC STR DATASEC + CELL.BIASSEC.SOURCE STR HEADER + CELL.BIASSEC STR BIASSEC + END +END + + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.FILTERID STR FILTERID + FPA.FILTER STR FILTERID + FPA.RA STR RA + FPA.DEC STR DEC + # FPA.RA STR COMRA + # FPA.DEC STR COMDEC + FPA.RADECSYS STR RADECSYS + FPA.OBSTYPE STR OBSTYPE + FPA.OBJECT STR OBJECT + FPA.COMMENT STR CMMTOBS + FPA.AIRMASS STR AIRMASS + FPA.POSANGLE STR POSANGLE + FPA.FOCUS STR M2Z + FPA.TIME STR MJD-OBS + FPA.ALT STR ALT + FPA.AZ STR AZ + FPA.TEMP STR DETTEM + FPA.M1X STR M1X + FPA.M1Y STR M1Y + FPA.M1Z STR M1Z + FPA.M1TIP STR M1TIP + FPA.M1TILT STR M1TILT + FPA.M2X STR M2X + FPA.M2Y STR M2Y + FPA.M2Z STR M2Z + FPA.M2TIP STR M2TIP + FPA.M2TILT STR M2TILT + FPA.ENV.TEMP STR ENVTEM + FPA.ENV.HUMID STR ENVHUM + FPA.ENV.WIND STR ENVWIN + FPA.ENV.DIR STR ENVDIR + + FPA.TELTEMP.M1 STR TELTEMM1 # Primary mirror temps (C) + FPA.TELTEMP.M1CELL STR TELTEMMS # Primary mirror support temps (C) + FPA.TELTEMP.M2 STR TELTEMM2 # Secondary mirror temps (C + FPA.TELTEMP.SPIDER STR TELTEMSP # Spider temperatures (C) + FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C + FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.PON.TIME STR PONTIME + + CHIP.ID STR DETECTOR + CELL.XBIN STR CCDSUM + CELL.YBIN STR CCDSUM + CELL.X0 STR IMNPIX1 + CELL.Y0 STR IMNPIX2 + CELL.XPARITY STR LTM1_1 + CELL.YPARITY STR LTM2_2 +# CELL.SATURATION STR SATURATE + +# CHIP.TEMP STR DETTEM # XY24 is bad, so get it from the database; all others from header + CHIP.TEMP.DEPEND STR CHIP.NAME + CHIP.TEMP METADATA + XY01 STR DETTEM + XY02 STR DETTEM + XY03 STR DETTEM + XY04 STR DETTEM + XY05 STR DETTEM + XY06 STR DETTEM + XY10 STR DETTEM + XY11 STR DETTEM + XY12 STR DETTEM + XY13 STR DETTEM + XY14 STR DETTEM + XY15 STR DETTEM + XY16 STR DETTEM + XY17 STR DETTEM + XY20 STR DETTEM + XY21 STR DETTEM + XY22 STR DETTEM + XY23 STR DETTEM +# XY24 STR DETTEM # This is currently bad --- get it from the database + XY25 STR DETTEM + XY26 STR DETTEM + XY27 STR DETTEM + XY30 STR DETTEM + XY31 STR DETTEM + XY32 STR DETTEM + XY33 STR DETTEM + XY34 STR DETTEM + XY35 STR DETTEM + XY36 STR DETTEM + XY37 STR DETTEM + XY40 STR DETTEM + XY41 STR DETTEM + XY42 STR DETTEM + XY43 STR DETTEM + XY44 STR DETTEM + XY45 STR DETTEM + XY46 STR DETTEM + XY47 STR DETTEM + XY50 STR DETTEM + XY51 STR DETTEM + XY52 STR DETTEM + XY53 STR DETTEM + XY54 STR DETTEM + XY55 STR DETTEM + XY56 STR DETTEM + XY57 STR DETTEM + XY60 STR DETTEM + XY61 STR DETTEM + XY62 STR DETTEM + XY63 STR DETTEM + XY64 STR DETTEM + XY65 STR DETTEM + XY66 STR DETTEM + XY67 STR DETTEM + XY71 STR DETTEM + XY72 STR DETTEM + XY73 STR DETTEM + XY74 STR DETTEM + XY75 STR DETTEM + XY76 STR DETTEM + END + + FPA.EXPOSURE STR EXPREQ # Requested exposure time, presumably camera exposure time + CELL.EXPOSURE STR EXPTIME # Exposure time measured by shutter + CELL.DARKTIME STR DARKTIME # Exposure time for camera +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.TELESCOPE STR PS1 + FPA.INSTRUMENT STR GPC1 + FPA.DETECTOR STR GPC1 + FPA.TIMESYS STR UTC + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CHIP.X0.DEPEND STR CHIP.NAME + CHIP.X0 METADATA + XY01 S32 4971 + XY02 S32 4971 + XY03 S32 4971 + XY04 S32 4971 + XY05 S32 4971 + XY06 S32 4971 + XY10 S32 9942 + XY11 S32 9942 + XY12 S32 9942 + XY13 S32 9942 + XY14 S32 9942 + XY15 S32 9942 + XY16 S32 9942 + XY17 S32 9942 + XY20 S32 14913 + XY21 S32 14913 + XY22 S32 14913 + XY23 S32 14913 + XY24 S32 14913 + XY25 S32 14913 + XY26 S32 14913 + XY27 S32 14913 + XY30 S32 19884 + XY31 S32 19884 + XY32 S32 19884 + XY33 S32 19884 + XY34 S32 19884 + XY35 S32 19884 + XY36 S32 19884 + XY37 S32 19884 + XY40 S32 20041 + XY41 S32 20041 + XY42 S32 20041 + XY43 S32 20041 + XY44 S32 20041 + XY45 S32 20041 + XY46 S32 20041 + XY47 S32 20041 + XY50 S32 25012 + XY51 S32 25012 + XY52 S32 25012 + XY53 S32 25012 + XY54 S32 25012 + XY55 S32 25012 + XY56 S32 25012 + XY57 S32 25012 + XY60 S32 29983 + XY61 S32 29983 + XY62 S32 29983 + XY63 S32 29983 + XY64 S32 29983 + XY65 S32 29983 + XY66 S32 29983 + XY67 S32 29983 + XY71 S32 34954 + XY72 S32 34954 + XY73 S32 34954 + XY74 S32 34954 + XY75 S32 34954 + XY76 S32 34954 + END + CHIP.Y0.DEPEND STR CHIP.NAME + CHIP.Y0 METADATA + XY01 S32 10286 + XY02 S32 15429 + XY03 S32 20572 + XY04 S32 25715 + XY05 S32 30858 + XY06 S32 36001 + XY10 S32 5143 + XY11 S32 10286 + XY12 S32 15429 + XY13 S32 20572 + XY14 S32 25715 + XY15 S32 30858 + XY16 S32 36001 + XY17 S32 41144 + XY20 S32 5143 + XY21 S32 10286 + XY22 S32 15429 + XY23 S32 20572 + XY24 S32 25715 + XY25 S32 30858 + XY26 S32 36001 + XY27 S32 41144 + XY30 S32 5143 + XY31 S32 10286 + XY32 S32 15429 + XY33 S32 20572 + XY34 S32 25715 + XY35 S32 30858 + XY36 S32 36001 + XY37 S32 41144 + XY40 S32 306 + XY41 S32 5449 + XY42 S32 10592 + XY43 S32 15735 + XY44 S32 20878 + XY45 S32 26021 + XY46 S32 31164 + XY47 S32 36307 + XY50 S32 306 + XY51 S32 5449 + XY52 S32 10592 + XY53 S32 15735 + XY54 S32 20878 + XY55 S32 26021 + XY56 S32 31164 + XY57 S32 36307 + XY60 S32 306 + XY61 S32 5449 + XY62 S32 10592 + XY63 S32 15735 + XY64 S32 20878 + XY65 S32 26021 + XY66 S32 31164 + XY67 S32 36307 + XY71 S32 5449 + XY72 S32 10592 + XY73 S32 15735 + XY74 S32 20878 + XY75 S32 26021 + XY76 S32 31164 + END + CHIP.XPARITY.DEPEND STR CHIP.NAME + CHIP.XPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CHIP.YPARITY.DEPEND STR CHIP.NAME + CHIP.YPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CELL.GAIN F32 1.0 + CELL.READNOISE F32 15.0 + CELL.READDIR S32 1 + CELL.BAD S32 -100 +# CELL.TIME STR MJD-OBS +# CELL.TIMESYS STR TIMESYS + CELL.SATURATION F32 40000.0 +END + +# How to translation PS concepts into database lookups +DATABASE METADATA + CHIP.TEMP.DEPEND STR CHIP.NAME + CHIP.TEMP METADATA + XY24 STR SELECT AVG(ccd_temp) FROM rawImfile WHERE dateobs = '{FPA.TIME}' AND class_id != 'XY24' + END +END + + +# Where there might be some ambiguity, specify the format +FORMATS METADATA + FPA.RA STR DEGREES + FPA.DEC STR DEGREES + FPA.TIME STR MJD + CELL.TIME STR MJD + CELL.BINNING STR TOGETHER + CELL.X0 STR FORTRAN + CELL.Y0 STR FORTRAN +END + +# Recipe options +RECIPES METADATA +END + +# How to get the supplementary stuff: mask and weight +SUPPLEMENTARY METADATA + MASK.SOURCE STR FILE # Source type for mask: EXT | FILE + MASK.NAME STR %a_mask.fits # Name for mask extension or filename + WEIGHT.SOURCE STR FILE # Source type for weight: EXT | FILE + WEIGHT.NAME STR %a_weight.fits # Name for weight extension or filename +END Index: /branches/eam_branch_20080706/ippconfig/lulin/.cvsignore =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/.cvsignore (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/.cvsignore (revision 18467) @@ -0,0 +1,2 @@ +Makefile +Makefile.in Index: /branches/eam_branch_20080706/ippconfig/lulin/Makefile.am =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/Makefile.am (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/Makefile.am (revision 18467) @@ -0,0 +1,22 @@ + +installdir = $(datadir)/ippconfig/lulin + +install_files = \ + dvo.config \ + camera.config \ + format.config \ + ppImage.config \ + ppMerge.config \ + psastro.config \ + psphot.config \ + pswarp.config \ + rejections.config + +install_DATA = $(install_files) + +install-data-hook: + chmod 0755 $(installdir) + +EXTRA_DIST = $(install_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/eam_branch_20080706/ippconfig/lulin/camera.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/camera.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/camera.config (revision 18467) @@ -0,0 +1,78 @@ +# Camera configuration file for Lulin 1m Optical Telescope + +# File formats that we know about +FORMATS METADATA + SIMPLE STR lulin/format.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + Chip STR Cell +END + +# Lookup table for filter name to filter abstract name +FILTER.ID METADATA + UNKNOWN STR UNKNOWN +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa + CHIP STR {CHIP.NAME} + CELL STR {CHIP.NAME}:{CELL.NAME} + fpa STR fpa + chip STR {CHIP.NAME} + cell STR {CHIP.NAME}:{CELL.NAME} +END + +DVO.CAMERADIR STR lulin # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR LIGHT + science STR LIGHT +END + +# Recipe options +RECIPES METADATA + # Other recipes + PSPHOT STR lulin/psphot.config # psphot details + PSASTRO STR lulin/psastro.config # psastro details + PSWARP STR lulin/pswarp.config # pswarp details + PPIMAGE STR lulin/ppImage.config # ppImage recipe + REJECTIONS STR lulin/rejections.config # rejection recipe + PPMERGE STR lulin/ppMerge.config # ppMerge recipe +END + +# reduction classes (recipes which are grouped together) +REDUCTION STR recipes/reductionClasses.mdc + +FITSTYPES STR recipes/fitstypes.mdc + +FILERULES STR recipes/filerules-simple.mdc # File rules appropriate for MEF format + +EXTNAME.RULES METADATA + CMF.HEAD STR {CHIP.NAME}.hdr + CMF.DATA STR {CHIP.NAME}.psf # use .PSF and .EXT? + CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? + CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + + PSF.HEAD STR {CHIP.NAME}.hdr + PSF.TABLE STR {CHIP.NAME}.psf_model + PSF.RESID STR {CHIP.NAME}.psf_resid + + REF.ASTROM STR {CHIP.NAME}.ref_astrom +END + +BLANK.HEADERS METADATA + FPA.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS + FPA.NAME STR EXPNUM +END Index: /branches/eam_branch_20080706/ippconfig/lulin/dvo.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/dvo.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/dvo.config (revision 18467) @@ -0,0 +1,67 @@ + +# location of DVO database tables +CATDIR /data/alala.0/eugene/isp/catdir + +# keywords used by DVO to interpret the headers + +# keyword abstractions: +#DATE-KEYWORD DATE-OBS +#DATE-MODE yyyy-mm-dd +#UT-KEYWORD UTC-OBS +JD-KEYWORD NONE +MJD-KEYWORD MJD-OBS + +# other keyword abstractions +EXPTIME-KEYWORD EXPTIME +AIRMASS-KEYWORD AIRMASS +CCDNUM-KEYWORD EXTNAME +ST-KEYWORD NONE +OBSERVATORY-LATITUDE NONE +OBSERVATORY-LONGITUDE NONE +SUBPIX_DATAFILE NONE + +# instrumental magnitude range for calibration mode +CAL_INSTMAG_MAX 0 +CAL_INSTMAG_MIN 0 + +# exclude overscan region from the dB image boundaries +XOVERSCAN 0 +YOVERSCAN 0 + +# only upload stars within region; a value of 0 means ignore the limit +ADDSTAR_XMIN 0 +ADDSTAR_XMAX 0 +ADDSTAR_YMIN 0 +ADDSTAR_YMAX 0 + +# exclude stars with SN > SNLIMIT (ADDSTAR_SNLIMIT overrides old name MIN_SN_FSTAT) +ADDSTAR_SNLIMIT 0 + +# correlation radius (arcseconds) +ADDSTAR_RADIUS 1.0 + +# scaled correlation radius +ADDSTAR_NSIGMA 0 + +IMAGE_SCATTER 0.075 # mark images POOR if stdev(Mcal) > IMAGE_SCATTER +STAR_SCATTER 0.005 # mark stars POOR if stdev(Mrel) > STAR_SCATTER +IMAGE_OFFSET 0.100 # mark images POOR if abs(delta(Mcal)) > IMAGE_OFFSET +STAR_CHISQ 10.0 # mark stars POOR if Xm > STAR_CHISQ +STAR_TOOFEW 3 # mark star FEW if N(good) < STAR_TOOFEW +IMAGE_TOOFEW 10 # mark image FEW if N(good) < IMAGE_TOOFEW +IMAGE_GOOD_FRACTION 0.05 # mark image FEW if N(good) < IMAGE_GOOD_FRACTION * Nstars + +SCATTER_LIM 15.0 +MAG_LIM 20.0 # select stars brighter than this for relphot analysis +SIGMA_LIM 0.015 # select measurements with dM < SIGMA_LIM for relphot analysis +# INST_MAG_MIN -17 # optional constraints on magnitude ranges +# INST_MAG_MAX -13 # optional constraints on magnitude ranges + +RELPHOT_GRID_BINNING 256 +RELPHOT_GRID_X 6 +RELPHOT_GRID_Y 14 + +PM_DT_MIN 0.25 +PM_TOOFEW 4 +POS_TOOFEW 1 + Index: /branches/eam_branch_20080706/ippconfig/lulin/format.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/format.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/format.config (revision 18467) @@ -0,0 +1,101 @@ +# The LOT1m data is a single chip with single cell + +# How to recognise this type +RULE METADATA + TELESCOP STR LOT1m + INSTRUME STR Roper + OBSERVAT STR NCU Lulin observatory, Taiwan + CAMERA STR Princeton Instruments 1300B + DETECTOR STR EEV 1340x1300 CCD36-4 + NAXIS S32 2 + SWCREATE STR MaxIm DL Version 4.10 +END + +FILE METADATA + # How to read this data + PHU STR FPA # The FITS file represents an entire FPA + EXTENSIONS STR NONE # There are no extensions + FPA.OBS STR OWNER # A PHU keyword for unique identifier +END + +# What's in the FITS file? +CONTENTS STR Chip:Cell:Amplifier + +# Specify the cells +CELLS METADATA + # Cell type, concepts to use + Amplifier METADATA + CELL.BIASSEC.SOURCE STR NONE + CELL.TRIMSEC.SOURCE STR VALUE + CELL.TRIMSEC STR [0:0,0:0] + END +END + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.TELESCOPE STR TELESCOP + FPA.INSTRUMENT STR INSTRUME + FPA.DETECTOR STR DETECTOR + FPA.OBSTYPE STR IMAGETYP + FPA.TIME STR DATE-OBS TIME-OBS + FPA.TIMESYS STR TIMESYS + FPA.RA STR OBJCTRA + FPA.DEC STR OBJCTDEC + FPA.EXPOSURE STR EXPTIME + FPA.TEMP STR CCD-TEMP + CHIP.TEMP STR CCD-TEMP + CELL.EXPOSURE STR EXPTIME + CELL.DARKTIME STR EXPTIME + CELL.READNOISE STR SLOWRN + CELL.GAIN STR SLOWGAIN + CELL.TIME STR DATE-OBS TIME-OBS + CELL.TIMESYS STR TIMESYS + CELL.XBIN STR XBINNING + CELL.YBIN STR YBINNING + CELL.XWINDOW STR XORGSUBF + CELL.YWINDOW STR YORGSUBF +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.AIRMASS F32 0.0 + FPA.FILTERID STR UNKNOWN + FPA.FILTER STR UNKNOWN + FPA.OBJECT STR Unknown object + FPA.RADECSYS STR ICRS + FPA.POSANGLE STR 0.0 + FPA.FOCUS STR 0.0 + FPA.ALT STR 0.0 + FPA.AZ STR 0.0 + + CHIP.XSIZE S32 1340 + CHIP.YSIZE S32 1300 + CELL.XSIZE S32 1340 + CELL.YSIZE S32 1300 + CELL.READDIR S32 1 # Cell is read in x direction + CELL.SATURATION F32 50000.0 + CELL.BAD F32 0 + CELL.XPARITY S32 1 + CELL.YPARITY S32 1 + CELL.X0 S32 0 + CELL.Y0 S32 0 + CHIP.X0 S32 0 + CHIP.Y0 S32 0 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 +END + + +# How to translation PS concepts into database lookups +DATABASE METADATA +# CELL.GAIN STR SELECT gain FROM camera WHERE chip_id = '{CHIP.NAME}' AND cell_id = '{CELL.NAME}' +END + +# Where there might be some ambiguity, specify the format +FORMATS METADATA + FPA.RA STR HOURS + FPA.DEC STR DEGREES + FPA.TIME STR SEPARATE YEAR.FIRST + CELL.TIME STR SEPARATE YEAR.FIRST +END + Index: /branches/eam_branch_20080706/ippconfig/lulin/ppImage.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/ppImage.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/ppImage.config (revision 18467) @@ -0,0 +1,172 @@ +### ppImage recipe configuration file for Megacam + +# Overscan subtraction +OVERSCAN.SINGLE BOOL FALSE # Reduce overscan to a single value? +OVERSCAN.FIT STR NONE # NONE | POLYNOMIAL | SPLINE +OVERSCAN.ORDER S32 0 # Order of polynomial fit +OVERSCAN.STAT STR MEAN # MEAN | MEDIAN + +# for a test, turn off selected detrend types +OVERSCAN BOOL FALSE # Overscan subtraction +BIAS BOOL FALSE # Bias subtraction +DARK BOOL TRUE # Dark subtraction +FLAT BOOL TRUE # Flat-field normalisation + +# binned output image options +BIN1.XBIN S32 18 +BIN1.YBIN S32 18 +BIN2.XBIN S32 144 +BIN2.YBIN S32 144 + +# megacam needs fringe subtraction +FRINGE BOOL TRUE # Fringe subtraction + +# this table lists extra constraints which should be applied when +# selecting the detrend images. +# note: camera and time are always applied + +DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA +# EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + FILTER STR FPA.FILTERID +# DETTYPE STR DOMEFLAT.RAW +# VERSION STR SKYFLAT.RAW +# VERSION STR DOMEFLAT.CORR.XX +# VERSION STR SKYFLAT.CORR + END + FLAT_CORRECTION METADATA + FILTER STR FPA.FILTER + END + FRINGE METADATA + FILTER STR FPA.FILTER +# AIRMASS STR FPA.AIRMASS +# TWILIGHT STR FPA.TWILIGHT + END + SHUTTER METADATA + END + MASK METADATA + END +END + +# only apply the fringe for these filters +FRINGE.FILTERS MULTI +FRINGE.FILTERS STR i +FRINGE.FILTERS STR z + +PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.N} + + + +# Overscan, bias, dark, shutter +PPIMAGE_OBDS METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.WEIGHT.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field +PPIMAGE_OBDSF METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.WEIGHT.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field, fringe +PPIMAGE_OBDSFR METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.WEIGHT.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom +PPIMAGE_OBDSFRP METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom, astrom +PPIMAGE_OBDSFRA METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END Index: /branches/eam_branch_20080706/ippconfig/lulin/ppMerge.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/ppMerge.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/ppMerge.config (revision 18467) @@ -0,0 +1,40 @@ + +# Bias combination --- don't want min/max rejection +PPMERGE_BIAS METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 2 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Dark combination --- don't want min/max rejection +# More aggressive clipping than bias, so as to remove CRs +PPMERGE_DARK METADATA + REJ F32 2.0 # Rejection threshold (sigma) + ITER S32 4 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Flat combination --- use min/max rejection +PPMERGE_FLAT METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 1 # Number of rejection iterations + FRACHIGH F32 0.3 # Fraction of high pixels to reject immediately + FRACLOW F32 0.1 # Fraction of low pixels to reject immediately + NKEEP S32 5 # Minimum number of pixels in stack to keep + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR MEAN # Statistic to use for combination: +END + + +# Fringe combination --- already included in default, above +PPMERGE_FRINGE METADATA + FRACHIGH F32 0.1 # Fraction of high pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? +END + Index: /branches/eam_branch_20080706/ippconfig/lulin/psastro.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/psastro.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/psastro.config (revision 18467) @@ -0,0 +1,74 @@ + +# astrometry matching parameters + +# nominal plate scale (microns / pixel) +PSASTRO.PIXEL.SCALE F32 1.0 + +# pmAstromGridAngle +# max grid offset in FP units (microns) +# use plate-scale to make this in pixels? +PSASTRO.GRID.OFFSET F32 2500. +PSASTRO.GRID.SCALE F32 50. + +# extra field for ref stars: +PSASTRO.FIELD.PADDING F32 1.0 + +# these tweak are in FP units (pixels, currently) +PSASTRO.TWEAK.SCALE F32 1 +PSASTRO.TWEAK.RANGE F32 75 +PSASTRO.TWEAK.SMOOTH F32 2 +PSASTRO.TWEAK.NSIGMA F32 3 + +# single-chip radius match in pixels +PSASTRO.MATCH.FIT.NITER S32 3 +PSASTRO.MATCH.RADIUS F32 12.0 +PSASTRO.MATCH.RADIUS.N0 F32 15.0 +PSASTRO.MATCH.RADIUS.N1 F32 10.0 +PSASTRO.MATCH.RADIUS.N2 F32 5.0 + +PSASTRO.MAX.NRAW S32 1500 # max stars accepted for fitting (0 for all) +PSASTRO.MAX.NREF S32 1500 # max stars accepted for fitting (0 for all) + +PSASTRO.MIN.INST.MAG.RAW F32 -15.5 # min instrumental magnitude for stars accepted for fitting +PSASTRO.MAX.INST.MAG.RAW F32 -8.0 # max instrumental magnitude for stars accepted for fitting + +PSASTRO.MAX.ERROR F32 10.0 # max error in pixels +PSASTRO.MIN.NSTAR S32 10 # min fitted stars in solution + +PSASTRO.MOSAIC.MODE BOOL FALSE + +PSASTRO.MOSAIC.MAX.ERROR.N0 F32 1.50 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N1 F32 1.50 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N2 F32 0.90 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N3 F32 0.90 # max allow error for valid solution (arcsec) + +PSASTRO.MOSAIC.RADIUS.N0 F32 10.0 +PSASTRO.MOSAIC.RADIUS.N1 F32 8.0 # do not refine the match +PSASTRO.MOSAIC.RADIUS.N2 F32 8.0 # do not refine the match +PSASTRO.MOSAIC.RADIUS.N3 F32 4.0 + +PSASTRO.MOSAIC.CHIP.ORDER.N0 S32 0 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N1 S32 1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N2 S32 1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N3 S32 3 # fit order (-1 means use default) + +PSASTRO.MOSAIC.GRADIENT.NX S32 2 # number of x-dir cells per chip +PSASTRO.MOSAIC.GRADIENT.NY S32 4 # number of y-dir cells per chip + +# use this recipe to set a tight constraint +PSASTRO.FINE METADATA + PSASTRO.MOSAIC.MAX.ERROR.N0 F32 1.50 # max allow error for valid solution (arcsec) + PSASTRO.MOSAIC.MAX.ERROR.N1 F32 1.50 # max allow error for valid solution (arcsec) + PSASTRO.MOSAIC.MAX.ERROR.N2 F32 0.90 # max allow error for valid solution (arcsec) + PSASTRO.MOSAIC.MAX.ERROR.N3 F32 0.90 # max allow error for valid solution (arcsec) + + PSASTRO.MOSAIC.RADIUS.N0 F32 8.0 + PSASTRO.MOSAIC.RADIUS.N1 F32 0.0 + PSASTRO.MOSAIC.RADIUS.N2 F32 4.0 + PSASTRO.MOSAIC.RADIUS.N3 F32 4.0 + + PSASTRO.MOSAIC.CHIP.ORDER.N0 S32 0 # fit order (-1 means use default) + PSASTRO.MOSAIC.CHIP.ORDER.N1 S32 1 # fit order (-1 means use default) + PSASTRO.MOSAIC.CHIP.ORDER.N2 S32 1 # fit order (-1 means use default) + PSASTRO.MOSAIC.CHIP.ORDER.N3 S32 1 # fit order (-1 means use default) +END Index: /branches/eam_branch_20080706/ippconfig/lulin/psphot.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/psphot.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/psphot.config (revision 18467) @@ -0,0 +1,43 @@ + +# turn these on to see specific outputs +SAVE.OUTPUT BOOL TRUE +SAVE.BACKMDL BOOL TRUE +SAVE.PSF BOOL TRUE +SAVE.PLOTS BOOL TRUE + +BACKGROUND.XBIN S32 128 # size of background superpixels +BACKGROUND.YBIN S32 128 # size of background superpixels + +# image background parameters +IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: +SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background +SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background + +PSF_SN_LIM F32 100 # minimum S/N for stars used for PSF model +PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model + +# PSF model parameters : choose the PSF model desired +PSF_MODEL STR PS_MODEL_QGAUSS + +MOMENTS_SN_MIN F32 30.0 +EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit +FULL_FIT_SN_LIM F32 50.0 +AP_MIN_SN F32 50.0 + +# OUTPUT.FORMAT STR SMPDATA +OUTPUT.FORMAT STR PS1_DEV_1 + +PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation +PSF_MAX_CHI F32 50.0 # reject objects worse that this + +#PSF.TREND.MODE STR MAP +#PSF.TREND.NX S32 1 +#PSF.TREND.NY S32 1 + +DIAGNOSTIC.PLOTS METADATA + IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE + IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 -1 + IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 12 +END + +USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging Index: /branches/eam_branch_20080706/ippconfig/lulin/pswarp.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/pswarp.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/pswarp.config (revision 18467) @@ -0,0 +1,3 @@ +### Megacam recipe for pswarp + +ASTROM.SOURCE STR PSASTRO.OUTPUT # Source file rule for astrometry, or NULL Index: /branches/eam_branch_20080706/ippconfig/lulin/rejections.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/lulin/rejections.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/lulin/rejections.config (revision 18467) @@ -0,0 +1,38 @@ + +BIAS METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 2.0 + IMFILE.STDEV F32 10.0 + IMFILE.MEANSTDEV F32 0.5 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.5 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 1.0 + EXP.STDEV F32 10.0 + EXP.MEANSTDEV F32 0.5 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 3.0 + ENSEMBLE.STDEV F32 3.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +# FILTER is an additional qualifier, and may be "*" (or absent!), in which case it matches everything +# EXPECTED is the expected mean value +# IMFILE.MEAN is the maximum permitted mean value for an imfile, relative to the standard deviation +# IMFILE.STDEV is the maximum permitted standard deviation for an imfile +# EXP.MEAN is the maximum permitted mean value for an exposure, relative to the standard deviation +# EXP.STDEV is the maximum permitted standard deviation for an exposure +# EXP.MEANSTDEV is the maximum permitted mean standard deviation for an exposure relative to the mean +# ENSEMBLE.MEAN is the maximum permitted mean for an ensemble of exposures +# ENSEMBLE.STDEV is the maximum permitted standard deviation for an ensemble of exposures +# ENSEMBLE.MEANSTDEV is the maximum permitted mean standard deviation for an ensemble of exposures +# IMFILE.SNR is the minimum permitted signal-to-noise for an imfile +# EXP.SNR is the minimum permitted signal-to-noise for an exposure +# These values (all except FILTER) may be zero, in which case no clipping is applied. Index: /branches/eam_branch_20080706/ippconfig/recipes/rejections.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/recipes/rejections.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/recipes/rejections.config (revision 18467) @@ -0,0 +1,235 @@ + +BIAS METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 2.0 + IMFILE.STDEV F32 10.0 + IMFILE.MEANSTDEV F32 0.5 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.5 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 1.0 + EXP.STDEV F32 10.0 + EXP.MEANSTDEV F32 0.5 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 3.0 + ENSEMBLE.STDEV F32 3.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +DARK METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 2.0 + IMFILE.STDEV F32 10.0 + IMFILE.MEANSTDEV F32 0.5 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 1.0 + EXP.STDEV F32 10.0 + EXP.MEANSTDEV F32 0.5 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 3.0 + ENSEMBLE.STDEV F32 3.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +DARK_PREMASK METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +SHUTTER METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 20.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +FLAT METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 3.0 +END + +FLAT_PREMASK METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 3.0 +END + +FRINGE METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +DARKMASK METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +FLATMASK METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +# FILTER is an additional qualifier, and may be "*" (or absent!), in which case it matches everything +# EXPECTED is the expected mean value +# IMFILE.MEAN is the maximum permitted mean value for an imfile, relative to the standard deviation +# IMFILE.STDEV is the maximum permitted standard deviation for an imfile +# EXP.MEAN is the maximum permitted mean value for an exposure, relative to the standard deviation +# EXP.STDEV is the maximum permitted standard deviation for an exposure +# EXP.MEANSTDEV is the maximum permitted mean standard deviation for an exposure relative to the mean +# ENSEMBLE.MEAN is the maximum permitted mean for an ensemble of exposures +# ENSEMBLE.STDEV is the maximum permitted standard deviation for an ensemble of exposures +# ENSEMBLE.MEANSTDEV is the maximum permitted mean standard deviation for an ensemble of exposures +# IMFILE.SNR is the minimum permitted signal-to-noise for an imfile +# EXP.SNR is the minimum permitted signal-to-noise for an exposure +# These values (all except FILTER) may be zero, in which case no clipping is applied. + + +ITERATION BOOL TRUE # Are iterations permitted? +MASTER BOOL FALSE # Force stacks to be accepted as a master? + Index: /branches/eam_branch_20080706/ippconfig/simple/format.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/simple/format.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/simple/format.config (revision 18467) @@ -0,0 +1,75 @@ +# Drop-dead simple camera with single chip with single cell + +# How to identify this type +RULE METADATA + SIMPLE BOOL TRUE +### NAXIS S32 2 +END + +# How to read this data +FILE METADATA + PHU STR FPA # The FITS file represents an entire FPA + EXTENSIONS STR NONE # There are no extensions + FPA.OBS STR NAXIS # A PHU keyword for unique identifier within the hierarchy level +END + +# What's in the FITS file? +CONTENTS STR Chip:Cell:amplifier + +# Specify the cell data +CELLS METADATA + amplifier METADATA + CELL.TRIMSEC.SOURCE STR VALUE + CELL.TRIMSEC STR [0:0,0:0] + # alternatives for getting these from the headers + # CELL.TRIMSEC.SOURCE STR HEADER + # CELL.BIASSEC.SOURCE STR VALUE + # CELL.TRIMSEC STR DATASEC + # CELL.BIASSEC STR BIASSEC + END +END + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.TELESCOPE STR TELESCOP + FPA.INSTRUMENT STR INSTRUME + FPA.OBSTYPE STR OBSTYPE + FPA.AIRMASS STR AIRMASS + FPA.FILTER STR FILTER + FPA.POSANGLE STR POSANGLE + FPA.RA STR RA + FPA.DEC STR DEC + FPA.OBJECT STR OBJECT + FPA.TIME STR DATE-OBS UTC-OBS # Date and time + FPA.EXPOSURE STR EXPTIME + CELL.EXPOSURE STR EXPTIME + CELL.DARKTIME STR DARKTIME +# CELL.TIME STR DATE-OBS TIME-OBS # Date and time +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + CELL.XBIN S32 1 + CELL.YBIN S32 1 + FPA.TIMESYS STR UTC + FPA.RADECSYS STR ICRS + CELL.GAIN F32 1.0 + CELL.READNOISE F32 2.0 + CELL.BAD F32 0 + CELL.SATURATION F32 65535 +# CELL.BAD F32 -65535 + CELL.READDIR S32 1 + CELL.TIMESYS STR UTC + CELL.XPARITY S32 1 + CELL.YPARITY S32 1 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CELL.X0 S32 0 + CELL.Y0 S32 0 +END + +FORMATS METADATA + FPA.RA STR HOURS + FPA.DEC STR DEGREES + FPA.TIME STR SEPARATE,YEAR.FIRST +END Index: /branches/eam_branch_20080706/ippconfig/simtest/rejections.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/simtest/rejections.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/simtest/rejections.config (revision 18467) @@ -0,0 +1,78 @@ + +BIAS METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 2.0 # 2 sigma from expected mean + IMFILE.STDEV F32 20.0 # per-image sigma of < 20.0 (supplied read-noise is 10.0) + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.5 # binned stdev < 0.5 (binning is 64x64 : 10.0 / 64.0 = 0.15) + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 1.0 + EXP.STDEV F32 20.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 5.0 + ENSEMBLE.STDEV F32 5.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +DARK METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 2.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 1.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 1.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.5 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 5.0 + ENSEMBLE.STDEV F32 5.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + + +SHUTTER METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + + +ITERATION BOOL FALSE # Are iterations permitted? +MASTER BOOL TRUE # Force stacks to be accepted as a master? + Index: /branches/eam_branch_20080706/ippconfig/system.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/system.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/system.config (revision 18467) @@ -0,0 +1,65 @@ +## system-wide options : these are concepts not specific to any camera or recipe + +### Setups for each camera system +CAMERAS METADATA + MEGACAM STR megacam/camera.config # CFHT MegaCam + CFH12K STR cfh12k/camera.config # CFHT 12K + ISP-Apogee STR isp/camera.config # Pan-STARRS Imaging sky probe + MOSAIC2 STR mosaic2/camera.config # CTIO MOSAIC2 camera, for ESSENCE + SDSSMOSAIC STR sdssmosaic/camera.config # SDSS, mosaic version + SDSS STR sdss/camera.config # Sloan Digital Sky Survey + GPC1 STR gpc1/camera.config # Pan-STARRS GPC1 + ESOWFI STR esowfi/camera.config # ESO Wide-Field Imager + LBCRED STR lbc_red/camera.config # Large Binocular Camera Red + LULIN STR lulin/camera.config # Lulin Optical Telescope 1m + UH8K STR uh8k/camera.config # WFI/8k on UH88 + SIMMOSAIC STR simmosaic/camera.config # Simulated mosaic, for testing + SIMTEST STR simtest/camera.config # Simulation test camera + SIMPLE STR simple/camera.config # If all else fails.... +END + +## directories exist for other camera, introduce if they are tested and working +# UCAM STR ucam/camera.config +# LRIS_BLUE STR lris_blue/camera.config +# TC3 STR tc3/camera.config # Pan-STARRS Test Camera III +# LRIS_RED STR lris_red/camera.config + +### camera names as expected by DVO +DVO.CAMERAS METADATA + MEGACAM STR megacam + CFH12K STR cfh12k + ISP-Apogee STR isp + MOSAIC2 STR mosaic2 + SDSSMOSAIC STR sdssmosaic + SDSS STR sdss + GPC1 STR gpc1 + ESOWFI STR esowfi + LULIN STR lulin + UH8K STR uh8k + SIMMOSAIC STR simmosaic + SIMTEST STR simtest + SIMPLE STR simple +END + +# Header keywords for skycell concepts; required because DVO doesn't read HIERARCH +SKYCELLS METADATA + FPA.TIME STR MJD-OBS + CELL.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + CELL.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS +END + +RECIPES METADATA # Site-level recipes + MASKS STR recipes/masks.config # Mask values + REJECTIONS STR recipes/rejections.config # Rejection for detrend creation + PPIMAGE STR recipes/ppImage.config # Image reduction + PPMERGE STR recipes/ppMerge.config # Image combination + PPSTATS STR recipes/ppStats.config # Image statistics + PSPHOT STR recipes/psphot.config # Photometry + PSASTRO STR recipes/psastro.config # Astrometry + PPSTACK STR recipes/ppStack.config # Combination + PSWARP STR recipes/pswarp.config # Warping + PPSIM STR recipes/ppSim.config # Simulations + PPSUB STR recipes/ppSub.config # Subtraction +END Index: /branches/eam_branch_20080706/ippconfig/uh8k/.cvsignore =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/.cvsignore (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/.cvsignore (revision 18467) @@ -0,0 +1,2 @@ +Makefile +Makefile.in Index: /branches/eam_branch_20080706/ippconfig/uh8k/Makefile.am =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/Makefile.am (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/Makefile.am (revision 18467) @@ -0,0 +1,22 @@ + +installdir = $(datadir)/ippconfig/uh8k + +install_files = \ + dvo.config \ + camera.config \ + format_split.config \ + ppImage.config \ + ppMerge.config \ + psastro.config \ + psphot.config \ + pswarp.config \ + rejections.config + +install_DATA = $(install_files) + +install-data-hook: + chmod 0755 $(installdir) + +EXTRA_DIST = $(install_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/eam_branch_20080706/ippconfig/uh8k/camera.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/camera.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/camera.config (revision 18467) @@ -0,0 +1,85 @@ +# Camera configuration file for WFI/8k on UH88 + +# File formats that we know about +FORMATS METADATA + SPLIT STR uh8k/format_split.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + chip00 STR Cell + chip01 STR Cell + chip02 STR Cell + chip03 STR Cell + chip04 STR Cell + chip05 STR Cell + chip06 STR Cell + chip07 STR Cell +END + +# move this elsewhere? we need a lookup table to go from filter ID to abstract name +FILTER.ID METADATA + UKNOWN STR UNKNOWN +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa + CHIP STR {CHIP.NAME} + CELL STR {CHIP.NAME}:{CELL.NAME} + fpa STR fpa + chip STR {CHIP.NAME} + cell STR {CHIP.NAME}:{CELL.NAME} +END + +DVO.CAMERADIR STR uh8k # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END + +# Recipe options +RECIPES METADATA + # Other recipes + PSPHOT STR uh8k/psphot.config # psphot details + PSASTRO STR uh8k/psastro.config # psastro details + PSWARP STR uh8k/pswarp.config # pswarp details + PPIMAGE STR uh8k/ppImage.config # ppImage recipe + REJECTIONS STR uh8k/rejections.config # rejection recipe + PPMERGE STR uh8k/ppMerge.config # ppMerge recipe +END + +# reduction classes (recipes which are grouped together) +REDUCTION STR recipes/reductionClasses.mdc + +FITSTYPES STR recipes/fitstypes.mdc + +FILERULES STR recipes/filerules-split.mdc # File rules appropriate for MEF format + +EXTNAME.RULES METADATA + CMF.HEAD STR {CHIP.NAME}.hdr + CMF.DATA STR {CHIP.NAME}.psf # use .PSF and .EXT? + CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? + CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + + PSF.HEAD STR {CHIP.NAME}.hdr + PSF.TABLE STR {CHIP.NAME}.psf_model + PSF.RESID STR {CHIP.NAME}.psf_resid + + REF.ASTROM STR {CHIP.NAME}.ref_astrom +END + +BLANK.HEADERS METADATA + FPA.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS + FPA.NAME STR EXPNUM +END Index: /branches/eam_branch_20080706/ippconfig/uh8k/dvo.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/dvo.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/dvo.config (revision 18467) @@ -0,0 +1,67 @@ + +# location of DVO database tables +CATDIR /data/alala.0/eugene/isp/catdir + +# keywords used by DVO to interpret the headers + +# keyword abstractions: +#DATE-KEYWORD DATE-OBS +#DATE-MODE yyyy-mm-dd +#UT-KEYWORD UTC-OBS +JD-KEYWORD NONE +MJD-KEYWORD MJD-OBS + +# other keyword abstractions +EXPTIME-KEYWORD EXPTIME +AIRMASS-KEYWORD AIRMASS +CCDNUM-KEYWORD EXTNAME +ST-KEYWORD NONE +OBSERVATORY-LATITUDE NONE +OBSERVATORY-LONGITUDE NONE +SUBPIX_DATAFILE NONE + +# instrumental magnitude range for calibration mode +CAL_INSTMAG_MAX 0 +CAL_INSTMAG_MIN 0 + +# exclude overscan region from the dB image boundaries +XOVERSCAN 0 +YOVERSCAN 0 + +# only upload stars within region; a value of 0 means ignore the limit +ADDSTAR_XMIN 0 +ADDSTAR_XMAX 0 +ADDSTAR_YMIN 0 +ADDSTAR_YMAX 0 + +# exclude stars with SN > SNLIMIT (ADDSTAR_SNLIMIT overrides old name MIN_SN_FSTAT) +ADDSTAR_SNLIMIT 0 + +# correlation radius (arcseconds) +ADDSTAR_RADIUS 1.0 + +# scaled correlation radius +ADDSTAR_NSIGMA 0 + +IMAGE_SCATTER 0.075 # mark images POOR if stdev(Mcal) > IMAGE_SCATTER +STAR_SCATTER 0.005 # mark stars POOR if stdev(Mrel) > STAR_SCATTER +IMAGE_OFFSET 0.100 # mark images POOR if abs(delta(Mcal)) > IMAGE_OFFSET +STAR_CHISQ 10.0 # mark stars POOR if Xm > STAR_CHISQ +STAR_TOOFEW 3 # mark star FEW if N(good) < STAR_TOOFEW +IMAGE_TOOFEW 10 # mark image FEW if N(good) < IMAGE_TOOFEW +IMAGE_GOOD_FRACTION 0.05 # mark image FEW if N(good) < IMAGE_GOOD_FRACTION * Nstars + +SCATTER_LIM 15.0 +MAG_LIM 20.0 # select stars brighter than this for relphot analysis +SIGMA_LIM 0.015 # select measurements with dM < SIGMA_LIM for relphot analysis +# INST_MAG_MIN -17 # optional constraints on magnitude ranges +# INST_MAG_MAX -13 # optional constraints on magnitude ranges + +RELPHOT_GRID_BINNING 256 +RELPHOT_GRID_X 6 +RELPHOT_GRID_Y 14 + +PM_DT_MIN 0.25 +PM_TOOFEW 4 +POS_TOOFEW 1 + Index: /branches/eam_branch_20080706/ippconfig/uh8k/format_split.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/format_split.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/format_split.config (revision 18467) @@ -0,0 +1,156 @@ +# Camera format for WFI/8k on the UH88 + +# How to recognise this type +RULE METADATA + ORIGIN STR UH 2.2m Telescope + CCD STR 8k + DETECTOR STR 8k + INSTRUME STR 8k/WFI + IMAGESWV STR CFHT DetCom v3.41 (Jun 10 2008) + NAXIS S32 2 +END + +FILE METADATA + # How to read this data + PHU STR CHIP # The FITS file represents an entire FPA + EXTENSIONS STR NONE # The extensions represent chips + FPA.OBS STR EXPNUM # A PHU keyword for unique identifier + CONTENT STR EXTNAME # Key to the CONTENTS menu + CONTENT.RULE STR {CHIP.NAME} # How to derive the CONTENT when writing +END + +# What's in the FITS file? +CONTENTS METADATA + # Content name, chipType + chip00 STR chip00:SingleAmp + chip01 STR chip01:SingleAmp + chip02 STR chip02:SingleAmp + chip03 STR chip03:SingleAmp + chip04 STR chip04:SingleAmp + chip05 STR chip05:SingleAmp + chip06 STR chip06:SingleAmp + chip07 STR chip07:SingleAmp +END + +# Specify the chips +CHIPS METADATA + # Chip type, cellName:cellType + SingleAmp STR Cell:amplifier +END + + +# Specify the cells +CELLS METADATA + # Cell type, concepts for this type + amplifier METADATA + CELL.BIASSEC.SOURCE STR HEADER + CELL.TRIMSEC.SOURCE STR HEADER + CELL.BIASSEC STR BIASSEC + CELL.TRIMSEC STR DATASEC + END +END + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.INSTRUMENT STR INSTRUME + FPA.DETECTOR STR DETECTOR + FPA.AIRMASS STR AIRMASS + FPA.FILTER STR FILTER + FPA.FILTERID STR FILTER + FPA.RA STR RA + FPA.DEC STR DEC + FPA.OBSTYPE STR OBSTYPE + FPA.OBJECT STR OBJECT + FPA.FOCUS STR TELFOCUS + FPA.TIME STR DATE-OBS TIME-OBS + FPA.TIMESYS STR TIMESYS + FPA.EXPOSURE STR EXPTIME + CELL.EXPOSURE STR EXPTIME + CELL.DARKTIME STR DARKTIME + CELL.TIME STR DATE-OBS TIME-OBS + CELL.TIMESYS STR TIMESYS + CELL.XBIN STR CCDBIN1 + CELL.YBIN STR CCDBIN2 + CELL.SATURATION STR MAXLIN +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.POSANGLE F32 0.0 + FPA.RADECSYS STR ICRS + CELL.READNOISE F32 10.0 # A guess + CELL.GAIN F32 1.0 # A guess + + FPA.TELESCOPE STR UH 88-inch + CHIP.XSIZE S32 2048 + CHIP.YSIZE S32 4096 + CELL.XSIZE S32 2048 + CELL.YSIZE S32 4096 + CELL.XWINDOW S32 0 + CELL.YWINDOW S32 0 + CELL.READDIR S32 1 # Cell is read in x direction + CELL.BAD F32 0 + CELL.XPARITY S32 1 + CELL.YPARITY S32 1 + CELL.Y0 S32 0 + CELL.X0 S32 0 + CHIP.X0.DEPEND STR CHIP.NAME + CHIP.X0 METADATA + chip00 S32 0 + chip01 S32 2048 + chip02 S32 4096 + chip03 S32 6144 + chip04 S32 0 + chip05 S32 2048 + chip06 S32 4096 + chip07 S32 6144 + END + CHIP.Y0.DEPEND STR CHIP.NAME + CHIP.Y0 METADATA + chip00 S32 8192 + chip01 S32 8192 + chip02 S32 8192 + chip03 S32 8192 + chip04 S32 0 + chip05 S32 0 + chip06 S32 0 + chip07 S32 0 + END + CHIP.XPARITY.DEPEND STR CHIP.NAME + CHIP.XPARITY METADATA + chip00 S32 -1 + chip01 S32 -1 + chip02 S32 -1 + chip03 S32 -1 + chip04 S32 1 + chip05 S32 1 + chip06 S32 1 + chip07 S32 1 + END + CHIP.YPARITY.DEPEND STR CHIP.NAME + CHIP.YPARITY METADATA + chip00 S32 -1 + chip01 S32 -1 + chip02 S32 -1 + chip03 S32 -1 + chip04 S32 1 + chip05 S32 1 + chip06 S32 1 + chip07 S32 1 + END +END + + +# How to translation PS concepts into database lookups +DATABASE METADATA +END + + +# Where there might be some ambiguity, specify the format +FORMATS METADATA + FPA.RA STR HOURS + FPA.DEC STR DEGREES + FPA.TIME STR SEPARATE YEAR.FIRST + CELL.TIME STR SEPARATE YEAR.FIRST +END + Index: /branches/eam_branch_20080706/ippconfig/uh8k/ppImage.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/ppImage.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/ppImage.config (revision 18467) @@ -0,0 +1,172 @@ +### ppImage recipe configuration file for Megacam + +# Overscan subtraction +OVERSCAN.SINGLE BOOL FALSE # Reduce overscan to a single value? +OVERSCAN.FIT STR NONE # NONE | POLYNOMIAL | SPLINE +OVERSCAN.ORDER S32 0 # Order of polynomial fit +OVERSCAN.STAT STR MEAN # MEAN | MEDIAN + +# for a test, turn off selected detrend types +OVERSCAN BOOL FALSE # Overscan subtraction +BIAS BOOL FALSE # Bias subtraction +DARK BOOL TRUE # Dark subtraction +FLAT BOOL TRUE # Flat-field normalisation + +# binned output image options +BIN1.XBIN S32 18 +BIN1.YBIN S32 18 +BIN2.XBIN S32 144 +BIN2.YBIN S32 144 + +# megacam needs fringe subtraction +FRINGE BOOL TRUE # Fringe subtraction + +# this table lists extra constraints which should be applied when +# selecting the detrend images. +# note: camera and time are always applied + +DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA +# EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + FILTER STR FPA.FILTERID +# DETTYPE STR DOMEFLAT.RAW +# VERSION STR SKYFLAT.RAW +# VERSION STR DOMEFLAT.CORR.XX +# VERSION STR SKYFLAT.CORR + END + FLAT_CORRECTION METADATA + FILTER STR FPA.FILTER + END + FRINGE METADATA + FILTER STR FPA.FILTER +# AIRMASS STR FPA.AIRMASS +# TWILIGHT STR FPA.TWILIGHT + END + SHUTTER METADATA + END + MASK METADATA + END +END + +# only apply the fringe for these filters +FRINGE.FILTERS MULTI +FRINGE.FILTERS STR i +FRINGE.FILTERS STR z + +PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.N} + + + +# Overscan, bias, dark, shutter +PPIMAGE_OBDS METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.WEIGHT.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field +PPIMAGE_OBDSF METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.WEIGHT.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field, fringe +PPIMAGE_OBDSFR METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.WEIGHT.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom +PPIMAGE_OBDSFRP METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom, astrom +PPIMAGE_OBDSFRA METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END Index: /branches/eam_branch_20080706/ippconfig/uh8k/ppMerge.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/ppMerge.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/ppMerge.config (revision 18467) @@ -0,0 +1,40 @@ + +# Bias combination --- don't want min/max rejection +PPMERGE_BIAS METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 2 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Dark combination --- don't want min/max rejection +# More aggressive clipping than bias, so as to remove CRs +PPMERGE_DARK METADATA + REJ F32 2.0 # Rejection threshold (sigma) + ITER S32 4 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Flat combination --- use min/max rejection +PPMERGE_FLAT METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 1 # Number of rejection iterations + FRACHIGH F32 0.3 # Fraction of high pixels to reject immediately + FRACLOW F32 0.1 # Fraction of low pixels to reject immediately + NKEEP S32 5 # Minimum number of pixels in stack to keep + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR MEAN # Statistic to use for combination: +END + + +# Fringe combination --- already included in default, above +PPMERGE_FRINGE METADATA + FRACHIGH F32 0.1 # Fraction of high pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? +END + Index: /branches/eam_branch_20080706/ippconfig/uh8k/psastro.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/psastro.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/psastro.config (revision 18467) @@ -0,0 +1,74 @@ + +# astrometry matching parameters + +# nominal plate scale (microns / pixel) +PSASTRO.PIXEL.SCALE F32 1.0 + +# pmAstromGridAngle +# max grid offset in FP units (microns) +# use plate-scale to make this in pixels? +PSASTRO.GRID.OFFSET F32 2500. +PSASTRO.GRID.SCALE F32 50. + +# extra field for ref stars: +PSASTRO.FIELD.PADDING F32 1.0 + +# these tweak are in FP units (pixels, currently) +PSASTRO.TWEAK.SCALE F32 1 +PSASTRO.TWEAK.RANGE F32 75 +PSASTRO.TWEAK.SMOOTH F32 2 +PSASTRO.TWEAK.NSIGMA F32 3 + +# single-chip radius match in pixels +PSASTRO.MATCH.FIT.NITER S32 3 +PSASTRO.MATCH.RADIUS F32 12.0 +PSASTRO.MATCH.RADIUS.N0 F32 15.0 +PSASTRO.MATCH.RADIUS.N1 F32 10.0 +PSASTRO.MATCH.RADIUS.N2 F32 5.0 + +PSASTRO.MAX.NRAW S32 1500 # max stars accepted for fitting (0 for all) +PSASTRO.MAX.NREF S32 1500 # max stars accepted for fitting (0 for all) + +PSASTRO.MIN.INST.MAG.RAW F32 -15.5 # min instrumental magnitude for stars accepted for fitting +PSASTRO.MAX.INST.MAG.RAW F32 -8.0 # max instrumental magnitude for stars accepted for fitting + +PSASTRO.MAX.ERROR F32 10.0 # max error in pixels +PSASTRO.MIN.NSTAR S32 10 # min fitted stars in solution + +PSASTRO.MOSAIC.MODE BOOL FALSE + +PSASTRO.MOSAIC.MAX.ERROR.N0 F32 1.50 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N1 F32 1.50 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N2 F32 0.90 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N3 F32 0.90 # max allow error for valid solution (arcsec) + +PSASTRO.MOSAIC.RADIUS.N0 F32 10.0 +PSASTRO.MOSAIC.RADIUS.N1 F32 8.0 # do not refine the match +PSASTRO.MOSAIC.RADIUS.N2 F32 8.0 # do not refine the match +PSASTRO.MOSAIC.RADIUS.N3 F32 4.0 + +PSASTRO.MOSAIC.CHIP.ORDER.N0 S32 0 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N1 S32 1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N2 S32 1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N3 S32 3 # fit order (-1 means use default) + +PSASTRO.MOSAIC.GRADIENT.NX S32 2 # number of x-dir cells per chip +PSASTRO.MOSAIC.GRADIENT.NY S32 4 # number of y-dir cells per chip + +# use this recipe to set a tight constraint +PSASTRO.FINE METADATA + PSASTRO.MOSAIC.MAX.ERROR.N0 F32 1.50 # max allow error for valid solution (arcsec) + PSASTRO.MOSAIC.MAX.ERROR.N1 F32 1.50 # max allow error for valid solution (arcsec) + PSASTRO.MOSAIC.MAX.ERROR.N2 F32 0.90 # max allow error for valid solution (arcsec) + PSASTRO.MOSAIC.MAX.ERROR.N3 F32 0.90 # max allow error for valid solution (arcsec) + + PSASTRO.MOSAIC.RADIUS.N0 F32 8.0 + PSASTRO.MOSAIC.RADIUS.N1 F32 0.0 + PSASTRO.MOSAIC.RADIUS.N2 F32 4.0 + PSASTRO.MOSAIC.RADIUS.N3 F32 4.0 + + PSASTRO.MOSAIC.CHIP.ORDER.N0 S32 0 # fit order (-1 means use default) + PSASTRO.MOSAIC.CHIP.ORDER.N1 S32 1 # fit order (-1 means use default) + PSASTRO.MOSAIC.CHIP.ORDER.N2 S32 1 # fit order (-1 means use default) + PSASTRO.MOSAIC.CHIP.ORDER.N3 S32 1 # fit order (-1 means use default) +END Index: /branches/eam_branch_20080706/ippconfig/uh8k/psphot.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/psphot.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/psphot.config (revision 18467) @@ -0,0 +1,43 @@ + +# turn these on to see specific outputs +SAVE.OUTPUT BOOL TRUE +SAVE.BACKMDL BOOL TRUE +SAVE.PSF BOOL TRUE +SAVE.PLOTS BOOL TRUE + +BACKGROUND.XBIN S32 128 # size of background superpixels +BACKGROUND.YBIN S32 128 # size of background superpixels + +# image background parameters +IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: +SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background +SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background + +PSF_SN_LIM F32 100 # minimum S/N for stars used for PSF model +PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model + +# PSF model parameters : choose the PSF model desired +PSF_MODEL STR PS_MODEL_QGAUSS + +MOMENTS_SN_MIN F32 30.0 +EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit +FULL_FIT_SN_LIM F32 50.0 +AP_MIN_SN F32 50.0 + +# OUTPUT.FORMAT STR SMPDATA +OUTPUT.FORMAT STR PS1_DEV_1 + +PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation +PSF_MAX_CHI F32 50.0 # reject objects worse that this + +#PSF.TREND.MODE STR MAP +#PSF.TREND.NX S32 1 +#PSF.TREND.NY S32 1 + +DIAGNOSTIC.PLOTS METADATA + IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE + IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 -1 + IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 12 +END + +USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging Index: /branches/eam_branch_20080706/ippconfig/uh8k/pswarp.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/pswarp.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/pswarp.config (revision 18467) @@ -0,0 +1,3 @@ +### Megacam recipe for pswarp + +ASTROM.SOURCE STR PSASTRO.OUTPUT # Source file rule for astrometry, or NULL Index: /branches/eam_branch_20080706/ippconfig/uh8k/rejections.config =================================================================== --- /branches/eam_branch_20080706/ippconfig/uh8k/rejections.config (revision 18467) +++ /branches/eam_branch_20080706/ippconfig/uh8k/rejections.config (revision 18467) @@ -0,0 +1,38 @@ + +BIAS METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 2.0 + IMFILE.STDEV F32 10.0 + IMFILE.MEANSTDEV F32 0.5 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.5 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 1.0 + EXP.STDEV F32 10.0 + EXP.MEANSTDEV F32 0.5 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 3.0 + ENSEMBLE.STDEV F32 3.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +# FILTER is an additional qualifier, and may be "*" (or absent!), in which case it matches everything +# EXPECTED is the expected mean value +# IMFILE.MEAN is the maximum permitted mean value for an imfile, relative to the standard deviation +# IMFILE.STDEV is the maximum permitted standard deviation for an imfile +# EXP.MEAN is the maximum permitted mean value for an exposure, relative to the standard deviation +# EXP.STDEV is the maximum permitted standard deviation for an exposure +# EXP.MEANSTDEV is the maximum permitted mean standard deviation for an exposure relative to the mean +# ENSEMBLE.MEAN is the maximum permitted mean for an ensemble of exposures +# ENSEMBLE.STDEV is the maximum permitted standard deviation for an ensemble of exposures +# ENSEMBLE.MEANSTDEV is the maximum permitted mean standard deviation for an ensemble of exposures +# IMFILE.SNR is the minimum permitted signal-to-noise for an imfile +# EXP.SNR is the minimum permitted signal-to-noise for an exposure +# These values (all except FILTER) may be zero, in which case no clipping is applied. Index: /branches/eam_branch_20080706/psphot/src/Makefile.am =================================================================== --- /branches/eam_branch_20080706/psphot/src/Makefile.am (revision 18467) +++ /branches/eam_branch_20080706/psphot/src/Makefile.am (revision 18467) @@ -0,0 +1,114 @@ + +lib_LTLIBRARIES = libpsphot.la +libpsphot_la_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) + +bin_PROGRAMS = psphot + +psphot_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) +psphot_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PSPHOT_LIBS) +psphot_LDADD = libpsphot.la + +psphot_SOURCES = \ + psphot.c \ + psphotArguments.c \ + psphotCleanup.c \ + psphotImageLoop.c \ + psphotMosaicChip.c \ + psphotParseCamera.c + +libpsphot_la_SOURCES = \ + psphotErrorCodes.c \ + pmFootprint.c \ + pmFootprintArrayGrow.c \ + pmFootprintArraysMerge.c \ + pmFootprintIDs.c \ + pmFootprintsFind.c \ + pmFootprintsFindAtPoint.c \ + pmFootprintsAssignPeaks.c \ + pmDetections.c \ + pmSpan.c \ + psphotCullPeaks.c \ + psphotVersion.c \ + psphotModelGroupInit.c \ + psphotMaskReadout.c \ + psphotDefineFiles.c \ + psphotReadout.c \ + psphotModelBackground.c \ + psphotSubtractBackground.c \ + psphotFindDetections.c \ + psphotFindPeaks.c \ + psphotFindFootprints.c \ + psphotSignificanceImage.c \ + psphotSourceStats.c \ + psphotRoughClass.c \ + psphotBasicDeblend.c \ + psphotChoosePSF.c \ + psphotGuessModels.c \ + psphotFitSourcesLinear.c \ + psphotBlendFit.c \ + psphotReplaceUnfit.c \ + psphotApResid.c \ + psphotMagnitudes.c \ + psphotSkyReplace.c \ + psphotEvalPSF.c \ + psphotEvalFLT.c \ + psphotSourceFits.c \ + psphotRadiusChecks.c \ + psphotOutput.c \ + psphotFakeSources.c \ + psphotModelWithPSF.c \ + psphotExtendedSourceAnalysis.c \ + psphotExtendedSourceFits.c \ + psphotRadialProfile.c \ + psphotPetrosian.c \ + psphotIsophotal.c \ + psphotAnnuli.c \ + psphotKron.c \ + psphotKernelFromPSF.c \ + psphotPSFConvModel.c \ + psphotModelTest.c \ + psphotFitSet.c \ + psphotSourceFreePixels.c \ + psphotSummaryPlots.c \ + psphotMergeSources.c \ + psphotLoadPSF.c \ + psphotReadoutCleanup.c \ + psphotSourcePlots.c \ + psphotRadialPlot.c \ + psphotDeblendSatstars.c \ + psphotMosaicSubimage.c \ + psphotMakeResiduals.c \ + psphotSourceSize.c \ + psphotDiagnosticPlots.c \ + psphotMakeFluxScale.c \ + psphotCheckStarDistribution.c \ + psphotAddNoise.c + +include_HEADERS = \ + psphot.h \ + pmFootprint.h \ + psphotErrorCodes.h + +noinst_HEADERS = \ + psphotInternal.h \ + psphotStandAlone.h + +clean-local: + -rm -f TAGS + +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` + +# Error codes. +BUILT_SOURCES = psphotErrorCodes.h psphotErrorCodes.c +CLEANFILES = psphotErrorCodes.h psphotErrorCodes.c +EXTRA_DIST = psphotErrorCodes.dat psphotErrorCodes.c.in psphotErrorCodes.h.in \ + models/pmModel_STRAIL.c \ + models/pmModel_TEST1.c + +psphotErrorCodes.h : psphotErrorCodes.dat psphotErrorCodes.h.in + $(ERRORCODES) --data=psphotErrorCodes.dat --outdir=. psphotErrorCodes.h + +psphotErrorCodes.c : psphotErrorCodes.dat psphotErrorCodes.c.in psphotErrorCodes.h + $(ERRORCODES) --data=psphotErrorCodes.dat --outdir=. psphotErrorCodes.c