Index: /branches/czw_branch/20110406/Nebulous/bin/neb-shift =================================================================== --- /branches/czw_branch/20110406/Nebulous/bin/neb-shift (revision 31384) +++ /branches/czw_branch/20110406/Nebulous/bin/neb-shift (revision 31385) @@ -16,5 +16,5 @@ use Pod::Usage qw( pod2usage ); -my ($volume, $server); +my ($volume, $server, $nocull, $cull); $server = $ENV{'NEB_SERVER'} unless $server; @@ -23,9 +23,18 @@ 'volume=s' => \$volume, 'server|s=s' => \$server, + 'nocull' => \$nocull, + 'cull' => \$cull, ) || pod2usage( 2 ); my $src = shift; my $abandon_vol = shift; +my $do_cull = 0; +if ($nocull) { + $do_cull = 0; +} +if ($cull) { + $do_cull = 1; +} pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( -msg => "Required options: --server ", @@ -54,7 +63,11 @@ die "Replicate phase failed for $src ($abandon_vol) ($volume)" unless $status; -$status = $neb->cull($src,$abandon_vol); +if ($do_cull) { + $status = $neb->cull($src,$abandon_vol); -warn "Cull phase failed for $src from $abandon_vol" unless $status; + warn "Cull phase failed for $src from $abandon_vol" unless $status; +} + + __END__ Index: /branches/czw_branch/20110406/ippScripts/scripts/lap_science.pl =================================================================== --- /branches/czw_branch/20110406/ippScripts/scripts/lap_science.pl (revision 31385) +++ /branches/czw_branch/20110406/ippScripts/scripts/lap_science.pl (revision 31385) @@ -0,0 +1,654 @@ +#!/usr/bin/env perl + + +use warnings; +use strict; +use Carp; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP:Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use DateTime; + +# +# Set up +################################################################################ + +my $missing_tools = 0; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $laptool = can_run('laptool') or (warn "Can't find laptool" and $missing_tools = 1); + +my ( $camera, $dbname, $logfile, $verbose, $debug); +my ( $proj_cell, $filter); +my ( $chip_mode, $diff_mode, $stack_mode, $qstack_mode, $magic_mode, $clean_mode); + + + + +# +# Chip +################################################################################ +# This queues a chipRun for this exposure, and records the chip_id in the exposure structure and in the database +sub construct_chiptool_command { + my $lap_id = shift; + my $label = shift; + my $tess_id = shift; + my $projection_cell = shift; + my $data_group = shift; + my $workdir = shift; + my $exp_id = shift; + + my $cmd = "$chiptool "; + $cmd .= " -dbname $dbname " if defined $dbname; + + $cmd .= " -definebyquery -exp_id $exp_id -set_end_stage warp -set_tess_id $tess_id "; + $cmd .= " -set_label $label -set_workdir $workdir -set_dist_group $label -set_data_group $data_group "; + + return($cmd); +} +sub get_chip_id_from_metadata { + my $exp_id = shift; + my $label = shift; + my $data_group = shift; + # This is a puzzler... chiptool doesn't actually return a useful metadata. We'll just scrape it from the database for now. + + my $command = "$chiptool -listrun -exp_id $exp_id -label $label -data_group $data_group"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", $exp_id, $data_group); + } + my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from chiptool", $exp_id, $data_group); + # There should be only one. + my $chip = ${ $chips }[0]; + my $chip_id = $chip->{chip_id}; + + return($chip_id); +} +sub remake_this_exposure { + my $exposure = shift; + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $chip_cmd = construct_chiptool_command($exposure->{lap_id},$exposure->{label}, + $exposure->{tess_id},$exposure->{proj_cell}, + $workdir, $data_group, + $exposure->{exp_id}); + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + + $exposure->{chip_id} = get_chip_id_from_metadata($exposure->{exp_id},$exposure->{label},$data_group); #join "", @$stdout_buf); + $exposure->{data_state} = 'new'; + update_this_exposure($exposure); + return($exposure); +} + +# This is the "user level" subroutine. +sub queue_chips { + my $proj_cell = shift; + + my $lap_id = get_lap_id($proj_cell); + + my $command = "$laptool -pendingrun -lap_id $lap_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool: $error_code", $re_id, $proj_cell); + } + + if ($@stdout_buf == 0) { + # Nothing to do. + return(0); + } + + $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool", $re_id, $proj_cell); + + # Preliminary scan of exposures to set up diff structures. + my $counter = 0; + my %matching = (); + my %indexing = (); + foreach my $exposure (@$exposures) { + my $lap_id = $exposure->{lap_id}; + my $tess_id = $exposure->{tess_id}; + my $exp_id = $exposure->{exp_id}; + my $chip_id = $exposure->{chip_id}; + my $pair_id = $exposure->{pair_id}; + my $private = $exposure->{private}; + my $pairwise = $exposure->{pairwise}; + my $active = $exposure->{active}; + my $data_state = $exposure->{data_state}; + my $dateobs = $exposure->{dateobs}; + my $object = $exposure->{object}; + my $comment = $exposure->{comment}; + + if ($chip_id) { # We already have a defined chip_id + if (($pairwise) && !($pair_id)) { + # This is an error. + &my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id, $proj_cell); + } + $counter++; + next; + } + + $exposure = remake_this_exposure($exposure); +# my $chip_cmd = construct_chiptool_command($lap_id,$proj_cell,$exp_id); +# my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform chiptool: $error_code", $lap_id, $proj_cell); +# } + +# $exposure->{chip_id} = get_chip_id_from_metadata(join "", @$stdout_buf); + + # We will need to define a chipRun for this exposure. However, let's do diff matching before we do so. + + $matching{$object}{$comment} = $exp_id; + $indexing{$exp_id} = $counter; + $counter++; + } + +# Pair up exposures to diff. + foreach my $object (keys %matching) { + my @exp_ids_to_diff = (); + foreach my $comment (keys %{ $matching{$object} }) { + push @exp_ids_to_diff, $matching{$object}{$comment}; + } + @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; + + if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering + pop(@exp_ids_to_diff); # dump the last entry. + } + + while ($#{ $exp_ids_to_diff } > -1) { # Save the pair information to the opposite exposure. + my $exp_id_A = shift(@exp_ids_to_diff); + my $exp_id_B = shift(@exp_ids_to_diff); + + my $exp_A = ${ $exposures }[$indexing{$exp_id_A}]; + my $exp_B = ${ $exposures }[$indexing{$exp_id_B}]; + + $exp_A->{pairwise} = 1; + $exp_A->{pair_id} = $exp_B->{chip_id}; + + $exp_B->{pairwise} = 1; + $exp_B->{pair_id} = $exp_A->{chip_id}; + } + } + +# Queue chipRuns and update lapExp table. + foreach my $exposure (@$exposures) { + if ($exposure->{pairwise} && !($exposure->{pair_id})) { + $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. + } + if (!($exposure->{pairwise})) { + $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private + } + + update_this_exposure($exposure); +# my $lapE_cmd = construct_laptool_updateExp_command($lap_id, +# $exposure->{exp_id}, +# $exposure->{chip_id}, +# $exposure->{private}, +# $exposure->{pairwise}, +# $exposure->{pair_id}); +# my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die_with_fault("Unable to perform chiptool: $error_code", $lap_id, $proj_cell); +# } + } +} + + +# +# Decide Things +################################################################################ + +sub check_stack_status { # look at lapRun associated stacks, and confirm their states. + my $lap_id = shift; + + my ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack); + + my $command = "$laptool -stacks -lap_id $lap_id"; + $command .= " -dbname $dbname" if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool: $error_code", $lap_id, ""); + } + + ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack) = (0,0,0,0); + if ($@stdout_buf == 0) { + # Nothing to do. Possibly an error, but for now, just accept it + return($defined_qstack,$have_qstack,$defined_fstack,$have_fstack); + } + + $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool", $lap_id, ""); + + my $total_qstacks = 0; + my $complete_qstacks = 0; + my $total_fstacks = 0; + my $complete_fstacks = 0; + foreach my $stack (@$stacks) { + if ($stack->{quick_stack_id}) { + $defined_qstack = 1; + $total_qstacks++; + } + if ($stack->{final_stack_id}) { + $defiend_fstack = 1; + $total_fstacks++; + } + + if ($stack->{quick_state} eq 'full') { + $complete_qstacks++; + } + if ($stack->{final_state} eq 'full') { + $complete_fstacks++; + } + } + if ($complete_qstacks == $total_qstacks) { + $have_qstack = 1; + } + if ($complete_fstacks == $total_fstacks) { + $have_fstack = 1; + } + + return($defined_qstack,$have_qstack,$defined_fstack,$have_fstack); +} +sub check_status { + my $proj_cell = shift; + my $lap_id = get_lap_id($proj_cell); + + my ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack) = check_stack_status($lap_id); + + my $command = "$laptool -exposures -lap_id $lap_id"; + $command .= " -dbname $dbname" if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool: $error_code", $re_id, $proj_cell); + } + + if ($@stdout_buf == 0) { + # Nothing to do. However, this is likely an error. + return(0); + } + + $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool", $re_id, $proj_cell); + + # Need to prescan to see who matches whom. + %match_hash = (); + my $index = 0; + foreach my $exposure (@$exposures) { + if ($exposure->{pair_id}) { + $match_hash{$exposure->{pair_id}} = $index; # Tell the companion who we are, so they can find us next time. + } + $index++; + } + + # Things I want to know before I'm through + my $needs_qstack = 0; + my $needs_something_remade = 0; + my $needs_something_private = 0; + my $can_qstack = 0; + my $can_diff = 0; + my $can_fstack = 0; + my $total_exposures = 0; + foreach my $exposure (@$exposures) { + $total_exposures++; + my $companion; + + if ($exposure->{pair_id}) { # Load companion exposure information + if (exists($match_hash{$exposure->{chip_id}})) { + $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; # Match! + } + } + + if ($exposure->{private}) { # I've declared this exposure private to this lapRun. + $needs_qstack = 1; + } + + if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun + $needs_something_remade = 1; + $exposure = remake_this_exposure($exposure); + } + if ($companion) { # Validate that there are no problems with the companion exposure + if ($companion->{cam_quality}) { # Maybe other things here? + $exposure->{private} = 1; + $exposure->{data_state} = 'drop'; + &update_this_exposure($exposure); + $needs_qstack = 1; + } + } + if (($exposure->{warpRun_state} eq 'full')|| # This exposure's warp is either done or impossible. + ($exposure->{data_state} eq 'drop')) { + $can_qstack ++; + $can_diff ++; + } + if (($exposure->{magicked})|| # This exposure has been magicked, so it is through with diff. + ($exposure->{data_state} eq 'drop')) { + $can_fstack ++; + } + } + + if (($needs_qstack == 1)&& # Do we need the quick stack? + ($defined_qstack == 0)&& # Have we not made it already? + ($can_qstack == $total_exposures)&& # Are all warps done? + ($needs_something_remade == 0) # Do we need to wait for a stolen exposure? + ) { + &queue_quickstack($exposures); + } + + if (($can_diff == $total_exposures)&& # Are all warps done? + ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? + (($needs_qstack == 0)|| # Are we just doing pairwise diffs? + ($needs_qstack == 1)&&($have_qstack == 1)) ) { # If we need a quickstack, do we have it? + &queue_diffs($exposures); + } + + if (($can_fstack == $total_exposures)&& # Are all warps done? + ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? + ($defined_fstack == 0)) { # Have we not made it already? + &queue_finalstack($exposures); + } + + if (($have_fstack == 1)) { # Are we all done making the final product? + &deactivate_exposures($exposures); + $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state full "; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool: $error_code", $re_id, $proj_cell); + } + } +} + +sub queue_quickstack { + my $lap_id = shift; + my $label = shift; + my $filter = shift; + my $proj_cell = shift; + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $command = "$stacktool "; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -set_label ${label}.quick -set_workdir $workdir -set_data_group ${proj_cell}.quick.${date} "; + $command .= " -min_num 2 -set_reduction QUICKSTACK "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool", $lap_id, ""); + + my $stack = ${ $stacks }[0]; + my $sass_id = $stack->{sass_id}; + + $command = "$laptool -updaterun -lap_id $lap_id -set_quick_sass_id $sass_id"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + + +} +sub queue_finalstack { + my $lap_id = shift; + my $label = shift; + my $filter = shift; + my $proj_cell = shift; + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $command = "$stacktool "; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -set_label ${label}.final -set_workdir $workdir -set_data_group ${proj_cell}.final.${date} "; + $command .= " -min_num 2 -set_reduction THREEPI_STACK "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool", $lap_id, ""); + + my $stack = ${ $stacks }[0]; + my $sass_id = $stack->{sass_id}; + + $command = "$laptool -updaterun -lap_id $lap_id -set_final_sass_id $sass_id"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + +} +sub queue_diffs { + my $exposures = shift; + + # Need to prescan to see who matches whom. + %match_hash = (); + my $index = 0; + foreach my $exposure (@$exposures) { + if ($exposure->{pair_id}) { + $match_hash{$exposure->{pair_id}} = $index; # Tell the companion who we are, so they can find us next time. + } + $index++; + } + + my %already_queued = (); + foreach my $exposure (@$exposures) { + if ($exposure->{data_state} eq 'drop') { # Kick out unusable exposures + next; + } + if ($exposure->{diff_id}) { # Not sure how this would happen, but still. + next; + } + + my $command = "$difftoool "; + my $command .= " -dbname $dbname " if defined $dbname; + my $command .= " -set_label $label -set_workdir $workdir -set_data_group $data_group "; + my $command .= " -set_dist_group $dist_group "; + + if ($exposure->{pairwise}) { # warpwarp + $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; + $command .= " -definewarpwarp "; + $command .= "-input_label $label -template_label $label -backwards "; + $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} "; + + } + else { # warp-qstack + $command .= " -definewarpstack -set_reduction WARPQSTACK -available -good_frac 0.2 "; + $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; + } + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform chiptool: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + } +} + +# Deactivate all exposures in this run. +sub deactivate_exposures { + my $exposures = shift; + foreach my $exposure (@$exposures) { + $exposure->{active} = 0; + $exposure = update_this_exposure($exposure); + } +} + +# Take the current exposure structure, and save it to the database. +sub update_this_exposure { + my $exposure = shift; + + # These are required + my $lap_id = $exposure->{lap_id}; + my $exp_id = $exposure->{exp_id}; + + my $command = "$laptool -updateexp -lap_id $lap_id -exp_id $exp_id "; + if ($exposure->{chip_id}) { + $command .= " -set_chip_id $exposure->{chip_id} "; + } + if ($exposure->{pair_id}) { + $command .= " -set_pair_id $exposure->{pair_id} "; + } + + if ($exposure->{private}) { + $command .= " -private "; + } + else { + $command .= " -public "; + } + if ($exposure->{pairwise}) { + $command .= " -pairwise "; + } + else { + $command .= " -nopairwise "; + } + if ($exposure->{active}) { + $command .= " -active "; + } + else { + $command .= " -inactive "; + } + if ($exposure->{data_state}) { + $command .= " -set_data_state $exposure->{data_state} "; + } + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + +} + +# +# Cleanup +################################################################################ + +sub cleanup_exposures { + my $command = "$laptool -inactive "; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool: $error_code", "none", "none"); + } + if ($@stdout_buf == 0) { + # Nothing to do. + return(0); + } + + foreach my $exposure (@$exposures) { + foreach my $clean_mode (@clean_modes) { + $command = $clean_mode->{COMMAND}; + $clean_id = $clean_mode->{CLEAN_ID}; + $clean_state = $clean_mode->{CLEAN_STATE}; + if (($exposure->{$clean_id} != 0)&& + (($exposure->{$clean_state} eq 'full')|| + ($exposure->{$clean_state} eq 'drop'))) { + + $command =~ s/@DBNAME@/$dbname/g; + $command =~ s/@CHIP_ID@/$exposure->{chip_id}/; + $command =~ s/@CHIP_STATE@/$exposure->{chip_state}/; + $command =~ s/@CHIP_LABEL@/$exposure->{chip_label}/; + + $command =~ s/@WARP_ID@/$exposure->{chip_id}/; + $command =~ s/@WARP_STATE@/$exposure->{chip_state}/; + $command =~ s/@WARP_LABEL@/$exposure->{chip_label}/; + + $command =~ s/@DIFF_ID@/$exposure->{diff_id}/; + $command =~ s/@DIFF_STATE@/$exposure->{diff_state}/; + $command =~ s/@DIFF_LABEL@/$exposure->{diff_label}/; + + $command =~ s/@MAGICDS_ID@/$exposure->{magicDS_id}/; + $command =~ s/@MAGICDS_STATE@/$exposure->{magicDS_state}/; + $command =~ s/@MAGICDS_LABEL@/$exposure->{magicDS_label}/; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool: $error_code", "none", "none"); + } + } + } + $exposure->{data_state} = 'cleaned'; + update_this_exposure($exposure); + } +} + + + + +# +# Utilities +################################################################################ + Index: /branches/czw_branch/20110406/ippTasks/lap.pro =================================================================== --- /branches/czw_branch/20110406/ippTasks/lap.pro (revision 31385) +++ /branches/czw_branch/20110406/ippTasks/lap.pro (revision 31385) @@ -0,0 +1,270 @@ +## reprocessing.pro : -*- sh -*- + +check.globals + +macro reprocessing.on + re.chip.on + re.diff.on + re.stack.on + re.magic.on + re.clean.on +end + +macro reprocessing.off + re.chip.off + re.diff.off + re.stack.off + re.magic.off + re.clean.off +end + +macro re.on + reprocessing.on +end + +macro re.off + reprocessing.off +end + +macro re.chip.on + task re.chip.load + active true + end +end + +macro re.diff.on + task re.diff.load + active true + end +end + +macro re.stack.on + task re.stack.load + active true + end +end + +macro re.magic.on + task re.magic.load + active true + end +end + +macro re.clean.on + task re.clean.load + active true + end +end + +macro re.chip.off + task re.chip.load + active false + end +end + +macro re.diff.off + task re.diff.load + active false + end +end + +macro re.stack.off + task re.stack.load + active false + end +end + +macro re.magic.off + task re.magic.load + active false + end +end + +macro re.clean.off + task re.clean.load + active false + end +end + +book init reData + +task re.chip.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDEIR/re.chip.log + + $run = echo no command yet + command $run + end + # success + task.exit 0 + book delpage reData $options:0 + ipptool2book stdout reData -uniq -key proj + + if ($VERBOSE > 2) + book listbook reData + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + +task re.diff.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDEIR/re.diff.log + + $run = echo no command yet + command $run + end + # success + task.exit 0 + book delpage reData $options:0 + ipptool2book stdout reData -uniq -key proj + + if ($VERBOSE > 2) + book listbook reData + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task re.magic.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDEIR/re.magic.log + + $run = echo no command yet + command $run + end + # success + task.exit 0 + book delpage reData $options:0 + ipptool2book stdout reData -uniq -key proj + + if ($VERBOSE > 2) + book listbook reData + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task re.stack.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDEIR/re.stack.log + + $run = echo no command yet + command $run + end + # success + task.exit 0 + book delpage reData $options:0 + ipptool2book stdout reData -uniq -key proj + + if ($VERBOSE > 2) + book listbook reData + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task re.clean.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDEIR/re.clean.log + + $run = echo no command yet + command $run + end + # success + task.exit 0 + book delpage reData $options:0 + ipptool2book stdout reData -uniq -key proj + + if ($VERBOSE > 2) + book listbook reData + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/czw_branch/20110406/ippTools/share/laptool_definerun.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/laptool_definerun.sql (revision 31385) +++ /branches/czw_branch/20110406/ippTools/share/laptool_definerun.sql (revision 31385) @@ -0,0 +1,8 @@ +SELECT exp_id, chip_id, false as private, true as active + FROM rawExp + LEFT JOIN + (SELECT exp_id, chip_id + FROM reprocExp + where private IS FALSE + AND data_state = 'full') AS old USING(exp_id) +-- Position restriction goes here. Index: /branches/czw_branch/20110406/ippTools/share/laptool_exposures.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/laptool_exposures.sql (revision 31385) +++ /branches/czw_branch/20110406/ippTools/share/laptool_exposures.sql (revision 31385) @@ -0,0 +1,30 @@ +SELECT DISTINCT + D.*,diffRun.state,sum(others.private) AS needs_remade FROM ( + SELECT DISTINCT + W.*,IFNULL(diff1.diff_id,diff2.diff_id) AS diff_id FROM ( + SELECT DISTINCT + lap_id,tess_id,projection_cell,filter,lapRun.state as lapRun_state, lapRun.registered, lapRun.fault, labRun.label, + lapExp.exp_id,lapExp.chip_id,lapExp.pair_id,private,pairwise,active,lapExp.data_state, + chipRun.state as chipRun_state, sum(chipProcessedImfile.fault) as chip_faults, sum(chipProcessedImfile.quality) as chip_quality, + camRun.state as camRun_state, sum(camProcessedExp.fault) AS cam_faults, sum(camProcessedExp.quality) AS cam_quality, + fakeRun.state as fakeRun_state, sum(fakeProcessedImfile.fault) as fake_faults, + warpRun.state as warpRun_state, sum(warpSkyfile.fault) as warp_faults, sum(warpSkyfile.quality) as warp_quality, + warpRun.magicked + FROM lapRun JOIN lapExp USING(lap_id) + JOIN chipRun USING(chip_id,exp_id) + FROM chipRun JOIN chipProcessedImfile USING(chip_id) + LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id) + LEFT JOIN fakeRun USING(cam_id) LEFT JOIN fakeProcessedImfile USING(fake_id) + LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id) + @WHERE@ + AND warpSkyfile.quality != 8007 -- known cases where quality != 0, but everything's fine. + AND warpSkyfile.quality != 3006 -- known cases where quality != 0, but everything's fine. + + ) AS W +-- This was unreasonably slow in testing, so that's why I'm using a subquery here. + LEFT JOIN diffInputSkyfile AS diff1 ON (W.warp_id = diff1.warp1) + LEFT JOIN diffInputSkyfile AS diff2 ON (W.warp_id = diff2.warp2) +) AS D +LEFT JOIN diffRun USING(diff_id) +JOIN lapExp AS others ON (D.chip_id = lapExp.chip_id AND D.lap_id != lapExp.lap_id) +GROUP BY chip_id Index: /branches/czw_branch/20110406/ippTools/share/laptool_pendingchipexp.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/laptool_pendingchipexp.sql (revision 31385) +++ /branches/czw_branch/20110406/ippTools/share/laptool_pendingchipexp.sql (revision 31385) @@ -0,0 +1,11 @@ +SELECT re_id,projection_cell,registered,reprocRun.state as reproc_state, tess_id, + exp_id,private,active, + chip_id,chipRun.state AS chip_state, + cam_id,camRun.state AS cam_state, + fake_id,fakeRun.state AS fake_state, + warp_id,warpRun.state AS warp_state + FROM reprocRun JOIN reprocExp USING(re_id) + LEFT JOIN chipRun USING(chip_id,tess_id) + LEFT JOIN camRun USING(chip_id,tess_id) + LEFT JOIN fakeRun USING(fake_id,tess_id) + LEFT JOIN warpRun USING(fake_id,tess_id) Index: /branches/czw_branch/20110406/ippTools/share/laptool_pendingrun.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/laptool_pendingrun.sql (revision 31385) +++ /branches/czw_branch/20110406/ippTools/share/laptool_pendingrun.sql (revision 31385) @@ -0,0 +1,8 @@ +select lap_id, projection_cell, tess_id, registered, state, fault, + exp_id, chip_id, pair_id, private, pairwise, active, data_state, + dateobs, object, comment + FROM lapRun JOIN lapExp USING(lap_id) + JOIN rawExp USING(exp_id) +WHERE active IS TRUE AND fault = 0 +-- lap_id restriction here. +-- This probably needs to be sorted by dateobs. Index: /branches/czw_branch/20110406/ippTools/share/laptool_stacks.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/laptool_stacks.sql (revision 31385) +++ /branches/czw_branch/20110406/ippTools/share/laptool_stacks.sql (revision 31385) @@ -0,0 +1,84 @@ +SELECT lapRun.*,stacks.* FROM + lapRun LEFT JOIN + (SELECT DISTINCT * FROM ( + + (select lapRun.lap_id, + stackAssociation.sass_id AS quick_sass_id, + stackAssociation.data_group AS quick_data_group + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssocaition.filter, + stackRun.stack_id AS quick_stack_id, + stackRun.skycell_id, + stackRun.state AS quick_state, + stackSumSkyfile.fault AS quick_fault, + stackSumSkyfile.quality AS quick_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.quick_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + where lapRun.lap_id = @LAP_ID@ + AND sass_id = lapRun.quick_sass_id + ) AS quick USING(lap_id,projection_cell,tess_id,filter) LEFT JOIN + (select lapRun.lap_id, + stackAssociation.sass_id AS final_sass_id, + stackAssociation.data_group AS final_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssocaition.filter, + stackRun.stack_id AS final_stack_id, + stackRun.skycell_id, + stackRun.state AS final_state, + stackSumSkyfile.fault AS final_fault, + stackSumSkyfile.quality AS final_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.final_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + where lapRun.lap_id = @LAP_ID@ + AND sass_id = lapRun.final_sass_id + ) AS final USING(lap_id,projection_cell,tess_id,filter,skycell_id) + + UNION + + (select lapRun.lap_id, + stackAssociation.sass_id AS quick_sass_id, + stackAssociation.data_group AS quick_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssocaition.filter, + stackRun.stack_id AS quick_stack_id, + stackRun.skycell_id, + stackRun.state AS quick_state, + stackSumSkyfile.fault AS quick_fault, + stackSumSkyfile.quality AS quick_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.quick_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + where lapRun.lap_id = @LAP_ID@ + AND sass_id = lapRun.quick_sass_id + ) AS quick USING(lap_id,projection_cell,tess_id,filter) RIGHT JOIN + (select lapRun.lap_id, + stackAssociation.sass_id AS final_sass_id, + stackAssociation.data_group AS final_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssocaition.filter, + stackRun.stack_id AS final_stack_id, + stackRun.skycell_id, + stackRun.state AS final_state, + stackSumSkyfile.fault AS final_fault, + stackSumSkyfile.quality AS final_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.final_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + where lapRun.lap_id = @LAP_ID@ + AND sass_id = lapRun.final_sass_id + ) AS final USING(lap_id,projection_cell,tess_id,filter,skycell_id) + ) stacks USING(lap_id,projection_cell,tess_id,filter) Index: /branches/czw_branch/20110406/ippTools/src/laptool.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/laptool.c (revision 31384) +++ /branches/czw_branch/20110406/ippTools/src/laptool.c (revision 31385) @@ -1,4 +1,4 @@ /* - * reproctool.c + * laptool.c */ @@ -13,5 +13,5 @@ #include "pxtools.h" #include "pxdata.h" -#include "reproctool.h" +#include "laptool.h" @@ -38,5 +38,5 @@ psLibInit(NULL); - pxConfig *config = reproctoolConfig(NULL, argc, argv); + pxConfig *config = laptoolConfig(NULL, argc, argv); if (!config) { psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); @@ -45,17 +45,17 @@ switch (config->mode) { - MODECASE(REPROCTOOL_MODE_DEFINERUN, definerunMode); - MODECASE(REPROCTOOL_MODE_PENDINGRUN, pendingrunMode); - MODECASE(REPROCTOOL_MODE_UPDATERUN, updaterunMode); - - MODECASE(REPROCTOOL_MODE_PENDINGCHIPEXP, pendingchipexpMode); - MODECASE(REPROCTOOL_MODE_PENDINGQUICKSTACK,pendingquickstackMode); - MODECASE(REPROCTOOL_MODE_PENDINGDIFF, pendingdiffMode); - MODECASE(REPROCTOOL_MODE_PENDINGFINALSTACK,pendingfinalstackMode); - MODECASE(REPROCTOOL_MODE_UPDATEEXP, updateexpMode); + MODECASE(LAPTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(LAPTOOL_MODE_PENDINGRUN, pendingrunMode); + MODECASE(LAPTOOL_MODE_UPDATERUN, updaterunMode); + + MODECASE(LAPTOOL_MODE_PENDINGCHIPEXP, pendingchipexpMode); + MODECASE(LAPTOOL_MODE_PENDINGQUICKSTACK,pendingquickstackMode); + MODECASE(LAPTOOL_MODE_PENDINGDIFF, pendingdiffMode); + MODECASE(LAPTOOL_MODE_PENDINGFINALSTACK,pendingfinalstackMode); + MODECASE(LAPTOOL_MODE_UPDATEEXP, updateexpMode); default: psAbort("invalid option (this should not happen)"); } - psTrace("reproctool", 9, "Attempting to free config\n"); + psTrace("laptool", 9, "Attempting to free config\n"); psFree(config); pmConfigDone(); @@ -86,6 +86,6 @@ // Insert the run - if (!reprocRunInsert(config->dbh, - NULL, // re_id + if (!lapRunInsert(config->dbh, + NULL, // lap_id projection_cell, tess_id, @@ -105,9 +105,9 @@ return false; } - psS64 re_id = psDBLastInsertID(config->dbh); + psS64 lap_id = psDBLastInsertID(config->dbh); // Find the input exposures - psString query = pxDataGet("reproctool_definerun.sql"); + psString query = pxDataGet("laptool_definerun.sql"); if (!query) { psError(PXTOOL_ERR_SYS, false, "failed to retrieve SQL statement"); @@ -135,5 +135,5 @@ } if (!psArrayLength(output)) { - psTrace("reproctool", PS_LOG_INFO, "no rows found"); + psTrace("laptool", PS_LOG_INFO, "no rows found"); psFree(output); return(true); @@ -145,8 +145,8 @@ psMetadata *row = output->data[i]; // Row from select bool status; - reprocExpRow *reExp = reprocExpObjectFromMetadata(row); - reExp->re_id = re_id; - - if (!reprocExpInsertObject(config->dbh,reExp)) { + lapExpRow *reExp = lapExpObjectFromMetadata(row); + reExp->lap_id = lap_id; + + if (!lapExpInsertObject(config->dbh,reExp)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -169,8 +169,8 @@ psmetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-re_id", "re_id", "=="); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); - psString query = pxDataGet("reproctool_pendingrun.sql"); + psString query = pxDataGet("laptool_pendingrun.sql"); if (!query) { psError(PXTOOL_ERR_SYS, false, "failed to retrieve SQL statement"); @@ -205,10 +205,10 @@ } if (!psArrayLength(output)) { - psTrace("reproctool", PS_LOG_INFO, "no rows found"); - psFree(output); - return(true); - } - - if (!ippdbPrintMetadatas(stdout, output, "reprocExp", !simple)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + if (!ippdbPrintMetadatas(stdout, output, "lapExp", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -225,5 +225,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_LOOKUP_S64(re_id, config->args, "-re_id", true, false); + PXOPT_LOOKUP_S64(lap_id, config->args, "-lap_id", true, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); @@ -235,10 +235,10 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-re_id", "re_id", "=="); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); // Set a fault if (fault != INT16_MAX) { // this is fairly dangerous : can set all if the where is not set... - if (!pxSetFaultCode(config->dbh, "reprocRun", where, fault, 0)) { + if (!pxSetFaultCode(config->dbh, "lapRun", where, fault, 0)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree (where); @@ -255,5 +255,5 @@ // Deactivate exposures - psString query = pxDataGet("reproctool_updaterun_deactivate.sql"); + psString query = pxDataGet("laptool_updaterun_deactivate.sql"); if (!query) { psError(PXTOOL_ERR_SYS, false, "failed to retrieve SQL statement"); @@ -270,8 +270,8 @@ } - char *query = "UPDATE reprocRun SET state = '%s' WHERE re_id = %"PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, re_id)) { + char *query = "UPDATE lapRun SET state = '%s' WHERE lap_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, lap_id)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for re_id %"PRId64, re_id); + "failed to change state for lap_id %"PRId64, lap_id); return(false); } @@ -280,15 +280,14 @@ } // Exposure level -static bool pendingchipexpMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - + +static bool exposuresMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); psmetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-re_id", "re_id", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); - - psString query = pxDataGet("reproctool_pendingchipexp.sql"); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); + + psString query = pxDataGet("laptool_exposures.sql"); if (!query) { psError(PXTOOL_ERR_SYS, false, "failed to retrieve SQL statement"); @@ -323,10 +322,10 @@ } if (!psArrayLength(output)) { - psTrace("reproctool", PS_LOG_INFO, "no rows found"); - psFree(output); - return(true); - } - - if (!ippdbPrintMetadatas(stdout, output, "reprocExp", !simple)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + if (!ippdbPrintMetadatas(stdout, output, "lapExp", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -337,16 +336,18 @@ return(true); - -} -static bool pendingquickstackMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); +} + + +static bool pendingchipexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); psmetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-re_id", "re_id", "=="); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); - psString query = pxDataGet("reproctool_pendingquickstack.sql"); + psString query = pxDataGet("laptool_pendingchipexp.sql"); if (!query) { psError(PXTOOL_ERR_SYS, false, "failed to retrieve SQL statement"); @@ -381,10 +382,10 @@ } if (!psArrayLength(output)) { - psTrace("reproctool", PS_LOG_INFO, "no rows found"); - psFree(output); - return(true); - } - - if (!ippdbPrintMetadatas(stdout, output, "reprocExp", !simple)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + if (!ippdbPrintMetadatas(stdout, output, "lapExp", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -397,5 +398,5 @@ } -static bool pendingdiffMode(pxConfig *config) +static bool pendingquickstackMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -403,8 +404,8 @@ psmetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-re_id", "re_id", "=="); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); - psString query = pxDataGet("reproctool_pendingdiff.sql"); + psString query = pxDataGet("laptool_pendingquickstack.sql"); if (!query) { psError(PXTOOL_ERR_SYS, false, "failed to retrieve SQL statement"); @@ -439,10 +440,10 @@ } if (!psArrayLength(output)) { - psTrace("reproctool", PS_LOG_INFO, "no rows found"); - psFree(output); - return(true); - } - - if (!ippdbPrintMetadatas(stdout, output, "reprocExp", !simple)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + if (!ippdbPrintMetadatas(stdout, output, "lapExp", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -455,5 +456,5 @@ } -static bool pendingfinalstackMode(pxConfig *config) +static bool pendingdiffMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -461,8 +462,8 @@ psmetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-re_id", "re_id", "=="); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); - psString query = pxDataGet("reproctool_pendingfinalstack.sql"); + psString query = pxDataGet("laptool_pendingdiff.sql"); if (!query) { psError(PXTOOL_ERR_SYS, false, "failed to retrieve SQL statement"); @@ -497,10 +498,10 @@ } if (!psArrayLength(output)) { - psTrace("reproctool", PS_LOG_INFO, "no rows found"); - psFree(output); - return(true); - } - - if (!ippdbPrintMetadatas(stdout, output, "reprocExp", !simple)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + if (!ippdbPrintMetadatas(stdout, output, "lapExp", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -513,4 +514,62 @@ } +static bool pendingfinalstackMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psmetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + psString query = pxDataGet("laptool_pendingfinalstack.sql"); + if (!query) { + psError(PXTOOL_ERR_SYS, false, "failed to retrieve SQL statement"); + return(false); + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (whereClause) { + psStringPrepend(&whereClause, "\n AND "); + } + + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + return(false); + } + psFree(limitString); + psFree(query); + psFree(whereClause); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + if (!ippdbPrintMetadatas(stdout, output, "lapExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return(false); + } + + psFree(output); + return(true); + + +} static bool updateexpMode(pxConfig *config) { @@ -518,5 +577,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_LOOKUP_S64(re_id, config->args, "-re_id", true, false); + PXOPT_LOOKUP_S64(lap_id, config->args, "-lap_id", true, false); PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false); @@ -532,5 +591,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-re_id", "re_id", "=="); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); @@ -550,5 +609,5 @@ } - long rows = psDBUpdateRows(config->dbh,"reprocExp",where,values); + long rows = psDBUpdateRows(config->dbh,"lapExp",where,values); if (rows) { return(true); Index: /branches/czw_branch/20110406/ippTools/src/laptool.h =================================================================== --- /branches/czw_branch/20110406/ippTools/src/laptool.h (revision 31384) +++ /branches/czw_branch/20110406/ippTools/src/laptool.h (revision 31385) @@ -1,25 +1,26 @@ /* - * reproctool.h + * laptool.h */ -#ifndef REPROCTOOL_H -#define REPROCTOOL_H 1 +#ifndef LAPTOOL_H +#define LAPTOOL_H 1 #include "pxtools.h" typedef enum { - REPROCTOOL_MODE_DEFINERUN = 0x0, - REPROCTOOL_MODE_PENDINGRUN, - REPROCTOOL_MODE_PENDINGCHIPEXP, - REPROCTOOL_MODE_PENDINGQUICKSTACK, - REPROCTOOL_MODE_PENDINGDIFF, - REPROCTOOL_MODE_PENDINGFINALSTACK, - REPROCTOOL_MODE_UPDATERUN, - REPROCTOOL_MODE_UPDATEEXP -} reproctoolMode; + LAPTOOL_MODE_DEFINERUN = 0x0, + LAPTOOL_MODE_PENDINGRUN, + LAPTOOL_MODE_EXPOSURES, + LAPTOOL_MODE_PENDINGCHIPEXP, + LAPTOOL_MODE_PENDINGQUICKSTACK, + LAPTOOL_MODE_PENDINGDIFF, + LAPTOOL_MODE_PENDINGFINALSTACK, + LAPTOOL_MODE_UPDATERUN, + LAPTOOL_MODE_UPDATEEXP +} laptoolMode; -pxConfig *reproctoolConfig(pxConfig *config, int argc, char **argv); +pxConfig *laptoolConfig(pxConfig *config, int argc, char **argv); -#endif // REPROCTOOL_H +#endif // LAPTOOL_H Index: /branches/czw_branch/20110406/ippTools/src/laptoolConfig.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/laptoolConfig.c (revision 31384) +++ /branches/czw_branch/20110406/ippTools/src/laptoolConfig.c (revision 31385) @@ -1,4 +1,4 @@ /* - * reproctoolConfig.c + * laptoolConfig.c */ @@ -16,5 +16,5 @@ #define ADD_OPT(TYPE,TARG,NAME,COMMENT,DEFAULT) psMetadataAdd##TYPE(TARG, PS_LIST_TAIL, NAME, 0, COMMENT, DEFAULT) -pxConfig *reproctoolConfig(pxConfig *config, int argc, char **argv) +pxConfig *laptoolConfig(pxConfig *config, int argc, char **argv) { if (!config) { @@ -43,17 +43,23 @@ psMetadata *pendingrunArgs = psMetadataAlloc(); ADD_OPT(Str, pendingrunArgs, "-projection_cell", "projection cell to consider", NULL); - ADD_OPT(S64, pendingrunArgs, "-re_id", "reprocessing run ID", 0); + ADD_OPT(S64, pendingrunArgs, "-lap_id", "lap run ID", 0); ADD_OPT(U64, pendingrunArgs, "-limit", "limit result set to N items", 0); ADD_OPT(Bool,pendingrunArgs, "-simple", "use the simple output format", false); - - // -pendingchipexp + + // -exposures + psMetadata *exposuresArgs = psMetadataAlloc(); + ADD_OPT(S64, exposuresArgs, "-lap_id", "search by lap run ID", 0); + ADD_OPT(Bool,exposuresArgs, "-simple", "use the simple output format", false); + + // -pendingexp psMetadata *pendingchipexpArgs = psMetadataAlloc(); - ADD_OPT(S64, pendingchipexpArgs, "-re_id", "search by reprocessing run ID", 0); + ADD_OPT(S64, pendingchipexpArgs, "-lap_id", "search by lap run ID", 0); ADD_OPT(Str, pendingchipexpArgs, "-filter", "search by filter", 0); ADD_OPT(Bool,pendingchipexpArgs, "-simple", "use the simple output format", false); + // -pendingquickstack psMetadata *pendingquickstackArgs = psMetadataAlloc(); - ADD_OPT(S64, pendingquickstackArgs, "-re_id", "search by reprocessing run ID", 0); + ADD_OPT(S64, pendingquickstackArgs, "-lap_id", "search by lap run ID", 0); ADD_OPT(Str, pendingquickstackArgs, "-filter", "search by filter", 0); ADD_OPT(Bool,pendingquickstackArgs, "-simple", "use the simple output format", false); @@ -61,5 +67,5 @@ // -pendingdiff psMetadata *pendingdiffArgs = psMetadataAlloc(); - ADD_OPT(S64, pendingdiffArgs, "-re_id", "search by reprocessing run ID", 0); + ADD_OPT(S64, pendingdiffArgs, "-lap_id", "search by lap run ID", 0); ADD_OPT(Str, pendingdiffArgs, "-filter", "search by filter", 0); ADD_OPT(Bool,pendingdiffArgs, "-simple", "use the simple output format", false); @@ -67,5 +73,5 @@ // -pendingfinalstack psMetadata *pendingfinalstackArgs = psMetadataAlloc(); - ADD_OPT(S64, pendingfinalstackArgs, "-re_id", "search by reprocessing run ID", 0); + ADD_OPT(S64, pendingfinalstackArgs, "-lap_id", "search by lap run ID", 0); ADD_OPT(Str, pendingfinalstackArgs, "-filter", "search by filter", 0); ADD_OPT(Bool,pendingfinalstackArgs, "-simple", "use the simple output format", false); @@ -73,5 +79,5 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - ADD_OPT(S64, updaterunArgs, "-re_id", "search by reprocessing run ID", 0); + ADD_OPT(S64, updaterunArgs, "-lap_id", "search by lap run ID", 0); ADD_OPT(Str, updaterunArgs, "-set_state", "set state", NULL); ADD_OPT(S16, updaterunArgs, "-fault", "set fault code", INT16_MAX); @@ -79,5 +85,5 @@ // -updateexp psMetadata *updateexpArgs = psMetadataAlloc(); - ADD_OPT(S64, updateexpArgs, "-re_id", "search by reprocessing run ID", 0); + ADD_OPT(S64, updateexpArgs, "-lap_id", "search by lap run ID", 0); ADD_OPT(S64, updateexpArgs, "-exp_id", "search by exposure ID", 0); ADD_OPT(S64, updateexpArgs, "-chip_id", "search by chip ID", 0); @@ -92,12 +98,13 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definerun", "", REPROCTOOL_MODE_DEFINERUN, definerunArgs); - PXOPT_ADD_MODE("-pendingrun", "", REPROCTOOL_MODE_PENDINGRUN, pendingrunArgs); - PXOPT_ADD_MODE("-pendingchipexp", "", REPROCTOOL_MODE_PENDINGCHIPEXP, pendingchipexpArgs); - PXOPT_ADD_MODE("-pendingquickstack", "", REPROCTOOL_MODE_PENDINGQUICKSTACK,pendingquickstackArgs); - PXOPT_ADD_MODE("-pendingdiff", "", REPROCTOOL_MODE_PENDINGDIFF, pendingdiffArgs); - PXOPT_ADD_MODE("-pendingfinalstack", "", REPROCTOOL_MODE_PENDINGFINALSTACK,pendingfinalstackArgs); - PXOPT_ADD_MODE("-updaterun", "", REPROCTOOL_MODE_UPDATERUN, updaterunArgs); - PXOPT_ADD_MODE("-updateexp", "", REPROCTOOL_MODE_UPDATEEXP, updateexpArgs); + PXOPT_ADD_MODE("-definerun", "", LAPTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-pendingrun", "", LAPTOOL_MODE_PENDINGRUN, pendingrunArgs); + PXOPT_ADD_MODE("-exposures", "", LAPTOOL_MODE_EXPOSURES, exposuresArgs); + PXOPT_ADD_MODE("-pendingchipexp", "", LAPTOOL_MODE_PENDINGCHIPEXP, pendingchipexpArgs); + PXOPT_ADD_MODE("-pendingquickstack", "", LAPTOOL_MODE_PENDINGQUICKSTACK,pendingquickstackArgs); + PXOPT_ADD_MODE("-pendingdiff", "", LAPTOOL_MODE_PENDINGDIFF, pendingdiffArgs); + PXOPT_ADD_MODE("-pendingfinalstack", "", LAPTOOL_MODE_PENDINGFINALSTACK,pendingfinalstackArgs); + PXOPT_ADD_MODE("-updaterun", "", LAPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-updateexp", "", LAPTOOL_MODE_UPDATEEXP, updateexpArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/czw_branch/20110406/pstamp/scripts/detectability_respond.pl =================================================================== --- /branches/czw_branch/20110406/pstamp/scripts/detectability_respond.pl (revision 31384) +++ /branches/czw_branch/20110406/pstamp/scripts/detectability_respond.pl (revision 31385) @@ -217,4 +217,5 @@ $query{$fpa_id}{FAULT}[$i] = 'no_fault'; $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate'; + $query{$fpa_id}{PROC_ERROR}[$i] = 0; } @@ -227,4 +228,5 @@ } elsif ($fpa_id eq 'Not_Set') { + next; } @@ -413,5 +415,6 @@ my $stage = $query{$fpa_id}{STAGE}[$index]; # if there's a fault, then we can't process this image. - if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)) { + if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)|| + ($fpa_id eq 'Not_Set')) { $query{$fpa_id}{PROC_ERROR}[$index] = 23; @@ -713,5 +716,14 @@ # Fill the table columns with the data, making sure the flux is defined - foreach my $fpa_id (keys %query) { + foreach my $fpa_id (sort {$query{$a}{ROWNUM}[0] <=> $query{$b}{ROWNUM}[0] } (keys %query)) { + if ($fpa_id eq 'Not_Set') { + foreach my $index (@{ $query{$a}{ROWNUM} }) { + push @{ $query{$fpa_id}{PROC_ERROR} }, 23; + push @{ $query{$fpa_id}{NPIX} }, -1; + push @{ $query{$fpa_id}{QFACTOR} }, -1; + push @{ $query{$fpa_id}{FLUX} }, -9999; + push @{ $query{$fpa_id}{FLUX_SIG} }, -9999; + } + } $inHeader->{QUERY_ID}->{value} = $query{$fpa_id}{QUERY_ID}[0]; if ($EXTVER_IS_1 == 1) { @@ -720,4 +732,5 @@ } + print "BADPLACE: $fpa_id\n"; push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} }; push @{$colData{'ERROR_CODE'}}, @{ $query{$fpa_id}{PROC_ERROR} };