Changeset 30102
- Timestamp:
- Dec 17, 2010, 10:05:43 AM (15 years ago)
- Location:
- branches/eam_branches/ipp-20101205
- Files:
-
- 65 edited
- 8 copied
-
ippScripts/Build.PL (modified) (1 diff)
-
ippScripts/MANIFEST (modified) (1 diff)
-
ippScripts/scripts/automate_stacks.pl (modified) (1 diff)
-
ippScripts/scripts/camera_exp.pl (modified) (1 diff)
-
ippScripts/scripts/chip_imfile.pl (modified) (4 diffs)
-
ippScripts/scripts/dist_bundle.pl (modified) (2 diffs)
-
ippScripts/scripts/ipp_apply_burntool_single.pl (copied) (copied from trunk/ippScripts/scripts/ipp_apply_burntool_single.pl )
-
ippScripts/scripts/magic_destreak.pl (modified) (1 diff)
-
ippScripts/scripts/magic_destreak_defineruns.pl (modified) (1 diff)
-
ippScripts/scripts/magic_destreak_revert.pl (modified) (3 diffs)
-
ippScripts/scripts/nightly_science.pl (copied) (copied from trunk/ippScripts/scripts/nightly_science.pl )
-
ippScripts/scripts/publish_file.pl (modified) (5 diffs)
-
ippScripts/scripts/receive_file.pl (modified) (2 diffs)
-
ippScripts/scripts/register_exp.pl (modified) (4 diffs)
-
ippScripts/scripts/register_imfile.pl (modified) (7 diffs)
-
ippTasks/Makefile.am (modified) (1 diff)
-
ippTasks/nightly_science.pro (copied) (copied from trunk/ippTasks/nightly_science.pro )
-
ippTasks/nightly_stacks.pro (modified) (15 diffs)
-
ippTasks/publish.pro (modified) (2 diffs)
-
ippTasks/register.pro (modified) (6 diffs)
-
ippToPsps/docs (copied) (copied from trunk/ippToPsps/docs )
-
ippToPsps/docs/loadingSummary.txt (copied) (copied from trunk/ippToPsps/docs/loadingSummary.txt )
-
ippToPsps/perl/checkOdmStatus.pl (modified) (2 diffs)
-
ippToPsps/perl/exposureSummary.pl (copied) (copied from trunk/ippToPsps/perl/exposureSummary.pl )
-
ippToPsps/perl/ippToPsps/DetectionBatch.pm (modified) (1 diff)
-
ippToPsps/perl/ippToPsps/IppToPspsDb.pm (modified) (5 diffs)
-
ippToPsps/perl/ippToPsps_run.pl (modified) (2 diffs)
-
ippTools/configure.ac (modified) (2 diffs)
-
ippTools/share/Makefile.am (modified) (2 diffs)
-
ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
-
ippTools/share/chiptool_find_rawexp.sql (modified) (1 diff)
-
ippTools/share/pubtool_definerun.sql (modified) (4 diffs)
-
ippTools/share/pubtool_pending.sql (modified) (3 diffs)
-
ippTools/share/pxadmin_create_tables.sql (modified) (3 diffs, 1 prop)
-
ippTools/share/regtool_checkburntoolimfile.sql (copied) (copied from trunk/ippTools/share/regtool_checkburntoolimfile.sql )
-
ippTools/share/regtool_pendingburntoolimfile.sql (copied) (copied from trunk/ippTools/share/regtool_pendingburntoolimfile.sql )
-
ippTools/share/regtool_pendingexp.sql (modified) (1 diff)
-
ippTools/src (modified) (1 prop)
-
ippTools/src/chiptool.c (modified) (2 diffs)
-
ippTools/src/chiptoolConfig.c (modified) (1 diff)
-
ippTools/src/difftool.c (modified) (1 diff)
-
ippTools/src/difftoolConfig.c (modified) (1 diff)
-
ippTools/src/magicdstool.c (modified) (1 diff)
-
ippTools/src/magicdstoolConfig.c (modified) (1 diff)
-
ippTools/src/pstamptool.c (modified) (1 diff)
-
ippTools/src/pstamptoolConfig.c (modified) (1 diff)
-
ippTools/src/pubtool.c (modified) (2 diffs)
-
ippTools/src/pubtoolConfig.c (modified) (1 diff)
-
ippTools/src/pztool.c (modified) (1 diff)
-
ippTools/src/regtool.c (modified) (8 diffs)
-
ippTools/src/regtool.h (modified) (1 diff)
-
ippTools/src/regtoolConfig.c (modified) (5 diffs)
-
ippconfig (modified) (1 prop)
-
ippconfig/gpc1/format_20080925.config (modified) (1 diff)
-
ippconfig/gpc1/format_20080929.config (modified) (1 diff)
-
ippconfig/gpc1/format_20081011.config (modified) (1 diff)
-
ippconfig/gpc1/format_20090120.config (modified) (1 diff)
-
ippconfig/gpc1/format_20090220.config (modified) (1 diff)
-
ippconfig/gpc1/format_20100122.config (modified) (1 diff)
-
ippconfig/gpc1/format_orig.config (modified) (1 diff)
-
ippconfig/gpc1/ppImage.config (modified) (3 diffs)
-
ippconfig/gpc1/psastro.config (modified) (1 diff)
-
ippconfig/recipes/nightly_science.config (modified) (10 diffs)
-
ippconfig/recipes/ppImage.config (modified) (3 diffs)
-
ippconfig/recipes/ppStats.config (modified) (1 diff)
-
ippconfig/recipes/ppStatsFromMetadata.config (modified) (1 diff)
-
ippconfig/recipes/psastro.config (modified) (1 diff)
-
ippconfig/recipes/psphot.config (modified) (1 diff)
-
ippconfig/recipes/reductionClasses.mdc (modified) (1 diff)
-
tools/czarpoll.pl (modified) (3 diffs)
-
tools/czartool/DayMetrics.pm (modified) (1 diff)
-
tools/czartool/Metrics.pm (modified) (1 diff)
-
tools/makedistdest (modified) (2 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20101205/ippScripts/Build.PL
r29530 r30102 105 105 scripts/whichimage 106 106 scripts/automate_stacks.pl 107 scripts/nightly_science.pl 107 108 scripts/lossy_compress_imfile.pl 108 109 scripts/ipp_apply_burntool.pl 110 scripts/ipp_apply_burntool_single.pl 109 111 scripts/make_burntool_pcontrol.pl 110 112 scripts/bundle_detrends.pl -
branches/eam_branches/ipp-20101205/ippScripts/MANIFEST
r28375 r30102 37 37 scripts/ds9_cmf_regions.pl 38 38 scripts/automate_stacks.pl 39 scripts/nightly_science.pl 39 40 scripts/lossy_compress_imfile.pl 40 41 scripts/ipp_apply_burntool.pl 42 scripts/ipp_apply_burntool_single.pl 41 43 scripts/make_burntool_pcontrol.pl 42 44 scripts/bundle_detrends.pl -
branches/eam_branches/ipp-20101205/ippScripts/scripts/automate_stacks.pl
r29919 r30102 1355 1355 my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); 1356 1356 1357 if ( $target eq 'OSS') {1357 if (($target eq 'OSS')||($target eq 'SweetSpot')) { 1358 1358 my $db = init_gpc_db(); 1359 1359 -
branches/eam_branches/ipp-20101205/ippScripts/scripts/camera_exp.pl
r29893 r30102 378 378 $ipprc->file_exists($file); 379 379 380 if ($replicate and (file_scheme($file) eq 'neb')) { 380 my $scheme = file_scheme($file); 381 if ($replicate and $scheme and (file_scheme($file) eq 'neb')) { 381 382 $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $cam_id, $PS_EXIT_SYS_ERROR); 382 383 } -
branches/eam_branches/ipp-20101205/ippScripts/scripts/chip_imfile.pl
r29950 r30102 176 176 $ipprc->delete_destreak_backup_file($outputWeight) 177 177 or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR); 178 179 # don't do binned images when updating unless we are starting from scratch. 180 $do_binned_images = 0 unless $dump_config; 178 181 } 179 182 if ($do_binned_images) { … … 490 493 check_output($configuration, 1) if $dump_config; 491 494 check_output($backmdl, 1) if $outputBackmdlExpect; 492 check_output($pattern, 1) if $outputPatternExpect; 495 # allow the pattern file to be missing if run state is update older data doesn't have one 496 # I should parse the config dump file to calculate the 'Expect' variables 497 check_output($pattern, 1, $run_state eq 'update') if $outputPatternExpect; 493 498 if ($do_photom) { 494 499 check_output($outputSources, 1); … … 701 706 my $file = shift; 702 707 my $replicate = shift; 708 my $allow_missing = shift; 703 709 704 710 if (!defined $file) { … … 706 712 } 707 713 708 &my_die("Couldn't find expected output file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); 714 my $exists = $ipprc->file_exists($file); 715 716 if (!$exists) { 717 if ($allow_missing) { 718 carp("Couldn't find expected output_file: $file but continuing anyways\n"); 719 return 1; 720 } 721 &my_die("Couldn't find expected output file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); 722 } 709 723 710 724 if ($replicate and $neb) { -
branches/eam_branches/ipp-20101205/ippScripts/scripts/dist_bundle.pl
r29063 r30102 62 62 my $streaksrelease = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1); 63 63 my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); 64 my $file_cmd = can_run('file') or (warn "can't find program file" and $missing_tools = 1); 65 my $zcat = can_run('zcat') or (warn "can't find program zcat" and $missing_tools = 1); 64 66 if ($missing_tools) { 65 67 warn("Can't find required tools."); … … 526 528 $PS_EXIT_CONFIG_ERROR) if (!$resolved); 527 529 530 &my_die("config dump file resolved but not accessible: $config_file_rule", $component, 531 $PS_EXIT_CONFIG_ERROR) if !$ipprc->file_exists($resolved); 532 533 my $mdc_compressed; 534 { 535 my $command = "$file_cmd $resolved"; 536 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 537 run(command => $command, verbose => $verbose); 538 unless ($success) { 539 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 540 &my_die("Unable to perform $command: $error_code", $component, $error_code); 541 } 542 my $output = join "", @$stdout_buf; 543 # XXX: may need to to make this more robust 544 $mdc_compressed = ($output =~ /gzip/); 545 } 546 my $inName; 547 if ($mdc_compressed) { 548 my $tmpfile; 549 ($tmpfile, $inName) = tempfile( "/tmp/bundle.XXXX", UNLINK => !$save_temps ); 550 close($tmpfile) or &my_die("failed to close $inName", $component, $PS_EXIT_UNKNOWN_ERROR); 551 552 my $command = "$zcat $resolved > $inName"; 553 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 554 run(command => $command, verbose => $verbose); 555 unless ($success) { 556 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 557 &my_die("Unable to perform $command: $error_code", $component, $error_code); 558 } 559 560 561 } else { 562 $inName = $resolved; 563 } 564 my $in = open_with_retries($inName); 565 528 566 # we don't use the mdc parser because the perl parser is way is too slow for complicated config 529 567 # files like this 530 my $in = open_with_retries($resolved);531 532 568 my $line; 533 569 while ($line = <$in>) { -
branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak.pl
r29571 r30102 309 309 } 310 310 311 $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id); 311 # only destreak cmf file if this is a new run 312 if ($run_state eq 'new') { 313 $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id); 314 } 312 315 313 316 } elsif ($stage eq "chip_bg") { -
branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_defineruns.pl
r27718 r30102 78 78 foreach my $label (@labels) { 79 79 my $command = "$magicdstool -definebyquery -stage $stage -workdir $workdir -label $label"; 80 $command .= " - dry_run" if $no_update;80 $command .= " -pretend" if $no_update; 81 81 $command .= " -limit $stage_limit" if $stage_limit; 82 82 $command .= " -set_label $label"; -
branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_revert.pl
r29573 r30102 191 191 $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id); 192 192 $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); 193 $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id);194 193 195 194 # we use the mask output from the camera stage for input and replace … … 204 203 } 205 204 205 # only revert sources if this is a new run, not one being updated 206 if ($run_state eq 'new') { 207 $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id); 208 $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $class_id); 209 } 210 206 211 $bimage = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id); 207 212 # This is somewhat kludgey but it works whether the mask is camera mask or chip mask … … 209 214 $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id); 210 215 $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id); 211 $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $class_id);212 216 } elsif ($stage eq "camera") { 213 217 $astrom = $ipprc->filename("PSASTRO.OUTPUT", $path_base); -
branches/eam_branches/ipp-20101205/ippScripts/scripts/publish_file.pl
r29622 r30102 44 44 my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir ); 45 45 my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect ); 46 my ( $output_format ); 46 47 47 48 GetOptions( … … 59 60 'save-temps' => \$save_temps, # Save temporary files? 60 61 'redirect-output' => \$redirect, # Redirect output to log file? 62 'output_format=i' => \$output_format, # Output format for ppMops 61 63 ) or pod2usage( 2 ); 62 64 … … 191 193 warp_id => $comp->{warp1}, 192 194 diff_id => $comp->{diff_id}, 195 output_format => $comp->{output_format}, 193 196 direction => 1, 194 197 }; … … 323 326 324 327 my $command = "$ppMops $input $output"; 325 $command .= " -version 1"; # XXX : NOTE: for now, MOPS just wants PS1_DV1 (remove this eventually...)326 328 $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name}; 327 329 $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id}; … … 335 337 $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err}; 336 338 $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom}; 339 $command .= " -version " . $data->{output_format} if defined $data->{output_format}; 337 340 338 341 unless ($no_op) { -
branches/eam_branches/ipp-20101205/ippScripts/scripts/receive_file.pl
r29574 r30102 32 32 my $receivetool = can_run('receivetool') or (warn "Can't find receivetool" and $missing_tools = 1); 33 33 my $dsproductls = can_run('dsfilesetls') or (warn "Can't find dsfilesetls" and $missing_tools = 1); 34 my $gunzip = can_run('gunzip') or (warn "Can't find gunzip" and $missing_tools = 1); 34 35 if ($missing_tools) { 35 36 warn("Can't find required tools."); … … 404 405 my $workdir = shift; 405 406 407 my $filecmd_output = `file $src`; 408 &my_die("failed to determinte file type of $src", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$filecmd_output; 409 chomp $filecmd_output; 410 411 my $tmpName; 412 if ($filecmd_output =~ /gzip/) { 413 my $tmpfile; 414 ($tmpfile, $tmpName) = tempfile( "/tmp/receive.XXXX", UNLINK => !$save_temps ); 415 close($tmpfile) or &my_die("failed to close $tmpName", $file_id, $PS_EXIT_UNKNOWN_ERROR); 416 417 my $command = "$gunzip -c $src > $tmpName"; 418 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 419 run(command => $command, verbose => $verbose); 420 unless ($success) { 421 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 422 &my_die("Unable to perform $command: $error_code", $component, $error_code); 423 } 424 $src = $tmpName; 425 } 426 406 427 open my $IN, "<$src" or &my_die("failed to open $src for input", $file_id, $PS_EXIT_UNKNOWN_ERROR); 407 428 open my $OUT, ">$dest" or &my_die("failed to open $dest for output", $file_id, $PS_EXIT_UNKNOWN_ERROR); -
branches/eam_branches/ipp-20101205/ippScripts/scripts/register_exp.pl
r27891 r30102 30 30 my $regtool = can_run('regtool') or (warn "can't find regtool" and $missing_tools = 1); 31 31 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); 32 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); 32 33 if ($missing_tools) { 33 34 warn ("Can't find required tools"); … … 118 119 if (uc($exp_type) eq "NULL") { &my_die ("exp_type not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } 119 120 121 my ($data_group,$dist_group,$chip_workdir,$reduction); 122 ($label,$data_group,$dist_group,$end_stage, 123 $tess_id,$chip_workdir,$reduction) = advance_decisions(&value_for_flag($cmdflags,"-dateobs"), 124 &value_for_flag($cmdflags,"-exp_type"), 125 &value_for_flag($cmdflags,"-obs_mode"), 126 &value_for_flag($cmdflags,"-object"), 127 &value_for_flag($cmdflags,"-comment"), 128 $dbname); 120 129 my $command = "$regtool -addprocessedexp"; 121 130 $command .= " -exp_id $exp_id"; … … 127 136 $command .= " -end_stage $end_stage" if defined $end_stage; 128 137 $command .= " -tess_id $tess_id" if defined $tess_id; 138 $command .= " -data_group $data_group" if defined $data_group; 139 $command .= " -dist_group $dist_group" if defined $dist_group; 140 $command .= " -chip_workdir $chip_workdir" if defined $chip_workdir; 141 $command .= " -reduction $reduction" if defined $reduction; 129 142 $command .= " -state full"; 130 143 $command .= " $cmdflags"; … … 156 169 157 170 ### Pau. 171 sub advance_decisions { 172 my $dateobs = shift; 173 my $exptype = shift; 174 my $obsmode = shift; 175 my $object = shift; 176 my $comment = shift; 177 my $dbname = shift; 178 179 # The strings come pre-quoted. 180 $dateobs =~ s/\'//g; 181 $exptype =~ s/\'//g; 182 $obsmode =~ s/\'//g; 183 $object =~ s/\'//g; 184 $comment =~ s/\'//g; 185 186 print "Inside decision engine\n"; 187 my ($label,$data_group,$dist_group,$end_stage,$tess_id,$chip_workdir,$reduction); 188 my $target; 189 190 if ($exptype ne 'OBJECT') { 191 print "Aborting because exptype doesn't claim to be OBJECT: $exptype\n"; 192 return(undef,undef,undef,"reg",undef,undef,undef); 193 } 194 if ($obsmode eq 'ENGINEERING') { 195 print "Aborting because we don't queue engineering data to chip\n"; 196 return(undef,undef,undef,"reg",undef,undef,undef); 197 } 198 # Grab the configuration data. 199 my %nightlyscience_config = (); 200 my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; 201 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 202 run(command => $conf_cmd, verbose => $verbose); 203 unless ($success) { 204 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 205 &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR); 206 } 207 print "data loaded\n"; 208 my $mdcParser = PS::IPP::Metadata::Config->new; 209 my $metadata = $mdcParser->parse(join "", @$stdout_buf); 210 foreach my $entry (@{ $metadata }) { 211 if (${ $entry }{name} eq 'TARGETS') { 212 my @target_data = @{ ${ $entry }{value} }; 213 my $this_target = ''; 214 foreach my $tentry (@target_data) { 215 if (${ $tentry }{name} eq 'NAME') { 216 $this_target = ${ $tentry }{value}; 217 $nightlyscience_config{$this_target}{DISTRIBUTION} = $this_target; 218 } 219 elsif (${ $tentry }{name} eq 'DISTRIBUTION') { 220 $nightlyscience_config{$this_target}{DISTRIBUTION} = ${ $tentry }{value}; 221 } 222 elsif (${ $tentry }{name} eq 'TESS') { 223 $nightlyscience_config{$this_target}{TESS_ID} = ${ $tentry }{value}; 224 } 225 elsif (${ $tentry }{name} eq 'OBSMODE') { 226 $nightlyscience_config{$this_target}{OBSMODE} = ${ $tentry }{value}; 227 $nightlyscience_config{$this_target}{OBSMODE} =~ s/%/\.\*/g; 228 } 229 elsif (${ $tentry }{name} eq 'OBJECT') { 230 $nightlyscience_config{$this_target}{OBJECT} = ${ $tentry }{value}; 231 $nightlyscience_config{$this_target}{OBJECT} =~ s/%/\.\*/g; 232 } 233 elsif (${ $tentry }{name} eq 'COMMENT') { 234 $nightlyscience_config{$this_target}{COMMENT} = ${ $tentry }{value}; 235 $nightlyscience_config{$this_target}{COMMENT} =~ s/%/\.\*/g; 236 } 237 elsif (${ $tentry }{name} eq 'STACKABLE') { 238 $nightlyscience_config{$this_target}{STACKABLE} = ${ $tentry }{value}; 239 } 240 elsif (${ $tentry }{name} eq 'DIFFABLE') { 241 $nightlyscience_config{$this_target}{DIFFABLE} = ${ $tentry }{value}; 242 } 243 elsif (${ $tentry }{name} eq 'REDUCTION') { 244 $nightlyscience_config{$this_target}{REDUCTION} = ${ $tentry }{value}; 245 } 246 } 247 } 248 } 249 print "data parsed\n"; 250 my $found_target; 251 foreach $target (keys %nightlyscience_config) { 252 # print "TEST: $target ($obsmode $object $comment) ($nightlyscience_config{$target}{OBSMODE} $nightlyscience_config{$target}{OBJECT} $nightlyscience_config{$target}{COMMENT}\t"; 253 my $match = 0; 254 my $possible = 0; 255 if (exists($nightlyscience_config{$target}{OBSMODE})) { 256 $possible++; 257 if ($obsmode =~ /$nightlyscience_config{$target}{OBSMODE}/) { 258 $match++; 259 } 260 } 261 if (exists($nightlyscience_config{$target}{OBJECT})) { 262 $possible++; 263 if ($object =~ /$nightlyscience_config{$target}{OBJECT}/) { 264 $match++; 265 } 266 } 267 if (exists($nightlyscience_config{$target}{COMMENT})) { 268 $possible++; 269 if ($comment =~ /$nightlyscience_config{$target}{COMMENT}/) { 270 $match++; 271 } 272 } 273 print "$possible $match\n"; 274 if (($possible > 0)&&($match == $possible)) { 275 $found_target = $target; 276 last; 277 } 278 } 279 unless ($found_target) { 280 print "No acceptible target found.\n"; 281 return(undef,undef,undef,"reg",undef,undef,undef); 282 } 283 print "Found this target: $found_target\n"; 284 print "$dateobs\n"; 285 $target = $found_target; 286 287 my $date = $dateobs; 288 $date =~ s/T.*//; 289 my $workdir_date = $date; 290 $date =~ s/-//g; 291 $workdir_date =~ s%-%/%g; 292 print "$dateobs $date $workdir_date\n"; 293 $label = "${target}.nightlyscience"; 294 $data_group = "${target}.${date}"; 295 $dist_group = $nightlyscience_config{$target}{DISTRIBUTION}; 296 $end_stage = 'warp'; 297 $tess_id = $nightlyscience_config{$target}{TESS_ID}; 298 $chip_workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}/"; 299 if (exists($nightlyscience_config{$target}{REDUCTION})) { 300 $reduction = $nightlyscience_config{$target}{REDUCTION}; 301 } 302 else { 303 undef($reduction); 304 } 305 306 307 return ($label,$data_group,$dist_group,$end_stage,$tess_id,$chip_workdir,$reduction); 308 } 309 310 311 158 312 159 313 sub cache_run -
branches/eam_branches/ipp-20101205/ippScripts/scripts/register_imfile.pl
r27891 r30102 19 19 use File::Basename qw( basename); 20 20 use IPC::Run; 21 use IPC::Cmd 0.36 qw( can_run );21 use IPC::Cmd 0.36 qw( can_run run ); 22 22 use PS::IPP::Config 1.01 qw( :standard ); 23 use PS::IPP::Metadata::List qw( parse_md_list ); 23 24 use PS::IPP::Metadata::Config; 24 25 use Math::Trig; … … 35 36 my $ppStatsFromMetadata = can_run( 'ppStatsFromMetadata' ) or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); 36 37 my $ppConfigDump = can_run( 'ppConfigDump' ) or (warn "Can't find ppConfigDump" and $missing_tools = 1); 38 my $ippApplyBurntoolSingle = can_run( 'ipp_apply_burntool_single.pl' ) or (warn "Can't find ipp_apply_burntool_single" and $missing_tools = 1); 37 39 38 40 my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $bytes, $md5sum, $dbname, $verbose, $no_update, $no_op, $logfile); … … 82 84 83 85 my $cmdflags; 86 my $burntoolStateGood; 84 87 # Run ppStats on the input file 85 88 { … … 114 117 my $burntoolState = 0; 115 118 my $isGPC1 = 0; 116 my$burntoolStateGood = 0;119 $burntoolStateGood = 0; 117 120 foreach my $line (split /\n/, $out1) { 118 121 if ($line =~ /FPA.BURNTOOL.APPLIED/) { … … 174 177 $command .= " -hostname $host" if defined $host; 175 178 $command .= " -dbname $dbname" if defined $dbname; 176 $command .= " -data_state full";179 $command .= " -data_state check_burntool"; 177 180 $command .= " $cmdflags"; 178 181 182 print "$command\n"; 179 183 # determine solar-system parameters 180 184 my $longitude = &value_for_flag($cmdflags, 0.0, "-longitude"); … … 206 210 } 207 211 } 212 213 # This might have a race condition 214 # $date_end = $dateobs; 215 # $date_start = $dateobs - 30 minutes ? dateobs_UTC_midnight? 216 # lock file? 217 # if exp_type = DARK and date > MIDNIGHT HST { wait } 218 # system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log"); 208 219 209 220 $now_time = localtime(); … … 222 233 print "skipping command: $command\n"; 223 234 } 235 236 # We now have an imfile in the database, check if we can burntool it. If not, continue on. 237 238 { 239 my $mdcParser = PS::IPP::Metadata::Config->new; 240 241 my $class_id = &value_for_flag($cmdflags, 0.0, "-class_id"); 242 my $check_date = &value_for_flag($cmdflags, 0.0, "-dateobs"); 243 $check_date =~ s/T.*$//; 244 my $exp_name = $tmp_exp_name; 245 246 247 my $bt_check_command = "$regtool -checkburntoolimfile "; 248 $bt_check_command .= " -class_id $class_id "; 249 $bt_check_command .= " -date $check_date "; 250 $bt_check_command .= " -valid_burntool $burntoolStateGood "; 251 $bt_check_command .= " -exp_name $exp_name "; 252 $bt_check_command .= " -dbname $dbname" if defined $dbname; 253 254 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 255 run ( command => $bt_check_command, verbose => $verbose); 256 unless ($success) { 257 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 258 &my_die("Unable to perform regtool: $error_code", 259 $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR); 260 } 261 262 my @whole = split /\n/, (join "", @$stdout_buf); 263 my @single = (); 264 my $exposures; 265 while ( scalar @whole > 0 ) { 266 my $value = shift @whole; 267 push @single, $value; 268 if ($value =~ /^\s*END\s*$/) { 269 push @single, "\n"; 270 271 my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); 272 &my_die("Unable to parse output from regtool", 273 $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR) unless 274 defined $list; 275 push @{ $exposures }, @$list; 276 @single = (); 277 } 278 } 279 280 # We only care about the final entry, as that contains the exposure we are. 281 282 my $regtool_update = "$regtool -updateprocessedimfile "; 283 $regtool_update .= "-dbname $dbname " if defined $dbname; 284 $regtool_update .= "-exp_id $exp_id -class_id $class_id "; 285 286 my $burntool_data = pop(@{ $exposures }); 287 if ($burntool_data->{burnable} == 0) { 288 $regtool_update .= " -burntool_state 0 -set_state pending_burntool "; 289 } 290 else { 291 my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname "; 292 $apply_command .= " --class_id $class_id --exp_id $exp_id "; 293 $apply_command .= " --this_uri $burntool_data->{uri} "; 294 $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri}; 295 $apply_command .= " --imfile_state $burntool_data->{imfile_state} "; 296 $apply_command .= " --verbose " if $verbose; 297 print "$apply_command\n"; 298 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 299 IPC::Cmd::run(command => $apply_command, verbose => $verbose); 300 unless ($success) { 301 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 302 warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code"); 303 exit($error_code); 304 } 305 } 306 307 unless ($no_update) { 308 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 309 IPC::Cmd::run(command => $regtool_update, verbose => $verbose); 310 unless ($success) { 311 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 312 warn ("Unable to perform regtool -addprocessedimfile: $error_code"); 313 exit($error_code); 314 } 315 } else { 316 print "skipping command: $command\n"; 317 } 318 319 } 320 321 224 322 225 323 $now_time = localtime(); -
branches/eam_branches/ipp-20101205/ippTasks/Makefile.am
r29614 r30102 36 36 minidvodb.pro \ 37 37 minidvodbcopy.pro \ 38 nightly_s tacks.pro \38 nightly_science.pro \ 39 39 burntool.pro \ 40 40 lossy_compress.pro \ -
branches/eam_branches/ipp-20101205/ippTasks/nightly_stacks.pro
r29656 r30102 7 7 ns.detrends.off 8 8 ns.dqstats.on 9 ns.sweetspot.on10 ns.registration.on11 ns.burntool.on12 ns.chips.on9 # ns.sweetspot.on 10 # ns.registration.on 11 # ns.burntool.on 12 # ns.chips.on 13 13 ns.stacks.on 14 ns.diffs.on 14 15 end 15 16 … … 18 19 ns.detrends.off 19 20 ns.dqstats.off 20 ns.sweetspot.off21 ns.registration.off22 ns.burntool.off23 ns.chips.off21 # ns.sweetspot.off 22 # ns.registration.off 23 # ns.burntool.off 24 # ns.chips.off 24 25 ns.stacks.off 26 ns.diffs.off 25 27 end 26 28 … … 51 53 end 52 54 53 macro ns.sweetspot.on54 task ns.sweetspot.load55 active true56 end57 end58 59 macro ns.registration.on60 task ns.registration.load61 active true62 end63 end64 65 macro ns.burntool.on66 task ns.burntool.load67 active true68 end69 task ns.burntool.run70 active true71 end72 end73 74 macro ns.chips.on75 task ns.chips.load76 active true77 end78 task ns.chips.run79 active true80 end81 end55 # macro ns.sweetspot.on 56 # task ns.sweetspot.load 57 # active true 58 # end 59 # end 60 61 # macro ns.registration.on 62 # task ns.registration.load 63 # active true 64 # end 65 # end 66 67 # macro ns.burntool.on 68 # task ns.burntool.load 69 # active true 70 # end 71 # task ns.burntool.run 72 # active true 73 # end 74 # end 75 76 # macro ns.chips.on 77 # task ns.chips.load 78 # active true 79 # end 80 # task ns.chips.run 81 # active true 82 # end 83 # end 82 84 83 85 macro ns.stacks.on … … 93 95 end 94 96 97 macro ns.diffs.on 98 task ns.diffs.load 99 active false 100 end 101 task ns.diffs.run 102 active true 103 end 104 end 105 95 106 macro ns.initday.off 96 107 task ns.initday.load … … 111 122 end 112 123 113 macro ns.sweetspot.off114 task ns.sweetspot.load115 active false116 end117 end118 119 macro ns.registration.off120 task ns.registration.load121 active false122 end123 end124 125 macro ns.burntool.off126 task ns.burntool.load127 active false128 end129 task ns.burntool.run130 active false131 end132 end133 134 macro ns.chips.off135 task ns.chips.load136 active false137 end138 task ns.chips.run139 active false140 end141 end124 # macro ns.sweetspot.off 125 # task ns.sweetspot.load 126 # active false 127 # end 128 # end 129 130 # macro ns.registration.off 131 # task ns.registration.load 132 # active false 133 # end 134 # end 135 136 # macro ns.burntool.off 137 # task ns.burntool.load 138 # active false 139 # end 140 # task ns.burntool.run 141 # active false 142 # end 143 # end 144 145 # macro ns.chips.off 146 # task ns.chips.load 147 # active false 148 # end 149 # task ns.chips.run 150 # active false 151 # end 152 # end 142 153 143 154 macro ns.stacks.off … … 150 161 end 151 162 152 $ns_regPAGE = 0 153 $ns_burnPAGE = 0 154 $ns_RburnPAGE = 0 155 $ns_RburnCELL = 0 156 $ns_chipPAGE = 0 157 $ns_RchipPAGE = 0 163 macro ns.diffs.off 164 task ns.diffs.load 165 active false 166 end 167 task ns.diffs.run 168 active false 169 end 170 end 171 172 # $ns_regPAGE = 0 173 # $ns_burnPAGE = 0 174 # $ns_RburnPAGE = 0 175 # $ns_RburnCELL = 0 176 # $ns_chipPAGE = 0 177 # $ns_RchipPAGE = 0 158 178 $ns_stackPAGE = 0 159 179 $ns_RstackPAGE = 0 160 180 $ns_CstackPAGE = 0 181 $ns_diffPAGE = 0 182 $ns_RdiffPAGE = 0 161 183 162 184 book init nsData 163 book init nsBurntool185 # book init nsBurntool 164 186 # 165 187 # Macros to control the book. … … 177 199 echo $date $state 178 200 end 179 book npages nsBurntool -var Npages201 # book npages nsBurntool -var Npages 180 202 end 181 203 … … 198 220 periods -exec $LOADEXEC 199 221 periods -timeout 30 200 trange 16:30:00 17:30:00 -nmax 1222 trange 0:00:00 1:00:00 -nmax 1 201 223 npending 1 202 224 … … 204 226 stdout $LOGDIR/ns.initday.log 205 227 stderr $LOGDIR/ns.initday.log 206 $today = `date +%Y-%m-%d`228 $today = `date -u +%Y-%m-%d` 207 229 book newpage nsData $today 208 230 book setword nsData $today nsState NEW … … 242 264 stdout $LOGDIR/ns.detrends.log 243 265 stdout $LOGDIR/ns.detrends.log 244 $today = `date +%Y-%m-%d`266 $today = `date -u +%Y-%m-%d` 245 267 246 268 command automate_stacks.pl --queue_detrends --date $today … … 278 300 stdout $LOGDIR/ns.dqstats.log 279 301 stderr $LOGDIR/ns.dqstats.log 280 $today = `date +%Y-%m-%d`302 $today = `date -u +%Y-%m-%d` 281 303 282 304 command automate_stacks.pl --queue_dqstats --date $today … … 313 335 stdout $LOGDIR/ns.sweetspot.log 314 336 stderr $LOGDIR/ns.sweetspot.log 315 $today = `date +%Y-%m-%d`337 $today = `date -u +%Y-%m-%d` 316 338 317 339 command automate_stacks.pl --queue_sweetspot --date $today … … 337 359 # Check to see if registration is complete 338 360 # 339 task ns.registration.load340 host local341 periods -poll $LOADPOLL342 periods -exec $LOADEXEC343 periods -timeout 30344 npending 1345 346 task.exec347 stdout NULL348 stderr $LOGDIR/ns.registration.log349 350 book getpage nsData $ns_regPAGE -var date351 book getword nsData $date nsState -var ns_STATE352 book npages nsData -var Npages353 354 if ($VERBOSE > 5)355 echo "ns.registration.load: " $ns_regPAGE $date $ns_STATE $Npages356 end357 358 $ns_regPAGE ++359 if ($ns_regPAGE >= $Npages) set ns_regPAGE = 0360 option $date361 362 if ("$ns_STATE" != "NEW") break363 $run = automate_stacks.pl --check_registration --date $date364 command $run365 end366 367 # success368 task.exit 0369 # convert 'stdout' to book format370 book delpage nsData $options:0371 ipptool2book stdout nsData -uniq -key date372 373 if ($VERBOSE > 2)374 book listbook nsData375 end376 end377 378 # locked list379 task.exit default380 showcommand failure381 end382 task.exit crash383 showcommand crash384 end385 #operation times out?386 task.exit timeout387 showcommand timeout388 end389 end390 391 # 392 # Check to see if the chips have been properly burntooled and are ready for queueing393 # 394 task ns.chips.load395 host local396 periods -poll $LOADPOLL397 periods -exec $LOADEXEC398 periods -timeout 300399 npending 1400 401 task.exec402 stdout NULL403 stderr $LOGDIR/ns.chips.log404 405 book getpage nsData $ns_chipPAGE -var date406 book getword nsData $date nsState -var ns_STATE407 book npages nsData -var Npages408 409 if ($VERBOSE > 5)410 echo "ns.chips.load: " $ns_chipPAGE $date $ns_STATE $Npages411 end412 413 $ns_chipPAGE ++414 if ($ns_chipPAGE >= $Npages) set ns_chipPAGE = 0415 option $date416 417 if (("$ns_STATE" != "REGISTERED")&&("$ns_STATE" != "BURNING")) break418 $run = automate_stacks.pl --check_chips --date $date419 420 if ("$ns_STATE" == "BURNING")421 $run = $run --isburning422 end423 command $run424 end425 # success426 task.exit 0427 # convert 'stdout' to book format428 book delpage nsData $options:0429 ipptool2book stdout nsData -uniq -key date430 431 # remove the burntool page if we're done with it.432 book getword nsData $options:0 nsState -var ns_STATE433 if ("$ns_STATE" == "QUEUECHIPS")434 book delpage nsBurntool $options:0435 end436 437 book getword nsData $options:0 nsNmacros -var ns_Nmacros438 if ("$ns_Nmacros" != "NULL")439 for i 0 $ns_Nmacros440 sprintf macroName "ns%dMacro" $i441 book getword nsData $options:0 $macroName -var macroCmd442 $macroCmd443 end444 end445 446 if ($VERBOSE > 2)447 book listbook nsData448 end449 end450 451 # locked list452 task.exit default453 showcommand failure454 end455 task.exit crash456 showcommand crash457 end458 #operation times out?459 task.exit timeout460 showcommand timeout461 end462 end463 464 # 465 # Check to see if the chips have been properly burntooled and are ready for queueing466 # 467 task ns.chips.run468 host local469 periods -poll $LOADPOLL470 periods -exec $LOADEXEC471 periods -timeout 120472 npending 1473 474 task.exec475 stdout NULL476 stderr $LOGDIR/ns.chips.log477 478 book getpage nsData $ns_RchipPAGE -var date479 book getword nsData $date nsState -var ns_STATE480 book npages nsData -var Npages481 482 if ($VERBOSE > 5)483 echo "ns.chips.run: " $ns_RchipPAGE $date $ns_STATE $Npages484 end361 # task ns.registration.load 362 # host local 363 # periods -poll $LOADPOLL 364 # periods -exec $LOADEXEC 365 # periods -timeout 30 366 # npending 1 367 368 # task.exec 369 # stdout NULL 370 # stderr $LOGDIR/ns.registration.log 371 372 # book getpage nsData $ns_regPAGE -var date 373 # book getword nsData $date nsState -var ns_STATE 374 # book npages nsData -var Npages 375 376 # if ($VERBOSE > 5) 377 # echo "ns.registration.load: " $ns_regPAGE $date $ns_STATE $Npages 378 # end 379 380 # $ns_regPAGE ++ 381 # if ($ns_regPAGE >= $Npages) set ns_regPAGE = 0 382 # option $date 383 384 # if ("$ns_STATE" != "NEW") break 385 # $run = automate_stacks.pl --check_registration --date $date 386 # command $run 387 # end 388 389 # # success 390 # task.exit 0 391 # # convert 'stdout' to book format 392 # book delpage nsData $options:0 393 # ipptool2book stdout nsData -uniq -key date 394 395 # if ($VERBOSE > 2) 396 # book listbook nsData 397 # end 398 # end 399 400 # # locked list 401 # task.exit default 402 # showcommand failure 403 # end 404 # task.exit crash 405 # showcommand crash 406 # end 407 # #operation times out? 408 # task.exit timeout 409 # showcommand timeout 410 # end 411 # end 412 413 # # 414 # # Check to see if the chips have been properly burntooled and are ready for queueing 415 # # 416 # task ns.chips.load 417 # host local 418 # periods -poll $LOADPOLL 419 # periods -exec $LOADEXEC 420 # periods -timeout 300 421 # npending 1 422 423 # task.exec 424 # stdout NULL 425 # stderr $LOGDIR/ns.chips.log 426 427 # book getpage nsData $ns_chipPAGE -var date 428 # book getword nsData $date nsState -var ns_STATE 429 # book npages nsData -var Npages 430 431 # if ($VERBOSE > 5) 432 # echo "ns.chips.load: " $ns_chipPAGE $date $ns_STATE $Npages 433 # end 434 435 # $ns_chipPAGE ++ 436 # if ($ns_chipPAGE >= $Npages) set ns_chipPAGE = 0 437 # option $date 438 439 # if (("$ns_STATE" != "REGISTERED")&&("$ns_STATE" != "BURNING")) break 440 # $run = automate_stacks.pl --check_chips --date $date 441 442 # if ("$ns_STATE" == "BURNING") 443 # $run = $run --isburning 444 # end 445 # command $run 446 # end 447 # # success 448 # task.exit 0 449 # # convert 'stdout' to book format 450 # book delpage nsData $options:0 451 # ipptool2book stdout nsData -uniq -key date 452 453 # # remove the burntool page if we're done with it. 454 # book getword nsData $options:0 nsState -var ns_STATE 455 # if ("$ns_STATE" == "QUEUECHIPS") 456 # book delpage nsBurntool $options:0 457 # end 458 459 # book getword nsData $options:0 nsNmacros -var ns_Nmacros 460 # if ("$ns_Nmacros" != "NULL") 461 # for i 0 $ns_Nmacros 462 # sprintf macroName "ns%dMacro" $i 463 # book getword nsData $options:0 $macroName -var macroCmd 464 # $macroCmd 465 # end 466 # end 467 468 # if ($VERBOSE > 2) 469 # book listbook nsData 470 # end 471 # end 472 473 # # locked list 474 # task.exit default 475 # showcommand failure 476 # end 477 # task.exit crash 478 # showcommand crash 479 # end 480 # #operation times out? 481 # task.exit timeout 482 # showcommand timeout 483 # end 484 # end 485 486 # # 487 # # Check to see if the chips have been properly burntooled and are ready for queueing 488 # # 489 # task ns.chips.run 490 # host local 491 # periods -poll $LOADPOLL 492 # periods -exec $LOADEXEC 493 # periods -timeout 120 494 # npending 1 495 496 # task.exec 497 # stdout NULL 498 # stderr $LOGDIR/ns.chips.log 499 500 # book getpage nsData $ns_RchipPAGE -var date 501 # book getword nsData $date nsState -var ns_STATE 502 # book npages nsData -var Npages 503 504 # if ($VERBOSE > 5) 505 # echo "ns.chips.run: " $ns_RchipPAGE $date $ns_STATE $Npages 506 # end 485 507 486 $ns_RchipPAGE ++487 if ($ns_RchipPAGE >= $Npages) set ns_RchipPAGE = 0488 option $date489 490 if ("$ns_STATE" != "QUEUECHIPS") break491 $BURNTOOLING = 0492 $run = automate_stacks.pl --queue_chips --date $date493 command $run494 end495 # success496 task.exit 0497 # convert 'stdout' to book format498 book delpage nsData $options:0499 ipptool2book stdout nsData -uniq -key date500 501 book getword nsData $options:0 nsNmacros -var ns_Nmacros502 if ("$ns_Nmacros" != "NULL")503 for i 0 $ns_Nmacros504 sprintf macroName "ns%dMacro" $i505 book getword nsData $options:0 $macroName -var macroCmd506 $macroCmd507 end508 end509 510 if ($VERBOSE > 2)511 book listbook nsData512 end513 end514 515 # locked list516 task.exit default517 showcommand failure518 end519 task.exit crash520 showcommand crash521 end522 #operation times out?523 task.exit timeout524 showcommand timeout525 end526 end508 # $ns_RchipPAGE ++ 509 # if ($ns_RchipPAGE >= $Npages) set ns_RchipPAGE = 0 510 # option $date 511 512 # if ("$ns_STATE" != "QUEUECHIPS") break 513 # $BURNTOOLING = 0 514 # $run = automate_stacks.pl --queue_chips --date $date 515 # command $run 516 # end 517 # # success 518 # task.exit 0 519 # # convert 'stdout' to book format 520 # book delpage nsData $options:0 521 # ipptool2book stdout nsData -uniq -key date 522 523 # book getword nsData $options:0 nsNmacros -var ns_Nmacros 524 # if ("$ns_Nmacros" != "NULL") 525 # for i 0 $ns_Nmacros 526 # sprintf macroName "ns%dMacro" $i 527 # book getword nsData $options:0 $macroName -var macroCmd 528 # $macroCmd 529 # end 530 # end 531 532 # if ($VERBOSE > 2) 533 # book listbook nsData 534 # end 535 # end 536 537 # # locked list 538 # task.exit default 539 # showcommand failure 540 # end 541 # task.exit crash 542 # showcommand crash 543 # end 544 # #operation times out? 545 # task.exit timeout 546 # showcommand timeout 547 # end 548 # end 527 549 528 550 … … 557 579 command $run 558 580 end 581 559 582 # success 560 583 task.exit 0 … … 718 741 end 719 742 720 # 721 # Generate a list of date ranges and chips that need to be processed with burntool722 # 723 task ns.burntool.load724 host local725 periods -poll $LOADPOLL726 periods -exec $LOADEXEC727 periods -timeout 30728 npending 1729 730 task.exec731 stdout NULL732 stderr $LOGDIR/ns.burntool.log733 734 book getpage nsData $ns_burnPAGE -var date735 book getword nsData $date nsState -var ns_STATE736 book npages nsData -var Npages737 738 if ($VERBOSE > 5)739 echo "ns.burntool.load: " $ns_burnPAGE $date $ns_STATE $Npages740 end741 742 $ns_burnPAGE ++743 if ($ns_burnPAGE >= $Npages) set ns_burnPAGE = 0744 option $date743 # # 744 # # Generate a list of date ranges and chips that need to be processed with burntool 745 # # 746 # task ns.burntool.load 747 # host local 748 # periods -poll $LOADPOLL 749 # periods -exec $LOADEXEC 750 # periods -timeout 30 751 # npending 1 752 753 # task.exec 754 # stdout NULL 755 # stderr $LOGDIR/ns.burntool.log 756 757 # book getpage nsData $ns_burnPAGE -var date 758 # book getword nsData $date nsState -var ns_STATE 759 # book npages nsData -var Npages 760 761 # if ($VERBOSE > 5) 762 # echo "ns.burntool.load: " $ns_burnPAGE $date $ns_STATE $Npages 763 # end 764 765 # $ns_burnPAGE ++ 766 # if ($ns_burnPAGE >= $Npages) set ns_burnPAGE = 0 767 # option $date 745 768 746 if ("$ns_STATE" != "NEEDSBURNING") break747 748 $run = automate_stacks.pl --define_burntool --date $date749 command $run750 end751 # success752 task.exit 0753 # convert 'stdout' to book format754 # book delpage nsBurntool $options:0755 ipptool2book stdout nsBurntool -uniq -key date756 book setword nsData $options:0 nsState "QUEUEBURNING"757 758 book getword nsData $options:0 nsNmacros -var ns_Nmacros759 if ("$ns_Nmacros" != "NULL")760 for i 0 $ns_Nmacros761 sprintf macroName "ns%dMacro" $i762 book getword nsData $options:0 $macroName -var macroCmd763 $macroCmd764 end765 end766 767 if ($VERBOSE > 2)768 book listbook nsData769 end770 end771 772 # locked list773 task.exit default774 showcommand failure775 end776 task.exit crash777 showcommand crash778 end779 #operation times out?780 task.exit timeout781 showcommand timeout782 end783 end784 785 # 786 # Magically run burntool on the data, based on the information stored in our book.787 # 788 task ns.burntool.run789 periods -poll $LOADPOLL790 periods -exec $LOADEXEC791 periods -timeout 10800792 npending 30793 task.exec794 stdout NULL795 stderr $LOGDIR/ns.burntool.log796 797 book getpage nsData $ns_RburnPAGE -var date798 book getword nsData $date nsState -var ns_STATE799 book npages nsData -var Npages800 801 if ($VERBOSE > 5)802 echo "ns.burntool.run: " $ns_RburnPAGE $date $ns_STATE $Npages803 end804 805 $ns_RburnPAGE ++806 if ($ns_RburnPAGE >= $Npages) set ns_RburnPAGE = 0807 if (("$ns_STATE" != "QUEUEBURNING")&&("$ns_STATE" != "BURNING")) break808 $BURNTOOLING = 1809 # Find out where in the list of jobs we are810 book getword nsBurntool $date btN -var btN811 book getword nsBurntool $date btNCounter -var btNcounter812 813 if ($VERBOSE > 5)814 echo "ns.burntool.run: Status: " $btNcounter $date815 end816 817 if ("$ns_STATE" == "QUEUEBURNING")818 $new_state = "QUEUEBURNING"819 end820 if ("$ns_STATE" == "BURNING")821 $new_state = "BURNING"822 end823 if ($btNcounter + 1 > $btN)824 $new_state = "BURNING"825 end826 if ($btNcounter > $btN)827 $btNcounter = 0828 end829 if ($VERBOSE > 5)830 echo "ns.burntool.run: Status: " $btNcounter $new_state831 end769 # if ("$ns_STATE" != "NEEDSBURNING") break 770 771 # $run = automate_stacks.pl --define_burntool --date $date 772 # command $run 773 # end 774 # # success 775 # task.exit 0 776 # # convert 'stdout' to book format 777 # # book delpage nsBurntool $options:0 778 # ipptool2book stdout nsBurntool -uniq -key date 779 # book setword nsData $options:0 nsState "QUEUEBURNING" 780 781 # book getword nsData $options:0 nsNmacros -var ns_Nmacros 782 # if ("$ns_Nmacros" != "NULL") 783 # for i 0 $ns_Nmacros 784 # sprintf macroName "ns%dMacro" $i 785 # book getword nsData $options:0 $macroName -var macroCmd 786 # $macroCmd 787 # end 788 # end 789 790 # if ($VERBOSE > 2) 791 # book listbook nsData 792 # end 793 # end 794 795 # # locked list 796 # task.exit default 797 # showcommand failure 798 # end 799 # task.exit crash 800 # showcommand crash 801 # end 802 # #operation times out? 803 # task.exit timeout 804 # showcommand timeout 805 # end 806 # end 807 808 # # 809 # # Magically run burntool on the data, based on the information stored in our book. 810 # # 811 # task ns.burntool.run 812 # periods -poll $LOADPOLL 813 # periods -exec $LOADEXEC 814 # periods -timeout 10800 815 # npending 30 816 # task.exec 817 # stdout NULL 818 # stderr $LOGDIR/ns.burntool.log 819 820 # book getpage nsData $ns_RburnPAGE -var date 821 # book getword nsData $date nsState -var ns_STATE 822 # book npages nsData -var Npages 823 824 # if ($VERBOSE > 5) 825 # echo "ns.burntool.run: " $ns_RburnPAGE $date $ns_STATE $Npages 826 # end 827 828 # $ns_RburnPAGE ++ 829 # if ($ns_RburnPAGE >= $Npages) set ns_RburnPAGE = 0 830 # if (("$ns_STATE" != "QUEUEBURNING")&&("$ns_STATE" != "BURNING")) break 831 # $BURNTOOLING = 1 832 # # Find out where in the list of jobs we are 833 # book getword nsBurntool $date btN -var btN 834 # book getword nsBurntool $date btNCounter -var btNcounter 835 836 # if ($VERBOSE > 5) 837 # echo "ns.burntool.run: Status: " $btNcounter $date 838 # end 839 840 # if ("$ns_STATE" == "QUEUEBURNING") 841 # $new_state = "QUEUEBURNING" 842 # end 843 # if ("$ns_STATE" == "BURNING") 844 # $new_state = "BURNING" 845 # end 846 # if ($btNcounter + 1 > $btN) 847 # $new_state = "BURNING" 848 # end 849 # if ($btNcounter > $btN) 850 # $btNcounter = 0 851 # end 852 # if ($VERBOSE > 5) 853 # echo "ns.burntool.run: Status: " $btNcounter $new_state 854 # end 832 855 833 # Increment the counter in the book for the next job.834 $counter_update = $btNcounter + 1835 book setword nsBurntool $date btNCounter $counter_update836 # book setword nsData $options:0 nsState $options:1837 book setword nsData $date nsState $new_state838 839 # Get the current status of this job, and skip if it doesn't need to process.840 sprintf status_label "bt%dStatus" $btNcounter841 book getword nsBurntool $date $status_label -var status842 843 if ($VERBOSE > 5)844 echo "ns.burntool.run: Status: " $btNcounter $status $date $status_label845 end846 if (("$status" == "FINISHED")||("$status" == "RUN")) break856 # # Increment the counter in the book for the next job. 857 # $counter_update = $btNcounter + 1 858 # book setword nsBurntool $date btNCounter $counter_update 859 # # book setword nsData $options:0 nsState $options:1 860 # book setword nsData $date nsState $new_state 861 862 # # Get the current status of this job, and skip if it doesn't need to process. 863 # sprintf status_label "bt%dStatus" $btNcounter 864 # book getword nsBurntool $date $status_label -var status 865 866 # if ($VERBOSE > 5) 867 # echo "ns.burntool.run: Status: " $btNcounter $status $date $status_label 868 # end 869 # if (("$status" == "FINISHED")||("$status" == "RUN")) break 847 870 848 # Continue loading information to process this job849 sprintf start_date_label "bt%dBegin" $btNcounter850 sprintf end_date_label "bt%dEnd" $btNcounter851 sprintf class_count_label "bt%dClass" $btNcounter852 853 book getword nsBurntool $date $start_date_label -var start_date854 book getword nsBurntool $date $end_date_label -var end_date855 book getword nsBurntool $date $class_count_label -var chip_counter856 857 # Lookup class_id/host pairs858 list word -split $hostmatch:$chip_counter859 $class_id = $word:0860 $host = $word:1861 host $host862 # set.host.for.camera GPC1 $class_id863 $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log"864 865 $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile871 # # Continue loading information to process this job 872 # sprintf start_date_label "bt%dBegin" $btNcounter 873 # sprintf end_date_label "bt%dEnd" $btNcounter 874 # sprintf class_count_label "bt%dClass" $btNcounter 875 876 # book getword nsBurntool $date $start_date_label -var start_date 877 # book getword nsBurntool $date $end_date_label -var end_date 878 # book getword nsBurntool $date $class_count_label -var chip_counter 879 880 # # Lookup class_id/host pairs 881 # list word -split $hostmatch:$chip_counter 882 # $class_id = $word:0 883 # $host = $word:1 884 # host $host 885 # # set.host.for.camera GPC1 $class_id 886 # $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log" 887 888 # $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile 866 889 867 echo "ns.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run868 echo "ns.burntool.run: " $date $btN $btNcounter $chip_counter $new_state869 870 book setword nsBurntool $date $status_label RUN871 option $date $new_state $status_label872 873 command $run874 # command /bin/true875 end876 # success877 task.exit 0878 # convert 'stdout' to book format879 # Set data state based on if we're queueing or waiting880 # Set the job state for success.881 book setword nsBurntool $options:0 $options:2 FINISHED882 883 book getword nsData $options:0 nsNmacros -var ns_Nmacros884 if ("$ns_Nmacros" != "NULL")885 for i 0 $ns_Nmacros886 sprintf macroName "ns%dMacro" $i887 book getword nsData $options:0 $macroName -var macroCmd888 $macroCmd889 end890 end891 892 if ($VERBOSE > 2)893 book listbook nsData894 end895 end896 897 # locked list898 task.exit default899 book setword nsBurntool $options:0 $options:2 FAIL900 showcommand failure901 end902 task.exit crash903 book setword nsBurntool $options:0 $options:2 FAIL904 showcommand crash905 end906 #operation times out?907 task.exit timeout908 book setword nsBurntool $options:0 $options:2 FAIL909 showcommand timeout910 end911 end912 913 914 # 915 # This is all burntool related stuff.916 # 917 918 macro burntool919 if ($0 != 3)920 echo "USAGE: burntool (dateobs_begin) (dateobs_end)"921 break922 end923 924 for i 0 $hostmatch:n925 job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile926 end927 end928 macro loadhosts929 for i 0 $allhosts:n930 controller host add $allhosts:$i931 end932 end933 934 # sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time.935 936 list hostmatch937 XY01 ipp014938 XY03 ipp038939 XY05 ipp023940 XY10 ipp039941 XY12 ipp024942 XY15 ipp040943 XY17 ipp020944 XY21 ipp041945 XY23 ipp042946 XY25 ipp043947 XY27 ipp028948 XY31 ipp044949 XY33 ipp029950 XY35 ipp045951 XY37 ipp030952 XY41 ipp046953 XY43 ipp031954 XY45 ipp047955 XY47 ipp032956 XY51 ipp048957 XY53 ipp033958 XY55 ipp049959 XY57 ipp034960 XY61 ipp050961 XY63 ipp035962 XY65 ipp051963 XY67 ipp036964 XY72 ipp052965 XY74 ipp015966 XY76 ipp053967 XY02 ipp014968 XY04 ipp038969 XY06 ipp023970 XY11 ipp039971 XY13 ipp024972 XY14 ipp040973 XY16 ipp020974 XY20 ipp041975 XY22 ipp042976 XY24 ipp043977 XY26 ipp028978 XY30 ipp044979 XY32 ipp029980 XY34 ipp045981 XY36 ipp030982 XY40 ipp046983 XY42 ipp031984 XY44 ipp047985 XY46 ipp032986 XY50 ipp048987 XY52 ipp033988 XY54 ipp049989 XY56 ipp034990 XY60 ipp050991 XY62 ipp035992 XY64 ipp051993 XY66 ipp036994 XY71 ipp052995 XY73 ipp015996 XY75 ipp053997 end998 999 list allhosts1000 ipp0431001 ipp0141002 ipp0151003 ipp0231004 ipp0241005 ipp0201006 ipp0281007 ipp0291008 ipp0301009 ipp0311010 ipp0321011 ipp0331012 ipp0341013 ipp0351014 ipp0361015 ipp0381016 ipp0391017 ipp0401018 ipp0411019 ipp0421020 ipp0441021 ipp0451022 ipp0461023 ipp0471024 ipp0481025 ipp0491026 ipp0501027 ipp0511028 ipp0521029 ipp0531030 end890 # echo "ns.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run 891 # echo "ns.burntool.run: " $date $btN $btNcounter $chip_counter $new_state 892 893 # book setword nsBurntool $date $status_label RUN 894 # option $date $new_state $status_label 895 896 # command $run 897 # # command /bin/true 898 # end 899 # # success 900 # task.exit 0 901 # # convert 'stdout' to book format 902 # # Set data state based on if we're queueing or waiting 903 # # Set the job state for success. 904 # book setword nsBurntool $options:0 $options:2 FINISHED 905 906 # book getword nsData $options:0 nsNmacros -var ns_Nmacros 907 # if ("$ns_Nmacros" != "NULL") 908 # for i 0 $ns_Nmacros 909 # sprintf macroName "ns%dMacro" $i 910 # book getword nsData $options:0 $macroName -var macroCmd 911 # $macroCmd 912 # end 913 # end 914 915 # if ($VERBOSE > 2) 916 # book listbook nsData 917 # end 918 # end 919 920 # # locked list 921 # task.exit default 922 # book setword nsBurntool $options:0 $options:2 FAIL 923 # showcommand failure 924 # end 925 # task.exit crash 926 # book setword nsBurntool $options:0 $options:2 FAIL 927 # showcommand crash 928 # end 929 # #operation times out? 930 # task.exit timeout 931 # book setword nsBurntool $options:0 $options:2 FAIL 932 # showcommand timeout 933 # end 934 # end 935 936 937 # # 938 # # This is all burntool related stuff. 939 # # 940 941 # macro burntool 942 # if ($0 != 3) 943 # echo "USAGE: burntool (dateobs_begin) (dateobs_end)" 944 # break 945 # end 946 947 # for i 0 $hostmatch:n 948 # job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile 949 # end 950 # end 951 # macro loadhosts 952 # for i 0 $allhosts:n 953 # controller host add $allhosts:$i 954 # end 955 # end 956 957 # # sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time. 958 959 # list hostmatch 960 # XY01 ipp014 961 # XY03 ipp038 962 # XY05 ipp023 963 # XY10 ipp039 964 # XY12 ipp024 965 # XY15 ipp040 966 # XY17 ipp020 967 # XY21 ipp041 968 # XY23 ipp042 969 # XY25 ipp043 970 # XY27 ipp028 971 # XY31 ipp044 972 # XY33 ipp029 973 # XY35 ipp045 974 # XY37 ipp030 975 # XY41 ipp046 976 # XY43 ipp031 977 # XY45 ipp047 978 # XY47 ipp032 979 # XY51 ipp048 980 # XY53 ipp033 981 # XY55 ipp049 982 # XY57 ipp034 983 # XY61 ipp050 984 # XY63 ipp035 985 # XY65 ipp051 986 # XY67 ipp036 987 # XY72 ipp052 988 # XY74 ipp015 989 # XY76 ipp053 990 # XY02 ipp014 991 # XY04 ipp038 992 # XY06 ipp023 993 # XY11 ipp039 994 # XY13 ipp024 995 # XY14 ipp040 996 # XY16 ipp020 997 # XY20 ipp041 998 # XY22 ipp042 999 # XY24 ipp043 1000 # XY26 ipp028 1001 # XY30 ipp044 1002 # XY32 ipp029 1003 # XY34 ipp045 1004 # XY36 ipp030 1005 # XY40 ipp046 1006 # XY42 ipp031 1007 # XY44 ipp047 1008 # XY46 ipp032 1009 # XY50 ipp048 1010 # XY52 ipp033 1011 # XY54 ipp049 1012 # XY56 ipp034 1013 # XY60 ipp050 1014 # XY62 ipp035 1015 # XY64 ipp051 1016 # XY66 ipp036 1017 # XY71 ipp052 1018 # XY73 ipp015 1019 # XY75 ipp053 1020 # end 1021 1022 # list allhosts 1023 # ipp043 1024 # ipp014 1025 # ipp015 1026 # ipp023 1027 # ipp024 1028 # ipp020 1029 # ipp028 1030 # ipp029 1031 # ipp030 1032 # ipp031 1033 # ipp032 1034 # ipp033 1035 # ipp034 1036 # ipp035 1037 # ipp036 1038 # ipp038 1039 # ipp039 1040 # ipp040 1041 # ipp041 1042 # ipp042 1043 # ipp044 1044 # ipp045 1045 # ipp046 1046 # ipp047 1047 # ipp048 1048 # ipp049 1049 # ipp050 1050 # ipp051 1051 # ipp052 1052 # ipp053 1053 # end -
branches/eam_branches/ipp-20101205/ippTasks/publish.pro
r29724 r30102 130 130 book getword publishRun $pageName stage_id -var STAGE_ID 131 131 book getword publishRun $pageName dbname -var DBNAME 132 book getword publishRun $pageName output_format -var OUTPUT_FORMAT 132 133 133 134 stdout $LOGDIR/publish.run.log … … 137 138 strsub $WORKDIR_TEMPLATE @HOST@ $default_host -var WORKDIR 138 139 139 $run = publish_file.pl --pub_id $PUB_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID -- redirect-output140 $run = publish_file.pl --pub_id $PUB_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID --output_format $OUTPUT_FORMAT --redirect-output 140 141 add_standard_args run 141 142 -
branches/eam_branches/ipp-20101205/ippTasks/register.pro
r25418 r30102 8 8 book init regPendingImfile 9 9 book init regPendingExp 10 book init regPendingBurntoolImfile 10 11 11 12 macro register.reset 12 13 book init regPendingImfile 13 14 book init regPendingExp 15 book init regPendingBurntoolImfile 14 16 end 15 17 … … 18 20 book listbook regPendingImfile 19 21 book listbook regPendingExp 22 book listbook regPendingBurntoolImfile 20 23 end 21 24 … … 33 36 active true 34 37 end 38 task register.burntool.load 39 active true 40 end 41 task register.burntool.run 42 active true 43 end 35 44 end 36 45 … … 48 57 active false 49 58 end 50 end 59 task register.burntool.load 60 active false 61 end 62 task register.burntool.run 63 active false 64 end 65 end 66 67 $valid_burntool_value = 14 51 68 52 69 macro replace.survey.values … … 129 146 # these variables will cycle through the known database names 130 147 $regPendingImfile_DB = 0 148 $regPendingBurntoolImfile_DB = 0 131 149 $regPendingExp_DB = 0 132 150 … … 429 447 end 430 448 449 450 # select imfiles that can now be burntooled. 451 task register.burntool.load 452 host local 453 454 periods -poll $LOADPOLL 455 periods -exec $LOADEXEC 456 periods -timeout 30 457 npending 1 458 459 # silently drop stdout 460 stdout NULL 461 stderr $LOGDIR/register.burntool.load.log 462 463 # select entried from the current DB; cycle to the next DB, if it exists 464 task.exec 465 $run = regtool -pendingburntoolimfile 466 $today = `date -u +%Y-%m-%d` 467 # debugging purposes 468 # $today = "2010-12-06" 469 $run = $run -date $today -valid_burntool $valid_burntool_value 470 if ($DB:n == 0) 471 option DEFAULT 472 else 473 # save the DB name for the exit tasks 474 option $DB:$regPendingBurntoolImfile_DB 475 $run = $run -dbname $DB:$regPendingBurntoolImfile_DB 476 $regPendingBurntoolImfile_DB ++ 477 if ($regPendingBurntoolImfile_DB >= $DB:n) set regPendingBurntoolImfile_DB = 0 478 end 479 480 # echo $run 481 add_poll_args run 482 command $run 483 end 484 485 # success 486 task.exit $EXIT_SUCCESS 487 # convert 'stdout' to book format 488 ipptool2book stdout regPendingBurntoolImfile -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT 489 book shuffle regPendingBurntoolImfile 490 if ($VERBOSE > 2) 491 book listbook regPendingBurntoolImfile 492 end 493 494 # delete existing entries in the appropriate pantasksStates 495 process_cleanup regPendingBurntoolImfile 496 end 497 498 # locked list 499 task.exit default 500 showcommand failure 501 end 502 task.exit crash 503 showcommand crash 504 end 505 task.exit timeout 506 showcommand timeout 507 end 508 end 509 510 # run the ipp_apply_burntool_single.pl script on the pending images 511 task register.burntool.run 512 periods -poll $RUNPOLL 513 periods -exec $RUNEXEC 514 periods -timeout 30 515 516 task.exec 517 if ($NETWORK == 0) break 518 book npages regPendingBurntoolImfile -var N 519 if ($N == 0) 520 periods -exec $RUNEXEC 521 break 522 end 523 periods -exec 0.05 524 525 # look for new images to burn in regPendingBurntoolImfile 526 book getpage regPendingBurntoolImfile 0 -var pageName -key pantaskState INIT 527 if ("$pageName" == "NULL") break 528 529 book setword regPendingBurntoolImfile $pageName pantaskState RUN 530 531 book getword regPendingBurntoolImfile $pageName exp_id -var EXP_ID 532 book getword regPendingBurntoolImfile $pageName class_id -var CLASS_ID 533 book getword regPendingBurntoolImfile $pageName uri -var THIS_URI 534 book getword regPendingBurntoolImfile $pageName previous_uri -var PREVIOUS_URI 535 book getword regPendingBurntoolImfile $pageName camera -var CAMERA 536 book getword regPendingBurntoolImfile $pageName dbname -var DBNAME 537 set.host.for.camera $CAMERA $CLASS_ID 538 539 stdout $LOGDIR/register.burntool.run.log 540 stderr $LOGDIR/register.burntool.run.log 541 542 $run = ipp_apply_burntool_single.pl --exp_id $EXP_ID --class_id $CLASS_ID --this_uri $THIS_URI 543 if ("$PREVIOUS_URI" != "NULL") 544 $run = $run --previous_uri $PREVIOUS_URI 545 end 546 547 add_standard_args run 548 549 options $pageName 550 551 if ($VERBOSE > 1) 552 echo command $run 553 end 554 command $run 555 end 556 557 # default exit status 558 task.exit default 559 process_exit regPendingBurntoolImfile $options:0 $JOB_STATUS 560 end 561 562 # locked list 563 task.exit crash 564 showcommand crash 565 echo "hostname: $JOB_HOSTNAME" 566 book setword regPendingBurntoolImfile $options:0 pantaskState CRASH 567 end 568 569 # operation timed out 570 task.exit timeout 571 showcommand timeout 572 book setword regPendingBurntoolImfile $options:0 pantaskState TIMEOUT 573 end 574 end 575 576 577 578 431 579 ## XXX add a global path to output files -
branches/eam_branches/ipp-20101205/ippToPsps/perl/checkOdmStatus.pl
r29652 r30102 39 39 ); 40 40 41 if (@ARGV) { 42 print "* UNKNKOWN: option @ARGV\n"; 43 } 41 44 if (!defined $product) { 42 45 print "* OPTIONAL: a datastore product name -p <name>\n"; … … 159 162 $numBatchesToCheck++; 160 163 161 # if not merged then update by polling ODM for status164 # if not merged or failed load, then update by polling ODM for status 162 165 if (!$merged && !$loadFailed) { 163 164 166 if (checkODM($batch->getName(), \$loadedToOdm, \$loadFailed, \$mergeWorthy, \$merged)) { 165 167 -
branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps/DetectionBatch.pm
r29236 r30102 78 78 # get neb path of smf file 79 79 my $nebPath = $self->{_gpc1Db}->getCameraStageSmfForThisDvoDb($self->{_dvoDb}, $self->{_expId}); 80 if (!$nebPath) { return 0; }80 if (!$nebPath) { return undef; } 81 81 82 82 # get real filename from neb 'key' -
branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps/IppToPspsDb.pm
r29345 r30102 11 11 ########################################################################### 12 12 # 13 # Returns a list of batches that have been processed and loaded to datastore13 # Returns a list of batches created within the provided dates 14 14 # 15 15 ########################################################################### … … 39 39 $query->execute; 40 40 ${$batches} = $query->fetchall_arrayref(); 41 my $count = scalar @{${$batches}}; 42 43 return $count; 41 return scalar @{${$batches}}; 44 42 } 45 43 … … 72 70 $query->execute; 73 71 ${$batches} = $query->fetchall_arrayref(); 74 my $count = scalar @{${$batches}}; 75 76 return $count; 72 return scalar @{${$batches}}; 73 } 74 75 ########################################################################### 76 # 77 # Returns a count of all distinct exposures between provided limits 78 # 79 ########################################################################### 80 sub countExposures { 81 my ($self, $fromExp, $toExp, $datastoreProduct, $batchType, $dvoDb) = @_; 82 83 my $query = $self->{_db}->prepare(<<SQL); 84 SELECT COUNT(distinct exp_id) 85 FROM batches 86 WHERE exp_id >= $fromExp 87 AND exp_id <= $toExp 88 AND dvo_db = '$dvoDb' 89 AND batch_type = '$batchType' 90 AND datastore_product = '$datastoreProduct' 91 SQL 92 93 $query->execute; 94 return scalar $query->fetchrow_array(); 95 } 96 97 ########################################################################### 98 # 99 # Returns a count of exposures (between provided limits) that have been merged 100 # 101 ########################################################################### 102 sub countMergedExposures { 103 my ($self, $fromExp, $toExp, $datastoreProduct, $batchType, $dvoDb) = @_; 104 105 my $query = $self->{_db}->prepare(<<SQL); 106 SELECT COUNT(distinct exp_id) 107 FROM batches 108 WHERE exp_id >= $fromExp 109 AND exp_id <= $toExp 110 AND dvo_db = '$dvoDb' 111 AND batch_type = '$batchType' 112 AND datastore_product = '$datastoreProduct' 113 AND merged 114 SQL 115 116 $query->execute; 117 return scalar $query->fetchrow_array(); 118 } 119 120 ########################################################################### 121 # 122 # Returns a list of exposures (between provided limits) that have not been merged 123 # 124 ########################################################################### 125 sub getUnmergedExposures { 126 my ($self, $exposures, $fromExp, $toExp, $datastoreProduct, $batchType, $dvoDb) = @_; 127 128 my $query = $self->{_db}->prepare(<<SQL); 129 SELECT batch_id, exp_id 130 FROM batches 131 WHERE exp_id >= $fromExp 132 AND exp_id <= $toExp 133 AND dvo_db = '$dvoDb' 134 AND batch_type = '$batchType' 135 AND datastore_product = '$datastoreProduct' 136 AND !merged 137 GROUP BY exp_id 138 SQL 139 140 $query->execute; 141 ${$exposures} = $query->fetchall_arrayref(); 142 return scalar @{${$exposures}}; 77 143 } 78 144 … … 166 232 ####################################################################################### 167 233 # 234 # Checks whether this exposure failed to load to the ODM 235 # 236 ######################################################################################## 237 sub didLoadFail { 238 my ($self, $expId) = @_; 239 240 my $query = $self->{_db}->prepare(<<SQL); 241 SELECT COUNT(*) 242 FROM batches 243 WHERE exp_id = $expId 244 AND load_failed 245 SQL 246 247 $query->execute; 248 249 return scalar $query->fetchrow_array(); 250 } 251 252 ####################################################################################### 253 # 254 # Checks whether this exposure has a min object ID under old PSPS hard limit of 72010000000000001 255 # 256 ######################################################################################## 257 sub isMinObjIdUnderLimit { 258 my ($self, $expId) = @_; 259 260 my $query = $self->{_db}->prepare(<<SQL); 261 SELECT COUNT(*) 262 FROM batches 263 WHERE exp_id = $expId 264 AND min_obj_id < 72010000000000001 265 SQL 266 267 $query->execute; 268 269 return scalar $query->fetchrow_array(); 270 } 271 ####################################################################################### 272 # 273 # Checks whether this exposure has been processed 274 # 275 ######################################################################################## 276 sub isExposureProcessed { 277 my ($self, $expId) = @_; 278 279 my $query = $self->{_db}->prepare(<<SQL); 280 SELECT COUNT(*) 281 FROM batches 282 WHERE exp_id = $expId 283 AND processed 284 SQL 285 286 $query->execute; 287 288 return scalar $query->fetchrow_array(); 289 } 290 291 ####################################################################################### 292 # 293 # Checks whether this exposure has been merged 294 # 295 ######################################################################################## 296 sub isExposureMerged { 297 my ($self, $expId) = @_; 298 299 my $query = $self->{_db}->prepare(<<SQL); 300 SELECT COUNT(*) 301 FROM batches 302 WHERE exp_id = $expId 303 AND merged 304 SQL 305 306 $query->execute; 307 308 return scalar $query->fetchrow_array(); 309 } 310 311 ####################################################################################### 312 # 313 # Checks whether this exposure has been loaded to ODM 314 # 315 ######################################################################################## 316 sub isExposureLoadedToOdm { 317 my ($self, $expId) = @_; 318 319 my $query = $self->{_db}->prepare(<<SQL); 320 SELECT COUNT(*) 321 FROM batches 322 WHERE exp_id = $expId 323 AND loaded_to_ODM 324 SQL 325 326 $query->execute; 327 328 return scalar $query->fetchrow_array(); 329 } 330 331 ####################################################################################### 332 # 168 333 # Checks whether we have successfully processed this exposure and loaded it to the datastore 169 334 # … … 184 349 my $processed = $query->fetchrow_array(); 185 350 351 # TODO can use return scalar $query->fetchrow_array(); 186 352 return $processed; 187 353 } -
branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps_run.pl
r29089 r30102 15 15 use ippToPsps::IppToPspsDb; 16 16 use ippToPsps::Datastore; 17 #use ippToPsps::BatchManager; 17 18 18 19 # globals … … 112 113 my $datastore = undef; 113 114 if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);} 115 #my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $output, $verbose, $save_temps); 114 116 115 117 # check we can run programs and get camera config -
branches/eam_branches/ipp-20101205/ippTools/configure.ac
r29499 r30102 1 1 AC_PREREQ(2.61) 2 2 3 AC_INIT([ipptools], [1.1.6 5], [ipp-support@ifa.hawaii.edu])3 AC_INIT([ipptools], [1.1.66], [ipp-support@ifa.hawaii.edu]) 4 4 AC_CONFIG_SRCDIR([autogen.sh]) 5 5 … … 18 18 PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) 19 19 PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) 20 PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.6 5])20 PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.66]) 21 21 PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) 22 22 -
branches/eam_branches/ipp-20101205/ippTools/share/Makefile.am
r29717 r30102 316 316 receivetool_revert.sql \ 317 317 receivetool_toadvance.sql \ 318 regtool_checkburntoolimfile.sql \ 318 319 regtool_create_dup_table.sql \ 319 320 regtool_export_exp.sql \ … … 331 332 regtool_updatebyquery.sql \ 332 333 regtool_updatebyqueryimfile.sql \ 334 regtool_pendingburntoolimfile.sql \ 333 335 stacktool_associationdefine_select.sql \ 334 336 stacktool_definebyquery_insert.sql \ -
branches/eam_branches/ipp-20101205/ippTools/share/camtool_find_pendingimfile.sql
- Property svn:mergeinfo changed (with no actual effect on merging)
-
branches/eam_branches/ipp-20101205/ippTools/share/chiptool_find_rawexp.sql
r23917 r30102 1 1 -- this query is used to find potental rawExps to be queued for chiptool 2 2 -- processeing 3 SELECT 3 SELECT DISTINCT 4 4 rawExp.*, 5 5 newExp.label 6 6 FROM rawExp 7 7 JOIN newExp using (exp_id) 8 LEFT JOIN chipRun USING (exp_id) 8 9 WHERE 9 10 rawExp.fault = 0 -
branches/eam_branches/ipp-20101205/ippTools/share/pubtool_definerun.sql
r28428 r30102 3 3 client_id, 4 4 stage_id, 5 src_label 5 src_label, 6 output_format 6 7 FROM ( 7 8 -- Get diffs to publish … … 9 10 client_id, 10 11 diff_id AS stage_id, 11 diffRun.label AS src_label 12 diffRun.label AS src_label, 13 output_format 12 14 FROM publishClient 13 15 JOIN diffRun … … 28 30 client_id, 29 31 cam_id AS stage_id, 30 camRun.label AS src_label 32 camRun.label AS src_label, 33 output_format 31 34 FROM publishClient 32 35 JOIN camRun … … 43 46 client_id, 44 47 diff_phot_id AS stage_id, 45 diffPhotRun.label AS src_label 48 diffPhotRun.label AS src_label, 49 output_format 46 50 FROM publishClient 47 51 JOIN diffPhotRun -
branches/eam_branches/ipp-20101205/ippTools/share/pubtool_pending.sql
r28491 r30102 9 9 publishClient.stage, 10 10 publishClient.workdir, 11 publishClient.output_format, 11 12 diffRun.diff_id AS stage_id, 12 13 rawExp.camera, … … 35 36 publishClient.stage, 36 37 publishClient.workdir, 38 publishClient.output_format, 37 39 camRun.cam_id AS stage_id, 38 40 rawExp.camera, … … 56 58 publishClient.stage, 57 59 publishClient.workdir, 60 publishClient.output_format, 58 61 diffPhotRun.diff_phot_id AS stage_id, 59 62 rawExp.camera, -
branches/eam_branches/ipp-20101205/ippTools/share/pxadmin_create_tables.sql
- Property svn:mergeinfo changed
/branches/czw_branch/20101203/ippTools/share/pxadmin_create_tables.sql (added) merged: 29965,29982
r29923 r30102 9 9 uri VARCHAR(255), 10 10 epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 11 use_compress SMALLINT, 11 12 PRIMARY KEY(camera, telescope) 12 13 ) ENGINE=innodb DEFAULT CHARSET=latin1; … … 251 252 md5sum VARCHAR(32), 252 253 burntool_state SMALLINT, 254 video_cells TINYINT DEFAULT 0, 253 255 PRIMARY KEY(exp_id, class_id), 254 256 KEY(tmp_class_id), … … 1643 1645 workdir VARCHAR(255) NOT NULL, -- working directory 1644 1646 comment VARCHAR(255), -- for human memory 1645 PRIMARY KEY(client_id) 1647 name varchar(64) default NULL, -- unique client_id verbose identifier 1648 output_format SMALLINT NOT NULL default 1, -- format output versioning 1649 PRIMARY KEY(client_id), 1650 UNIQUE KEY name (name) 1646 1651 ) ENGINE=innodb DEFAULT CHARSET=latin1; 1647 1652 - Property svn:mergeinfo changed
-
branches/eam_branches/ipp-20101205/ippTools/share/regtool_pendingexp.sql
r20759 r30102 1 1 SELECT DISTINCT 2 exp_id, 3 tmp_exp_name, 4 tmp_camera, 5 tmp_telescope, 6 state, 7 workdir, 8 workdir_state, 9 reduction, 10 dvodb, 11 tess_id, 12 end_stage, 13 label, 14 camera, 15 filter, 16 obs_mode, 17 obs_group, 18 epoch, 19 dateobs 20 FROM 21 (SELECT 22 newExp.*, 23 newImfile.tmp_class_id, 24 rawImfile.tmp_class_id as raw_tmp_class_id, 25 rawImfile.camera, 26 rawImfile.filter, 27 rawImfile.dateobs, 28 rawImfile.obs_mode, 29 rawImfile.obs_group 30 FROM newExp 31 JOIN newImfile 32 USING(exp_id) 33 LEFT JOIN rawExp 34 USING(exp_id) 35 LEFT JOIN rawImfile 36 ON newImfile.exp_id = rawImfile.exp_id 37 AND newImfile.tmp_class_id = rawImfile.tmp_class_id 38 WHERE 39 newExp.state = 'run' 40 AND rawExp.exp_id IS NULL 41 -- where hook %s 42 GROUP BY 43 newExp.exp_id 44 HAVING 45 COUNT(newImfile.tmp_class_id) = COUNT(rawImfile.tmp_class_id) 46 AND SUM(rawImfile.fault) = 0 2 exp_id, 3 tmp_exp_name, 4 tmp_camera, 5 tmp_telescope, 6 state, 7 workdir, 8 workdir_state, 9 reduction, 10 dvodb, 11 tess_id, 12 end_stage, 13 label, 14 camera, 15 filter, 16 obs_mode, 17 obs_group, 18 epoch, 19 dateobs 20 FROM 21 ( 22 select DISTINCT * from 23 ( 24 select newExp.*,count(newImfile.tmp_class_id) AS new_count 25 from newExp 26 LEFT JOIN newImfile USING (exp_id) 27 LEFT JOIN rawExp USING (exp_id) 28 where 29 newExp.state = 'run' AND rawExp.exp_id IS NULL 30 GROUP BY newExp.exp_id 31 ) AS NEWEXPOSURES 32 JOIN 33 ( 34 select DISTINCT newExp.exp_id, 35 rawImfile.camera, 36 rawImfile.filter, 37 rawImfile.dateobs, 38 rawImfile.obs_mode, 39 rawImfile.obs_group, 40 count(rawImfile.tmp_class_id) AS raw_count 41 from newExp 42 LEFT JOIN rawExp USING (exp_id) 43 LEFT JOIN rawImfile USING (exp_id) 44 where 45 newExp.state = 'run' AND rawExp.exp_id IS NULL 46 AND rawImfile.data_state = 'full' 47 -- where hook %s 48 GROUP BY newExp.exp_id 49 HAVING SUM(rawImfile.fault) = 0 50 ) AS RAWEXPOSURES 51 USING (exp_id) 52 WHERE raw_count = new_count 53 ) AS Foo 47 54 -- limit hook %s 48 ) as Foo -
branches/eam_branches/ipp-20101205/ippTools/src
- Property svn:mergeinfo changed
/branches/czw_branch/20101203/ippTools/src (added) merged: 29908,29965,29982,30014
- Property svn:mergeinfo changed
-
branches/eam_branches/ipp-20101205/ippTools/src/chiptool.c
r29902 r30102 271 271 272 272 // default 273 PXOPT_LOOKUP_BOOL(unique, config->args, "-unique", false); 273 274 PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); 274 275 PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); 275 276 276 277 // find the exp_id of all the exposures that we want to queue up. 277 278 psString query = pxDataGet("chiptool_find_rawexp.sql"); … … 288 289 psFree(where); 289 290 291 if (unique) { 292 psStringAppend(&query, "AND chip_id IS NULL"); 293 } 294 290 295 if (!p_psDBRunQuery(config->dbh, query)) { 291 296 psError(PS_ERR_UNKNOWN, false, "database error"); -
branches/eam_branches/ipp-20101205/ippTools/src/chiptoolConfig.c
r29902 r30102 61 61 psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define distribution group", NULL); 62 62 psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); 63 psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-unique", 0, "only queue exposures that have no previous chipRun", false); 63 64 psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); 64 65 psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); -
branches/eam_branches/ipp-20101205/ippTools/src/difftool.c
r29958 r30102 1978 1978 PXOPT_COPY_STR(config->args, stack2Where, "-skycell_id", "stackRun.skycell_id", "=="); 1979 1979 PXOPT_COPY_STR(config->args, stack1Where, "-input_label", "stackRun.label","=="); 1980 PXOPT_COPY_STR(config->args, stack1Where, "-input_data_group", "stackRun.data_group","=="); 1980 1981 PXOPT_COPY_STR(config->args, stack2Where, "-template_label", "stackRun.label","=="); 1981 1982 PXOPT_COPY_F32(config->args, stack1Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); -
branches/eam_branches/ipp-20101205/ippTools/src/difftoolConfig.c
r29553 r30102 336 336 psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_label", 0, "search by stack label for input", NULL); 337 337 psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-template_label", 0, "search by stack label for template", NULL); 338 psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_data_group", 0, "search by stack data_group for input", NULL); 338 339 psMetadataAddF32(definestackstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); 339 340 psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); -
branches/eam_branches/ipp-20101205/ippTools/src/magicdstool.c
r29611 r30102 1155 1155 1156 1156 psMetadata *where = psMetadataAlloc(); 1157 // new state 1158 PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false);1157 // new state (required 1158 PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false); 1159 1159 // old state (required) 1160 1160 PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); -
branches/eam_branches/ipp-20101205/ippTools/src/magicdstoolConfig.c
r29611 r30102 163 163 // -clearstatefaults 164 164 psMetadata *clearstatefaultsArgs = psMetadataAlloc(); 165 psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state ", NULL);165 psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); 166 166 psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); 167 167 psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0); -
branches/eam_branches/ipp-20101205/ippTools/src/pstamptool.c
r29569 r30102 950 950 PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); 951 951 PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); 952 953 PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); 954 PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); 955 952 956 PXOPT_COPY_S32(config->args, where, "-fault", "pstampDependent.fault", "=="); 957 953 958 pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); 954 // if (fault_count) { 955 PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">="); 956 // } 959 PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">="); 957 960 958 961 // XXX: How about selecting by pstampRequest.label? No. That is too dangerous by itself. -
branches/eam_branches/ipp-20101205/ippTools/src/pstamptoolConfig.c
r29328 r30102 178 178 psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id of jobs to update", 0); 179 179 psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL, "-dep_id", 0, "dep_id of jobs to update", 0); 180 psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL, "-stage_id", 0, "stage_id of jobs to update", 0); 181 psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-component", 0, "component of jobs to update", NULL); 180 182 psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL, "-fault", 0, "current value for dependent fault", 0); 181 183 psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL, "-fault_count", 0, "select by fault_count (>=)", 0); -
branches/eam_branches/ipp-20101205/ippTools/src/pubtool.c
r29321 r30102 96 96 PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); 97 97 PXOPT_LOOKUP_BOOL(unmagicked, config->args, "-unmagicked", false); 98 99 if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment)) { 98 PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); 99 PXOPT_LOOKUP_S16(output_format, config->args, "-output_format", false, false); 100 101 if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment, name, output_format)) { 100 102 psError(PS_ERR_UNKNOWN, false, "Database error"); 101 103 return false; … … 313 315 PXOPT_COPY_STR(config->args, where, "-stage", "publishClient.stage", "=="); 314 316 PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE"); 317 PXOPT_COPY_STR(config->args, where, "-name", "publishClient.name", "LIKE"); 315 318 pxAddLabelSearchArgs(config, where, "-label", "publishRun.label", "=="); 316 319 -
branches/eam_branches/ipp-20101205/ippTools/src/pubtoolConfig.c
r28371 r30102 83 83 psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use simple output format?", false); 84 84 psMetadataAddU64(pendingArgs, PS_LIST_TAIL, "-limit", 0, "limit result set", 0); 85 psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-name", 0, "search on client name", NULL); 85 86 86 87 // -add -
branches/eam_branches/ipp-20101205/ippTools/src/pztool.c
r27053 r30102 112 112 PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); 113 113 PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); 114 114 115 115 if (!pzDataStoreInsert(config->dbh, 116 116 camera, -
branches/eam_branches/ipp-20101205/ippTools/src/regtool.c
r27748 r30102 34 34 static bool pendingimfileMode(pxConfig *config); 35 35 static bool addprocessedimfileMode(pxConfig *config); 36 static bool checkburntoolimfileMode(pxConfig *config); 37 static bool pendingburntoolimfileMode(pxConfig *config); 36 38 static bool processedimfileMode(pxConfig *config); 37 39 static bool revertprocessedimfileMode(pxConfig *config); … … 72 74 // imfile 73 75 MODECASE(REGTOOL_MODE_PENDINGIMFILE, pendingimfileMode); 76 MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileMode); 77 MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode); 74 78 MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); 75 79 MODECASE(REGTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); … … 176 180 } 177 181 182 static bool checkburntoolimfileMode(pxConfig *config) 183 { 184 PS_ASSERT_PTR_NON_NULL(config,false); 185 186 // required 187 PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); 188 PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); 189 PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); 190 PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); 191 // optional 192 PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); 193 PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); 194 PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); 195 psString query = pxDataGet("regtool_checkburntoolimfile.sql"); 196 if (!query) { 197 psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); 198 return false; 199 } 200 psString rep = psStringCopy(query); 201 psFree(query); 202 query = rep; 203 204 psStringSubstitute(&query,exp_name,"@EXP_NAME@"); 205 psStringSubstitute(&query,class_id,"@CLASS_ID@"); 206 psStringSubstitute(&query,date,"@DATE@"); 207 208 fprintf(stderr,"%s",query); 209 210 if (!p_psDBRunQuery(config->dbh, query)) { 211 // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) 212 psError(PXTOOLS_ERR_PROG, false, "database error"); 213 psFree(query); 214 return false; 215 } 216 psFree(query); 217 218 psArray *output = p_psDBFetchResult(config->dbh); 219 if (!output) { 220 psErrorCode err = psErrorCodeLast(); 221 switch (err) { 222 case PS_ERR_DB_CLIENT: 223 psError(PXTOOLS_ERR_SYS, false, "database error"); 224 case PS_ERR_DB_SERVER: 225 psError(PXTOOLS_ERR_PROG, false, "database error"); 226 default: 227 psError(PXTOOLS_ERR_PROG, false, "unknown error"); 228 } 229 230 return false; 231 } 232 if (!psArrayLength(output)) { 233 psTrace("regtool", PS_LOG_INFO, "no rows found"); 234 psFree(output); 235 return true; 236 } 237 238 psString previous_uri = NULL; 239 bool ok_to_burn = true; 240 bool already_burned = true; 241 for (long i = 0; i < output->n; i++) { 242 psMetadata *row = output->data[i]; 243 244 if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| 245 (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { 246 ok_to_burn = false; 247 } 248 if (already_burned == false) { 249 ok_to_burn = false; 250 } 251 if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { 252 already_burned = true; 253 } 254 else { 255 already_burned = false; 256 } 257 258 if (previous_uri) { 259 psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); 260 psFree(previous_uri); 261 } 262 psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn); 263 psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned); 264 previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); 265 } 266 267 // negate simple so the default is true 268 if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) { 269 psError(PXTOOLS_ERR_PROG, false, "failed to print array"); 270 psFree(output); 271 return false; 272 } 273 274 psFree(output); 275 276 277 return(true); 278 279 } 280 281 static bool pendingburntoolimfileMode(pxConfig *config) 282 { 283 PS_ASSERT_PTR_NON_NULL(config,false); 284 285 // required 286 PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); 287 PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); 288 // optional 289 PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); 290 PXOPT_LOOKUP_BOOL(ignore_state, config->args, "-ignore_state", false); 291 psString query = pxDataGet("regtool_pendingburntoolimfile.sql"); 292 if (!query) { 293 psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); 294 return false; 295 } 296 psString rep = psStringCopy(query); 297 psFree(query); 298 query = rep; 299 300 psStringSubstitute(&query,date,"@DATE@"); 301 302 // fprintf(stderr,"%s",query); 303 304 if (!p_psDBRunQuery(config->dbh, query)) { 305 // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) 306 psError(PXTOOLS_ERR_PROG, false, "database error"); 307 psFree(query); 308 return false; 309 } 310 psFree(query); 311 312 psArray *output = p_psDBFetchResult(config->dbh); 313 if (!output) { 314 psErrorCode err = psErrorCodeLast(); 315 switch (err) { 316 case PS_ERR_DB_CLIENT: 317 psError(PXTOOLS_ERR_SYS, false, "database error"); 318 case PS_ERR_DB_SERVER: 319 psError(PXTOOLS_ERR_PROG, false, "database error"); 320 default: 321 psError(PXTOOLS_ERR_PROG, false, "unknown error"); 322 } 323 324 return false; 325 } 326 if (!psArrayLength(output)) { 327 psTrace("regtool", PS_LOG_INFO, "no rows found"); 328 psFree(output); 329 return true; 330 } 331 332 psString previous_uri = NULL; 333 psString this_uri = NULL; 334 bool ok_to_burn = true; 335 bool already_burned = true; 336 psString previous_class_id = NULL; 337 psString this_class_id = NULL; 338 psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool. 339 340 for (long i = 0; i < output->n; i++) { 341 psMetadata *row = output->data[i]; 342 343 // Add the information about this row and the previous, if it exists. 344 // Write the class_id stuff for debugging. 345 this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); 346 psStringSubstitute(&this_class_id,"XY","ota"); 347 psMetadataAddStr(row,PS_LIST_TAIL,"this_class_id",PS_META_REPLACE,"",this_class_id); 348 if (previous_class_id) { 349 psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id); 350 } 351 // class_id = NULL sorts to the top of the list, so skip those (incomplete downloads) 352 if (!this_class_id) { 353 continue; 354 } 355 // Determine if we've crossed a class_id boundary, as this resets the bits. 356 if (previous_class_id) { 357 if (strcmp(this_class_id,previous_class_id) != 0) { 358 ok_to_burn = true; 359 already_burned = true; 360 previous_uri = NULL; 361 } 362 } 363 364 // Write the URIs as well. 365 this_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Duplicate, but helpful for my debugging. 366 psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); 367 if (previous_uri) { 368 psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); 369 } 370 371 // Check the two status variables 372 // Convert bits of the SQL query into booleans describing the data state 373 if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| 374 (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { 375 // printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id); 376 ok_to_burn = false; 377 } 378 if (already_burned == false) { 379 // printf("already_burned looks false %s %s\n",this_uri,this_class_id); 380 ok_to_burn = false; 381 } 382 if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { 383 already_burned = true; 384 } 385 else { 386 already_burned = false; 387 } 388 psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn); 389 psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned); 390 391 // Check the uri for this exposure 392 if (!this_uri) { 393 ok_to_burn = false; 394 already_burned = false; 395 396 // Save this round for next round. 397 psFree(previous_class_id); 398 psFree(previous_uri); 399 previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); 400 psStringSubstitute(&previous_class_id,"XY","ota"); 401 previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. 402 continue; 403 } 404 405 // printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded")); 406 407 // If the state of this imfile is not "pending_burntool" then we can't burn it. 408 if (!ignore_state) { 409 psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state"); 410 411 if (!imfile_state) { // imfile state is NULL, so we probably aren't registered. 412 ok_to_burn = false; 413 414 // Save this round for next round. 415 psFree(previous_class_id); 416 psFree(previous_uri); 417 previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); 418 psStringSubstitute(&previous_class_id,"XY","ota"); 419 previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. 420 continue; 421 } 422 if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states 423 // Save this round for next round. 424 psFree(previous_class_id); 425 psFree(previous_uri); 426 previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); 427 psStringSubstitute(&previous_class_id,"XY","ota"); 428 previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. 429 continue; 430 } 431 } 432 433 // Determine if we've already suggested an entry for this ota, and if not, copy this 434 // suggestion into our output result list. 435 if (!ok_to_burn || already_burned) { 436 // Save this round for next round. 437 psFree(previous_class_id); 438 psFree(previous_uri); 439 previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); 440 psStringSubstitute(&previous_class_id,"XY","ota"); 441 previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. 442 continue; 443 } 444 // If we're here, then we think we could potentially burntool this file. 445 psArrayAdd(results,results->n,row); 446 447 // Save this round for next round. 448 psFree(previous_class_id); 449 psFree(previous_uri); 450 previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); 451 psStringSubstitute(&previous_class_id,"XY","ota"); 452 previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. 453 } 454 455 // negate simple so the default is true 456 if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) { 457 psError(PXTOOLS_ERR_PROG, false, "failed to print array"); 458 psFree(output); 459 psFree(results); 460 return false; 461 } 462 463 psFree(output); 464 psFree(results); 465 466 467 return(true); 468 469 } 470 471 178 472 179 473 static bool addprocessedimfileMode(pxConfig *config) … … 250 544 PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); 251 545 PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); 252 546 PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false); 547 253 548 PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); 254 549 PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); … … 323 618 bytes, 324 619 md5sum, 620 video_cells, 325 621 0 // burntool_state 326 622 )) { … … 809 1105 PXOPT_LOOKUP_F32(moon_phase, config->args, "-moon_phase", false, false); 810 1106 PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); 1107 PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); 1108 PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); 1109 PXOPT_LOOKUP_STR(chip_workdir,config->args, "-chip_workdir", false, false); 811 1110 PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); 812 1111 … … 1008 1307 // else continue on... 1009 1308 1010 1309 chip_workdir = chip_workdir ? chip_workdir : workdir; 1011 1310 // insert an entry into the chipPendingExp table 1012 1311 // this can only be run as the newExp's state has been set to stop 1013 1312 if (!pxchipQueueByExpTag(config, 1014 1313 exp_id, 1015 1314 chip_workdir, 1016 1315 label, 1017 NULL, // data_group 1018 NULL, // dist_group 1316 data_group, 1317 dist_group, 1019 1318 reduction, 1020 1319 NULL, // expgroup … … 1092 1391 PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); 1093 1392 PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); 1094 PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); 1393 PXOPT_COPY_STR(config->args, where, "-object", "object", "LIKE"); 1394 PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "LIKE"); 1395 PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); 1095 1396 PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); 1096 1397 PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); -
branches/eam_branches/ipp-20101205/ippTools/src/regtool.h
r27149 r30102 26 26 REGTOOL_MODE_NONE = 0x0, 27 27 REGTOOL_MODE_PENDINGIMFILE, 28 REGTOOL_MODE_CHECKBURNTOOLIMFILE, 29 REGTOOL_MODE_PENDINGBURNTOOLIMFILE, 28 30 REGTOOL_MODE_ADDPROCESSEDIMFILE, 29 31 REGTOOL_MODE_PROCESSEDIMFILE, -
branches/eam_branches/ipp-20101205/ippTools/src/regtoolConfig.c
r27748 r30102 53 53 ADD_OPT(Bool, pendingimfileArgs, "-simple", "use the simple output format", false); 54 54 55 // -checkburntoolimfile 56 psMetadata *checkburntoolimfileArgs = psMetadataAlloc(); 57 ADD_OPT(Str, checkburntoolimfileArgs, "-exp_name", "define the exp_name (required)", NULL); 58 ADD_OPT(Str, checkburntoolimfileArgs, "-class_id", "define class ID (required)", NULL); 59 ADD_OPT(Str, checkburntoolimfileArgs, "-date", "select the date to process (required)", NULL); 60 ADD_OPT(S32, checkburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); 61 ADD_OPT(Str, checkburntoolimfileArgs, "-inst", "define the camera name", NULL); 62 ADD_OPT(Str, checkburntoolimfileArgs, "-telescope", "define the telescope name", NULL); 63 ADD_OPT(Bool, checkburntoolimfileArgs, "-simple", "use the simple output format", false); 64 65 // -pendingburntoolimfile 66 psMetadata *pendingburntoolimfileArgs = psMetadataAlloc(); 67 ADD_OPT(Str, pendingburntoolimfileArgs, "-date", "select the date to process (required)", NULL); 68 ADD_OPT(S32, pendingburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); 69 ADD_OPT(Bool, pendingburntoolimfileArgs, "-simple", "use the simple output format", false); 70 ADD_OPT(Bool, pendingburntoolimfileArgs, "-ignore_state", "ignore the data_state when deciding what to work on", false); 71 ADD_OPT(U64, pendingburntoolimfileArgs, "-limit", "limit result set to N items", 0); 72 55 73 // -addprocessedimfile 56 74 psMetadata *addprocessedimfileArgs = psMetadataAlloc(); … … 128 146 ADD_OPT(S32, addprocessedimfileArgs, "-bytes", "define bytes", 0); 129 147 ADD_OPT(S16, addprocessedimfileArgs, "-burntool_state", "set burntool state", 0); 148 ADD_OPT(Bool, addprocessedimfileArgs, "-video_cells", "define if chip has video cells", false); 130 149 ADD_OPT(S16, addprocessedimfileArgs, "-fault", "set fault code", 0); 131 150 ADD_OPT(S16, addprocessedimfileArgs, "-quality", "set quality flag", 0); … … 249 268 ADD_OPT(F32, addprocessedexpArgs, "-moon_phase", "define moon phase (0.0 = new)", NAN); 250 269 ADD_OPT(Str, addprocessedexpArgs, "-label", "define label for chip stage (non-detrend data only)", NULL); 270 ADD_OPT(Str, addprocessedexpArgs, "-data_group", "define data_group for chip stage (non-detrend data only)", NULL); 271 ADD_OPT(Str, addprocessedexpArgs, "-dist_group", "define dist_group for chip stage (non-detrend data only)", NULL); 272 ADD_OPT(Str, addprocessedexpArgs, "-chip_workdir", "define workdir for chip stage (non-detrend data only)", NULL); 251 273 ADD_OPT(Str, addprocessedexpArgs, "-hostname", "define host name", NULL); 252 274 ADD_OPT(S16, addprocessedexpArgs, "-fault", "set fault code", 0); … … 291 313 psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); 292 314 psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); 315 psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by exposure obs_mode", NULL); 316 psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-comment", 0, "search by exposure comment", NULL); 317 293 318 psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); 294 319 psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); … … 395 420 396 421 PXOPT_ADD_MODE("-pendingimfile", "", REGTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); 422 PXOPT_ADD_MODE("-checkburntoolimfile", "", REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileArgs); 423 PXOPT_ADD_MODE("-pendingburntoolimfile", "", REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileArgs); 397 424 PXOPT_ADD_MODE("-addprocessedimfile", "", REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); 398 425 PXOPT_ADD_MODE("-processedimfile", "", REGTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); -
branches/eam_branches/ipp-20101205/ippconfig
- Property svn:mergeinfo changed
/branches/czw_branch/20101203/ippconfig (added) merged: 29965,30014,30017-30018,30048
- Property svn:mergeinfo changed
-
branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20080925.config
r27664 r30102 235 235 CELL.YPARITY STR ATM2_2 236 236 237 CHIP.VIDEOCELL STR CELLMODE 237 238 # XY24 is bad, so get it from the database; all others from header 238 239 CHIP.TEMP.DEPEND STR CHIP.NAME -
branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20080929.config
r27664 r30102 230 230 CELL.YPARITY STR ATM2_2 231 231 232 CHIP.VIDEOCELL STR CELLMODE 232 233 # XY24 is bad, so get it from the database; all others from header 233 234 CHIP.TEMP.DEPEND STR CHIP.NAME -
branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20081011.config
r27664 r30102 221 221 CELL.YPARITY STR ATM2_2 222 222 223 CHIP.VIDEOCELL STR CELLMODE 223 224 # XY24 is bad, so get it from the database; all others from header 224 225 CHIP.TEMP.DEPEND STR CHIP.NAME -
branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20090120.config
r27664 r30102 209 209 210 210 FPA.BURNTOOL.APPLIED STR BTOOLAPP 211 CHIP.VIDEOCELL STR CELLMODE 211 212 212 213 FPA.PON.TIME STR PONTIME -
branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20090220.config
r27664 r30102 209 209 210 210 FPA.BURNTOOL.APPLIED STR BTOOLAPP 211 CHIP.VIDEOCELL STR CELLMODE 211 212 212 213 FPA.PON.TIME STR PONTIME -
branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20100122.config
r28552 r30102 209 209 210 210 FPA.BURNTOOL.APPLIED STR BTOOLAPP 211 CHIP.VIDEOCELL STR CELLMODE 211 212 212 213 FPA.PON.TIME STR PONTIME -
branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_orig.config
r27664 r30102 217 217 218 218 FPA.BURNTOOL.APPLIED STR BTOOLAPP 219 CHIP.VIDEOCELL STR CELLMODE 219 220 220 221 FPA.PON.TIME STR PONTIME -
branches/eam_branches/ipp-20101205/ippconfig/gpc1/ppImage.config
r29921 r30102 13 13 NORM.CLASS STR CHIP # How to find the per-class normalizations 14 14 15 NONLIN BOOL TRUE # apply non-linearity correction15 NONLIN BOOL FALSE # apply non-linearity correction 16 16 OLDDARK BOOL FALSE 17 17 … … 280 280 CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? 281 281 OVERSCAN BOOL TRUE # Overscan subtraction 282 NONLIN BOOL TRUE # apply non-linearity correction282 NONLIN BOOL FALSE # apply non-linearity correction 283 283 BIAS BOOL FALSE # Bias subtraction 284 284 DARK BOOL TRUE # Dark subtraction … … 310 310 CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? 311 311 OVERSCAN BOOL TRUE # Overscan subtraction 312 NONLIN BOOL TRUE # apply non-linearity correction312 NONLIN BOOL FALSE # apply non-linearity correction 313 313 BIAS BOOL FALSE # Bias subtraction 314 314 DARK BOOL TRUE # Dark subtraction -
branches/eam_branches/ipp-20101205/ippconfig/gpc1/psastro.config
r29591 r30102 291 291 REFSTAR_MASK BOOL FALSE 292 292 END 293 294 SAS_REFERENCE METADATA 295 PSASTRO.CATDIR STR SAS.REF.V1 296 END -
branches/eam_branches/ipp-20101205/ippconfig/recipes/nightly_science.config
r29801 r30102 1 RETENTION_TIME U16 90002 3 1 CLEAN_MODES MULTI 4 2 CLEAN_MODES METADATA 5 MODE STR CHIP6 COMMAND STR chiptool3 MODE STR CHIP 4 COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ 7 5 RETENTION_TIME U16 30 8 6 END 9 7 CLEAN_MODES METADATA 10 MODE STR WARP11 COMMAND STR warptool8 MODE STR WARP 9 COMMAND STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ 12 10 RETENTION_TIME U16 7 13 11 END 14 12 CLEAN_MODES METADATA 15 MODE STR DIFF16 COMMAND STR difftool13 MODE STR DIFF 14 COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ 17 15 RETENTION_TIME S16 30 18 # RETENTION_TIME S16 -1 19 END 20 CLEAN_MODES METADATA 21 MODE STR DIST 22 COMMAND STR disttool 16 END 17 CLEAN_MODES METADATA 18 MODE STR DIST 19 COMMAND STR disttool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -label @LABEL@ -time_stamp_end @CURRENT_DATE@ 23 20 RETENTION_TIME S16 10 24 ALTERNATE_CMD STR A 25 END 26 CLEAN_MODES METADATA 27 MODE STR MAGICDS 28 COMMAND STR magicdstool 21 END 22 CLEAN_MODES METADATA 23 MODE STR MAGICDS 24 COMMAND STR magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ 29 25 RETENTION_TIME S16 1 30 ALTERNATE_CMD STR B 26 END 27 28 END_OF_NIGHT MULTI 29 END_OF_NIGHT METADATA 30 NAME STR MORNING_DARKS 31 OBSMODE STR ENGINEERING 32 EXPTYPE STR DARK 33 OBJECT STR ENGINEERING 34 COMMENT STR darks_morn% 31 35 END 32 36 … … 43 47 44 48 MACRO_DEFINITIONS MULTI 45 #MACRO_DEFINITIONS METADATA46 # PROC_MODE STR check_chips47 # MACRO STR echo survey.del.destreak @LABEL@48 #END49 49 MACRO_DEFINITIONS METADATA 50 50 PROC_MODE STR queue_chips … … 55 55 MACRO STR survey.add.destreak @LABEL@ /data/ipp053.0/gpc1_destreak 56 56 END 57 #MACRO_DEFINITIONS METADATA 58 # PROC_MODE STR check_confirm_stacks 59 # MACRO STR echo survey.add.destreak @LABEL@ /data/ipp053.0/gpc1_destreak 60 #END 61 # MACRO_DEFINITIONS METADATA 62 # PROC_MODE STR clean_old 63 # MACRO STR 57 58 TARGETS MULTI 59 TARGETS METADATA 60 NAME STR M31 61 DISTRIBUTION STR M31 62 TESS STR M31 63 OBSMODE STR M31 64 OBJECT STR M31% 65 STACKABLE BOOL TRUE 66 MIN_STACK U16 8 67 DIFFABLE BOOL FALSE 68 DIST S16 30 69 END 70 TARGETS METADATA 71 NAME STR MD01 72 DISTRIBUTION STR MD01 73 TESS STR MD01.V2 74 OBSMODE STR MD 75 OBJECT STR MD01% 76 STACKABLE BOOL TRUE 77 MIN_STACK U16 8 78 DIFFABLE BOOL FALSE 79 DIST S16 30 80 END 81 TARGETS METADATA 82 NAME STR MD02 83 DISTRIBUTION STR MD02 84 TESS STR MD02.V2 85 OBSMODE STR MD 86 OBJECT STR MD02% 87 STACKABLE BOOL TRUE 88 MIN_STACK U16 8 89 DIFFABLE BOOL FALSE 90 DIST S16 30 91 END 92 TARGETS METADATA 93 NAME STR MD03 94 DISTRIBUTION STR MD03 95 TESS STR MD03 96 OBSMODE STR MD 97 OBJECT STR MD03% 98 STACKABLE BOOL TRUE 99 MIN_STACK U16 8 100 DIFFABLE BOOL FALSE 101 DIST S16 30 102 END 103 TARGETS METADATA 104 NAME STR MD04 105 DISTRIBUTION STR MD04 106 TESS STR MD04.V2 107 OBSMODE STR MD 108 OBJECT STR MD04% 109 STACKABLE BOOL TRUE 110 MIN_STACK U16 8 111 DIFFABLE BOOL FALSE 112 DIST S16 30 113 END 114 TARGETS METADATA 115 NAME STR MD05 116 DISTRIBUTION STR MD05 117 TESS STR MD05 118 OBSMODE STR MD 119 OBJECT STR MD05% 120 STACKABLE BOOL TRUE 121 MIN_STACK U16 8 122 DIFFABLE BOOL FALSE 123 DIST S16 30 124 END 125 TARGETS METADATA 126 NAME STR MD06 127 DISTRIBUTION STR MD06 128 TESS STR MD06 129 OBSMODE STR MD 130 OBJECT STR MD06% 131 STACKABLE BOOL TRUE 132 MIN_STACK U16 8 133 DIFFABLE BOOL FALSE 134 DIST S16 30 135 END 136 TARGETS METADATA 137 NAME STR MD07 138 DISTRIBUTION STR MD07 139 TESS STR MD07 140 OBSMODE STR MD 141 OBJECT STR MD07% 142 STACKABLE BOOL TRUE 143 MIN_STACK U16 8 144 DIFFABLE BOOL FALSE 145 DIST S16 30 146 END 147 TARGETS METADATA 148 NAME STR MD08 149 DISTRIBUTION STR MD08 150 TESS STR MD08.V2 151 OBSMODE STR MD 152 OBJECT STR MD08% 153 STACKABLE BOOL TRUE 154 MIN_STACK U16 8 155 DIFFABLE BOOL FALSE 156 DIST S16 30 157 END 158 TARGETS METADATA 159 NAME STR MD09 160 DISTRIBUTION STR MD09 161 TESS STR MD09.V2 162 OBSMODE STR MD 163 OBJECT STR MD09% 164 STACKABLE BOOL TRUE 165 MIN_STACK U16 8 166 DIFFABLE BOOL FALSE 167 DIST S16 30 168 END 169 TARGETS METADATA 170 NAME STR MD10 171 DISTRIBUTION STR MD10 172 TESS STR MD10.V2 173 OBSMODE STR MD 174 OBJECT STR MD10% 175 STACKABLE BOOL TRUE 176 MIN_STACK U16 8 177 DIFFABLE BOOL FALSE 178 DIST S16 30 179 END 180 TARGETS METADATA 181 NAME STR MD11 182 DISTRIBUTION STR MD11 183 TESS STR MD11.V2 184 OBSMODE STR MD 185 OBJECT STR MD11% 186 STACKABLE BOOL TRUE 187 MIN_STACK U16 8 188 DIFFABLE BOOL FALSE 189 DIST S16 30 190 END 191 TARGETS METADATA 192 NAME STR STS 193 DISTRIBUTION STR STS 194 TESS STR STS 195 OBSMODE STR STS% 196 OBJECT STR STS% 197 STACKABLE BOOL FALSE 198 DIFFABLE BOOL FALSE 199 END 200 TARGETS METADATA 201 NAME STR SweetSpot 202 DISTRIBUTION STR SweetSpot 203 TESS STR RINGS.V0 204 OBSMODE STR SS 205 STACKABLE BOOL FALSE 206 DIFFABLE BOOL TRUE 207 WARP S16 60 208 END 209 TARGETS METADATA 210 NAME STR OSS 211 DISTRIBUTION STR SweetSpot 212 TESS STR RINGS.V0 213 OBSMODE STR OSS 214 STACKABLE BOOL FALSE 215 DIFFABLE BOOL TRUE 216 END 217 TARGETS METADATA 218 NAME STR CNP 219 DISTRIBUTION STR CNP 220 TESS STR RINGS.V0 221 OBSMODE STR CNP 222 STACKABLE BOOL FALSE 223 DIFFABLE BOOL FALSE 224 END 225 TARGETS METADATA 226 NAME STR ThreePi 227 DISTRIBUTION STR ThreePi 228 TESS STR RINGS.V0 229 OBSMODE STR 3PI 230 STACKABLE BOOL FALSE 231 DIFFABLE BOOL TRUE 232 CHIP S16 14 233 DIFF S16 14 234 END 235 # This is a hack, and I freely admit it. 236 # Removed the rest because we do not regularly run microtests. 237 # TARGETS METADATA 238 # NAME STR microtestMD01 239 # TESS STR MD01 240 # OBJECT STR MD01% 241 # COMMENT STR Microtest Obs% 242 # STACKABLE BOOL TRUE 64 243 # END 65 # MACRO_DEFINITIONS METADATA 66 # PROC_MODE STR check_registration 67 # MACRO STR 244 # TARGETS METADATA 245 # NAME STR microtestMD01.noPattern 246 # TESS STR MD01 247 # OBJECT STR MD01% 248 # COMMENT STR Microtest Obs% 249 # REDUCTION STR MICROTEST_NOPATTERN 250 # STACKABLE BOOL TRUE 68 251 # END 69 # MACRO_DEFINITIONS METADATA70 # PROC_MODE STR define_burntool71 # MACRO STR72 # END73 # MACRO_DEFINITIONS METADATA74 # PROC_MODE STR queue_burntool75 # MACRO STR76 # END77 # MACRO_DEFINITIONS METADATA78 # PROC_MODE STR check_stacks79 # MACRO STR80 # END81 # MACRO_DEFINITIONS METADATA82 # PROC_MODE STR queue_stacks83 # MACRO STR84 # END85 # MACRO_DEFINITIONS METADATA86 # PROC_MODE STR check_sweetspot87 # MACRO STR88 # END89 # MACRO_DEFINITIONS METADATA90 # PROC_MODE STR queue_sweetspot91 # MACRO STR92 # END93 # MACRO_DEFINITIONS METADATA94 # PROC_MODE STR check_dqstats95 # MACRO STR96 # END97 # MACRO_DEFINITIONS METADATA98 # PROC_MODE STR queue_dqstats99 # MACRO STR100 # END101 # MACRO_DEFINITIONS METADATA102 # PROC_MODE STR check_detrends103 # MACRO STR104 # END105 # MACRO_DEFINITIONS METADATA106 # PROC_MODE STR queue_detrends107 # MACRO STR108 # END109 110 TARGETS MULTI111 TARGETS METADATA112 NAME STR M31113 TESS STR M31114 OBSMODE STR M31115 OBJECT STR M31%116 # COMMENT STR M31%117 STACKABLE BOOL TRUE118 DIST S16 30119 END120 TARGETS METADATA121 NAME STR MD01122 TESS STR MD01.V2123 OBSMODE STR MD124 OBJECT STR MD01%125 # COMMENT STR %MD01%126 STACKABLE BOOL TRUE127 DIST S16 30128 END129 TARGETS METADATA130 NAME STR MD02131 TESS STR MD02132 OBSMODE STR MD133 OBJECT STR MD02%134 # COMMENT STR %MD02%135 STACKABLE BOOL TRUE136 DIST S16 30137 END138 TARGETS METADATA139 NAME STR MD03140 TESS STR MD03141 OBSMODE STR MD142 OBJECT STR MD03%143 # COMMENT STR %MD03%144 STACKABLE BOOL TRUE145 DIST S16 30146 END147 TARGETS METADATA148 NAME STR MD04149 TESS STR MD04150 OBSMODE STR MD151 OBJECT STR MD04%152 # COMMENT STR %MD04%153 STACKABLE BOOL TRUE154 DIST S16 30155 END156 TARGETS METADATA157 NAME STR MD05158 TESS STR MD05159 OBSMODE STR MD160 OBJECT STR MD05%161 # COMMENT STR %MD05%162 STACKABLE BOOL TRUE163 DIST S16 30164 END165 TARGETS METADATA166 NAME STR MD06167 TESS STR MD06168 OBSMODE STR MD169 OBJECT STR MD06%170 # COMMENT STR %MD06%171 STACKABLE BOOL TRUE172 DIST S16 30173 END174 TARGETS METADATA175 NAME STR MD07176 TESS STR MD07177 OBSMODE STR MD178 OBJECT STR MD07%179 # COMMENT STR %MD07%180 STACKABLE BOOL TRUE181 DIST S16 30182 END183 TARGETS METADATA184 NAME STR MD08185 TESS STR MD08.V2186 OBSMODE STR MD187 OBJECT STR MD08%188 # COMMENT STR %MD08%189 STACKABLE BOOL TRUE190 DIST S16 30191 END192 TARGETS METADATA193 NAME STR MD09194 TESS STR MD09.V2195 OBSMODE STR MD196 OBJECT STR MD09%197 # COMMENT STR %MD09%198 STACKABLE BOOL TRUE199 DIST S16 30200 END201 TARGETS METADATA202 NAME STR MD10203 TESS STR MD10.V2204 OBSMODE STR MD205 OBJECT STR MD10%206 # COMMENT STR %MD10%207 STACKABLE BOOL TRUE208 DIST S16 30209 END210 TARGETS METADATA211 NAME STR MD11212 TESS STR MD11.V2213 OBSMODE STR MD214 OBJECT STR MD11%215 # COMMENT STR %MD11%216 STACKABLE BOOL TRUE217 DIST S16 30218 END219 TARGETS METADATA220 NAME STR STS221 TESS STR STS222 OBSMODE STR STS%223 OBJECT STR STS%224 # COMMENT STR Stellar Transit%225 STACKABLE BOOL FALSE226 END227 TARGETS METADATA228 NAME STR SweetSpot229 TESS STR RINGS.V0230 OBSMODE STR SS231 # OBJECT STR SS232 STACKABLE BOOL FALSE233 EXTRA_PROCESSING BOOL TRUE234 # NOCLEAN BOOL TRUE235 WARP S16 60236 END237 TARGETS METADATA238 NAME STR OSS239 DISTRIBUTION STR SweetSpot240 TESS STR RINGS.V0241 OBSMODE STR OSS242 STACKABLE BOOL FALSE243 EXTRA_PROCESSING BOOL TRUE244 END245 TARGETS METADATA246 NAME STR CNP247 DISTRIBUTION STR CNP248 TESS STR RINGS.V0249 OBSMODE STR CNP250 STACKABLE BOOL FALSE251 EXTRA_PROCESSING BOOL FALSE252 END253 TARGETS METADATA254 NAME STR ThreePi255 TESS STR RINGS.V0256 OBSMODE STR 3PI257 # OBJECT STR 3PI258 STACKABLE BOOL FALSE259 CHIP S16 14260 DIFF S16 14261 END262 # This is a hack, and I freely admit it.263 TARGETS METADATA264 NAME STR microtestMD01265 TESS STR MD01266 OBJECT STR MD01%267 COMMENT STR Microtest Obs%268 STACKABLE BOOL TRUE269 END270 TARGETS METADATA271 NAME STR microtestMD01.noPattern272 TESS STR MD01273 OBJECT STR MD01%274 COMMENT STR Microtest Obs%275 REDUCTION STR MICROTEST_NOPATTERN276 STACKABLE BOOL TRUE277 END278 TARGETS METADATA279 NAME STR microtestMD02280 TESS STR MD02281 OBJECT STR MD02%282 COMMENT STR Microtest Obs%283 STACKABLE BOOL TRUE284 END285 TARGETS METADATA286 NAME STR microtestMD02.noPattern287 TESS STR MD02288 OBJECT STR MD02%289 COMMENT STR Microtest Obs%290 REDUCTION STR MICROTEST_NOPATTERN291 STACKABLE BOOL TRUE292 END293 TARGETS METADATA294 NAME STR microtestMD03295 TESS STR MD03296 OBJECT STR MD03%297 COMMENT STR Microtest Obs%298 STACKABLE BOOL TRUE299 END300 TARGETS METADATA301 NAME STR microtestMD03.noPattern302 TESS STR MD03303 OBJECT STR MD03%304 COMMENT STR Microtest Obs%305 REDUCTION STR MICROTEST_NOPATTERN306 STACKABLE BOOL TRUE307 END308 TARGETS METADATA309 NAME STR microtestMD04310 TESS STR MD04311 OBJECT STR MD04%312 COMMENT STR Microtest Obs%313 STACKABLE BOOL TRUE314 END315 TARGETS METADATA316 NAME STR microtestMD04.noPattern317 TESS STR MD04318 OBJECT STR MD04%319 COMMENT STR Microtest Obs%320 REDUCTION STR MICROTEST_NOPATTERN321 STACKABLE BOOL TRUE322 END323 TARGETS METADATA324 NAME STR microtestMD05325 TESS STR MD05326 OBJECT STR MD05%327 COMMENT STR Microtest Obs%328 STACKABLE BOOL TRUE329 END330 TARGETS METADATA331 NAME STR microtestMD05.noPattern332 TESS STR MD05333 OBJECT STR MD05%334 COMMENT STR Microtest Obs%335 REDUCTION STR MICROTEST_NOPATTERN336 STACKABLE BOOL TRUE337 END338 TARGETS METADATA339 NAME STR microtestMD06340 TESS STR MD06341 OBJECT STR MD06%342 COMMENT STR Microtest Obs%343 STACKABLE BOOL TRUE344 END345 TARGETS METADATA346 NAME STR microtestMD06.noPattern347 TESS STR MD06348 OBJECT STR MD06%349 COMMENT STR Microtest Obs%350 REDUCTION STR MICROTEST_NOPATTERN351 STACKABLE BOOL TRUE352 END353 TARGETS METADATA354 NAME STR microtestMD07355 TESS STR MD07356 OBJECT STR MD07%357 COMMENT STR Microtest Obs%358 STACKABLE BOOL TRUE359 END360 TARGETS METADATA361 NAME STR microtestMD07.noPattern362 TESS STR MD07363 OBJECT STR MD07%364 COMMENT STR Microtest Obs%365 REDUCTION STR MICROTEST_NOPATTERN366 STACKABLE BOOL TRUE367 END368 TARGETS METADATA369 NAME STR microtestMD08370 TESS STR MD08371 OBJECT STR MD08%372 COMMENT STR Microtest Obs%373 STACKABLE BOOL TRUE374 END375 TARGETS METADATA376 NAME STR microtestMD08.noPattern377 TESS STR MD08378 OBJECT STR MD08%379 COMMENT STR Microtest Obs%380 REDUCTION STR MICROTEST_NOPATTERN381 STACKABLE BOOL TRUE382 END383 TARGETS METADATA384 NAME STR microtestMD09385 TESS STR MD09386 OBJECT STR MD09%387 COMMENT STR Microtest Obs%388 STACKABLE BOOL TRUE389 END390 TARGETS METADATA391 NAME STR microtestMD09.noPattern392 TESS STR MD09393 OBJECT STR MD09%394 COMMENT STR Microtest Obs%395 REDUCTION STR MICROTEST_NOPATTERN396 STACKABLE BOOL TRUE397 END398 TARGETS METADATA399 NAME STR microtestMD10400 TESS STR MD10401 OBJECT STR MD10%402 COMMENT STR Microtest Obs%403 STACKABLE BOOL TRUE404 END405 TARGETS METADATA406 NAME STR microtestMD10.noPattern407 TESS STR MD10408 OBJECT STR MD10%409 COMMENT STR Microtest Obs%410 REDUCTION STR MICROTEST_NOPATTERN411 STACKABLE BOOL TRUE412 END413 252 414 253 # Detrend verification data … … 420 259 REF_ID S64 299 421 260 REF_ITER S32 0 422 # FILTER STR 423 MAX S32 10 261 MAX_EXP S32 10 424 262 END 425 263 … … 431 269 REF_ITER S32 0 432 270 FILTER STR g.00000 433 MAX 271 MAX_EXP S32 10 434 272 END 435 273 … … 441 279 REF_ITER S32 0 442 280 FILTER STR r.00000 443 MAX 281 MAX_EXP S32 10 444 282 END 445 283 … … 451 289 REF_ITER S32 0 452 290 FILTER STR i.00000 453 MAX 291 MAX_EXP S32 10 454 292 END 455 293 … … 461 299 REF_ITER S32 0 462 300 FILTER STR z.00000 463 MAX 301 MAX_EXP S32 10 464 302 END 465 303 … … 471 309 REF_ITER S32 0 472 310 FILTER STR y.00000 473 MAX 311 MAX_EXP S32 10 474 312 END 475 313 … … 481 319 REF_ITER S32 0 482 320 FILTER STR w.00000 483 MAX 484 END 485 321 MAX_EXP S32 10 322 END 323 -
branches/eam_branches/ipp-20101205/ippconfig/recipes/ppImage.config
r29921 r30102 947 947 BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? 948 948 BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? 949 MASK.SATURATED BOOL FALSE # DO NOT Mask the saturated pixels 950 MASK.LOW BOOL FALSE # DO NOT Mask the saturated pixels 949 951 END 950 952 … … 977 979 BIN2.XBIN S32 1 # Image is already binned 978 980 BIN2.YBIN S32 1 # Image is already binned 981 MASK.SATURATED BOOL FALSE # DO NOT Mask the saturated pixels 982 MASK.LOW BOOL FALSE # DO NOT Mask the saturated pixels 979 983 END 980 984 … … 1007 1011 BIN2.XBIN S32 1 # Image is already binned 1008 1012 BIN2.YBIN S32 1 # Image is already binned 1013 MASK.SATURATED BOOL FALSE # DO NOT Mask the saturated pixels 1014 MASK.LOW BOOL FALSE # DO NOT Mask the saturated pixels 1009 1015 END 1010 1016 -
branches/eam_branches/ipp-20101205/ippconfig/recipes/ppStats.config
r29902 r30102 236 236 CONCEPT STR FPA.BURNTOOL.APPLIED 237 237 238 238 239 CONCEPT STR FPA.PON.TIME # time since last power on 240 241 CONCEPT STR CHIP.VIDEOCELL 239 242 240 243 STAT STR ROBUST_MEDIAN # Background estimator -
branches/eam_branches/ipp-20101205/ippconfig/recipes/ppStatsFromMetadata.config
r29902 r30102 45 45 ENTRY VAL FPA.PON.TIME F32 CONSTANT -pon_time # time since last power on 46 46 # ENTRY VAL FPA.BURNTOOL.APPLIED S32 CONSTANT -burntool_state # 47 ENTRY VAL CHIP.VIDEOCELL BOOL CONSTANT -video_cells # This chip contains a video cell 47 48 ENTRY VAL CHIP.TEMP F32 SAMPLE_MEAN -ccd_temp # CCD temperature 48 49 ENTRY VAL CELL.EXPOSURE F32 SAMPLE_MEAN -exp_time # Exposure time -
branches/eam_branches/ipp-20101205/ippconfig/recipes/psastro.config
r28641 r30102 238 238 MOPS.TEST METADATA 239 239 END 240 241 SAS_REFERENCE METADATA 242 END 243 -
branches/eam_branches/ipp-20101205/ippconfig/recipes/psphot.config
r30085 r30102 49 49 PEAKS_NMAX S32 5000 # on first pass, only keep NMAX peaks (0 == all) 50 50 PEAKS_MIN_GAUSS F32 0.5 # On quick convolution, mask pixels for which the 51 PEAKS_POS2_NSIGMA_LIMIT F32 25.0 # peak signficance threshold for POS2 sources. (ppSub) 51 52 # input pixels contribute less than this fraction of the flux 52 53 # parameters which adjust the footprint analysis -
branches/eam_branches/ipp-20101205/ippconfig/recipes/reductionClasses.mdc
r29941 r30102 543 543 END 544 544 545 SAS_REFERENCE METADATA 546 CHIP_PPIMAGE STR CHIP 547 CHIP_PSPHOT STR CHIP 548 WARP_PSWARP STR WARP 549 STACK_PPSTACK STR STACK 550 STACK_PPSUB STR STACK 551 STACK_PSPHOT STR STACK 552 DIFF_PPSUB STR DIFF 553 DIFF_PSPHOT STR DIFF 554 JPEG_BIN1 STR PPIMAGE_J1 555 JPEG_BIN2 STR PPIMAGE_J2 556 FAKEPHOT STR FAKEPHOT 557 ADDSTAR STR ADDSTAR 558 PSASTRO STR SAS_REFERENCE 559 STACKPHOT STR STACKPHOT 560 END 561 -
branches/eam_branches/ipp-20101205/tools/czarpoll.pl
r29953 r30102 108 108 my $newState = undef; 109 109 my $nsStatus = undef; 110 my $lastDay = strftime('%Y-%m-%d', localtime);111 110 my $today = undef; 112 my $doneMetricsToday = 1; 113 111 my $yesterday = undef; 112 my $newDayTime = "18:00"; 113 my $lastDayDailyTasks = "2010-01-01"; 114 115 # main polling loop 114 116 while (1) { 115 117 116 # check whether day has changed. if so, cleanup tables from previous day and optimize118 # sort out times 117 119 $today = strftime('%Y-%m-%d', localtime); 118 if ($czarDb->isBefore($lastDay, $today)) { 119 120 $yesterday = $czarDb->subtractInterval($today, "1 DAY"); 121 $end = $czarDb->getNowTimestamp(); 122 123 # if before 18:00 today, then start plots from 18:00 yesterday 124 if ($czarDb->isBefore($end, "$today $newDayTime")) {$begin = "$yesterday $newDayTime";} 125 # if after 18:00 today, then perform some daily tasks and start plots from 18:00 today 126 else { 127 128 $begin = "$today $newDayTime"; 129 130 # check whether yesterday was cleaned. if not, cleanup tables and optimize 131 if ($lastDayDailyTasks ne $yesterday) { 132 133 print "* performing daily tasks (clean-up, metrics)"; 134 135 # create metrics for last 24 hours 136 print "* Creating metrics for last 24 hours\n"; 137 # TODO hardcoded path needs to be in config 138 my $dayMetrics = new czartool::DayMetrics($gpc1Db, 139 $czarDb, 140 "/data/ipp004.0/ipp/ippMetrics/", 141 1, 0, $today); 142 $dayMetrics->writeHTML(); 143 144 # now update metrics index page 145 my $metricsIndex = new czartool::MetricsIndex($gpc1Db, 146 $czarDb, 147 "/data/ipp004.0/ipp/ippMetrics/", 148 1, 0); 149 $metricsIndex->writeHTML(); 150 151 # now cleanup tables from yesterday and optimize 120 152 print "* New day - performing cleanup\n"; 121 $czarDb->cleanupDateRange($ lastDay, $lastDay, "30 MINUTE");153 $czarDb->cleanupDateRange($yesterday, $yesterday, "30 MINUTE"); 122 154 $czarDb->optimize(); 123 $lastDay = $today; 124 $doneMetricsToday = 0; 125 } 126 127 # sort out times 128 $begin = strftime('%Y-%m-%d 06:30', localtime); 129 $end = $czarDb->getNowTimestamp(); 130 131 # if time now is after 06:35am, then create metrics for past 24 hours 132 if (!$doneMetricsToday && $czarDb->isBefore($begin, $end)) { 133 134 # create metrics for last 24 hours 135 print "* Creating metrics for last 24 hours\n"; 136 my $yesterday = $czarDb->subtractInterval($today, "1 DAY"); 137 # TODO hardcoded path needs to be in config 138 my $dayMetrics = new czartool::DayMetrics($gpc1Db, 139 $czarDb, 140 "/data/ipp004.0/ipp/ippMetrics/", 141 1, 0, $yesterday); 142 $dayMetrics->writeHTML(); 143 $doneMetricsToday = 1; 144 145 # now update metrics index page 146 my $metricsIndex = new czartool::MetricsIndex($gpc1Db, 147 $czarDb, 148 "/data/ipp004.0/ipp/ippMetrics/", 149 1, 0); 150 } 151 152 # if time now is before 06:35am, include data from previous day 153 if ($czarDb->isBefore($end, $begin)) { 154 155 $begin = $czarDb->subtractInterval($begin, "1 DAY"); 155 $lastDayDailyTasks = $yesterday; 156 } 156 157 } 157 158 … … 247 248 248 249 $plotter->createLogAndLinearTimeSeries($allServerLabels, $stage, $begin, $end); # TODO must be a neater way... 249 $plotter->createRateTimeSeries($allServerLabels, $stage, $begin, $end, undef, 0);250 $plotter->createRateTimeSeries($allServerLabels, $stage, $begin, $end, undef, 0); 250 251 } 251 252 } … … 271 272 my $server = undef; 272 273 my $state = undef; 273 274 274 275 foreach $stage (@stages) { 275 276 -
branches/eam_branches/ipp-20101205/tools/czartool/DayMetrics.pm
r29863 r30102 37 37 $self->{day} = $_[6]; 38 38 39 my $yesterday = $self->{czarDb}->subtractInterval($self->{day}, "1 DAY"); 40 39 41 # sort out times 40 $self->{begin} = "$ self->{day} 06:00";42 $self->{begin} = "$yesterday 18:00"; 41 43 $self->{end} = $self->{czarDb}->addInterval($self->{begin}, "1 DAY"); 42 44 $self->{burntoolEnd} = $self->{czarDb}->addInterval($self->{begin}, "12 HOUR"); -
branches/eam_branches/ipp-20101205/tools/czartool/Metrics.pm
r29867 r30102 35 35 36 36 # instantiate a plotter object 37 $self->{plotter} = new czartool::Plotter(37 $self->{plotter} = czartool::Plotter->new_file( 38 38 $self->{gpc1Db}, 39 39 $self->{czarDb}, 40 "%Y%m%d-%H%M%S",41 "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8",42 40 ".", 43 41 $self->{save_temps}); -
branches/eam_branches/ipp-20101205/tools/makedistdest
r29237 r30102 13 13 14 14 my $prod_name; 15 my $dbhost; 15 16 16 17 GetOptions( 17 18 'prod_name=s' => \$prod_name, 19 'dbhost=s' => \$dbhost, 18 20 ) or pod2usage( 2 ); 19 21 20 22 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; 21 pod2usage( -msg => "Required options: --prod_name ",23 pod2usage( -msg => "Required options: --prod_name --dbhost", 22 24 -exitval => 3) 23 unless defined $prod_name; 25 unless defined $prod_name 26 and defined $dbhost; 24 27 25 28 my $cmd = "dsprodtool --add $prod_name --type ipp-dist --description $prod_name"; … … 31 34 die "dsprodtool failed: $rc" if $rc; 32 35 33 $cmd = "disttool -dbname gpc1 -definedestination -ds_dbname ippRequestServer -ds_dbhost ippdb02-name $prod_name";36 $cmd = "disttool -dbname gpc1 -definedestination -ds_dbname ippRequestServer -ds_dbhost $dbhost -name $prod_name"; 34 37 35 38 $rc = system $cmd;
Note:
See TracChangeset
for help on using the changeset viewer.
