IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 23777


Ignore:
Timestamp:
Apr 9, 2009, 1:56:50 PM (17 years ago)
Author:
bills
Message:

change disttool -pendingcomponent to output "$outroot.$dist_id" as outdir to help make it unique
allow 'clean' raw stage distributions. (Database information only.) This is required to
support mirrors with chip stage and beyond images but not raw data

Location:
trunk
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/dist_advancerun.pl

    r23744 r23777  
    2828
    2929# Parse the command-line arguments
    30 my ($dist_id, $stage, $stage_id, $outroot);
     30my ($dist_id, $stage, $stage_id, $outdir);
    3131my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
    3232
    … …  
    3535           'stage=s'        => \$stage,      # raw, chip, warp, or diff
    3636           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
    37            'outroot=s'      => \$outroot,    # "directory" for outputs
     37           'outdir=s'       => \$outdir,    # "directory" for outputs
    3838           'save-temps'     => \$save_temps, # Save temporary files?
    3939           'dbname=s'       => \$dbname,     # Database name
    … …  
    4545
    4646pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    47 pod2usage( -msg => "Required options: --dist_id --stage --stage_id --outroot",
     47pod2usage( -msg => "Required options: --dist_id --stage --stage_id --outdir",
    4848           -exitval => 3) unless
    4949    defined $dist_id and
    5050    defined $stage and
    5151    defined $stage_id and
    52     defined $outroot;
     52    defined $outdir;
    5353
    5454$ipprc->redirect_output($logfile) if $logfile;
    … …  
    9090
    9191# XXX we should create a file rule for this
    92 my $outfile = "$outroot/dbinfo.$stage.$stage_id.mdc";
     92my $outfile = "$outdir/dbinfo.$stage.$stage_id.mdc";
    9393
    9494{

  • trunk/ippScripts/scripts/dist_component.pl

    r23743 r23777  
    5858# Parse the command-line arguments
    5959my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean);
    60 my ($outroot, $run_state, $data_state, $magicked, $no_magic, $poor_quality);
     60my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality);
    6161my ($dbname, $save_temps, $verbose, $no_update, $logfile);
    6262
    … …  
    7171           'state=s'        => \$run_state,  # state of the run
    7272           'data_state=s'   => \$data_state, # data_state for this component
    73            'poor_quality'      => \$poor_quality,  # the processing for this component did not produced images
     73           'poor_quality'   => \$poor_quality,  # the processing for this component did not produced images
    7474           'no_magic'       => \$no_magic,   # magic is not required for this distribution run
    7575           'magicked'       => \$magicked,   # magicked state for this component
    76            'outroot=s'      => \$outroot,    # "directory" for outputs
     76           'outdir=s'       => \$outdir,     # "directory" for outputs
    7777           'clean'          => \$clean,      # create clean distribution
    7878           'save-temps'     => \$save_temps, # Save temporary files?
    … …  
    8484
    8585pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    86 pod2usage( -msg => "Required options: --dist_id --camera --stage --stage_id --component --path_base --outroot",
     86pod2usage( -msg => "Required options: --dist_id --camera --stage --stage_id --component --path_base --outdir",
    8787           -exitval => 3) unless
    8888    defined $dist_id and
    … …  
    9292    defined $component and
    9393    defined $path_base and
    94     defined $outroot;
     94    defined $outdir;
    9595
    9696$ipprc->redirect_output($logfile) if $logfile;
    9797
    98 if (($stage eq 'raw') and !defined $chip_path_base) {
     98if (($stage eq 'raw') and !$clean and !defined $chip_path_base) {
    9999    pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3);
    100100}
    … …  
    105105
    106106# making a clean bundle of raw images doesn't make sense
    107 if (($stage eq "raw") and $clean) {
    108     # well I suppose we could ship the registration log file. naw.
    109     &my_die("cannot create clean run at raw stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    110 }
    111107
    112108# create the output directories if it is not a nebulous path and it doesn't exist
    113 if (index($outroot, "neb://") != 0) {
    114     if (! -e $outroot ) {
    115         my $code = system "mkdir -p $outroot";
    116         &my_die("cannot create output directory $outroot", $dist_id, $component,
     109if (index($outdir, "neb://") != 0) {
     110    if (! -e $outdir ) {
     111        my $code = system "mkdir -p $outdir";
     112        &my_die("cannot create output directory $outdir", $dist_id, $component,
    117113                $code >> 8) if $code;
    118114    }
    … …  
    122118# note: We my_die in get_file_list if something goes wrong.
    123119
    124 my $file_list = get_file_list($stage, $component, $path_base);
    125 
    126 if ($stage ne 'fake') {
     120my $file_list = get_file_list($stage, $component, $path_base, $clean);
     121
     122if (($stage ne 'raw') and ($stage ne 'fake')) {
    127123    # If the file list is empty it is an error because we should at least get a config dump file
    128124    # except for fake stage which doesn't do anything yet
    … …  
    132128# set up directory for temporary files
    133129
    134 my $tmpdir  = "$outroot/tmpdir.$dist_id.$component";
     130my $tmpdir  = "$outdir/tmpdir.$dist_id.$component";
    135131if (-e $tmpdir) {
    136132    if (-d $tmpdir) {
    … …  
    155151
    156152# foreach my $file_rule (keys %$file_list) {
     153my $num_files = 0;
    157154foreach my $file (@$file_list) {
    158155    # check whether this file rule refers to an image, mask, or variance fits image
    … …  
    188185    # we need to pre-process the image before adding to the bundle. Save the path names.
    189186    # the images will be created below
     187    $num_files++;
    190188    if ($image_type && $nan_masked_pixels) {
    191189        # save the
    … …  
    251249my $bytes;
    252250my $md5sum;
    253 if ($stage eq 'fake') {
     251if (0) {
     252if (($stage eq 'fake')  or (($stage eq 'raw') and $clean)) {
    254253    open DUMMY, ">$tmpdir/fake.$component" or &my_die("Unable to create placeholder for fake component",
    255254        $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    … …  
    257256    close DUMMY;
    258257}
    259 # create the tarfile
    260 {
     258}
     259if ($num_files) {
     260    # create the tarfile
    261261    # XXX TODO: create a file rule for the tar file name
    262262    my $tbase = basename($path_base);
    263263    $tbase .= ".$component" if $component;
    264264    $file_name = "$tbase.tgz";
    265     my $tarfile = "$outroot/$file_name";
     265    my $tarfile = "$outdir/$file_name";
    266266
    267267    my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";
    … …  
    279279    &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum;
    280280
    281     my @finfo = stat($tarfile);
    282     &my_die("unable to stat $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !@finfo;
    283     $bytes = $finfo[7];
     281    $bytes = -s $tarfile;
    284282
    285283    delete_tmpdir($tmpdir);
     284} else {
     285    # no files for this component
     286    $file_name = "none";
     287    $bytes = 0;
     288    $md5sum = "0";
    286289}
    287290{
    … …  
    360363    my $component = shift;
    361364    my $path_base = shift;
     365    my $clean = shift;
    362366
    363367    my @file_list;
    … …  
    366370        # eventually rawImfile will have a path_base that we'll need to add '.fits'
    367371        # XXX do we want to distribute the registration log files?
    368         my %file;
    369         $file{file_rule} = "RAW.IMAGE";
    370         $file{name} = $path_base;
    371         push @file_list, \%file;
     372        if (!$clean) {
     373            my %file;
     374            $file{file_rule} = "RAW.IMAGE";
     375            $file{name} = $path_base;
     376            push @file_list, \%file;
     377        }
    372378        return \@file_list;
    373379    }

  • trunk/ippTasks/dist.pro

    r23702 r23777  
    131131    book getword distToProcess $pageName state -var STATE
    132132    book getword distToProcess $pageName data_state -var DATA_STATE
     133    book getword distToProcess $pageName quality -var QUALITY
    133134    book getword distToProcess $pageName no_magic -var NO_MAGIC
    134135    book getword distToProcess $pageName magicked -var MAGICKED
    135     book getword distToProcess $pageName outroot -var OUTROOT
     136    book getword distToProcess $pageName outdir -var OUTDIR
    136137    book getword distToProcess $pageName dbname -var DBNAME
    137138
    138     $EXTRA_FLAGS = ""
     139    $EXTRA_ARGS = ""
    139140    if ("$CLEAN" == "T")
    140         $EXTRA_FLAGS = --clean
     141        $EXTRA_ARGS = --clean
    141142    end
    142143    # magicked is output as integer due to the union in the sql
    143144    if ($MAGICKED)
    144         $EXTRA_FLAGS = $EXTRA_FLAGS --magicked
     145        $EXTRA_ARGS = $EXTRA_ARGS --magicked
    145146    end
    146147    # no_magic is output as integer due to the union in the sql
    147148    if ($NO_MAGIC)
    148         $EXTRA_FLAGS = $EXTRA_FLAGS --no_magic
     149        $EXTRA_ARGS = $EXTRA_ARGS --no_magic
     150    end
     151    if ($QUALITY)
     152        $EXTRA_ARGS = $EXTRA_ARGS --poor_quality
    149153    end
    150154
    … …  
    153157    host anyhost
    154158
    155     sprintf logfile "%s/dist.%s.%s.log" $OUTROOT $DIST_ID $COMPONENT
     159    sprintf logfile "%s.%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT
    156160    stdout $logfile
    157161    stderr $logfile
    158162
    159     $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_FLAGS --outroot $OUTROOT --logfile $logfile
     163    $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile
    160164
    161165    add_standard_args run
    … …  
    251255    book getword distToAdvance $pageName stage   -var STAGE
    252256    book getword distToAdvance $pageName stage_id -var STAGE_ID
    253     book getword distToAdvance $pageName outroot -var OUTROOT
     257    book getword distToAdvance $pageName outdir -var OUTDIR
    254258
    255259    host anyhost
    256260
    257     sprintf logfile "%s/dist.advance.%s.log" $OUTROOT $DIST_ID
     261    sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
    258262    stdout $logfile
    259263    stderr $logfile
    260264
    261     $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outroot $OUTROOT --logfile $logfile
     265    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR --logfile $logfile
    262266    add_standard_args run
    263267

  • trunk/ippTools/share/disttool_pendingcomponent.sql

    r23737 r23777  
    11SELECT * FROM (
     2    -- raw stage
    23SELECT
    34    distRun.dist_id,
    … …  
    89    clean,
    910    rawExp.camera,
    10     outroot,
     11    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    1112    rawImfile.uri as path_base,         -- change this once rawImfile has path_base
    1213    chipProcessedImfile.path_base as chip_path_base,
    … …  
    4142WHERE
    4243    distRun.state = 'new'
     44    AND distRun.clean = 0
    4345    AND distRun.stage = 'raw'
    4446    AND distComponent.dist_id IS NULL
    4547    AND (rawExp.magicked OR distRun.no_magic)
    4648    -- where hook 1 %s
     49UNION
     50    -- raw stage clean (dbinfo only)
     51SELECT
     52    distRun.dist_id,
     53    distRun.label,
     54    stage,
     55    stage_id,
     56    'exposure' AS component,
     57    clean,
     58    rawExp.camera,
     59    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
     60    NULL,
     61    NULL,
     62    NULL as state,
     63    NULL as data_state,
     64    0 as quality,
     65    distRun.no_magic,
     66    rawExp.magicked
     67FROM distRun
     68JOIN rawExp ON exp_id = stage_id
     69LEFT JOIN distComponent
     70    ON distRun.dist_id = distComponent.dist_id
     71WHERE
     72    distRun.state = 'new'
     73    AND distRun.stage = 'raw'
     74    AND distRun.clean
     75    AND distComponent.dist_id IS NULL
     76    -- where hook 2 %s
    4777
    4878-- chip stage
    … …  
    5686    clean,
    5787    rawExp.camera,
    58     outroot,
     88    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    5989    chipProcessedImfile.path_base,
    6090    chipProcessedImfile.path_base as chip_path_base,
    … …  
    77107    AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
    78108    AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned'))
    79     -- where hook 2 %s
     109    -- where hook 3 %s
    80110UNION
    81111SELECT
    … …  
    88118    clean,
    89119    rawExp.camera,
    90     outroot,
     120    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    91121    camProcessedExp.path_base,
    92122    NULL as chip_path_base,
    … …  
    111141    AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
    112142    AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))
    113     -- where hook 3 %s
     143    -- where hook 4 %s
    114144UNION
    115145SELECT
    … …  
    121151    clean,
    122152    rawExp.camera,
    123     outroot,
     153    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    124154    fakeProcessedImfile.path_base,
    125155    NULL,
    … …  
    142172    AND distRun.stage = 'fake'
    143173    AND distComponent.dist_id IS NULL
    144     -- where hook 4 %s
     174    -- where hook 5 %s
    145175UNION
    146176SELECT
    … …  
    152182    clean,
    153183    rawExp.camera,
    154     outroot,
     184    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    155185    warpSkyfile.path_base,
    156186    NULL as chip_path_base,
    … …  
    176206    AND (distRun.clean OR warpRun.magicked OR distRun.no_magic)
    177207    AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned'))
    178     -- where hook 5 %s
     208    -- where hook 6 %s
    179209UNION
    180210SELECT
    … …  
    186216    clean,
    187217    rawExp.camera,
    188     outroot,
     218    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    189219    diffSkyfile.path_base,
    190220    NULL as chip_path_base,
    … …  
    209239    AND (distRun.clean OR diffRun.magicked OR distRun.no_magic)
    210240    AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned'))
    211     -- where hook 6 %s
     241    -- where hook 7 %s
    212242UNION
    213243SELECT DISTINCT
    … …  
    219249    clean,
    220250    rawExp.camera,
    221     outroot,
     251    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
    222252    stackSumSkyfile.path_base,
    223253    NULL as chip_path_base,
    … …  
    258288    AND distComponent.dist_id IS NULL
    259289    AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned'))
    260     -- where hook 7 %s
     290    -- where hook 8 %s
    261291) as Foo

  • trunk/ippTools/share/disttool_toadvance.sql

    r23742 r23777  
    33    stage,
    44    stage_id,
    5     outroot,
     5    CONCAT_WS('.', outroot, CONVERT(dist_id, CHAR)) as outdir,
    66    label,
    77    clean
    88FROM
    99    (
    10 -- raw stage
     10-- raw stage not clean
    1111SELECT
    1212    distRun.dist_id,
    … …  
    2323    WHERE
    2424        distRun.state = 'new'
     25        AND distRun.clean = 0
    2526        AND distRun.fault = 0
    2627        AND distRun.stage = 'raw'
    … …  
    3132        COUNT(rawImfile.class_id) = COUNT(distComponent.component)
    3233        AND SUM(distComponent.fault) = 0
     34UNION
     35-- clean distribution of raw files (dbinfo only) only 1 component
     36SELECT
     37    distRun.dist_id,
     38    stage,
     39    stage_id,
     40    outroot,
     41    label,
     42    clean
     43    FROM distRun
     44    LEFT JOIN distComponent
     45        ON distRun.dist_id = distComponent.dist_id
     46    WHERE
     47        distRun.state = 'new'
     48        AND distRun.clean
     49        AND distRun.fault = 0
     50        AND distRun.stage = 'raw'
     51        AND distComponent.component IS NOT NULL
     52        AND distComponent.fault = 0
    3353UNION
    3454-- chip stage

  • trunk/ippTools/src/disttool.c

    r23765 r23777  
    564564    // will get blocked by pending earlier stages
    565565    psString    raw_where = "";
     566    psString    raw_clean_where = "";
    566567    psString    chip_where = "";
    567568    psString    camera_where = "";
    … …  
    574575            query,
    575576            raw_where,
     577            raw_clean_where,
    576578            chip_where,
    577579            camera_where,
Note: See TracChangeset for help on using the changeset viewer.