IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 20973


Ignore:
Timestamp:
Dec 13, 2008, 10:17:34 AM (17 years ago)
Author:
bills
Message:

Rework the diff stage to process the skycells from an exposure in a single
diffRun

Location:
trunk
Files:
4 added
38 edited

Legend:

Unmodified
Added
Removed

  • trunk/dbconfig/changes.txt

    r20855 r20973  
    717717alter table magicDSRun add column cam_id BIGINT after stage_id;
    718718
     719--  Version: 1.1.47
     720
     721-- WARNING WARNING WARNING
     722-- Updating a database which has populated diffRun's requires editing
     723-- the contents of the tables in a way that cannot be expresed here.
     724-- The script tools/collapsdiffinputs.pl does the appropriate edits
     725-- and should be run at the spot indicated below
     726
     727-- fix order of workdir to match pxadmin_create_tables.sql
     728ALTER TABLE diffRun CHANGE COLUMN workdir workdir VARCHAR(255) AFTER state;
     729
     730-- add exposure id to diffRun
     731ALTER TABLE diffRun ADD COLUMN exp_id BIGINT;
     732ALTER TABLE diffRun ADD FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id);
     733ALTER TABLE diffRun DROP COLUMN skycell_id;
     734
     735
     736-- drop exiting foreign key constraints
     737ALTER TABLE diffInputSkyfile drop FOREIGN KEY diffInputSkyfile_ibfk_1;
     738ALTER TABLE diffInputSkyfile drop FOREIGN KEY diffInputSkyfile_ibfk_2;
     739ALTER TABLE diffInputSkyfile drop FOREIGN KEY diffInputSkyfile_ibfk_3;
     740
     741ALTER TABLE diffInputSkyfile DROP COLUMN kind;
     742ALTER TABLE diffInputSkyfile CHANGE COLUMN skycell_id skycell_id VARCHAR(64) AFTER diff_id;
     743
     744-- WARNING WARNING
     745-- If you have existing diff runs they must be converted
     746
     747-- run the script collapse_diffinputs.pl here.
     748--        collapse_diffinputs.pl dbname dbuser dbpass dbhost
     749
     750DELETE FROM diffInputSkyfile where template = 1;
     751
     752ALTER TABLE diffInputSkyfile DROP PRIMARY KEY;
     753ALTER TABLE diffInputSkyfile ADD PRIMARY KEY (diff_id, skycell_id);
     754ALTER TABLE diffInputSkyfile DROP COLUMN template;
     755
     756ALTER TABLE diffInputSkyfile CHANGE COLUMN warp_id warp1 BIGINT;
     757ALTER TABLE diffInputSkyfile ADD COLUMN stack1 BIGINT AFTER warp1, ADD KEY(stack1);
     758ALTER TABLE diffInputSkyfile ADD COLUMN warp2 BIGINT AFTER stack1, ADD KEY(warp2);
     759ALTER TABLE diffInputSkyfile CHANGE COLUMN stack_id stack2 BIGINT AFTER warp2;
     760ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id);
     761
     762# need to check these
     763ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (warp1, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id);
     764ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (warp2, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id);
     765
     766ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (stack1) REFERENCES stackRun(stack_id);
     767ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (stack2) REFERENCES stackRun(stack_id);
     768
     769-- add unique ids for each image and flags that indicate whether the image has been
     770-- magic destreaked
     771ALTER TABLE diffSkyfile ADD COLUMN skycell_id VARCHAR(64) AFTER diff_id;
     772ALTER TABLE diffSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY (diff_id, skycell_id);
     773ALTER TABLE diffSkyfile ADD COLUMN diff_image_id BIGINT NOT NULL AUTO_INCREMENT, ADD KEY(diff_image_id);
     774ALTER TABLE diffSkyfile ADD COLUMN magicked TINYINT;
     775
     776ALTER TABLE warpRun DROP COLUMN magiced;
     777
     778ALTER TABLE warpSkyfile ADD COLUMN warp_image_id BIGINT NOT NULL AUTO_INCREMENT, ADD KEY(warp_image_id);
     779ALTER TABLE warpSkyfile ADD COLUMN magicked TINYINT;
     780
     781ALTER TABLE chipProcessedImfile ADD COLUMN chip_image_id BIGINT NOT NULL AUTO_INCREMENT, ADD KEY(chip_image_id);
     782ALTER TABLE chipProcessedImfile ADD COLUMN magicked TINYINT;
     783
     784ALTER TABLE rawImfile ADD COLUMN raw_image_id BIGINT NOT NULL AUTO_INCREMENT, ADD KEY(raw_image_id);
     785ALTER TABLE rawImfile ADD COLUMN magicked TINYINT;
     786
     787-- magic adaptations to diff reorginization
     788ALTER TABLE magicRun ADD COLUMN diff_id BIGINT AFTER exp_id;
     789ALTER TABLE magicRun ADD CONSTRAINT FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id);
     790ALTER TABLE magicRun ADD CONSTRAINT FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id);
     791ALTER TABLE magicInputSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY(magic_id, diff_id, node);
     792

  • trunk/dbconfig/chip.md

    r20240 r20973  
    8181    path_base       STR     255
    8282    fault           S16     0       # Key NOT NULL
     83    chip_image_id   S64     0       # Key AUTO_INCREMENT
     84    magicked        BOOL    f
    8385END
    8486

  • trunk/dbconfig/config.md

    r20275 r20973  
    22    pkg_name        STR     ippdb
    33    pkg_namespace   STR     ippdb
    4     pkg_version     STR     1.1.46
     4    pkg_version     STR     1.1.47
    55END

  • trunk/dbconfig/diff.md

    r20061 r20973  
    1 # $Id: diff.md,v 1.14 2008-10-11 02:33:40 price Exp $
     1# $Id: diff.md,v 1.15 2008-12-13 20:17:34 bills Exp $
    22
    33diffRun METADATA
    44    diff_id     S64         0       # Primary Key AUTO_INCREMENT
    55    state       STR         64      # Key
     6    workdir     STR         255
    67    label       STR         64      # Key
    78    reduction   STR         64      # Reduction class
    8     workdir     STR         255
    99    dvodb       STR         255
    1010    registered  TAI         NULL
    11     skycell_id  STR         64      # Key
    1211    tess_id     STR         64      # Key
     12    exp_id      S64         0       # fkey(exp_id) ref rawExp(exp_id)
    1313END
    1414
    … …  
    1717#
    1818
    19 # only ever 2 per run - one template / one not
    2019diffInputSkyfile METADATA
    2120    diff_id     S64         0       # Primary Key fkey(diff_id) ref diffRun(diff_id)
    22     template    BOOL        f       # Primary Key
    23     stack_id    S64         0       # fkey(stack_id) ref stackSumSkyfile(stack_id)
    24     warp_id     S64         0       # fkey(warp_id, skycell_id, tess_id) ref warpSkyfile(warp_id, skycell_id, tess_id)
    25     skycell_id  STR         64      # Key
     21    skycell_id  STR         64      # Primary Key
     22    warp1       S64         0       # fkey(warp1, skycell_id, tess_id) ref warpSkyfile(warp_id, skycell_id, tess_id)
     23    stack1      S64         0       # fkey(stack1) ref stackSumSkyfile(stack_id)
     24    warp2       S64         0       # fkey(warp2, skycell_id, tess_id) ref warpSkyfile(warp_id, skycell_id, tess_id)
     25    stack2      S64         0       # fkey(stack2) ref stackSumSkyfile(stack_id)
    2626    tess_id     STR         64      # Key
    27 # either a input or a template
    28     kind        STR         64      # Key
    2927END
    3028
    3129diffSkyfile METADATA
    3230    diff_id      S64        0       # Primary Key fkey(diff_id) ref diffRun(diff_id)
     31    skycell_id   STR        64      #
    3332    uri          STR        255
    3433    path_base    STR        255
    … …  
    5352    good_frac    F32        0.0     # Key
    5453    fault        S16        0       # Key
     54    diff_image_id S64       0       # Key NOT NULL AUTO_INCREMENT
     55    magicked     BOOL       f
    5556END

  • trunk/dbconfig/magic.md

    r20841 r20973  
    1 # $Id: magic.md,v 1.12 2008-11-26 03:21:13 bills Exp $
     1# $Id: magic.md,v 1.13 2008-12-13 20:17:34 bills Exp $
    22
    33### Fault in magicRun indicates that the processing tree failed
    … …  
    55    magic_id    S64         0       # Primary Key AUTO_INCREMENT
    66    exp_id      S64         0       # Key
     7    diff_id     S64         0       # Key
    78    state       STR         64      # Key
    89    workdir     STR         255

  • trunk/dbconfig/tasks.md

    r20608 r20973  
    1 # $Id: tasks.md,v 1.158 2008-11-09 23:51:01 price Exp $
     1# $Id: tasks.md,v 1.159 2008-12-13 20:17:34 bills Exp $
    22
    33# this table records all exposure ID ever seen from the summit
    … …  
    239239    fault       S16         0       # Key NOT NULL
    240240    epoch       UTC         0001-01-01T00:00:00Z
    241 END
     241    magicked    BOOL        f
     242END

  • trunk/dbconfig/warp.md

    r19930 r20973  
    1 # $Id: warp.md,v 1.19 2008-10-07 00:01:28 price Exp $
     1# $Id: warp.md,v 1.20 2008-12-13 20:17:34 bills Exp $
    22
    33#
    … …  
    1111warpRun METADATA
    1212    warp_id     S64         0       # Primary Key AUTO_INCREMENT
    13     fake_id      S64         0       # Key INDEX(warp_id, fake_id) fkey(fake_id) ref camProcessedExp(fake_id)
     13    fake_id     S64         0       # Key INDEX(warp_id, fake_id) fkey(fake_id) ref camProcessedExp(fake_id)
    1414    mode        STR         64      # Key
    1515    state       STR         64      # Key
    … …  
    2222    registered  TAI         NULL
    2323# if magic is T then look for the exp_id in the magic output tables
    24     magiced     BOOL        f       # Key
    2524END
    2625
    … …  
    5251    ignored        BOOL     f       # Key
    5352    fault          S16      0       # Key
     53    warp_image_id  S64      0       # Key AUTO_INCREMENT
     54    magicked       BOOL     f
    5455END
    5556

  • trunk/ippScripts/scripts/diff_skycell.pl

    r20928 r20973  
    2727use Pod::Usage qw( pod2usage );
    2828
    29 my ($diff_id, $diff_skyfile_id, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect);
     29my ($diff_id, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect);
     30my $skycell_id;                 # Skycell identifier
    3031GetOptions(
    31     'diff_id|d=s'         => \$diff_id, # Diff identifier
    32     'diff_skyfile_id|d=s' => \$diff_skyfile_id, # Unique skyfile identifier
    33     'dbname|d=s'          => \$dbname, # Database name
    34     'threads=s'           => \$threads,   # Number of threads to use
    35     'outroot=s'           => \$outroot, # Output root name
    36     'reduction=s'         => \$reduction, # Reduction class
    37     'verbose'             => \$verbose,   # Print to stdout
    38     'no-update'           => \$no_update, # Don't update the database?
    39     'no-op'               => \$no_op, # Don't do any operations?
    40     'redirect-output'     => \$redirect,
     32    'diff_id=s'         => \$diff_id, # Diff identifier
     33    'skycell_id=s'      => \$skycell_id, # Diff identifier
     34    'dbname|d=s'        => \$dbname, # Database name
     35    'threads=s'         => \$threads,   # Number of threads to use
     36    'outroot=s'         => \$outroot, # Output root name
     37    'reduction=s'       => \$reduction, # Reduction class
     38    'verbose'           => \$verbose,   # Print to stdout
     39    'no-update'         => \$no_update, # Don't update the database?
     40    'no-op'             => \$no_op, # Don't do any operations?
     41    'redirect-output'   => \$redirect,
    4142) or pod2usage( 2 );
    4243
    4344pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    4445pod2usage(
    45     -msg => "Required options: --diff_id --outroot",
     46    -msg => "Required options: --diff_id --skycell_id --outroot",
    4647    -exitval => 3,
    4748          ) unless defined $diff_id
     49    and defined $skycell_id
    4850    and defined $outroot;
    4951
    … …  
    5355my $logDest = "$outroot.log";
    5456$ipprc->redirect_output($logDest) if $redirect;
    55 
    56 my $image_id = $diff_skyfile_id;
    57 my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF);
    5857
    5958# Look for programs we need
    … …  
    7170my $files;
    7271{
    73     my $command = "$difftool -inputskyfile -diff_id $diff_id";
     72    my $command = "$difftool -inputskyfile -diff_id $diff_id -skycell_id $skycell_id";
    7473    $command .= " -dbname $dbname" if defined $dbname;
    7574    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    9392my ($template, $templateMask, $templateWeight, $templatePath, $templateSources); # Template files and path
    9493my $tess_id;                    # Tesselation identifier
    95 my $skycell_id;                 # Skycell identifier
    9694my $camera;                     # Camera
    9795foreach my $file (@$files) {
    … …  
    214212    $command .= " -dumpconfig $configuration";
    215213    $command .= " -dbname $dbname" if defined $dbname;
    216     # $command .= " -image_id $image_id" if defined $image_id;
    217     # $command .= " -source_id $source_id" if defined $source_id;
    218214
    219215    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    226222    &my_die("Couldn't find expected output file: $outputMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
    227223    &my_die("Couldn't find expected output file: $outputWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
    228     &my_die("Couldn't find expected output file: $outputSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
     224#    &my_die("Couldn't find expected output file: $outputSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
    229225#    &my_die("Couldn't find expected output file: $bin1Name",    $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name);
    230226#    &my_die("Couldn't find expected output file: $bin2Name",    $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name);
    … …  
    251247    # Add the subtraction result
    252248    {
    253         my $command = "$difftool -adddiffskyfile -diff_id $diff_id -uri $outputName -path_base $outroot";
     249        my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -uri $outputName -path_base $outroot";
    254250        $command .= " $cmdflags";
    255251        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    … …  
    275271    warn($msg);
    276272    if (defined $diff_id and not $no_update) {
    277         my $command = "$difftool -adddiffskyfile -diff_id $diff_id -code $exit_code";
     273        my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -code $exit_code";
    278274        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    279275        $command .= " -hostname $host" if defined $host;

  • trunk/ippTasks/diff.pro

    r20932 r20973  
    9898  task.exit    0
    9999    # convert 'stdout' to book format
    100     ipptool2book stdout diffSkyfile -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     100    ipptool2book stdout diffSkyfile -key diff_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    101101    if ($VERBOSE > 2)
    102102      book listbook diffSkyfile
    … …  
    157157    stderr $LOGDIR/diff.skycell.log
    158158
    159     $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --diff_skyfile_id $DIFF_SKYFILE_ID --outroot $outroot --redirect-output
     159    $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --skycell_id $SKYCELL_ID --outroot $outroot --redirect-output
    160160    add_standard_args run
    161161
    … …  
    215215  task.exit    0
    216216    # convert 'stdout' to book format
    217     ipptool2book stdout diffCleanup -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     217    ipptool2book stdout diffCleanup -key diff_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    218218    if ($VERBOSE > 2)
    219219      book listbook diffCleanup

  • trunk/ippTasks/simtest.auto

    r20909 r20973  
    7070END
    7171
    72 #### Magic automation???
     72### Magic automation???
    7373#automate METADATA
    7474#  name       STR MAGIC
    75 #  regular    STR "magictool -definebyquery -workdir file://@CWD@/magic -good_frac 0.2 -dbname @DBNAME@"
     75#  regular    STR "magictool -definebyquery -workdir file://@CWD@/magic -dbname @DBNAME@"
    7676#END
    77 
    7877
    7978

  • trunk/ippTools/configure.ac

    r20276 r20973  
    1818PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
    1919PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
    20 PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.46])
     20PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.47])
    2121
    2222PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}"

  • trunk/ippTools/share/Makefile.am

    r20841 r20973  
    7070     dettool_toresidimfile.sql \
    7171     dettool_tostacked.sql \
    72      difftool_definebyquery.sql \
     72     difftool_completed_runs.sql \
     73     difftool_definebyquery_part1.sql \
     74     difftool_definebyquery_part2.sql \
     75     difftool_definebyquery_temp_create.sql \
    7376     difftool_donecleanup.sql \
    7477     difftool_inputskyfile.sql \

  • trunk/ippTools/share/difftool_definebyquery.sql

    r20682 r20973  
    99    warpsToDiff.good_frac,
    1010    warpsToDiff.diff_id,
    11     warpsToDiff.kind,
    1211    current_stack_id,
    13     best_stack_id
     12    best_stack_id,
     13    exp_id
    1414FROM (
    1515    -- Get list of warps that can be diffed, with any associated diff
    … …  
    2323        warpRun.label as warp_label,
    2424        diffInputs.diff_id,
    25         diffInputs.kind,
    26         diffTemplates.stack_id AS current_stack_id
     25        diffInputs.stack2 AS current_stack_id,
     26        rawExp.exp_id
    2727    FROM warpSkyfile
    2828    JOIN warpRun USING(warp_id)
    … …  
    3333    -- Check if it has an associated diff
    3434    LEFT JOIN diffInputSkyfile AS diffInputs
    35         ON diffInputs.warp_id = warpSkyfile.warp_id
     35        ON diffInputs.warp1 = warpSkyfile.warp_id
    3636        AND diffInputs.skycell_id = warpSkyfile.skycell_id
    37         AND diffInputs.template = 0 -- only join input files
    38     -- Get the stack_id currently used as a template, if any
    39     LEFT JOIN diffInputSkyfile AS diffTemplates
    40         ON diffTemplates.diff_id = diffInputs.diff_id
    41         AND diffTemplates.template != 0 -- only join template files
     37        AND diffInputs.stack2 IS NOT NULL
    4238    WHERE
    4339        warpSkyfile.fault = 0

  • trunk/ippTools/share/difftool_inputskyfile.sql

    r19582 r20973  
    11SELECT * FROM
    2     (SELECT
     2    (SELECT
     3        -- warp input
    34        diffRun.diff_id,
    4         diffRun.skycell_id,
    5         diffRun.tess_id,
     5        diffInputSkyfile.skycell_id,
     6        diffInputSkyfile.tess_id,
    67        0 as stack_id,
    78        warpSkyfile.warp_id,
    89        warpSkyfile.uri,
    910        warpSkyfile.path_base,
    10         diffInputSkyfile.template,
     11        0 as template,
    1112        rawExp.camera
    1213    FROM diffRun
    … …  
    1415        USING(diff_id)
    1516    JOIN warpSkyfile
    16         ON  diffInputSkyfile.warp_id    = warpSkyfile.warp_id
     17        ON  diffInputSkyfile.warp1      = warpSkyfile.warp_id
    1718        AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
    1819        AND diffInputSkyfile.tess_id    = warpSkyfile.tess_id
    1920    JOIN warpRun
    20         ON diffInputSkyfile.warp_id = warpRun.warp_id
     21        ON diffInputSkyfile.warp1 = warpRun.warp_id
     22    JOIN fakeRun
     23        USING(fake_id)
     24    JOIN camRun
     25        USING(cam_id)
     26    JOIN chipRun
     27        USING(chip_id)
     28    JOIN chipProcessedImfile
     29        USING(chip_id)
     30    JOIN rawExp
     31        ON chipRun.exp_id = rawExp.exp_id
     32    WHERE
     33        diffRun.state = 'new'
     34        AND warpRun.state = 'full'
     35        AND fakeRun.state = 'full'
     36        AND camRun.state = 'full'
     37        AND chipRun.state = 'full'
     38        -- where hook %s
     39    UNION
     40    SELECT
     41        -- warp template
     42        diffRun.diff_id,
     43        diffInputSkyfile.skycell_id,
     44        diffInputSkyfile.tess_id,
     45        0 as stack_id,
     46        warpSkyfile.warp_id,
     47        warpSkyfile.uri,
     48        warpSkyfile.path_base,
     49        0 as template,
     50        rawExp.camera
     51    FROM diffRun
     52    JOIN diffInputSkyfile
     53        USING(diff_id)
     54    JOIN warpSkyfile
     55        ON  diffInputSkyfile.warp2      = warpSkyfile.warp_id
     56        AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
     57        AND diffInputSkyfile.tess_id    = warpSkyfile.tess_id
     58    JOIN warpRun
     59        ON diffInputSkyfile.warp2 = warpRun.warp_id
    2160    JOIN fakeRun
    2261        USING(fake_id)
    … …  
    3877    UNION
    3978    SELECT
     79        -- stack input
    4080        diffRun.diff_id,
    41         diffRun.skycell_id,
    42         diffRun.tess_id,
     81        diffInputSkyfile.skycell_id,
     82        diffInputSkyfile.tess_id,
    4383        stackSumSkyfile.stack_id,
    4484        0 as warp_id,
    4585        stackSumSkyfile.uri,
    4686        stackSumSkyfile.path_base,
    47         diffInputSkyfile.template,
     87        1 as template,
    4888        rawExp.camera
    4989    FROM diffRun
    … …  
    5191        USING(diff_id)
    5292    JOIN stackSumSkyfile
    53         ON  diffInputSkyfile.stack_id = stackSumSkyfile.stack_id
    54     JOIN stackInputSkyfile
    55         ON diffInputSkyfile.stack_id = stackInputSkyfile.stack_id
    56     JOIN warpRun
    57         ON stackInputSkyfile.warp_id = warpRun.warp_id
    58     JOIN fakeRun
    59         USING(fake_id)
    60     JOIN camRun
    61         USING(cam_id)
    62     JOIN chipRun
    63         USING(chip_id)
    64     JOIN chipProcessedImfile
    65         USING(chip_id)
     93        ON  diffInputSkyfile.stack1 = stackSumSkyfile.stack_id
    6694    JOIN rawExp
    67         ON chipRun.exp_id = rawExp.exp_id
     95        USING(exp_id)
    6896    WHERE
    6997        diffRun.state = 'new'
    70         AND warpRun.state = 'full'
    71         AND fakeRun.state = 'full'
    72         AND camRun.state = 'full'
    73         AND chipRun.state = 'full'
     98        -- where hook %s
     99    UNION
     100    SELECT
     101        -- stack template
     102        diffRun.diff_id,
     103        diffInputSkyfile.skycell_id,
     104        diffInputSkyfile.tess_id,
     105        stackSumSkyfile.stack_id,
     106        0 as warp_id,
     107        stackSumSkyfile.uri,
     108        stackSumSkyfile.path_base,
     109        1 as template,
     110        rawExp.camera
     111    FROM diffRun
     112    JOIN diffInputSkyfile
     113        USING(diff_id)
     114    JOIN stackSumSkyfile
     115        ON  diffInputSkyfile.stack2 = stackSumSkyfile.stack_id
     116    JOIN rawExp
     117        USING(exp_id)
     118    WHERE
     119        diffRun.state = 'new'
    74120        -- where hook %s
    75121    ) as Foo

  • trunk/ippTools/share/difftool_skyfile.sql

    r20694 r20973  
    11SELECT
    2     diffRun.skycell_id,
     2    diffSkyfile.*,
    33    diffRun.tess_id,
    44    diffRun.state,
    5     diffSkyfile.*,
    6     (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
    7         AND diffInputSkyfile.template = 0 ) AS warp_id_temp_0,
    8     (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
    9         AND diffInputSkyfile.template = 0 ) AS stack_id_temp_0,
    10     (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
    11         AND diffInputSkyfile.template = 1 ) AS warp_id_temp_1,
    12     (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
    13         AND diffInputSkyfile.template = 1 ) AS stack_id_temp_1
     5    warp1,
     6    stack1,
     7    warp2,
     8    stack2
     9--    (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
     10--        AND diffInputSkyfile.template = 0 ) AS warp_id_temp_0,
     11--    (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
     12--        AND diffInputSkyfile.template = 0 ) AS stack_id_temp_0,
     13--    (SELECT warp_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
     14--        AND diffInputSkyfile.template = 1 ) AS warp_id_temp_1,
     15--    (SELECT stack_id FROM diffInputSkyfile WHERE diffInputSkyfile.diff_id = diffRun.diff_id
     16--        AND diffInputSkyfile.template = 1 ) AS stack_id_temp_1
    1417FROM diffRun
    1518JOIN diffSkyfile
    … …  
    1720JOIN diffInputSkyfile
    1821    ON diffInputSkyfile.diff_id = diffRun.diff_id
    19     AND diffInputSkyfile.template = 0
    2022JOIN warpRun
    21     USING(warp_id)
    22 JOIN fakeRun
    23     USING(fake_id)
    24 JOIN camRun
    25     USING(cam_id)
    26 JOIN chipRun
    27     USING(chip_id)
     23    ON warpRun.warp_id = diffInputSkyfile.warp1
    2824JOIN rawExp
    2925    USING(exp_id)

  • trunk/ippTools/share/difftool_todiffskyfile.sql

    r20934 r20973  
    66    0 as diff_skyfile_id,
    77    diffRun.workdir,
    8     diffRun.skycell_id,
     8    diffInputSkyfile.skycell_id,
    99    diffRun.tess_id,
    1010    diffRun.label,
    1111    diffRun.state
    1212FROM diffRun
     13JOIN diffInputSkyfile USING(diff_id)
     14
    1315-- Get list of templates for each diffRun
    14 JOIN diffInputSkyfile AS diffTemplateSkyfile
    15     ON diffRun.diff_id = diffTemplateSkyfile.diff_id
    16     AND diffRun.skycell_id = diffTemplateSkyfile.skycell_id
    17     AND diffTemplateSkyfile.template = 1
     16-- JOIN diffInputSkyfile AS diffTemplateSkyfile
     17--     ON diffRun.diff_id = diffTemplateSkyfile.diff_id
     18--     AND diffRun.skycell_id = diffTemplateSkyfile.skycell_id
     19--     AND diffTemplateSkyfile.template = 1
    1820-- Get list of inputs for each diffRun
    19 JOIN diffInputSkyfile
    20     ON diffRun.diff_id = diffInputSkyfile.diff_id
    21     AND diffRun.skycell_id = diffInputSkyfile.skycell_id
    22     AND diffInputSkyfile.template = 0
     21-- JOIN diffInputSkyfile
     22--     ON diffRun.diff_id = diffInputSkyfile.diff_id
     23--     AND diffRun.skycell_id = diffInputSkyfile.skycell_id
     24--     AND diffInputSkyfile.template = 0
     25
    2326-- Get warp templates
    2427LEFT JOIN warpRun AS warpTemplateRun
    25     ON warpTemplateRun.warp_id = diffTemplateSkyfile.warp_id
    26     AND diffTemplateSkyfile.warp_id IS NOT NULL
     28    ON warpTemplateRun.warp_id = diffInputSkyfile.warp2
     29    AND diffInputSkyfile.warp2 IS NOT NULL
    2730LEFT JOIN warpSkyfile AS warpTemplateSkyfile
    2831    ON warpTemplateSkyfile.warp_id = warpTemplateRun.warp_id
    29     AND warpTemplateSkyfile.skycell_id = diffTemplateSkyfile.skycell_id
     32    AND warpTemplateSkyfile.skycell_id = diffInputSkyfile.skycell_id
     33
    3034-- Get warp inputs
    3135LEFT JOIN warpRun
    32     ON warpRun.warp_id = diffInputSkyfile.warp_id
    33     AND diffInputSkyfile.warp_id IS NOT NULL
     36    ON warpRun.warp_id = diffInputSkyfile.warp1
     37    AND diffInputSkyfile.warp1 IS NOT NULL
    3438LEFT JOIN warpSkyfile
    3539    ON warpSkyfile.warp_id = warpRun.warp_id
    3640    AND warpSkyfile.skycell_id = diffInputSkyfile.skycell_id
     41
    3742-- Get stack templates
    3843LEFT JOIN stackRun AS stackTemplateRun
    39     ON stackTemplateRun.stack_id = diffTemplateSkyfile.stack_id
     44    ON stackTemplateRun.stack_id = diffInputSkyfile.stack2
     45    AND diffInputSkyfile.stack2 IS NOT NULL
    4046LEFT JOIN stackSumSkyfile AS stackTemplateSkyfile
    4147    ON stackTemplateSkyfile.stack_id = stackTemplateRun.stack_id
     48
    4249-- Get stack inputs
    4350LEFT JOIN stackRun
    44     ON stackRun.stack_id = diffInputSkyfile.stack_id
    45     AND diffInputSkyfile.warp_id IS NULL
     51    ON stackRun.stack_id = diffInputSkyfile.stack1
     52    AND diffInputSkyfile.stack1 IS NOT NULL
    4653LEFT JOIN stackSumSkyfile
    4754    ON stackSumSkyfile.stack_id = stackRun.stack_id
    48     AND diffInputSkyfile.warp_id IS NULL
     55
    4956-- Get what's already been processed
    5057LEFT JOIN diffSkyfile
    5158    ON diffInputSkyfile.diff_id = diffSkyfile.diff_id
     59    AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id
    5260WHERE
    5361-- Ready to be processed
    … …  
    5866    )
    5967-- Ensure input warps are available
    60     AND (diffInputSkyfile.warp_id IS NULL
     68    AND (diffInputSkyfile.warp1 IS NULL
    6169    OR (warpRun.state = 'full'
    6270    AND warpSkyfile.fault = 0
    6371    AND warpSkyfile.ignored = 0))
    6472-- Ensure input stacks are available
    65     AND (diffInputSkyfile.stack_id IS NULL
     73    AND (diffInputSkyfile.stack1 IS NULL
    6674    OR (stackRun.state = 'full'
    6775    AND stackSumSkyfile.fault = 0))
    6876-- Ensure template warps are available
    69     AND (diffTemplateSkyfile.warp_id IS NULL
     77    AND (diffInputSkyfile.warp2 IS NULL
    7078    OR (warpTemplateRun.state = 'full'
    7179    AND warpTemplateSkyfile.fault = 0
    7280    AND warpTemplateSkyfile.ignored = 0))
    7381-- Ensure template stacks are available
    74     AND (diffTemplateSkyfile.stack_id IS NULL
     82    AND (diffInputSkyfile.stack2 IS NULL
    7583    OR (stackTemplateRun.state = 'full'
    7684    AND stackTemplateSkyfile.fault = 0))

  • trunk/ippTools/share/magicdstool_completed_runs.sql

    r20841 r20973  
    6565    JOIN magicRun USING (magic_id)
    6666    JOIN magicInputSkyfile USING(magic_id)
    67     -- Do we really need to join back to diffInputSkyfile here?
    68     JOIN diffRun USING(diff_id)
    69     JOIN diffSkyfile USING(diff_id)
    70     JOIN diffInputSkyfile USING(diff_id)
     67    JOIN diffSkyfile
     68        ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
     69        AND magicInputSkyfile.node = diffSkyfile.skycell_id
    7170    LEFT JOIN magicDSFile
    7271        ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    73         AND magicDSFile.component = diffInputSkyfile.skycell_id
     72        AND magicDSFile.component = diffSkyfile.skycell_id
    7473    WHERE
    7574        magicDSRun.state = 'run'
    7675        AND magicDSRun.stage = 'diff'
    77         AND diffRun.state = 'full'
    7876        AND diffSkyfile.fault = 0
    7977    GROUP BY

  • trunk/ippTools/share/magicdstool_getrunids.sql

    r20846 r20973  
    88FROM magicRun
    99JOIN magicInputSkyfile USING(magic_id)
    10 JOIN diffInputSkyfile USING(diff_id)
    11 JOIN warpRun USING(warp_id)
     10JOIN diffSkyfile
     11    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
     12    AND magicInputSkyfile.node = diffSkyfile.skycell_id
     13JOIN diffInputSkyfile
     14    ON diffInputSkyfile.diff_id = diffSkyfile.diff_id
     15    AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id
     16JOIN warpRun
     17    ON diffInputSkyfile.warp1 = warp_id
    1218JOIN fakeRun USING(fake_id)
    1319JOIN camRun USING(cam_id)
    1420JOIN chipRun USING(chip_id)
    15 WHERE template  = 0
    16 AND magic_id = %ld
     21WHERE magic_id = %ld

  • trunk/ippTools/share/magicdstool_getskycells.sql

    r20782 r20973  
    11SELECT DISTINCT
    22    diffSkyfile.diff_id,
    3     diffRun.skycell_id,
     3    diffSkyfile.skycell_id,
    44    diffSkyfile.uri,
    55    diffSkyfile.path_base
    66FROM magicDSRun
    77JOIN magicRun USING(magic_id)
    8 JOIN magicInputSkyfile USING(magic_id)
    9 JOIN diffRun USING(diff_id)
    10 JOIN diffSkyfile USING(diff_id)
     8JOIN magicInputSkyfile USING(magic_id, diff_id)
     9JOIN diffSkyfile
     10    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
     11    AND magicInputSkyfile.node = diffSkyfile.skycell_id
    1112JOIN diffInputSkyfile
    12     ON diffInputSkyfile.diff_id = diffRun.diff_id
    13     AND diffInputSkyfile.skycell_id = diffRun.skycell_id
     13    ON diffInputSkyfile.diff_id = diffSkyfile.diff_id
     14    AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id
    1415    -- Want input warps only
    15     AND diffInputSkyfile.warp_id IS NOT NULL
    16     AND diffInputSkyfile.template = 0
     16    AND diffInputSkyfile.warp1 IS NOT NULL
    1717JOIN warpSkyCellMap
    18     ON warpSkyCellMap.warp_id = diffInputSkyfile.warp_id
     18    ON warpSkyCellMap.warp_id = diffInputSkyfile.warp1
    1919    AND warpSkyCellMap.skycell_id = diffInputSkyfile.skycell_id
    2020JOIN warpSkyfile
    … …  
    2323    AND warpSkyfile.ignored = 0
    2424WHERE
    25     diffRun.state = 'full'
    26     AND diffSkyfile.fault = 0
     25    diffSkyfile.fault = 0
    2726    AND magic_ds_id = %lld

  • trunk/ippTools/share/magicdstool_todestreak.sql

    r20841 r20973  
    106106    magicMask.uri as streaks_uri,
    107107    stage,
    108     diff_id as stage_id,
    109     diffInputSkyfile.skycell_id as component,
     108    magicRun.diff_id as stage_id,
     109    diffSkyfile.skycell_id as component,
    110110    diffSkyfile.uri,
    111111    diffSkyfile.path_base,
    … …  
    120120JOIN magicDSRun USING(magic_id)
    121121JOIN magicInputSkyfile USING(magic_id)
    122 JOIN diffRun USING(diff_id)
    123 JOIN diffSkyfile USING(diff_id)
    124 JOIN diffInputSkyfile USING(diff_id)
     122JOIN diffSkyfile
     123    ON  magicInputSkyfile.diff_id = diffSkyfile.diff_id
     124    AND magicInputSkyfile.node = diffSkyfile.skycell_id
    125125LEFT JOIN magicDSFile
    126126    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    127     AND magicDSFile.component = diffInputSkyfile.skycell_id
     127    AND magicDSFile.component = diffSkyfile.skycell_id
    128128WHERE
    129129    magicDSRun.state = 'run'
    130130    AND magicDSRun.stage = 'diff'
    131     AND diffRun.state = 'full'
    132131    AND diffSkyfile.fault = 0
    133132    AND magicDSFile.component IS NULL

  • trunk/ippTools/share/magictool_definebyquery_insert.sql

    r18700 r20973  
    55    diff_id,
    66    skycell_id
    7 FROM magicBestDiffs
     7FROM diffSkyfile
    88WHERE
    9     exp_id = @EXP_ID@ -- Update this with the appropriate exp_id
     9    diff_id = @DIFF_ID@ -- Update this with the appropriate diff_id
     10    AND fault = 0

  • trunk/ippTools/share/magictool_definebyquery_select.sql

    r20742 r20973  
    11-- Get a list of exposures on which magic may be performed
    2 SELECT DISTINCT
     2SELECT
    33    exp_id,
    4     filter,
    5     num_todo,
    6     num_done,
    7     MAX(magic_id)
    8 FROM (
    9     -- Number of skycells as a function of exposure
    10     SELECT
    11         exp_id,
    12         filter,
    13         COUNT(DISTINCT warpSkyfile.tess_id,warpSkyfile.skycell_id) AS num_todo
    14     FROM rawExp
    15     JOIN chipRun USING(exp_id)
    16     JOIN camRun USING(chip_id)
    17     JOIN fakeRun USING(cam_id)
    18     JOIN warpRun USING(fake_id)
    19     JOIN warpSkyCellMap USING(warp_id)
    20     JOIN warpSkyfile USING(warp_id, skycell_id)
    21     JOIN diffInputSkyfile USING(warp_id,skycell_id)
    22     JOIN diffRun USING(diff_id)
    23     WHERE
    24         warpSkyfile.ignored = 0
    25         -- magicSkycellNums WHERE hook %s
    26     GROUP BY
    27         exp_id
    28     ) AS magicSkycellNums
    29 JOIN (
    30     -- Number of completed diffs for an exposure
    31     SELECT
    32         exp_id,
    33         COUNT(diff_id) AS num_done
    34     FROM magicBestDiffs
    35     GROUP BY
    36         exp_id
    37     ) AS magicDiffNums USING(exp_id)
     4    MAX(diffRun.diff_id) AS diff_id
     5FROM diffRun
     6JOIN rawExp USING(exp_id)
    387LEFT JOIN magicRun USING(exp_id)
     8-- WHERE hook %s
     9GROUP BY exp_id

  • trunk/ippTools/share/magictool_definebyquery_temp_insert.sql

    r20738 r20973  
    1414JOIN warpSkyfile USING(warp_id, skycell_id)
    1515JOIN diffInputSkyfile
    16     ON diffInputSkyfile.warp_id = warpSkyfile.warp_id
     16    ON diffInputSkyfile.warp1 = warpSkyfile.warp_id
    1717    AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
    18     AND diffInputSkyfile.template = 0 -- selecting inputs only
    1918JOIN diffRun USING(diff_id)
    2019JOIN diffSkyfile USING(diff_id)

  • trunk/ippTools/share/magictool_inputs.sql

    r20695 r20973  
    1010    diffSkyfile.path_base,
    1111    diffSkyfile.fault
    12 FROM magicInputSkyfile
     12FROM magicRun
     13JOIN magicInputSkyfile
     14USING(magic_id)
    1315JOIN diffSkyfile
    14     USING(diff_id)
    15 JOIN magicRun
    16     USING(magic_id)
     16    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
     17    AND magicInputSkyfile.node = diffSkyfile.skycell_id
    1718UNION
    1819-- Merged skycells
    … …  
    2122    magicRun.state,
    2223    magicTree.node,
    23     0,
     24    0,   -- no diff_id
    2425    magicNodeResult.uri,
    2526    NULL, -- magicNodeResult doesn't have a path_base

  • trunk/ippTools/share/magictool_inputskyfile.sql

    r20696 r20973  
    22    magicInputSkyfile.*,
    33    diffSkyfile.uri
    4 FROM magicInputSkyfile
    5 JOIN magicRun
     4FROM magicRun
     5JOIN magicInputSkyfile
    66    USING(magic_id)
    77JOIN diffSkyfile
    8     USING(diff_id)
     8    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
     9    AND magicInputSkyfile.node = diffSkyfile.skycell_id
    910WHERE
    1011    magicRun.state = 'run'

  • trunk/ippTools/share/magictool_toprocess_inputs.sql

    r20697 r20973  
    1111JOIN magicRun USING(magic_id)
    1212JOIN magicInputSkyfile USING(magic_id, node)
    13 JOIN diffSkyfile USING(diff_id)
     13JOIN diffSkyfile
     14    ON magicRun.diff_id = diffSkyfile.diff_id
     15    AND magicInputSkyfile.node = diffSkyfile.skycell_id
    1416JOIN rawExp USING(exp_id)
    1517LEFT JOIN magicNodeResult

  • trunk/ippTools/share/pxadmin_create_tables.sql

    r20841 r20973  
    238238    fault SMALLINT NOT NULL,
    239239    epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     240    raw_image_id BIGINT AUTO_INCREMENT,
     241    magicked TINYINT,
    240242    PRIMARY KEY(exp_id, class_id),
    241243    KEY(tmp_class_id),
    242244    KEY(fault),
     245    KEY(raw_image_id),
    243246    UNIQUE KEY(exp_id, tmp_class_id),
    244247    FOREIGN KEY (exp_id, tmp_class_id)
    … …  
    330333    path_base VARCHAR(255),
    331334    fault SMALLINT NOT NULL,
     335    chip_image_id BIGINT AUTO_INCREMENT,
     336    magicked BIGINT,
    332337    PRIMARY KEY(chip_id, exp_id, class_id),
    333338    KEY(data_state),
    334339    KEY(fault),
     340    KEY(chip_image_id),
    335341    FOREIGN KEY (chip_id, exp_id)
    336342        REFERENCES  chipRun(chip_id, exp_id),
    … …  
    815821    end_stage VARCHAR(64),
    816822    registered DATETIME,
    817     magiced TINYINT,
    818823    PRIMARY KEY(warp_id),
    819824    KEY(warp_id),
    … …  
    824829    KEY(label),
    825830    KEY(end_stage),
    826     KEY(magiced),
    827831    INDEX(warp_id, fake_id),
    828832    FOREIGN KEY (fake_id)
    … …  
    861865    ignored TINYINT,
    862866    fault SMALLINT,
     867    warp_image_id BIGINT AUTO_INCREMENT,
     868    magicked TINYINT,
    863869    PRIMARY KEY(warp_id, skycell_id, tess_id),
    864870    KEY(good_frac),
    865871    KEY(ignored), KEY(fault),
     872    KEY(warp_image_id),
    866873    FOREIGN KEY (warp_id, skycell_id, tess_id)
    867874        REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id)
    … …  
    941948        dvodb VARCHAR(255),
    942949        registered DATETIME,
    943         skycell_id VARCHAR(64),
    944950        tess_id VARCHAR(64),
     951        exp_id  BIGINT,
    945952        PRIMARY KEY(diff_id),
    946953        KEY(diff_id),
    947954        KEY(state),
    948         KEY(skycell_id),
    949         KEY(tess_id)
     955        KEY(tess_id),
     956        FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id)
    950957) ENGINE=innodb DEFAULT CHARSET=latin1;
    951958
    952959CREATE TABLE diffInputSkyfile (
    953960        diff_id BIGINT,
    954         template TINYINT,
    955         stack_id BIGINT,
    956         warp_id BIGINT,
    957961        skycell_id VARCHAR(64),
     962        warp1 BIGINT,
     963        stack1 BIGINT,
     964        warp2 BIGINT,
     965        stack2 BIGINT,
    958966        tess_id VARCHAR(64),
    959         kind VARCHAR(64),
    960         PRIMARY KEY(diff_id, template),
    961         KEY(stack_id),
    962         KEY(warp_id),
     967        PRIMARY KEY(diff_id, skycell_id),
     968        KEY(warp1),
     969        KEY(warp2),
     970        KEY(stack1),
     971        KEY(stack2),
    963972        KEY(skycell_id),
    964973        KEY(tess_id),
    965         KEY(kind),
    966974        FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id),
    967         FOREIGN KEY (stack_id)  REFERENCES  stackSumSkyfile(stack_id),
    968         FOREIGN KEY (warp_id, skycell_id, tess_id)  REFERENCES  warpSkyfile(warp_id, skycell_id, tess_id)
     975        FOREIGN KEY (warp1, skycell_id, tess_id)  REFERENCES  warpSkyfile(warp_id, skycell_id, tess_id),
     976        FOREIGN KEY (warp2, skycell_id, tess_id)  REFERENCES  warpSkyfile(warp_id, skycell_id, tess_id),
     977        FOREIGN KEY (stack1)  REFERENCES  stackSumSkyfile(stack_id),
     978        FOREIGN KEY (stack2)  REFERENCES  stackSumSkyfile(stack_id)
    969979) ENGINE=innodb DEFAULT CHARSET=latin1;
    970980
    971981CREATE TABLE diffSkyfile (
    972982        diff_id BIGINT,
     983        skycell_id VARCHAR(64),
    973984        uri VARCHAR(255),
    974985        path_base VARCHAR(255),
    … …  
    9931004        good_frac FLOAT,
    9941005        fault SMALLINT,
    995         PRIMARY KEY(diff_id),
     1006        diff_image_id BIGINT AUTO_INCREMENT,
     1007        magicked TINYINT,
     1008        PRIMARY KEY(diff_id, skycell_id),
    9961009        KEY(good_frac),
    9971010        KEY(fault),
     1011        KEY(diff_image_id),
    9981012        FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id)
    9991013) ENGINE=innodb DEFAULT CHARSET=latin1;
    … …  
    10021016        magic_id BIGINT AUTO_INCREMENT,
    10031017        exp_id BIGINT,
     1018        diff_id BIGINT,
    10041019        state VARCHAR(64),
    10051020        workdir VARCHAR(255),
    … …  
    10151030        KEY(label),
    10161031        KEY(fault),
    1017         FOREIGN KEY (exp_id)  REFERENCES  rawExp(exp_id)
     1032        FOREIGN KEY (exp_id)  REFERENCES rawExp(exp_id),
     1033        FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id)
    10181034) ENGINE=innodb DEFAULT CHARSET=latin1;
    10191035
    … …  
    10221038        diff_id BIGINT,
    10231039        node VARCHAR(64),
    1024         PRIMARY KEY(magic_id, diff_id),
     1040        PRIMARY KEY(magic_id, diff_id, node),
    10251041        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id),
    1026         FOREIGN KEY (diff_id)  REFERENCES diffRun(diff_id)
     1042        FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id)
    10271043) ENGINE=innodb DEFAULT CHARSET=latin1;
    10281044

  • trunk/ippTools/share/warptool_tooverlap.sql

    r19092 r20973  
    77    rawExp.camera,
    88    exp_id,
    9     exp_tag,
    10     warpRun.magiced
     9    exp_tag
    1110FROM warpRun
    1211JOIN fakeRun

  • trunk/ippTools/src/chiptool.c

    r20244 r20973  
    484484                                   n_cr,
    485485                                   path_base,
    486                                    code
     486                                   code,
     487                                   0,   // chip_image_id
     488                                   0    // magic_ds_id
    487489            )) {
    488490        // rollback

  • trunk/ippTools/src/difftool.c

    r20719 r20973  
    4747
    4848static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state);
     49static bool diffRunComplete(pxConfig *config);
    4950
    5051# define MODECASE(caseName, func) \
    … …  
    108109    // required options
    109110    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
    110     PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    111111    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
    112112    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    113113    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
     114    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
    114115
    115116    // default
    … …  
    120121            0,          // ID
    121122            "reg",      // state
     123            workdir,
    122124            label,
    123125            reduction,
    124             workdir,
    125126            NULL,       // dvodb
    126127            registered,
    127             skycell_id,
    128             tess_id
     128            tess_id,
     129            exp_id
    129130    );
    130131    if (!run) {
    … …  
    178179
    179180    // optional
    180     PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false);
    181     PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false);
    182     PXOPT_LOOKUP_STR(kind, config->args, "-kind", false, false);
    183 
    184     // defaults to false
    185     PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
    186 
    187     // must provide either stack_id or warp_id but not BOTH
    188     if (!(stack_id || warp_id)) {
    189         psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified");
    190         return false;
    191     }
    192     if (stack_id && warp_id) {
    193         psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified");
    194         return false;
    195     }
    196 
    197     // if a warp_id was provided we need to lookup the skycell_id and tess_id
    198     // from the warpRun
    199     psString skycell_id = NULL;
     181    PXOPT_LOOKUP_S64(stack1, config->args, "-stack1", false, false);
     182    PXOPT_LOOKUP_S64(stack2, config->args, "-stack2", false, false);
     183    PXOPT_LOOKUP_S64(warp1, config->args, "-warp1", false, false);
     184    PXOPT_LOOKUP_S64(warp2, config->args, "-warp2", false, false);
     185    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
     186
     187    // must provide either stack1 or warp1 but not BOTH
     188    if (!(stack1 || warp1)) {
     189        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
     190        return false;
     191    }
     192    if (stack1 && warp1) {
     193        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
     194        return false;
     195    }
     196    // must provide either stack2 or warp2 but not BOTH
     197    if (!(stack2 || warp2)) {
     198        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
     199        return false;
     200    }
     201    if (stack2 && warp2) {
     202        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
     203        return false;
     204    }
     205
     206    // if a warp1 was provided we need to lookup the and tess_id from the diffRun
    200207    psString tess_id = NULL;
    201     if (warp_id) {
     208    if (warp1) {
    202209        if (!p_psDBRunQuery(config->dbh, "SELECT * from diffRun WHERE diff_id = %" PRId64, diff_id)) {
    203210            psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    217224
    218225        diffRunRow *run = diffRunObjectFromMetadata(output->data[0]);
    219         skycell_id = run->skycell_id;
    220226        tess_id = run->tess_id;
    221227    }
    … …  
    228234    if (!diffInputSkyfileInsert(config->dbh,
    229235            diff_id,
    230             template,
    231             stack_id ? stack_id : PS_MAX_S64, // defined or NULL
    232             warp_id ? warp_id : PS_MAX_S64, // defined or NULL
    233236            skycell_id,
    234             tess_id,
    235             kind
     237            warp1 ? warp1 : PS_MAX_S64, // defined or NULL
     238            stack1 ? stack1 : PS_MAX_S64, // defined or NULL
     239            warp2 ? warp2 : PS_MAX_S64, // defined or NULL
     240            stack2 ? stack2 : PS_MAX_S64, // defined or NULL
     241            tess_id
    236242        )) {
    237243        if (!psDBRollback(config->dbh)) {
    … …  
    298304    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diff_id", "==");
    299305    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
    300     PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
     306    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
    301307    PXOPT_COPY_STR(config->args, where,  "-tess_id", "tess_id", "==");
    302     PXOPT_COPY_STR(config->args, where,  "-kind", "kind", "==");
    303308
    304309    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    … …  
    349354    }
    350355
    351     if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause)) {
     356    if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause, whereClause, whereClause)) {
    352357        psError(PS_ERR_UNKNOWN, false, "database error");
    353358        psFree(whereClause);
    … …  
    471476
    472477    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required
     478    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    473479    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
    474480    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false);
    … …  
    503509    if (!diffSkyfileInsert(config->dbh,
    504510                           diff_id,
     511                           skycell_id,
    505512                           uri,
    506513                           path_base,
    … …  
    524531                           hostname,
    525532                           good_frac,
    526                            code
     533                           code,
     534                           0,       // diff_image_id
     535                           0        // magic_ds_id
    527536          )) {
    528537        if (!psDBRollback(config->dbh)) {
    … …  
    533542    }
    534543
    535     if (!setdiffRunState(config, diff_id, "full")) {
     544    if (!diffRunComplete(config)) {
    536545        if (!psDBRollback(config->dbh)) {
    537546            psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    546555        return false;
    547556    }
     557
    548558
    549559    return true;
    … …  
    762772                         psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
    763773                         psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
     774                         psS64 exp_id, // exposure id for input_warp_id (if defined)
    764775                         pxConfig *config // Configuration
    765776                         )
    … …  
    786797            0,          // ID
    787798            "reg",      // state
     799            workdir,
    788800            label,
    789801            reduction,
    790             workdir,
    791802            NULL,       // dvodb
    792803            registered,
    793             skycell_id,
    794             tess_id
     804            tess_id,
     805            exp_id
    795806    );
    796807
    … …  
    817828    if (!diffInputSkyfileInsert(config->dbh,
    818829            run->diff_id,
    819             true,
     830            skycell_id,
     831            input_warp_id,
     832            input_stack_id,
     833            template_warp_id,
    820834            template_stack_id,
    821             template_warp_id,
    822             skycell_id,
    823             tess_id,
    824             NULL    // kind
    825         )) {
    826         if (!psDBRollback(config->dbh)) {
    827             psError(PS_ERR_UNKNOWN, false, "database error");
    828         }
    829         psError(PS_ERR_UNKNOWN, false, "database error");
    830         return false;
    831     }
    832 
    833     // Input
    834     if (!diffInputSkyfileInsert(config->dbh,
    835             run->diff_id,
    836             false,
    837             input_stack_id,
    838             input_warp_id,
    839             skycell_id,
    840             tess_id,
    841             NULL    // kind
     835            tess_id
    842836        )) {
    843837        if (!psDBRollback(config->dbh)) {
    … …  
    887881    PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
    888882    PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
     883    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
    889884    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    890885
    … …  
    906901        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
    907902                "No input has been defined (-input_stack_id or -input_warp_id)");
     903        return false;
     904    }
     905    if (input_warp_id && !exp_id) {
     906        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
     907                "-exp_id is required with -input_warp_id.");
    908908        return false;
    909909    }
    … …  
    916916                      template_warp_id ? template_warp_id : PS_MAX_S64,
    917917                      template_stack_id ? template_stack_id : PS_MAX_S64,
     918                      exp_id,
    918919                      config)) {
    919920        psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
    … …  
    937938    PS_ASSERT_PTR_NON_NULL(config, false);
    938939
     940    psMetadata *expWhere = psMetadataAlloc();
    939941    psMetadata *warpWhere = psMetadataAlloc();
    940942    psMetadata *stackWhere = psMetadataAlloc();
    941943
     944    PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "==");
    942945    PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "==");
    943946    PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpRun.skycell_id", "==");
    … …  
    945948    PXOPT_COPY_STR(config->args, warpWhere, "-filter", "rawExp.filter", "==");
    946949    PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "==");
    947     PXOPT_COPY_STR(config->args, warpWhere,  "-kind", "warpsToDiff.kind", "==");
    948950    PXOPT_COPY_F32(config->args, warpWhere,  "-good_frac", "warpSkyfile.good_frac", ">=");
    949951    PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
    … …  
    955957    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    956958    PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false);
     959    PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
     960    PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
    957961
    958962    // find all things to queue
    959     psString query = pxDataGet("difftool_definebyquery.sql");
     963    psString query = pxDataGet("difftool_definebyquery_part1.sql");
    960964    if (!query) {
    961965        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    … …  
    963967    }
    964968
    965     if (newTemplates) {
    966         // Warps that haven't been diffed OR warps that can take a newer template in the diff
    967         psStringAppend(&query, " WHERE (diff_id IS NULL OR best_stack_id > current_stack_id)");
    968     } else {
    969         // Only warps that haven't been diffed
    970         psStringAppend(&query, " WHERE diff_id IS NULL");
    971     }
    972 
    973969    psString warpQuery = NULL;
    974970    psString stackQuery = NULL;
    975 
     971    psString expQuery = NULL;
     972
     973    if (psListLength(expWhere->list)) {
     974        psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL);
     975        psStringAppend(&expQuery, "\n AND %s", whereClause);
     976        psFree(whereClause);
     977    } else {
     978        expQuery = psStringCopy("\n");
     979    }
     980    psFree(expWhere);
    976981    if (psListLength(warpWhere->list)) {
    977982        psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL);
    … …  
    979984        psFree(whereClause);
    980985    } else {
    981         warpQuery = psStringCopy("");
     986        warpQuery = psStringCopy("\n");
    982987    }
    983988    psFree(warpWhere);
     989
     990    if (!available) {
     991        // diff what's available, even if warp run has some faults and is incomplete
     992        psStringAppend(&warpQuery, " AND warpRun.state = 'full'");
     993    }
     994
     995    // don't queue for exposures that have already been diff'd unless requested
     996    psString diffQuery = NULL;
     997    if (! (reRun || newTemplates) ) {
     998        psStringAppend(&diffQuery, "\nAND diff_id IS NULL");
     999    } else {
     1000        diffQuery = psStringCopy("\n");
     1001    }
    9841002
    9851003    if (psListLength(stackWhere->list)) {
    9861004        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
    987         psStringAppend(&stackQuery, "\n AND %s", whereClause);
     1005        psStringAppend(&stackQuery, "\nAND %s", whereClause);
    9881006        psFree(whereClause);
    9891007    } else {
    … …  
    9921010    psFree(stackWhere);
    9931011
    994     psTrace("difftool", 1, query, warpQuery, stackQuery);
    995 
    996     if (!p_psDBRunQuery(config->dbh, query, warpQuery, stackQuery)) {
     1012    psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery);
     1013
     1014
     1015    if (!p_psDBRunQuery(config->dbh, query, warpQuery, expQuery, diffQuery)) {
    9971016        psError(PS_ERR_UNKNOWN, false, "database error");
    9981017        psFree(query);
    … …  
    10211040    }
    10221041
     1042    // create temporary table
     1043    query = pxDataGet("difftool_definebyquery_temp_create.sql");
     1044    if (!p_psDBRunQuery(config->dbh, query)) {
     1045        psError(PS_ERR_UNKNOWN, false, "database error");
     1046        psFree(query);
     1047        return false;
     1048    }
     1049    psFree(query);
     1050    query = NULL;
     1051
     1052    psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql");
     1053
    10231054    psArray *list = psArrayAllocEmpty(16); // List of runs, to print
    10241055    long numGood = 0;                   // Number of good rows added
     1056    psS64 last_exp_id = 0;
    10251057    for (long i = 0; i < output->n; i++) {
    10261058        psMetadata *row = output->data[i]; // Output row from query
    1027 
    1028         // Selected parameters
    10291059        bool mdok;                      // Status of MD lookup
    1030         const char *skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id");
    1031         if (!mdok || !skycell_id) {
    1032             psWarning("skycell_id not found --- ignoring row %ld", i);
     1060
     1061        // Take the first warp for each exposure.
     1062        // The list is sorted by exposure id and warp_id and the warps are in descending
     1063        // order.
     1064        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
     1065        if (!mdok) {
     1066            psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
     1067            return false;
     1068        }
     1069        if (exp_id == last_exp_id) {
    10331070            continue;
    10341071        }
    1035         const char *tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
    1036         if (!mdok || !tess_id) {
    1037             psWarning("tess_id not found --- ignoring row %ld", i);
    1038             continue;
    1039         }
    1040         psS64 stack_id = psMetadataLookupS64(&mdok, row, "best_stack_id");
    1041         if (!mdok) {
    1042             psWarning("stack_id not found --- ignoring row %ld", i);
    1043             continue;
     1072        last_exp_id = exp_id;
     1073
     1074        // clear temporary table
     1075        if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) {
     1076            psError(PS_ERR_UNKNOWN, false, "database error");
     1077            psFree(warpQuery);
     1078            psFree(stackQuery);
     1079            psFree(skycell_query);
     1080            return false;
    10441081        }
    10451082        psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
    10461083        if (!mdok) {
    1047             psWarning("warp_id not found --- ignoring row %ld", i);
     1084            psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i);
     1085            psFree(warpQuery);
     1086            psFree(stackQuery);
     1087            psFree(skycell_query);
     1088            return false;
     1089        }
     1090        psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count");
     1091        if (!mdok) {
     1092            psError(PXTOOLS_ERR_PROG, false, "skycell_count not found");
     1093            psFree(warpQuery);
     1094            psFree(stackQuery);
     1095            psFree(skycell_query);
     1096            return false;
     1097        }
     1098        psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
     1099        if (!mdok) {
     1100            psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
     1101            psFree(warpQuery);
     1102            psFree(stackQuery);
     1103            psFree(skycell_query);
     1104            return false;
     1105        }
     1106        psString filter = psMetadataLookupStr(&mdok, row, "filter");
     1107        if (!mdok) {
     1108            psError(PXTOOLS_ERR_PROG, false, "filter not found");
     1109            psFree(warpQuery);
     1110            psFree(stackQuery);
     1111            psFree(skycell_query);
     1112            return false;
     1113        }
     1114        if (!p_psDBRunQuery(config->dbh, skycell_query, warp_id, filter, warpQuery, stackQuery)) {
     1115            psError(PS_ERR_UNKNOWN, false, "database error");
     1116            psFree(warpQuery);
     1117            psFree(stackQuery);
     1118            psFree(skycell_query);
     1119            return false;
     1120        }
     1121        psS64 num = psDBAffectedRows(config->dbh);
     1122
     1123        if (num == 0) {
     1124            psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id);
    10481125            continue;
    10491126        }
    10501127
    1051         if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id,
    1052                           PS_MAX_S64, PS_MAX_S64, stack_id, config)) {
    1053             psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id,
    1054                       warp_id, stack_id);
    1055             psErrorClear();
     1128        if (!available && (num != skycell_count)) {
     1129            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %"
     1130                PRId64 " need %" PRId64, num, skycell_count, warp_id);
    10561131            continue;
    10571132        }
    10581133
     1134        // ok we've got one create the diffRun
     1135        diffRunRow *run = diffRunRowAlloc(
     1136                0,          // ID
     1137                "reg",      // state
     1138                workdir,
     1139                label,
     1140                reduction,
     1141                NULL,       // dvodb
     1142                registered,
     1143                tess_id,
     1144                exp_id
     1145        );
     1146
     1147        if (!diffRunInsertObject(config->dbh, run)) {
     1148            psError(PS_ERR_UNKNOWN, false, "database error");
     1149            psFree(run);
     1150            return true;
     1151        }
     1152        run->diff_id = psDBLastInsertID(config->dbh);
     1153
     1154        psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id);
     1155        if (!p_psDBRunQuery(config->dbh, query)) {
     1156            psError(PS_ERR_UNKNOWN, false, "database error");
     1157            psFree(warpQuery);
     1158            psFree(stackQuery);
     1159            psFree(skycell_query);
     1160            psFree(query);
     1161            return false;
     1162        }
     1163        psFree(query);
     1164        query = NULL;
     1165        psStringAppend(&query, "INSERT INTO diffInputSkyfile SELECT * from skycellsToDiff");
     1166        if (!p_psDBRunQuery(config->dbh, query)) {
     1167            psError(PS_ERR_UNKNOWN, false, "database error");
     1168            psFree(warpQuery);
     1169            psFree(stackQuery);
     1170            psFree(skycell_query);
     1171            psFree(query);
     1172            return false;
     1173        }
     1174        psFree(query);
     1175        query = NULL;
     1176
     1177        if (!setdiffRunState(config, run->diff_id, "new")) {
     1178            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
     1179                run->diff_id);
     1180            psFree(warpQuery);
     1181            psFree(stackQuery);
     1182            psFree(skycell_query);
     1183            psFree(query);
     1184            return false;
     1185        }
     1186
     1187        psArrayAdd(list, list->n, run);
    10591188        numGood++;
    10601189    }
    10611190    psFree(output);
     1191    psFree(warpQuery);
     1192    psFree(stackQuery);
     1193    psFree(skycell_query);
    10621194
    10631195    if (!diffRunPrintObjects(stdout, list, !simple)) {
    … …  
    12741406}
    12751407
     1408static bool diffRunComplete(pxConfig *config)
     1409{
     1410    PS_ASSERT_PTR_NON_NULL(config, false);
     1411
     1412    // look for completed diffRuns
     1413    psString query = pxDataGet("difftool_completed_runs.sql");
     1414    if (!query) {
     1415        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
     1416        return false;
     1417    }
     1418
     1419    if (!p_psDBRunQuery(config->dbh, query)) {
     1420        psError(PS_ERR_UNKNOWN, false, "database error");
     1421        psFree(query);
     1422        return false;
     1423    }
     1424    psFree(query);
     1425
     1426    psArray *output = p_psDBFetchResult(config->dbh);
     1427    if (!output) {
     1428        psError(PS_ERR_UNKNOWN, false, "database error");
     1429        return false;
     1430    }
     1431    if (!psArrayLength(output)) {
     1432        psTrace("difftool", PS_LOG_INFO, "no rows found");
     1433        psFree(output);
     1434        return true;
     1435    }
     1436    for (long i = 0; i < psArrayLength(output); i++) {
     1437        psMetadata *row = output->data[i];
     1438
     1439        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");
     1440
     1441        // set diffRun.state to 'stop'
     1442        if (!setdiffRunState(config, diff_id, "full")) {
     1443            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
     1444                diff_id);
     1445            psFree(output);
     1446            return false;
     1447        }
     1448    }
     1449
     1450    return true;
     1451}
     1452

  • trunk/ippTools/src/difftoolConfig.c

    r20693 r20973  
    4848    psMetadata *definerunArgs = psMetadataAlloc();
    4949    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0,            "define workdir (required)", NULL);
    50     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id",  0,            "define skycell ID (required)", NULL);
    5150    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id",  0,            "define tessellation ID (required)", NULL);
    5251    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label",  0,            "define label", NULL);
    … …  
    6564    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,            "define stack ID", 0);
    6665    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,            "define warp ID", 0);
    67     psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-kind", 0,            "define kind", NULL);
    6866    psMetadataAddBool(addinputskyfileArgs, PS_LIST_TAIL, "-template",  0,            "this sky cell file is the subtrahend", false);
    6967
    … …  
    7472    psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,            "search by skycell ID", NULL);
    7573    psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", NULL);
    76     psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-kind", 0,            "search by kind", NULL);
    7774    psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-template",  0,            "find only subtrahend", false);
    7875    psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-input", 0, "find only minuend", false);
    … …  
    9087    psMetadata *adddiffskyfileArgs = psMetadataAlloc();
    9188    psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,            "define warp ID (required)", 0);
     89    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,       "define skycell of file (required)", 0);
    9290    psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
    9391    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0,            "define URI of file", 0);
    … …  
    147145    // -definebyquery
    148146    psMetadata *definebyqueryArgs = psMetadataAlloc();
    149     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0);
    150147    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
     148    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
    151149    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL);
    152150    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL);
    … …  
    155153    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
    156154    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
    157     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-kind", 0, "search by kind", NULL);
    158155    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
    159156    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);
    … …  
    161158    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
    162159    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
     160    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
     161    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false);
    163162    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
    164163

  • trunk/ippTools/src/magicdstool.c

    r20850 r20973  
    469469    if (psArrayLength(output)) {
    470470        // negative simple so the default is true
    471         if (!ippdbPrintMetadatas(stdout, output, "totree", !simple)) {
     471        if (!ippdbPrintMetadatas(stdout, output, "todestreak", !simple)) {
    472472            psError(PS_ERR_UNKNOWN, false, "failed to print array");
    473473            psFree(output);

  • trunk/ippTools/src/magictool.c

    r20783 r20973  
    128128    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    129129
    130     // Create temporary table of the best diffs
    131     {
    132         psString query = pxDataGet("magictool_definebyquery_temp_create.sql");
    133         if (!query) {
    134             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    135             return false;
    136         }
    137 
    138         if (!p_psDBRunQuery(config->dbh, query)) {
    139             psError(PS_ERR_UNKNOWN, false, "database error");
    140             return false;
    141         }
    142         psFree(query);
    143     }
    144 
    145     // Insert list of best diffs into temporary table
    146     {
    147         psString query = pxDataGet("magictool_definebyquery_temp_insert.sql");
    148         if (!query) {
    149             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    150             return false;
    151         }
    152 
    153         psMetadata *where = psMetadataAlloc();
    154         PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
    155         PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
    156         PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
    157 
    158         psString whereClause = NULL;    // WHERE conditions
    159         if (psListLength(where->list)) {
    160             whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    161             psStringPrepend(&whereClause, "\n AND ");
    162         }
    163         psFree(where);
    164 
    165         if (!p_psDBRunQuery(config->dbh, query, whereClause)) {
    166             psError(PS_ERR_UNKNOWN, false, "database error");
    167             psFree(whereClause);
    168             psFree(query);
    169             return false;
    170         }
    171         psFree(whereClause);
    172         psFree(query);
    173     }
     130    psMetadata *where = psMetadataAlloc();
     131    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
     132    PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
     133
    174134
    175135    // Get list of exposures ready to magic
    … …  
    181141        }
    182142
    183         psString magicSkyCellNumsWhere = NULL;    // WHERE conditions for magicSkyCellNums
    184         {
    185             psMetadata *where = psMetadataAlloc();
    186             PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
    187             PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
    188             PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
    189 
    190             if (psListLength(where->list)) {
    191                 magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL);
    192                 psStringPrepend(&magicSkyCellNumsWhere, "\n AND ");
    193             }
    194             psFree(where);
    195         }
    196 
    197         // "available" means only concern ourselves with exposures that have all diffs completed, unless we're
    198         // told to only take what's available.
    199         // "new" means we want a new run even if there's already a magic run defined
     143        // "available" means queue magic run even though the diffRun has skycells that did not complete
     144        // "rerun" means we want a new run even if there's already a magic run defined for the exposure
    200145        PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
    201         PXOPT_LOOKUP_BOOL(new, config->args, "-new", false);
     146        PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
    202147
    203148        psString queryWhere = NULL;     // WHERE conditions for entire query
    204         if (available) {
    205             psStringAppend(&queryWhere, " WHERE num_done = num_todo");
    206         }
    207         if (new) {
     149        if (!available) {
     150            psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'");
     151        } else {
     152            // what if no skycells for the diff run completed?
     153        }
     154        if (!rerun) {
    208155            const char *newWhere = " magic_id IS NULL"; // String to add
    209156            if (queryWhere) {
    210157                psStringAppend(&queryWhere, " AND %s", newWhere);
    211158            } else {
    212                 psStringAppend(&queryWhere, " WHERE %s", newWhere);
     159                psStringAppend(&queryWhere, "\nWHERE %s", newWhere);
    213160            }
    214161        }
    215         if (queryWhere) {
    216             psStringAppend(&query, " %s", queryWhere);
     162        // now add the user specified qualifiers
     163        if (psListLength(where->list)) {
     164            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     165            psStringAppend(&queryWhere, "\n%s %s", queryWhere == NULL ? "WHERE" : "AND", whereClause);
     166            psFree(whereClause);
     167        }
     168        psFree(where);
     169        if (!queryWhere) {
     170            psStringAppend(&queryWhere, " ");
     171        }
     172
     173        if (!p_psDBRunQuery(config->dbh, query, queryWhere)) {
     174            psError(PS_ERR_UNKNOWN, false, "database error");
    217175            psFree(queryWhere);
    218         }
    219 
    220 
    221         if (!p_psDBRunQuery(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) {
    222             psError(PS_ERR_UNKNOWN, false, "database error");
    223             psFree(magicSkyCellNumsWhere);
    224176            psFree(query);
    225177            return false;
    226178        }
    227         psFree(magicSkyCellNumsWhere);
     179        psFree(queryWhere);
    228180        psFree(query);
    229181    }
    … …  
    262214        psMetadata *row = output->data[i]; // Row of interest
    263215        psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
     216        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier
    264217
    265218        // create a new magicRun for this group
    266         magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0);
     219        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "run", workdir, "dirty", label, dvodb, registered, 0);
    267220        if (!run) {
    268221            psAbort("failed to alloc magicRun object");
    … …  
    297250        {
    298251            psString idString = NULL;
    299             psStringAppend(&idString, "%" PRId64, exp_id);
    300             psStringSubstitute(&thisInsert, idString, "@EXP_ID@");
     252            psStringAppend(&idString, "%" PRId64, diff_id);
     253            psStringSubstitute(&thisInsert, idString, "@DIFF_ID@");
    301254            psFree(idString);
    302255        }
    … …  
    340293    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
    341294    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false);
     295    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
    342296
    343297    // optional
    … …  
    350304            0,          // ID
    351305            exp_id,
     306            diff_id,
    352307            "reg",      // state
    353308            workdir,

  • trunk/ippTools/src/magictoolConfig.c

    r20744 r20973  
    5555    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
    5656    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL);
    57     psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN);
    5857    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
    59     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new", 0, "generate new run even if existing?", false);
     58    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false);
    6059    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
    6160

  • trunk/ippTools/src/pxwarp.c

    r19092 r20973  
    147147        tess_id,
    148148        end_stage,
    149         NULL,       // registered
    150         false       // magiced
     149        NULL       // registered
    151150    )) {
    152151        psError(PS_ERR_UNKNOWN, false, "database error");

  • trunk/ippTools/src/regtool.c

    r20608 r20973  
    299299        hostname,
    300300        code,
    301         NULL
     301        NULL,
     302        0
    302303    )) {
    303304        psError(PS_ERR_UNKNOWN, false, "database error");

  • trunk/ippTools/src/warptool.c

    r20723 r20973  
    134134    PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false);
    135135    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    136     PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false);
    137136    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    138137
    … …  
    154153            tess_id,
    155154            end_stage,
    156             registered,
    157             magiced
     155            registered
    158156    );
    159157    if (!warpRun) {
    … …  
    246244
    247245    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    248     PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false);
    249246    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    250247
    … …  
    942939                           ymax,
    943940                           !accept,
    944                            code
     941                           code,
     942                           0,           // warp_image_id
     943                           0            // magic_ds_id
    945944        )) {
    946945        if (!psDBRollback(config->dbh)) {

  • trunk/ippTools/src/warptoolConfig.c

    r20440 r20973  
    103103
    104104    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered",  0,            "time detrend run was registered", now);
    105     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-magiced",  0,            "has this exposure been magiced", false);
    106105    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    107106
    … …  
    116115    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-end_stage", 0,            "define end stage", NULL);
    117116    psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered",  0,            "time detrend run was registered", now);
    118     psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-magiced",  0,            "has this exposure been magiced", false);
    119117    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    120118
Note: See TracChangeset for help on using the changeset viewer.