IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 13994


Ignore:
Timestamp:
Jul 2, 2007, 3:46:36 PM (19 years ago)
Author:
jhoblitt
Message:

refactor camtool & db to preserve metadata history

Location:
branches/backtrack
Files:
3 added
13 edited

Legend:

Unmodified
Added
Removed

  • branches/backtrack/dbconfig/cam.md

    r13937 r13994  
    11# imfiles is not here... is it needed?
    22
    3 camPendingExp METADATA
     3camRun METADATA
    44    cam_id      S64         0       # Primary Key AUTO_INCREMENT
    5     chip_id     S64         0       # Primary Key fkey(chip_id) ref chipProcessedExp(chip_id)
     5    chip_id     S64         0       # Primary Key fkey(chip_id) ref chipRun(chip_id)
     6    state       STR         64      # key
    67    workdir     STR         255
     8    workdir_state STR       64      # key
    79    label       STR         64      # key
    810    reduction   STR         64
    … …  
    1416# the camPendingExp row gets deleted so we can not put a fkey on cam_id
    1517    cam_id         S64      0       # Primary Key
    16     chip_id        S64      0       # Primary Key fkey(chip_id) ref chipProcessedExp(chip_id)
    17     workdir        STR      255
    18     label          STR      64      # key
    19     reduction      STR      64
    20     expgroup       STR      64      # key
    21     dvodb          STR      255
     18    chip_id        S64      0       # Primary Key fkey(cam_id, chip_id) ref camRun(cam_id, chip_id)
    2219    uri            STR      255
    2320    bg             F32      0.0

  • branches/backtrack/ippTools/scripts/camtest.sh

    r13969 r13994  
    88camtool -pendingimfile || exit 1
    99
    10 camtool -addprocessedexp -cam_id 1 -chip_id 1 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -nastro 42 -path_base file:///foo -zp_mean 1 -zp_stdev 2 || exit 1
    11 camtool -addprocessedexp -cam_id 2 -chip_id 2 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -nastro 42 -path_base file:///foo -zp_mean 1 -zp_stdev 2 || exit 1
     10camtool -addprocessedexp -cam_id 1 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -zp_mean 10 -zp_stdev 2 -fwhm 42 -fwhm_range 100 -n_stars 2 -n_extended 0 -n_astrom 42 -n_cr 10000000 -path_base file:///foo || exit 1
     11camtool -addprocessedexp -cam_id 2 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -zp_mean 10 -zp_stdev 2 -fwhm 42 -fwhm_range 100 -n_stars 2 -n_extended 0 -n_astrom 42 -n_cr 10000000 -path_base file:///foo || exit 1

  • branches/backtrack/ippTools/share/camtool_find_pendingexp.sql

    r12259 r13994  
    33-- "cleaner" to use the same query for both cases
    44SELECT
    5     camPendingExp.*,
     5    camRun.*,
    66    rawExp.exp_tag,
    77    rawExp.exp_id,
    … …  
    99    rawExp.telescope,
    1010    rawExp.filelevel
    11 FROM camPendingExp
    12 JOIN chipProcessedExp
     11FROM camRun
     12JOIN chipRun
     13    USING(chip_id)
     14JOIN chipProcessedImfile
    1315    USING(chip_id)
    1416JOIN rawExp
    15     ON chipProcessedExp.exp_tag = rawExp.exp_tag
     17    ON chipProcessedImfile.exp_tag = rawExp.exp_tag
    1618LEFT JOIN camProcessedExp
    17     ON camPendingExp.cam_id = camProcessedExp.cam_id
     19    ON camRun.cam_id = camProcessedExp.cam_id
    1820LEFT JOIN camMask
    19     ON camPendingExp.label = camMask.label
     21    ON camRun.label = camMask.label
    2022WHERE
    21     camProcessedExp.cam_id IS NULL
     23    chipRun.state = 'stop'
     24    AND camRun.state = 'run'
    2225    AND camMask.label IS NULL
     26    AND camProcessedExp.cam_id IS NULL

  • branches/backtrack/ippTools/share/camtool_queue_chip_id.sql

    r12237 r13994  
    11-- camtool only operates on exposures so we can safely queue more then one at a
    22-- time without worrying about losing the track of the generated cam_id
    3 INSERT INTO camPendingExp
     3INSERT INTO camRun
    44    SElECT
    55        0,              -- cam_id
    66        chip_id,        -- chip_id
     7        '%s',           -- state
    78        '%s',           -- workdir
     9        '%s',           -- workdir_state
    810        '%s',           -- label
    9         '%s',           -- recipe
     11        '%s',           -- reduction
    1012        '%s',           -- expgroup
    1113        '%s'            -- dvodb
    12     FROM chipProcessedExp
     14    FROM chipRun
    1315    WHERE
    14         chipProcessedExp.chip_id = %lld
     16        chipRun.state = 'stop'
     17        AND chipRun.chip_id = %lld

  • branches/backtrack/ippTools/share/chiptool_completely_processed_exp.sql

    r13990 r13994  
    1010    dvodb
    1111FROM
    12     (SELECT DISTINCT
     12    (SELECT
    1313        chipRun.*,
    1414        rawExp.imfiles,
    15         chipProcessedImfile.exp_tag,
    1615        chipProcessedImfile.class_id
    1716    FROM chipRun
    … …  
    2019    JOIN rawExp
    2120        ON chipPendingImfile.exp_tag = rawExp.exp_tag
    22     LEFT JOIN chipProcessedImfile
    23         USING(chip_id)
     21    JOIN chipProcessedImfile
     22        ON chipPendingImfile.chip_id = chipProcessedImfile.chip_id
     23        AND chipPendingImfile.exp_tag = chipProcessedImfile.exp_tag
     24        AND chipPendingImfile.class_id = chipProcessedImfile.class_id
    2425    WHERE
    25         chipProcessedImfile.chip_id IS NOT NULL
    26         AND chipProcessedImfile.exp_tag IS NOT NULL
    27         AND chipProcessedImfile.class_id IS NOT NULL
     26        chipRun.state = 'run'
    2827        AND chipProcessedImfile.fault = 0
    2928    GROUP BY
    30         chipProcessedImfile.class_id,
    31         chipProcessedImfile.exp_tag
     29        chipPendingImfile.chip_id,
     30        chipPendingImfile.exp_tag
    3231    HAVING rawExp.imfiles = COUNT(chipProcessedImfile.class_id)) as Foo

  • branches/backtrack/ippTools/src/Makefile.am

    r13967 r13994  
    2222        pxtag.h \
    2323        pxregister.h \
     24        pxchip.h \
     25        pxcam.h \
    2426        pxconfig.h \
    2527        pxtools.h
    … …  
    5456        pxtag.c \
    5557        pxregister.c \
     58        pxchip.c \
     59        pxcam.c \
    5660        chipqueue.c \
    5761        camqueue.c

  • branches/backtrack/ippTools/src/camqueue.c

    r12237 r13994  
    5151    // different on 32/64
    5252    if (!p_psDBRunQuery(config->dbh, query,
     53                "run", // state
    5354                workdir  ? workdir  : "NULL",
     55                "dirty", //workdir_state
    5456                label    ? label    : "NULL",
    5557                recipe   ? recipe   : "NULL",

  • branches/backtrack/ippTools/src/camtool.c

    r13937 r13994  
    203203
    204204    // find the chipProcessedExp exposures that we want to queue up.
    205     psString query = psStringCopy("SELECT * FROM chipProcessedExp");
     205//    psString query = psStringCopy("SELECT * FROM chipRun WHERE chipRun.state = 'stop'");
     206    psString query = pxDataGet("camtool_find_chip_id.sql");
     207    if (!query) {
     208        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
     209        return false;
     210    }
    206211
    207212    if (where) {
    208         psString whereClause = psDBGenerateWhereSQL(where, "chipProcessedExp");
     213        psString whereClause = psDBGenerateWhereSQL(where, "");
    209214        psFree(where);
    210215        psStringAppend(&query, " %s", whereClause);
    … …  
    245250        psMetadata *md = output->data[i];
    246251
    247         chipProcessedExpRow *row = chipProcessedExpObjectFromMetadata(md);
     252        chipRunRow *row = chipRunObjectFromMetadata(md);
    248253        if (!row) {
    249             psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipProcessedExp");
     254            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun");
    250255            psFree(output);
    251256            return false;
    … …  
    601606    }
    602607
    603     // start a transaction so we don't end up with an exp_tag is both
    604     // camPendingExp & camProcessedExp
    605608    if (!psDBTransaction(config->dbh)) {
    606609        psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    609612    }
    610613
    611     camPendingExpRow *pendingRow = camPendingExpObjectFromMetadata(output->data[0]);
     614    camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]);
    612615    psFree(output);
    613616    // create a new camProcessedImfile object
    … …  
    615618        pendingRow->cam_id,
    616619        pendingRow->chip_id,
    617         pendingRow->workdir,
    618         pendingRow->label,
    619         pendingRow->reduction,
    620         pendingRow->expgroup,
    621         pendingRow->dvodb,
    622620        uri,
    623621        bg,
    … …  
    637635        code
    638636    );
     637    psFree(pendingRow);
    639638
    640639    // insert the new row into the camProcessedImfile table
    … …  
    646645        psError(PS_ERR_UNKNOWN, false, "database error");
    647646        psFree(row);
    648         psFree(pendingRow);
    649         return false;
    650     }
     647        return false;
     648    }
     649
     650    // since there is only one exp per 'run' set camRun.state = 'stop'
     651    if (!pxcamRunSetState(config, row->cam_id, "stop")) {
     652        psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id);
     653        psFree(row);
     654        return false;
     655    }
     656
    651657    psFree(row);
    652658
    653     // delete the camPendingExp row from the database
    654     if (!camPendingExpDeleteObject(config->dbh, pendingRow)) {
    655         // rollback
    656         if (!psDBRollback(config->dbh)) {
    657             psError(PS_ERR_UNKNOWN, false, "database error");
    658         }
    659         psError(PS_ERR_UNKNOWN, false, "database error");
    660         psFree(pendingRow);
    661         return false;
    662     }
    663 
    664     psFree(pendingRow);
    665 
    666     // point of no return for camPendingExp -> camProcessedExp
    667659    if (!psDBCommit(config->dbh)) {
    668660        psError(PS_ERR_UNKNOWN, false, "database error");

  • branches/backtrack/ippTools/src/chiptool.c

    r13990 r13994  
    3030#include "pxtools.h"
    3131#include "pxdata.h"
     32
    3233#include "chiptool.h"
    3334#include "camtool.h"
    … …  
    553554    }
    554555
     556
     557
    555558    // XXX I've decided to make the transaction cover the Exp migration as
    556559    // well.  Otherwise, if the last imfile in an exp is moved and the exp
    … …  
    827830    }
    828831
    829     // insert into camPendingExp
    830832    for (long i = 0; i < psArrayLength(output); i++) {
    831833        psMetadata *row = output->data[i];
    832834
    833835        chipRunRow *chipRun = chipRunObjectFromMetadata(row);
    834 
    835         // camQueueChipID() can only be run after the chipProcessedExp entry
    836         // has been inserted.
    837         // queue the chip_id in the camPendingExp table
     836        // set chipRun.state to 'stop'
     837        if (!pxchipRunSetState(config, chipRun->chip_id, "stop")) {
     838            psError(PS_ERR_UNKNOWN, false, "failed to change chipRun.state for chip_id: %" PRId64, chipRun->chip_id);
     839            psFree(chipRun);
     840            psFree(output);
     841            return false;
     842        }
     843
     844        // camQueueChipID() can only be run after chipRun.state has been set to
     845        // stop
    838846        if (!camQueueChipID(config,
    839847                    chipRun->chip_id,

  • branches/backtrack/ippTools/src/pxregister.c

    r13967 r13994  
    2727
    2828#include "pxtools.h"
    29 #include "pxtag.h"
    30 #include "pxinject.h"
     29#include "pxregister.h"
    3130
    3231bool pxnewExpSetState(pxConfig *config, const char *exp_tag, const char *state)

  • branches/backtrack/ippTools/src/pxregister.h

    r13967 r13994  
    11/*
    2  * register.h
     2 * pxregister.h
    33 *
    4  * Copyright (C) 2006  Joshua Hoblitt
     4 * Copyright (C) 2007  Joshua Hoblitt
    55 *
    66 * This program is free software; you can redistribute it and/or modify it
    … …  
    1818 */
    1919
    20 #ifndef REGISTER_H
    21 #define REGISTER_H 1
     20#ifndef PXREGISTER_H
     21#define PXREGISTER_H 1
     22
     23#include <pslib.h>
    2224
    2325#include "pxtools.h"
    … …  
    2527bool pxnewExpSetState(pxConfig *config, const char *exp_tag, const char *state);
    2628
    27 #endif // REGISTER_H
     29#endif // PXREGISTER_H

  • branches/backtrack/ippTools/src/pxtables.c

    r13990 r13994  
    6464    CREATE_TABLE(chipProcessedImfileCreateTable);
    6565    CREATE_TABLE(chipMaskCreateTable);
    66     CREATE_TABLE(camPendingExpCreateTable);
     66    CREATE_TABLE(camRunCreateTable);
    6767    CREATE_TABLE(camProcessedExpCreateTable);
    6868    CREATE_TABLE(camMaskCreateTable);
    … …  
    145145    DROP_TABLE(chipProcessedImfileDropTable);
    146146    DROP_TABLE(chipMaskDropTable);
    147     DROP_TABLE(camPendingExpDropTable);
     147    DROP_TABLE(camRunDropTable);
    148148    DROP_TABLE(camProcessedExpDropTable);
    149149    DROP_TABLE(camMaskDropTable);

  • branches/backtrack/ippTools/src/pxtools.h

    r13967 r13994  
    3333#include "pxtag.h"
    3434#include "pxregister.h"
     35#include "pxchip.h"
     36#include "pxcam.h"
    3537#include "pxconfig.h"
    3638
Note: See TracChangeset for help on using the changeset viewer.