IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 26857


Ignore:
Timestamp:
Feb 10, 2010, 3:52:15 PM (16 years ago)
Author:
eugene
Message:

various updates from trunk (mostly magic state handling, dqstats)

Location:
branches/eam_branches/20091201/ippTools
Files:
62 edited
9 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/20091201/ippTools/configure.ac

    r26194 r26857  
    11AC_PREREQ(2.61)
    22
    3 AC_INIT([ipptools], [1.1.57], [ipp-support@ifa.hawaii.edu])
     3AC_INIT([ipptools], [1.1.63], [ipp-support@ifa.hawaii.edu])
    44AC_CONFIG_SRCDIR([autogen.sh])
    55
    … …  
    1818PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
    1919PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
    20 PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.57])
     20PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.63])
    2121
    2222PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}"

  • branches/eam_branches/20091201/ippTools/share/Makefile.am

    r26193 r26857  
    140140     disttool_updateinterest.sql \
    141141     disttool_updatercrun.sql \
     142     dqstatstool_createbundle.sql \
     143     dqstatstool_definebyquery.sql \
     144     dqstatstool_get_contents.sql \
     145     dqstatstool_get_run.sql \
    142146     faketool_change_exp_state.sql \
    143147     faketool_change_imfile_data_state.sql \

  • branches/eam_branches/20091201/ippTools/share/camtool_find_pendingexp.sql

    r23617 r26857  
    99    rawExp.camera,
    1010    rawExp.telescope,
    11     rawExp.filelevel
     11    rawExp.filelevel,
     12    chipRun.magicked AS chip_magicked
    1213FROM camRun
    1314JOIN chipRun

  • branches/eam_branches/20091201/ippTools/share/camtool_queue_chip_id.sql

    r25835 r26857  
    1616        '%s',           -- tess_id
    1717        '%s',           -- end_stage
    18         0,              -- magicked
    19         '%s'           -- note
     18        %lld,           -- magicked
     19        '%s'            -- note
    2020    FROM chipRun
    2121    WHERE

  • branches/eam_branches/20091201/ippTools/share/chiptool_change_exp_state.sql

    r19702 r26857  
    44-- the chipProcessedImfile sub query
    55UPDATE chipRun
    6     SET state = '%s'
     6JOIN rawExp using(exp_id)
     7    SET chipRun.state = '%s'
     8    -- set magicked hook %s
    79    WHERE
    810    chipRun.chip_id = %lld

  • branches/eam_branches/20091201/ippTools/share/chiptool_completely_processed_exp.sql

    r25835 r26857  
    1414    tess_id,
    1515    end_stage,
    16     all_files_magicked as magicked,
     16    imfile_magicked as magicked,
    1717    note
    1818FROM
    … …  
    2121        rawImfile.class_id as rawimfile_class_id,
    2222        chipProcessedImfile.class_id,
    23         SUM(!chipProcessedImfile.magicked) = 0 as all_files_magicked
     23        -- XXX using chipProcessedImfile assumes that all imfile's have
     24        -- the same magicked value if that isn't right then more than one
     25        -- row will be returned. In practice this is the case but bugs could
     26        -- cause it to not be true.
     27        -- We could use rawExp.magicked but that would make it possible for
     28        -- the chipRun to have a different magicked value than the imfiles
     29        chipProcessedImfile.magicked AS imfile_magicked
    2430    FROM chipRun
    2531    JOIN rawImfile

  • branches/eam_branches/20091201/ippTools/share/chiptool_revertprocessedimfile.sql

    r19433 r26857  
    33WHERE
    44    rawExp.exp_id = chipProcessedImfile.exp_id
     5    AND chipRun.state = 'new'
    56    AND chipProcessedImfile.fault != 0

  • branches/eam_branches/20091201/ippTools/share/difftool_change_run_state.sql

    r25800 r26857  
    55UPDATE diffRun
    66    SET state = '%s'
     7    -- set magicked hook %s
    78    WHERE
    89    diffRun.diff_id = %lld

  • branches/eam_branches/20091201/ippTools/share/difftool_change_skyfile_data_state.sql

    r25800 r26857  
    55    SET
    66    data_state = '%s'
     7    -- magicked hook %s
    78WHERE
    89    diff_id = %lld

  • branches/eam_branches/20091201/ippTools/share/difftool_revertdiffskyfile_delete.sql

    r19462 r26857  
    33WHERE
    44    diffRun.diff_id = diffSkyfile.diff_id
     5    AND diffRun.state = 'new'
    56    AND fault != 0

  • branches/eam_branches/20091201/ippTools/share/difftool_todiffskyfile.sql

    r26750 r26857  
    6666    OR (diffRun.state = 'update'
    6767    AND diffSkyfile.fault = 0
    68     AND diffSkyfile.quality = 0)
     68    AND diffSkyfile.data_state = 'cleaned')
    6969    )
    7070-- Ensure input warps are available

  • branches/eam_branches/20091201/ippTools/share/disttool_definebyquery_camera.sql

    r26189 r26857  
    55    camRun.magicked,
    66    camRun.label,
     7    camRun.data_group,
    78    distTarget.dist_group,
    89    distTarget.target_id,

  • branches/eam_branches/20091201/ippTools/share/disttool_definebyquery_chip.sql

    r26189 r26857  
    55    rawExp.exp_name as run_tag,
    66    chipRun.label,
     7    chipRun.data_group,
    78    chipRun.dist_group,
    89    distTarget.target_id,

  • branches/eam_branches/20091201/ippTools/share/disttool_definebyquery_diff.sql

    r26188 r26857  
    55    rawExp.exp_name as run_tag,
    66    diffRun.label,
     7    diffRun.data_group,
    78    distTarget.dist_group,
    89    distTarget.target_id,

  • branches/eam_branches/20091201/ippTools/share/disttool_definebyquery_fake.sql

    r26189 r26857  
    55    rawExp.exp_name as run_tag,
    66    fakeRun.label,
     7    fakeRun.data_group,
    78    distTarget.dist_group,
    89    distTarget.target_id,

  • branches/eam_branches/20091201/ippTools/share/disttool_definebyquery_raw.sql

    r26189 r26857  
    55    rawExp.magicked,
    66    chipRun.label,
     7    chipRun.data_group,
    78    distTarget.dist_group,
    89    distTarget.target_id,

  • branches/eam_branches/20091201/ippTools/share/disttool_definebyquery_stack.sql

    r26189 r26857  
    66    CONCAT_WS('.', 'stack', stackRun.skycell_id, convert(stackRun.stack_id, CHAR)) as run_tag,
    77    stackRun.label,
     8    stackRun.data_group,
    89    distTarget.dist_group,
    910    distTarget.target_id,

  • branches/eam_branches/20091201/ippTools/share/disttool_definebyquery_warp.sql

    r26189 r26857  
    55    rawExp.exp_name as run_tag,
    66    warpRun.label,
     7    warpRun.data_group,
    78    distTarget.dist_group,
    89    distTarget.target_id,

  • branches/eam_branches/20091201/ippTools/share/disttool_pending_camera.sql

    r25838 r26857  
    3030    AND distRun.stage = 'camera'
    3131    AND distComponent.dist_id IS NULL
    32     AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic)
     32    AND (((chipRun.magicked > 0) AND (camRun.magicked > 0)) OR distRun.no_magic)
    3333    AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))

  • branches/eam_branches/20091201/ippTools/share/disttool_pending_chip.sql

    r25838 r26857  
    2828    AND distRun.stage = 'chip'
    2929    AND distComponent.dist_id IS NULL
    30     AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
     30    AND ((chipRun.magicked > 0) OR distRun.no_magic)
    3131    AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned'))

  • branches/eam_branches/20091201/ippTools/share/disttool_pending_diff.sql

    r25838 r26857  
    3636    AND distRun.stage = 'diff'
    3737    AND distComponent.dist_id IS NULL
    38     AND (distRun.clean OR diffRun.magicked OR distRun.no_magic)
     38    AND ((diffRun.magicked > 0) OR distRun.no_magic)
    3939    AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned'))

  • branches/eam_branches/20091201/ippTools/share/disttool_pending_warp.sql

    r25838 r26857  
    3131    AND distRun.stage = 'warp'
    3232    AND distComponent.dist_id IS NULL
    33     AND (distRun.clean OR warpRun.magicked OR distRun.no_magic)
     33    AND ((warpRun.magicked > 0) OR distRun.no_magic)
    3434    AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned'))

  • branches/eam_branches/20091201/ippTools/share/disttool_pendingfileset.sql

    r26170 r26857  
    55    distRun.outdir as dist_dir,
    66    stage_id,
    7     -- we need to get data_group here but we don't have
    8     -- the right tables joined. Perhaps we need to add that column to distRun
    9     distRun.label as data_group,
     7    distRun.data_group,
    108    distTarget.filter,
    119    rcDestination.name AS product_name,

  • branches/eam_branches/20091201/ippTools/share/magicdstool_getskycells.sql

    r24185 r26857  
    11SELECT DISTINCT
    22    diffSkyfile.diff_id,
     3    diffRun.tess_id,
    34    diffSkyfile.skycell_id,
    4     diffSkyfile.path_base
     5    diffSkyfile.path_base,
     6    diffSkyfile.data_state
    57FROM magicDSRun
    68JOIN magicRun USING(magic_id)
    79JOIN magicInputSkyfile USING(magic_id)
     10JOIN diffRun ON magicRun.diff_id = diffRun.diff_id
    811JOIN diffSkyfile
    912    ON magicRun.diff_id = diffSkyfile.diff_id

  • branches/eam_branches/20091201/ippTools/share/magictool_definebyquery_select.sql

    r26040 r26857  
    1515    WHERE diffInputSkyfile.warp1 IS NOT NULL
    1616        AND diffRun.exposure = 1
     17        AND diffRun.magicked = 0
    1718    -- diff WHERE hook %s
    1819    UNION
    … …  
    2728        AND diffRun.exposure = 1
    2829        AND diffRun.bothways = 1
     30        AND diffRun.magicked = 0
    2931    -- diff WHERE hook %s
    3032    ) AS diffWarps

  • branches/eam_branches/20091201/ippTools/share/pxadmin_create_tables.sql

    r26015 r26857  
    11811181    backup_path_base VARCHAR(255),
    11821182    recovery_path_base VARCHAR(255),
     1183    streak_frac FLOAT,
     1184    run_time FLOAT,
    11831185    fault SMALLINT,
    11841186    data_state VARCHAR(64),
    … …  
    13501352    time_stamp  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    13511353    fault       SMALLINT,
     1354    data_group  VARCHAR(64),
    13521355    note        VARCHAR(255),
    13531356    PRIMARY KEY(dist_id),
    … …  
    14281431    product VARCHAR(64) NOT NULL, -- product of interest
    14291432    workdir VARCHAR(255) NOT NULL, -- where to extract
     1433    state VARCHAR(64) NOT NULL, -- state 'enabled' or 'disabled'
    14301434    comment VARCHAR(255),       -- for human memory
    14311435    fileset_last VARCHAR(128),  -- last fileset seen

  • branches/eam_branches/20091201/ippTools/share/receivetool_list.sql

    r23880 r26857  
    11SELECT *
    22FROM receiveSource
     3WHERE receiveSource.state = 'enabled'

  • branches/eam_branches/20091201/ippTools/share/receivetool_pendingfile.sql

    r24788 r26857  
    1818    ON receiveResult.file_id = receiveFile.file_id
    1919WHERE
    20     ((receiveFileset.state = 'new')
     20    receiveSource.state = 'enabled'
     21    AND ((receiveFileset.state = 'new')
    2122    OR (receiveFileset.state = 'listed' AND receiveFile.component = 'dirinfo' AND receiveFileset.dirinfo IS NULL))
    2223    AND receiveFileset.fault = 0

  • branches/eam_branches/20091201/ippTools/share/receivetool_pendingfileset.sql

    r24788 r26857  
    1 SELECT *
     1SELECT receiveFileset.*,
     2    receiveSource.source,
     3    receiveSource.product,
     4    receiveSource.workdir
    25FROM receiveFileset
    36JOIN receiveSource USING(source_id)
    47LEFT JOIN receiveFile USING(fileset_id)
    5 WHERE receiveFileset.state = 'reg'
     8WHERE receiveSource.state = 'enabled'
     9    AND receiveFileset.state = 'reg'
    610    AND receiveFileset.fault = 0
    711    AND receiveFile.fileset_id IS NULL

  • branches/eam_branches/20091201/ippTools/share/receivetool_toadvance.sql

    r24125 r26857  
    1010    JOIN receiveFile USING(fileset_id)
    1111    LEFT JOIN receiveResult USING(file_id)
    12 WHERE receiveFileset.state = 'new'
     12WHERE receiveSource.state = 'enabled'
     13    AND receiveFileset.state = 'new'
    1314    AND receiveFileset.fault = 0
    1415GROUP BY fileset_id

  • branches/eam_branches/20091201/ippTools/share/stacktool_revertsumskyfile_delete.sql

    r19448 r26857  
    22USING stackSumSkyfile, stackRun
    33WHERE stackSumSkyfile.stack_id = stackRun.stack_id
     4    AND stackRun.state = 'new'
    45    AND fault != 0

  • branches/eam_branches/20091201/ippTools/share/warptool_change_run_state.sql

    r19521 r26857  
    44-- the chipProcessedImfile sub query
    55UPDATE warpRun
    6     SET state = '%s'
     6JOIN fakeRun USING(fake_id)
     7JOIN camRun USING(cam_id)
     8JOIN chipRun USING(chip_id)
     9    SET warpRun.state = '%s'
     10    -- set magicked  hook %s
    711    WHERE
    812    warpRun.warp_id = %lld
    … …  
    1216        WHERE
    1317            warpSkyfile.warp_id = warpRun.warp_id
    14             AND data_state != '%s'
     18            AND warpSkyfile.data_state != '%s'
    1519        ) = 0

  • branches/eam_branches/20091201/ippTools/share/warptool_change_skyfile_data_state.sql

    r23418 r26857  
    11-- handle changes in warpSkyfile.data_state.
    2 -- Used for the modes tocleanedskyfile and topurgedskyfile
    3 -- args are new data_state, warp_id, skycell_id and current expected state for warpRun
     2-- Used for the modes tofullskyfile, tocleanedskyfile and topurgedskyfile
     3-- args are new data_state, string for magic hook, warp_id, skycell_id and current expected state for warpRun
    44UPDATE warpSkyfile
     5JOIN warpRun USING(warp_id)
     6JOIN fakeRun USING(fake_id)
     7JOIN camRun USING(cam_id)
     8JOIN chipRun USING(chip_id)
    59    SET
    610    data_state = '%s'
     11    -- set magicked hook %s
    712WHERE
    813    warp_id = %lld

  • branches/eam_branches/20091201/ippTools/share/warptool_finish_run.sql

    r23430 r26857  
    11UPDATE warpRun
    22    SET state = 'full',
    3     magicked = %d
     3    magicked = %lld
    44WHERE warp_id = %lld
    55

  • branches/eam_branches/20091201/ippTools/share/warptool_finished_run_select.sql

    r24043 r26857  
    88       warpSkyCellMap.warp_id as foo,
    99       warpSkyfile.warp_id as bar,
    10        SUM(!warpSkyfile.magicked) = 0 as magicked
     10       warpSkyfile.magicked as magicked
    1111   FROM warpRun
    1212   JOIN warpSkyCellMap

  • branches/eam_branches/20091201/ippTools/share/warptool_towarped.sql

    r24487 r26857  
    4040        AND warpSkyfile.tess_id IS NULL)
    4141    OR (warpRun.state = 'update'
     42        AND chipRun.magicked >= 0
     43        AND warpSkyfile.fault = 0
    4244        AND warpSkyfile.data_state = 'cleaned')
    4345    )

  • branches/eam_branches/20091201/ippTools/src

    • Property svn:ignore

      •  

        old new  
        1616difftool
        1717disttool
         18dqstatstool
        1819faketool
        1920flatcorr
        … …  
        3637stamp-h1
        3738warptool
         39

  • branches/eam_branches/20091201/ippTools/src/Makefile.am

    r25773 r26857  
    99        disttool \
    1010        flatcorr \
     11        dqstatstool \
    1112        magictool \
    1213        magicdstool \
    … …  
    3233        pxdata.h \
    3334        pxfake.h \
     35        pxdqstats.h
    3436        pxmagic.h \
    3537        pxregister.h \
    … …  
    5254        flatcorr.h \
    5355        faketool.h \
     56        dqstatstool.h \
    5457        magictool.h \
    5558        magicdstool.h \
    … …  
    7780        pxfake.c \
    7881        pxfault.c \
     82        pxdqstats.c \
    7983        pxmagic.c \
    8084        pxregister.c \
    … …  
    142146    faketool.c \
    143147    faketoolConfig.c
     148
     149dqstatstool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
     150dqstatstool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
     151dqstatstool_SOURCES = \
     152    dqstatstool.c \
     153    dqstatstoolConfig.c
    144154
    145155magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)

  • branches/eam_branches/20091201/ippTools/src/camtool.c

    r26113 r26857  
    222222                    tess_id     ? tess_id   : row->tess_id,
    223223                    end_stage   ? end_stage : row->end_stage,
     224                    row->magicked,
    224225                    note
    225226        )) {
    … …  
    480481    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
    481482
    482     PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
     483//    Get this from the chipRun
     484//    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
    483485
    484486    // generate restrictions
    … …  
    523525        return false;
    524526    }
     527
     528    bool status;
     529    psS64 chip_magicked = psMetadataLookupS64(&status, output->data[0], "chip_magicked");
    525530
    526531    camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]);
    … …  
    603608    }
    604609
    605     // NULL for end_stage means go as far as possible
    606     // EAM : skip here if fault != 0
    607     // Also, we can run fake even if tess_id is not defined
    608     if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam"))) {
     610    if (fault) {
    609611        psFree(row);
    610612        psFree(pendingRow);
    … …  
    618620
    619621    // since there is only one exp per 'new' set camRun.state = 'full'
    620     if (!pxcamRunSetState(config, row->cam_id, "full", magicked)) {
     622    // propagate magicked state from chipRun
     623    if (!pxcamRunSetState(config, row->cam_id, "full", chip_magicked)) {
    621624        psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id);
    622625        psFree(row);
    … …  
    625628    }
    626629    psFree(row);
     630
     631    // EAM:  NULL for end_stage means go as far as possible
     632    // Also, we can run fake even if tess_id is not defined
     633    if (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam")) {
     634        psFree(pendingRow);
     635        if (!psDBCommit(config->dbh)) {
     636            psError(PS_ERR_UNKNOWN, false, "database error");
     637            return false;
     638        }
     639        return true;
     640    }
    627641
    628642    if (!pxfakeQueueByCamID(config,

  • branches/eam_branches/20091201/ippTools/src/chiptool.c

    r25935 r26857  
    612612    if (psListLength(where->list)) {
    613613        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    614         psStringAppend(&query, " WHERE %s", whereClause);
     614        psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : "");
    615615        psFree(whereClause);
    616616    } else if (psMetadataLookupBool(NULL, config->args, "-all") || (faulted || where2)) {
    … …  
    720720    PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
    721721    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL);
     722    if (psListLength(where->list) == 0) {
     723        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
     724        return false;
     725    }
    722726       
    723727    if (!state) {
    … …  
    12541258                    chipRun->tess_id,
    12551259                    chipRun->end_stage,
     1260                    chipRun->magicked,
    12561261                    NULL    // note does not propagate
    12571262        )) {
    … …  
    12951300    }
    12961301
    1297     psString set_magic = "";
     1302    char *set_magicked_imfile = "";
     1303    char *set_magicked_run = "";
    12981304    if (!strcmp(data_state, "full")) {
    1299         // copy the magicked state from the input to the output when transitioning to full state
    1300         set_magic = "\n , chipProcessedImfile.magicked = rawImfile.magicked";
    1301     }
    1302 
    1303     // note only updates if chipRun.state = run_state
    1304     // XXX note that we have removed this constraint for now
    1305     if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magic, chip_id, class_id)) {
     1305        // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will
     1306        // block warp processing until destreaking has been done
     1307        // otherwise copy magicked from the rawImfile
     1308        // Same thing for chipRun/rawExp
     1309        set_magicked_imfile = "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0 AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)";
     1310        set_magicked_run = "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), chipRun.magicked, rawExp.magicked)";
     1311    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
     1312        // if magicked is non-zero set it to -1
     1313        set_magicked_imfile = "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)";
     1314        set_magicked_run = "\n, chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)";
     1315    }
     1316
     1317    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) {
    13061318        psFree(query);
    13071319        psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    13201332
    13211333    query = pxDataGet("chiptool_change_exp_state.sql");
    1322     if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, data_state)) {
     1334    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) {
    13231335        psFree(query);
    13241336        // rollback

  • branches/eam_branches/20091201/ippTools/src/chiptoolConfig.c

    r25835 r26857  
    154154    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
    155155    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
    156     psMetadataAddBool(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", false);
     156    psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", 0);
    157157
    158158    // -processedimfile

  • branches/eam_branches/20091201/ippTools/src/dettool.c

    r25324 r26857  
    550550    PXOPT_COPY_F32(config->args, where, "-select_airmass_max", "airmass", "<=");
    551551    PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_max", "sat_pixel_frac", "<=");
     552    PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_min", "sat_pixel_frac", ">=");
    552553    PXOPT_COPY_F32(config->args, where, "-select_exp_time_min", "exp_time", ">=");
    553554    PXOPT_COPY_F32(config->args, where, "-select_exp_time_max", "exp_time", "<=");

  • branches/eam_branches/20091201/ippTools/src/dettoolConfig.c

    r25324 r26857  
    117117    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max",  0,            "define max airmass", NAN);
    118118    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max",  0,            "define max fraction of saturated pixels", NAN);
     119    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_min",  0,            "define min fraction of saturated pixels", NAN);
    119120    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min",  0,            "define min exposure time", NAN);
    120121    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max",  0,            "define max exposure time", NAN);

  • branches/eam_branches/20091201/ippTools/src/difftool.c

    r26181 r26857  
    5555static bool topurgedskyfileMode(pxConfig *config);
    5656static bool toscrubbedskyfileMode(pxConfig *config);
     57static bool tofullskyfileMode(pxConfig *config);
    5758
    5859
    … …  
    9495        MODECASE(DIFFTOOL_MODE_EXPORTRUN,             exportrunMode);
    9596        MODECASE(DIFFTOOL_MODE_IMPORTRUN,             importrunMode);
    96         MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE,   tocleanedskyfileMode);
    97         MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE,    topurgedskyfileMode);
    98         MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,  toscrubbedskyfileMode);
     97        MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE,      tocleanedskyfileMode);
     98        MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE,       topurgedskyfileMode);
     99        MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,     toscrubbedskyfileMode);
     100        MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE,         tofullskyfileMode);
    99101
    100102        default:
    … …  
    186188    psMetadata *where = psMetadataAlloc();
    187189
    188     PXOPT_COPY_S64(config->args, where, "-diff_id",  "stack_id",   "==");
     190    PXOPT_COPY_S64(config->args, where, "-diff_id",  "diff_id",   "==");
    189191    PXOPT_COPY_STR(config->args, where, "-label",     "label",     "==");
    190192    PXOPT_COPY_STR(config->args, where, "-state",     "state",     "==");
    … …  
    11461148    PXOPT_COPY_STR(config->args, warp1Where, "-warp_label", "warpRun.label", "==");
    11471149    PXOPT_COPY_STR(config->args, warp1Where, "-tess_id", "warpRun.tess_id", "==");
     1150    PXOPT_COPY_STR(config->args, warp1Where, "-data_group", "warpRun.data_group", "==");
    11481151    PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "==");
    11491152    PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "==");
     1153    PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "==");
    11501154    PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "==");
    11511155    PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "==");
    … …  
    13611365            return false;
    13621366        }
     1367
     1368
    13631369        psString filter = psMetadataLookupStr(&mdok, row, "filter");
    13641370        if (!mdok) {
    … …  
    20362042  PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    20372043
     2044
     2045  psS64 magicked = 0;
     2046  if (!strcmp(data_state, "full")) {
     2047      PXOPT_LOOKUP_S64(set_magicked, config->args, "-magicked", 0, false);
     2048      magicked = set_magicked;
     2049  }
     2050
    20382051  psString query = pxDataGet("difftool_change_skyfile_data_state.sql");
    20392052
    … …  
    20432056  }
    20442057
     2058  psString set_magicked_skyfile = psStringCopy("");
     2059  psString set_magicked_run = psStringCopy("");
     2060  if (magicked != 0 && !strcmp(data_state, "full")) {
     2061    psStringAppend(&set_magicked_skyfile,  "\n , diffSkyfile.magicked = %" PRId64, magicked);
     2062    psStringAppend(&set_magicked_run,      "\n , diffRun.magicked = %" PRId64, magicked);
     2063
     2064  } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
     2065    psStringAppend(&set_magicked_skyfile, "\n, diffSkyfile.magicked = IF(diffSkyfile.magicked = 0, 0, -1)");
     2066    psStringAppend(&set_magicked_run, "\n, diffRun.magicked = IF(diffRun.magicked = 0, 0, -1)");
     2067  }
     2068
    20452069  // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c
    20462070
    2047   if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, skycell_id)) {
     2071  if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, diff_id, skycell_id)) {
    20482072    psError(PS_ERR_UNKNOWN, false, "database error");
    20492073    // rollback
    … …  
    20522076    }
    20532077    psError(PS_ERR_UNKNOWN, false, "database error");
     2078    psFree(set_magicked_skyfile);
    20542079    return(false);
    20552080  }
     2081  psFree(set_magicked_skyfile);
    20562082  psFree(query);
    20572083
    20582084  query = pxDataGet("difftool_change_run_state.sql");
    2059   if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, data_state)) {
     2085  if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, diff_id, data_state)) {
    20602086    // rollback
    20612087    if (!psDBRollback(config->dbh)) {
    … …  
    20632089    }
    20642090    psError(PS_ERR_UNKNOWN, false, "database error");
     2091    psFree(set_magicked_run);
    20652092    return(false);
    20662093  }
    … …  
    20682095  if (!psDBCommit(config->dbh)) {
    20692096    psError(PS_ERR_UNKNOWN, false, "database error");
     2097    psFree(set_magicked_run);
    20702098    return(false);
    20712099  }
     2100  psFree(set_magicked_run);
    20722101
    20732102  return(true);
    … …  
    20822111static bool toscrubbedskyfileMode(pxConfig *config) {
    20832112  return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed");
     2113}
     2114static bool tofullskyfileMode(pxConfig *config) {
     2115  return change_skyfile_data_state(config, "full", "update");
    20842116}
    20852117

  • branches/eam_branches/20091201/ippTools/src/difftool.h

    r25800 r26857  
    4545    DIFFTOOL_MODE_TOCLEANEDSKYFILE,
    4646    DIFFTOOL_MODE_TOPURGEDSKYFILE,
    47     DIFFTOOL_MODE_TOSCRUBBEDSKYFILE
     47    DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,
     48    DIFFTOOL_MODE_TOFULLSKYFILE
    4849   
    4950} difftoolMode;

  • branches/eam_branches/20091201/ippTools/src/difftoolConfig.c

    r26177 r26857  
    122122    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0,   "define hostname", 0);
    123123    psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac",  0, "define %% of good pixels", NAN);
    124     psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", false);
     124    psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", 0);
    125125
    126126    // -advance
    … …  
    183183    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
    184184    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
     185    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL);
    185186    psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
    186187    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL);
    … …  
    253254    // -pendingcleanuprun
    254255    psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
    255     psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     256    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list runs for cleanup with specified label", NULL);
    256257    psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    257258    psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    … …  
    288289    psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0);
    289290    psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
     291
     292    // -tofullskyfile
     293    psMetadata *tofullskyfileArgs = psMetadataAlloc();
     294    psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0);
     295    psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
     296    psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", 0);
    290297
    291298    // -exportrun
    … …  
    329336    PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs);
    330337    PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs);
     338    PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs);
    331339
    332340    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

  • branches/eam_branches/20091201/ippTools/src/disttool.c

    r26210 r26857  
    358358        psS64 target_id = psMetadataLookupS64(NULL, md, "target_id");
    359359        psString stage_label = psMetadataLookupStr(NULL, md, "label");
     360        psString data_group = psMetadataLookupStr(NULL, md, "data_group");
    360361        bool clean = psMetadataLookupBool(NULL, md, "clean");
    361362
    … …  
    383384                NULL,    // time_stamp
    384385                0,       // fault
     386                data_group,
    385387                set_note // note does not propagate
    386388                );
    … …  
    16151617    PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "==");
    16161618    PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "LIKE");
     1619    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
    16171620    PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE");
    16181621    PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE");
    … …  
    16621665    PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE");
    16631666    PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE");
    1664     PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
     1667    PXOPT_COPY_STR(config->args, where, "-state", "rcInterest.state", "==");
    16651668
    16661669    PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);

  • branches/eam_branches/20091201/ippTools/src/disttoolConfig.c

    r26210 r26857  
    293293    psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0);
    294294    psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL);
     295    psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-stage",    0, "define stage", NULL);
    295296    psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-filter",    0, "define filter (LIKE comparison)", NULL);
    296297    psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-dest_name",    0, "define destination name", NULL);

  • branches/eam_branches/20091201/ippTools/src/flatcorr.c

    r25913 r26857  
    535535                row->tess_id,
    536536                "camera",
     537                row->magicked,
    537538                NULL // note does not propragate
    538539                )) {

  • branches/eam_branches/20091201/ippTools/src/magicdstool.c

    r25935 r26857  
    4444static bool tocleanupMode(pxConfig *config);
    4545
    46 static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psMetadata *where, const char *state);
     46static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state);
    4747
    4848# define MODECASE(caseName, func) \
    … …  
    326326    PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
    327327
     328    // optional
     329    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
     330    psString setString = NULL;
     331    if (set_label) {
     332        psStringAppend(&setString, ", label = '%s'", set_label);
     333    }
     334
    328335    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false);
    329336    if (magic_ds_id) {
    330337
    331         return setmagicDSRunState(config, magic_ds_id, NULL, state);
     338        return setmagicDSRunState(config, magic_ds_id, setString, NULL, state);
    332339
    333340    } else if (!strcmp(state, "full")) {
    … …  
    336343    }
    337344    // we can transition by query as well
     345
    338346
    339347    psMetadata *where = psMetadataAlloc();
    … …  
    342350    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
    343351
    344 
    345352    if (psListLength(where->list) < 2) {
    346353        psError(PS_ERR_UNKNOWN, true, "at least 2 search arguments are required");
    … …  
    353360        psMetadataAddS32(where, PS_LIST_TAIL, "re_place", 0, ">", 0);
    354361    }
    355     bool result = setmagicDSRunState(config, magic_ds_id, where, state);
     362    bool result = setmagicDSRunState(config, magic_ds_id, setString, where, state);
    356363    psFree(where);
    357364
    … …  
    445452{
    446453    // first query the magicDSRun to find the stage and the stage_id
    447     psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;
     454    psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64;
    448455
    449456    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
    … …  
    472479    psString stage= psMetadataLookupStr(NULL, row, "stage");
    473480    psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
     481    psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
    474482
    475483    ippStage stageNum = ippStringToStage(stage);
    … …  
    499507    }
    500508
    501     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) {
     509    if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id, component)) {
    502510        psError(PS_ERR_UNKNOWN, false, "database error");
    503511        return false;
    … …  
    518526{
    519527    // first query the magicDSRun to find the stage and the stage_id
    520     psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;
     528    psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64;
    521529
    522530    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
    … …  
    545553    psString stage = psMetadataLookupStr(NULL, row, "stage");
    546554    psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
     555    psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
    547556
    548557    ippStage stageNum = ippStringToStage(stage);
    … …  
    570579        return false;
    571580    }
    572     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) {
     581    if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id)) {
    573582        psError(PS_ERR_UNKNOWN, false, "database error");
    574583        return false;
    … …  
    598607    PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false);
    599608    PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false);
     609    PXOPT_LOOKUP_F32(streak_frac, config->args, "-streak_frac", false, false);
     610    PXOPT_LOOKUP_F32(run_time, config->args, "-run_time", false, false);
    600611
    601612    if (setmagicked && (fault != 0)) {
    … …  
    620631    }
    621632
    622     if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault, "full")) {
     633    if (!magicDSFileInsert(config->dbh,
     634            magic_ds_id,
     635            component,
     636            backup_path_base,
     637            recovery_path_base,
     638            streak_frac,
     639            run_time,
     640            fault,
     641            "full"  // data_state
     642        )) {
    623643            // rollback
    624644        if (!psDBRollback(config->dbh)) {
    … …  
    699719
    700720        // set magicDSRun.state to 'full'
    701         if (!setmagicDSRunState(config, magic_ds_id, NULL, "full")) {
     721        if (!setmagicDSRunState(config, magic_ds_id, NULL, NULL, "full")) {
    702722            psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64,
    703723                magic_ds_id);
    … …  
    901921    if (!((strcmp(state, "new") == 0) ||
    902922          (strcmp(state, "full") == 0) ||
     923          (strcmp(state, "failed_revert") == 0) ||
     924          (strcmp(state, "failed_cleanup") == 0) ||
    903925          (strcmp(state, "restored") == 0) ||
    904926          (strcmp(state, "censored") == 0) ||
    … …  
    914936}
    915937
    916 static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psMetadata *where, const char *state)
     938static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psString extraSetStr, psMetadata *where, const char *state)
    917939{
    918940    PS_ASSERT_PTR_NON_NULL(state, false);
    … …  
    924946    }
    925947
    926     psString query = psStringCopy("UPDATE magicDSRun SET state = '%s'");
     948    psString query = NULL;
     949    psStringAppend(&query, "UPDATE magicDSRun SET state = '%s' %s\n", state, extraSetStr ? extraSetStr : "");
    927950    if (magic_ds_id) {
    928951        psStringAppend(&query, " WHERE magic_ds_id = %" PRId64, magic_ds_id);
    … …  
    936959    }
    937960
    938     if (!p_psDBRunQueryF(config->dbh, query, state)) {
     961    if (!p_psDBRunQuery(config->dbh, query)) {
    939962        psError(PS_ERR_UNKNOWN, false,
    940963                "failed to change state for magic_id %" PRId64, magic_ds_id);

  • branches/eam_branches/20091201/ippTools/src/magicdstoolConfig.c

    r25843 r26857  
    7474    psMetadata *updaterunArgs = psMetadataAlloc();
    7575    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL);
     76    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL);
    7677    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID", 0);
    7778    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage",     0, "define stage", NULL);
    … …  
    102103    psMetadataAddStr(adddestreakedfileArgs, PS_LIST_TAIL, "-recovery_path_base", 0, "define recovery pixels URI", NULL);
    103104    psMetadataAddBool(adddestreakedfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false);
     105    psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-streak_frac", 0, "set fraction of pixels masked by streaks", 0);
     106    psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-run_time", 0, "set the streaksremove run time for component ", 0);
    104107    psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
    105108

  • branches/eam_branches/20091201/ippTools/src/pstamptool.c

    r26213 r26857  
    925925    PXOPT_LOOKUP_BOOL(no_create,  config->args, "-no_create", false);
    926926
     927    psMetadata *where = psMetadataAlloc();
     928    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
     929    PXOPT_COPY_STR(config->args, where, "-imagedb", "imagedb", "==");
     930    PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
     931
     932    // start a transaction eraly so it will contain any row level locks
     933    if (!psDBTransaction(config->dbh)) {
     934        psError(PS_ERR_UNKNOWN, false, "database error");
     935        return false;
     936    }
     937
    927938    psString query = pxDataGet("pstamptool_pendingdependent.sql");
    928939    if (!query) {
    … …  
    930941        return false;
    931942    }
     943    psString whereClause = psDBGenerateWhereSQL(where, "WHERE");
     944    if (!no_create) {
     945        // This will lock the row until the transaction is committed
     946        psStringAppend(&query, " %s FOR UPDATE", whereClause);
     947    } else {
     948        psStringAppend(&query, " %s", whereClause);
     949    }
     950    psFree(whereClause);
     951    psFree(where);
     952
    932953    if (!p_psDBRunQuery(config->dbh, query)) {
    933954        psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    947968        if (!dep_id) {
    948969            psError(PS_ERR_UNKNOWN, false, "database error");
     970            psFree(output);
    949971            return false;
    950972        }
    951973        printf("%" PRId64 "\n", dep_id);
     974        psFree(output);
     975        if (!psDBRollback(config->dbh)) {
     976            psError(PS_ERR_UNKNOWN, false, "database error");
     977        }
    952978        return true;
    953979    }
    954980    if (no_create) {
     981        if (!psDBRollback(config->dbh)) {
     982            psError(PS_ERR_UNKNOWN, false, "database error");
     983            return false;
     984        }
    955985        return true;
    956986    }
    957987    // no existing dependent that matches, insert one
     988    // Since we have multiple processes running jobs we have a
     989    // race condition here so that's why we need to lock the table
    958990
    959991    if (!pstampDependentInsert(
    … …  
    967999        no_magic
    9681000        )) {
     1001        if (!psDBRollback(config->dbh)) {
     1002            psError(PS_ERR_UNKNOWN, false, "database error");
     1003        }
    9691004        psError(PS_ERR_UNKNOWN, false, "failed to insert pstampDependent");
    9701005        return false;
    9711006    }
    9721007
     1008    if (!psDBCommit(config->dbh)) {
     1009        // rollback
     1010        if (!psDBRollback(config->dbh)) {
     1011            psError(PS_ERR_UNKNOWN, false, "database error");
     1012        }
     1013        psError(PS_ERR_UNKNOWN, false, "database error");
     1014        return false;
     1015    }
     1016
    9731017    psS64 dep_id = psDBLastInsertID(config->dbh);
    974 
    9751018    printf("%" PRId64 "\n", dep_id);
    9761019

  • branches/eam_branches/20091201/ippTools/src/pxcam.c

    r25835 r26857  
    152152                    char *tess_id,
    153153                    char *end_stage,
     154                    psS64 magicked,
    154155                    char *note)
    155156{
    … …  
    168169
    169170    // queue the exp
    170     // XXX chip_id is being cast here work around psS64 have a different type
     171    // XXX chip_id and magicked are being cast here work around psS64 have a different type
    171172    // different on 32/64
    172173    if (!p_psDBRunQueryF(config->dbh, query,
    … …  
    182183                tess_id  ? tess_id  : "NULL",
    183184                end_stage ? end_stage : "NULL",
     185                (long long) magicked,
    184186                note     ? note     : "NULL",
    185187                (long long)chip_id

  • branches/eam_branches/20091201/ippTools/src/pxcam.h

    r25835 r26857  
    4040                        char *tess_id,
    4141                        char *end_stage,
     42                        psS64 magicked,
    4243                        char *note);
    4344

  • branches/eam_branches/20091201/ippTools/src/pxchip.c

    r25835 r26857  
    129129}
    130130
    131 bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked)
     131bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked)
    132132{
    133133    PS_ASSERT_PTR_NON_NULL(config, false);
    … …  
    140140    }
    141141
    142     char *query = "UPDATE chipRun SET state = '%s', magicked = %d WHERE chip_id = %" PRId64;
     142    char *query = "UPDATE chipRun SET state = '%s', magicked = %" PRId64 " WHERE chip_id = %" PRId64;
    143143    if (!p_psDBRunQueryF(config->dbh, query, state, magicked, chip_id)) {
    144144        psError(PS_ERR_UNKNOWN, false,
    … …  
    174174
    175175    psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'");
     176
     177    if (!strcmp(state, "cleaned") || !strcmp(state, "purged")) {
     178        // if magicked is non-zero set it to -1
     179        psStringAppend(&query, ", chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)");
     180    }
    176181
    177182    if (where && psListLength(where->list) > 0) {
    … …  
    219224        psStringAppend(&query, " %s", whereClause);
    220225        psFree(whereClause);
     226    } else {
     227        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
     228        return false;
    221229    }
    222230

  • branches/eam_branches/20091201/ippTools/src/pxchip.h

    r25835 r26857  
    2727
    2828
    29 bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked);
     29bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked);
    3030bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state);
    3131

  • branches/eam_branches/20091201/ippTools/src/pxmagic.c

    r25771 r26857  
    8585    }
    8686    if (destreaked) {
    87         psStringAppend(out, " AND %s.magicked != 0", table);
     87        psStringAppend(out, " AND %s.magicked > 0", table);
    8888    }
    8989    return true;

  • branches/eam_branches/20091201/ippTools/src/pztool.c

    r25418 r26857  
    117117            telescope,
    118118            uri,
    119             NULL  // epoch
     119            NULL,  // epoch
     120            0      // use_compress                         
    120121        )) {
    121122        psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    783784    psMetadata *where = psMetadataAlloc();
    784785    PXOPT_COPY_STR(config->args, where,  "-exp_name", "exp_name", "==");
    785     PXOPT_COPY_STR(config->args, where, "-inst", "inst", "==");
     786    PXOPT_COPY_STR(config->args, where, "-inst", "camera", "==");
    786787    PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "==");
    787788    PXOPT_COPY_STR(config->args, where,  "-class", "class", "==");

  • branches/eam_branches/20091201/ippTools/src/receivetool.c

    r24784 r26857  
    105105    // optional
    106106    PXOPT_LOOKUP_STR(comment, config->args, "-comment",  false, false);
     107    PXOPT_LOOKUP_STR(state, config->args, "-state",  false, false);
    107108    PXOPT_LOOKUP_STR(last, config->args, "-last",  false, false);
    108109    PXOPT_LOOKUP_STR(status_product, config->args, "-status_product",  false, false);
    … …  
    110111    PXOPT_LOOKUP_STR(ds_dbhost, config->args, "-ds_dbhost",  false, false);
    111112
    112     if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, comment, last, status_product, ds_dbname, ds_dbhost)) {
     113    if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, state, comment, last, status_product, ds_dbname, ds_dbhost)) {
    113114        psError(PS_ERR_UNKNOWN, false, "Database error");
    114115        return false;
    … …  
    142143    if (psListLength(where->list)) {
    143144        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    144         psStringAppend(&query, " WHERE %s", whereClause);
     145        psStringAppend(&query, " AND %s", whereClause);
    145146        psFree(whereClause);
    146147    }

  • branches/eam_branches/20091201/ippTools/src/receivetoolConfig.c

    r24125 r26857  
    4949    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-workdir",   0, "define workdir (required)", NULL);
    5050    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL);
     51    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-state", 0, "define state", "enabled");
    5152    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-last", 0, "define last fileset", NULL);
    5253    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-status_product", 0, "define status_product", NULL);

  • branches/eam_branches/20091201/ippTools/src/warptool.c

    r25935 r26857  
    11931193    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs",  ">=");
    11941194    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawExp.dateobs",  "<=");
    1195     PXOPT_COPY_STR(config->args, where, "-filter",    "rawExp.filter", "==");
     1195    PXOPT_COPY_STR(config->args, where, "-filter",    "rawExp.filter", "LIKE");
    11961196    PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "==");
    11971197    pxAddLabelSearchArgs (config, where, "-label",   "warpRun.label", "LIKE");
     1198    pxAddLabelSearchArgs (config, where, "-data_group",   "warpRun.data_group", "LIKE");
    11981199
    11991200    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
    … …  
    16461647    }
    16471648
    1648     // note only updates if warpRun.state = run_state
    1649     // XXX note that we have removed this constraint for now
    1650     if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) {
     1649    char *set_magicked_skyfile = "";
     1650    char *set_magicked_run = "";
     1651    if (!strcmp(data_state, "full")) {
     1652        set_magicked_skyfile = "\n , warpSkyfile.magicked = IF((warpSkyfile.magicked < 0 AND chipRun.magicked = 0), warpSkyfile.magicked, chipRun.magicked)";
     1653        set_magicked_run = "\n,  warpRun.magicked = IF((warpRun.magicked < 0 AND chipRun.magicked = 0), warpRun.magicked, chipRun.magicked)";
     1654
     1655    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
     1656        set_magicked_skyfile = "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1)";
     1657        set_magicked_run = "\n, warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)";
     1658    }
     1659
     1660    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) {
    16511661        psError(PS_ERR_UNKNOWN, false, "database error");
    16521662        // rollback
    … …  
    16601670
    16611671    query = pxDataGet("warptool_change_run_state.sql");
    1662     if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, data_state)) {
     1672    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) {
    16631673        // rollback
    16641674        if (!psDBRollback(config->dbh)) {

  • branches/eam_branches/20091201/ippTools/src/warptoolConfig.c

    r25835 r26857  
    212212    psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", 0);
    213213    psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
    214     psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", false);
     214    psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", 0);
    215215    psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
    216216    psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
    … …  
    230230    psMetadataAddS16(warpedArgs, PS_LIST_TAIL,  "-fault",  0,           "search by fault code", 0);
    231231    psMetadataAddStr(warpedArgs, PS_LIST_TAIL,  "-label",  PS_META_DUPLICATE_OK, "search by warpRun label", NULL);
     232    psMetadataAddStr(warpedArgs, PS_LIST_TAIL,  "-data_group",  PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL);
    232233    // add magic related arguments
    233234    pxmagicAddArguments(warpedArgs);
    … …  
    310311    psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
    311312
    312     // -toupdateskyfile
     313    // -updateskyfile
    313314    psMetadata *updateskyfileArgs = psMetadataAlloc();
    314315    psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,    "warptool ID to update", 0);
Note: See TracChangeset for help on using the changeset viewer.