IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 24427


Ignore:
Timestamp:
Jun 15, 2009, 5:28:09 PM (17 years ago)
Author:
Paul Price
Message:

Adding script to do the actual publishing to DataStore. Decided to bundle all components into the same script (we're working with detections, it's serial, should be fast) to simply things (can remove the 'advance' mode for pubtool). Everything compiles, not tested yet.

Location:
branches/pap_mops
Files:
1 added
1 deleted
11 edited

Legend:

Unmodified
Added
Removed

  • branches/pap_mops/dbconfig/publish.md

    r24413 r24427  
    33publishClient   METADATA
    44    client_id   S64         0       # Primary Key AUTO_INCREMENT
     5    product     STR         64
    56    stage       STR         64
    6     destination STR         255
    7     format      STR         64
    87    comment     STR         255
    98END
    … …  
    1312    client_id   S64         0
    1413    stage_id    S64         0
     14    workdir     STR         255
    1515    label       STR         64
    1616    state       STR         64
    1717END
    1818
    19 publishFile     METADATA
     19publishDone     METADATA
    2020    pub_id      S64         0       # Primary Key
    21     component   STR         64      # Primary Key
     21    path_base   STR         255
    2222    fault       S16         0
    2323END

  • branches/pap_mops/ippScripts/Build.PL

    r24174 r24427  
    9090        scripts/receive_setstatus.pl
    9191        scripts/rcserver_checkstatus.pl
     92        scripts/publish_file.pl
    9293    )],
    9394    dist_abstract => 'Scripts for running the Pan-STARRS IPP',

  • branches/pap_mops/ippTools/doc/publish_flow.txt

    r24413 r24427  
    33
    44### Regularly run by pantasks to generate publishing runs based on data that has been processed
    5 pubtool -definerun
     5pubtool -definerun -workdir /path/to/somewhere
    66
    77### Regularly run by pantasks to get list of files to publish
    88pubtool -pending
    9 --> publish_file.pl --pub_id 12345 --component skycell.123.456 --destination DataStore --stage diff --stage_id 67890 --format MOPS
    10 ----> pubtool -add -pub_id 12345 --component skycell.123.456 --fault 0
     9--> publish_file.pl --pub_id 12345 --workdir /path/to/somewhere --product IPP-MOPS --stage diff --stage_id 67890
     10----> pubtool -add -pub_id 12345 -path_base /path/to/somewherenice -fault 0
    1111
    1212### Run as required
    1313pubtool -revert
    1414
    15 ### Regularly run by pantasks to mark running publish runs as 'full'
    16 pubtool -advance

  • branches/pap_mops/ippTools/share/Makefile.am

    r24174 r24427  
    167167     pxadmin_create_mirror_tables.sql \
    168168     pxadmin_drop_tables.sql \
     169     pubtool_definerun.sql \
     170     pubtool_pending.sql \
     171     pubtool_revert.sql \
    169172     pztool_find_completed_exp.sql \
    170173     pztool_pendingimfile.sql \

  • branches/pap_mops/ippTools/share/difftool_skyfile.sql

    r24416 r24427  
    66    diffRun.workdir,
    77    diffRun.bothways,
     8    camProcessedExp.zpt_obs,
     9    camProcessedExp.zpt_stdev,
     10    camProcessedExp.zpt_lq,
     11    camProcessedExp.zpt_uq,
    812    warp1,
    913    stack1,
    … …  
    1418JOIN diffInputSkyfile USING(diff_id, skycell_id)
    1519JOIN warpRun
    16     ON warpRun.warp_id = diffInputSkyfile.warp1
     20    ON warpRun.warp_id = diffInputSkyfile.warp1 -- NOTE: joining input only!
    1721JOIN fakeRun USING(fake_id)
    1822JOIN camRun USING(cam_id)
     23JOIN camProcessedExp USING(cam_id)
    1924JOIN chipRun USING(chip_id)
    2025JOIN rawExp USING(exp_id)

  • branches/pap_mops/ippTools/share/pubtool_pending.sql

    r24392 r24427  
    11SELECT
    2     publishRun.pub_id,
    3     publishRun.stage_id,
    4     publishClient.destination,
    5     publishClient.stage,
    6     publishClient.format,
    7     publishToDo.component
    8 FROM publishRun
    9 JOIN publishClient USING(client_id)
    10 JOIN ((
     2    publishToDo.*
     3FROM ((
    114    SELECT
    12         pub_id,
    13         diff_id AS stage_id,
    14         skycell_id AS component
     5        publishRun.pub_id,
     6        publishClient.product,
     7        publishClient.stage,
     8        diffRun.diff_id AS stage_id,
     9        publishRun.workdir
    1510    FROM publishRun
    1611    JOIN publishClient USING(client_id)
    … …  
    1914    JOIN diffSkyfile USING(diff_id)
    2015    WHERE publishClient.stage = 'diff'
     16        AND publishRun.state = 'new'
     17        AND diffRun.state = 'full'
    2118    ) AS publishDiffs UNION (
    2219    SELECT
    23         pub_id,
    24         cam_id AS stage_id,
    25         NULL AS component
     20        publishRun.pub_id,
     21        publishClient.product,
     22        publishClient.stage,
     23        camRun.cam_id AS stage_id,
     24        publishRun.workdir
    2625    FROM publishRun
    2726    JOIN publishClient USING(client_id)
    … …  
    2928        ON camRun.cam_id = publishRun.stage_id
    3029    WHERE publishClient.stage = 'camera'
     30        AND publishRun.state ='new'
     31        AND camRun.state = 'full'
    3132    ) AS publishCams
    32 ) AS publishToDo USING(pub_id, stage_id)
    33 LEFT JOIN publishFile USING(pub_id, component)
    34 WHERE publishRun.state = 'new'
    35     AND publishFile.pub_id IS NULL
     33) AS publishToDo
     34LEFT JOIN publishDone USING(pub_id)
     35WHERE publishDone.pub_id IS NULL

  • branches/pap_mops/ippTools/share/pubtool_revert.sql

    r24413 r24427  
    1 DELETE FROM publishFile
    2 USING publishFile, publishRun, publishClient
    3 WHERE publishFile.pub_id = publishRun.pub_id
     1DELETE FROM publishDone
     2USING publishDone, publishRun, publishClient
     3WHERE publishDone.pub_id = publishRun.pub_id
    44    AND publishRun.client_id = publishClient.client_id
    5     AND receiveResult.fault != 0
     5    AND publishDone.fault != 0

  • branches/pap_mops/ippTools/share/pxadmin_create_tables.sql

    r24377 r24427  
    13741374CREATE TABLE publishClient (
    13751375    client_id BIGINT AUTO_INCREMENT, -- unique identifier
     1376    product VARCHAR(64),             -- product name
     1377    stage VARCHAR(64) NOT NULL, -- stage of interest (chip, camera, diff, etc.)
    13761378    comment VARCHAR(255),            -- for human memory
    1377     stage VARCHAR(64) NOT NULL, -- stage of interest (chip, camera, diff, etc.)
    1378     format VARCHAR(64),              -- format of data
    1379     destination VARCHAR(255) NOT NULL, -- where to send data
    13801379    PRIMARY KEY(client_id)
    13811380) ENGINE=innodb DEFAULT CHARSET=latin1;
    … …  
    13861385    client_id BIGINT NOT NULL,  -- link to publishClient
    13871386    stage_id BIGINT NOT NULL,   -- link to various stage tables
     1387    workdir VARCHAR(255) NOT NULL, -- working directory
    13881388    label VARCHAR(64),          -- label for run
    13891389    state VARCHAR(64),          -- state of run (new, full, etc.)
    … …  
    13971397
    13981398-- Publishing a file within a set
    1399 CREATE TABLE publishFile (
     1399CREATE TABLE publishDone (
    14001400    pub_id BIGINT AUTO_INCREMENT, -- link to publishRun
    1401     component VARCHAR(64),        -- analagous to class_id or skycell_id
     1401    path_base VARCHAR(255),     -- base path of output
    14021402    fault SMALLINT NOT NULL DEFAULT 0, -- Fault code
    1403     PRIMARY KEY(pub_id, component),
    1404     KEY(component),
     1403    PRIMARY KEY(pub_id),
    14051404    KEY(fault),
    14061405    FOREIGN KEY(pub_id) REFERENCES publishRun(pub_id)

  • branches/pap_mops/ippTools/src/pubtool.c

    r24413 r24427  
    3636static bool addMode(pxConfig *config);
    3737static bool revertMode(pxConfig *config);
    38 static bool advanceMode(pxConfig *config);
    3938
    4039# define MODECASE(caseName, func) \
    … …  
    6261        MODECASE(PUBTOOL_MODE_ADD, addMode);
    6362        MODECASE(PUBTOOL_MODE_REVERT, revertMode);
    64         MODECASE(PUBTOOL_MODE_ADVANCE, advanceMode);
    6563      default:
    6664        psAbort("invalid option (this should not happen)");
    … …  
    8987
    9088    // required
     89    PXOPT_LOOKUP_STR(product, config->args, "-product",  true, false);
    9190    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
    92     PXOPT_LOOKUP_STR(destination, config->args, "-destination",  true, false);
    9391
    9492    // optional
    95     PXOPT_LOOKUP_STR(format, config->args, "-format",  false, false);
    9693    PXOPT_LOOKUP_STR(comment, config->args, "-comment",  false, false);
    9794
    98     if (!publishClientInsert(config->dbh, 0, stage, destination, format, comment)) {
     95    if (!publishClientInsert(config->dbh, 0, product, stage, comment)) {
    9996        psError(PS_ERR_UNKNOWN, false, "Database error");
    10097        return false;
    … …  
    109106
    110107    psMetadata *where = psMetadataAlloc(); // WHERE conditions
     108
     109    // required
     110    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir",  true, false);
    111111
    112112    // optional
    … …  
    141141        psFree(query);
    142142        psFree(whereClause);
    143         if (!psDBTransaction(config->dbh)) {
     143        if (!psDBRollback(config->dbh)) {
    144144            psError(PS_ERR_UNKNOWN, false, "Database error");
    145145        }
    … …  
    152152    if (!output) {
    153153        psError(PS_ERR_UNKNOWN, false, "Database error");
    154         if (!psDBTransaction(config->dbh)) {
     154        if (!psDBRollback(config->dbh)) {
    155155            psError(PS_ERR_UNKNOWN, false, "Database error");
    156156        }
    … …  
    160160        psTrace("pubtool", PS_LOG_INFO, "No rows found");
    161161        psFree(output);
    162         if (!psDBTransaction(config->dbh)) {
     162        if (!psDBRollback(config->dbh)) {
    163163            psError(PS_ERR_UNKNOWN, false, "Database error");
    164164        }
    … …  
    171171        psS64 stage = psMetadataLookupS64(NULL, row, "stage_id");   // Stage identifier
    172172
    173         if (!publishRunInsert(config->dbh, 0, client, stage, set_label, "new")) {
     173        if (!publishRunInsert(config->dbh, 0, client, stage, workdir, set_label, "new")) {
    174174            psError(PS_ERR_UNKNOWN, false, "Unable to add fileset");
    175175            psFree(output);
    176             if (!psDBTransaction(config->dbh)) {
     176            if (!psDBRollback(config->dbh)) {
    177177                psError(PS_ERR_UNKNOWN, false, "Database error");
    178178            }
    … …  
    182182    psFree(output);
    183183
    184     if (!psDBTransaction(config->dbh)) {
     184    if (!psDBCommit(config->dbh)) {
    185185        psError(PS_ERR_UNKNOWN, false, "Database error");
    186186        return false;
    … …  
    259259    // required
    260260    PXOPT_LOOKUP_S64(pub_id, config->args, "-pub_id", true, false);
    261     PXOPT_LOOKUP_STR(component, config->args, "-component", true, false);
     261    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
    262262
    263263    // optional
    264264    PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false);
    265265
    266     if (!publishFileInsert(config->dbh, pub_id, component, fault)) {
     266    if (!psDBTransaction(config->dbh)) {
     267        psError(PS_ERR_UNKNOWN, false, "Database error");
     268        return false;
     269    }
     270
     271    if (!publishDoneInsert(config->dbh, pub_id, path_base, fault)) {
    267272        psError(PS_ERR_UNKNOWN, false, "Unable to add file");
     273        if (!psDBRollback(config->dbh)) {
     274            psError(PS_ERR_UNKNOWN, false, "Database error");
     275        }
     276        return false;
     277    }
     278
     279    if (!p_psDBRunQueryF(config->dbh,
     280                         "UPDATE publishRun SET state = 'full' WHERE pub_id = %" PRId64,
     281                         pub_id)) {
     282        psError(PS_ERR_UNKNOWN, false, "Database error");
     283        if (!psDBRollback(config->dbh)) {
     284            psError(PS_ERR_UNKNOWN, false, "Database error");
     285        }
     286        return false;
     287    }
     288
     289    if (!psDBCommit(config->dbh)) {
     290        psError(PS_ERR_UNKNOWN, false, "Database error");
    268291        return false;
    269292    }
    … …  
    283306    PXOPT_COPY_S32(config->args, where, "-fault", "fault", "==");
    284307    PXOPT_COPY_STR(config->args, where, "-client_id", "client_id", "==");
    285     PXOPT_COPY_STR(config->args, where, "-component", "component", "==");
    286308    PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE");
    287309    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
    … …  
    310332    return true;
    311333}
    312 static bool advanceMode(pxConfig *config)
    313 {
    314     PS_ASSERT_PTR_NON_NULL(config, false);
    315 
    316     psString query = pxDataGet("pubtool_advance.sql");
    317     if (!query) {
    318         psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement");
    319         return false;
    320     }
    321 
    322     if (!p_psDBRunQuery(config->dbh, query)) {
    323         psError(PS_ERR_UNKNOWN, false, "Database error");
    324         psFree(query);
    325         return false;
    326     }
    327     psFree(query);
    328 
    329     return true;
    330 }
     334

  • branches/pap_mops/ippTools/src/pubtool.h

    r24377 r24427  
    3030    PUBTOOL_MODE_ADD,
    3131    PUBTOOL_MODE_REVERT,
    32     PUBTOOL_MODE_ADVANCE,
    3332} pubtoolMode;
    3433

  • branches/pap_mops/ippTools/src/pubtoolConfig.c

    r24413 r24427  
    4646    psMetadata *defineclientArgs = psMetadataAlloc();
    4747    psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL);
    48     psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-destination", 0, "define destination (required)", NULL);
    49     psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-format", 0, "define format", NULL);
     48    psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-product", 0, "define product (required)", NULL);
    5049    psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL);
    5150
    5251    // -definerun
    5352    psMetadata *definerunArgs = psMetadataAlloc();
     53    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
    5454    psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", 0);
    5555    psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-client_id", 0, "search by client_id", 0);
    … …  
    6868    psMetadata *addArgs = psMetadataAlloc();
    6969    psMetadataAddS64(addArgs, PS_LIST_TAIL, "-pub_id", 0, "define pub_id (required)", 0);
    70     psMetadataAddStr(addArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL);
    7170    psMetadataAddS32(addArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
    7271
    … …  
    7473    psMetadata *revertArgs = psMetadataAlloc();
    7574    psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-pub_id", 0, "search on pub_id", 0);
    76     psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-component", 0, "search on component", NULL);
    7775    psMetadataAddS32(revertArgs, PS_LIST_TAIL, "-fault", 0, "search on fault code", 0);
    7876    psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-client_id", 0, "search on client_id", NULL);
    … …  
    8078    psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL);
    8179
    82     // -advance
    83     psMetadata *advanceArgs = psMetadataAlloc();
    8480
    8581    psMetadata *argSets = psMetadataAlloc();
    … …  
    9187    PXOPT_ADD_MODE("-add", "", PUBTOOL_MODE_ADD, addArgs);
    9288    PXOPT_ADD_MODE("-revert", "", PUBTOOL_MODE_REVERT, revertArgs);
    93     PXOPT_ADD_MODE("-advance", "", PUBTOOL_MODE_ADVANCE, advanceArgs);
    9489
    9590    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {
Note: See TracChangeset for help on using the changeset viewer.