IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 30675


Ignore:
Timestamp:
Feb 17, 2011, 3:19:36 PM (15 years ago)
Author:
eugene
Message:

merge changes from the trunk

Location:
branches/eam_branches/ipp-20110213
Files:
20 edited
2 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20110213/DataStore/scripts/dsgetfileset

    r29128 r30675  
    113113                exit $status;
    114114            }
     115        } elsif (($file_type =~ /gzip/) and ($outfile =~ /tgz/) ){
     116            $command = "tar -x --directory $outdir -f $outfile";
     117            $rc = system $command;
     118            if ($rc) {
     119                my $status = $rc >> 8;
     120                print STDERR "failed to extract $outfile: rc: $rc status: $status\n";
     121                exit $status;
     122            }
    115123        }
    116124    }
    … …  
    158166=item * --unpack
    159167
    160 Uncompress fits files if compressed.
     168Uncompress fits files if compressed or extract files from gzipped tar (tgz) files.
    161169
    162170=item * --first-file <fileid>

  • branches/eam_branches/ipp-20110213/ippTools/share/camtool_find_pendingimfile.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20110213/ippTools/share/disttool_pending_raw.sql

    r30387 r30675  
    1212    TRIM(TRAILING '.fits' FROM rawImfile.uri) AS path_base,
    1313    CAST(NULL AS CHAR(255)) AS alt_path_base,
    14     chipProcessedImfile.path_base as chip_path_base,
     14    -- pass camera stage path base since we want the camera mask file. The script knows what to do
     15    camProcessedExp.path_base as chip_path_base,
    1516    CAST(NULL AS CHAR(255)) AS state,
    1617    CAST(NULL AS CHAR(255)) AS data_state,
    … …  
    2627JOIN chipProcessedImfile
    2728    USING(chip_id, class_id)
     29JOIN camRun USING(chip_id)
     30JOIN camProcessedExp using(cam_id)
    2831LEFT JOIN distComponent
    2932    ON distRun.dist_id = distComponent.dist_id
    … …  
    3639    AND distComponent.dist_id IS NULL
    3740    AND (rawExp.magicked OR distRun.no_magic)
     41    -- need to have magicked the chip image which makes the camera mask
     42    AND chipProcessedImfile.magicked != 0
     43    AND camRun.magicked > 0
    3844    AND (Label.active OR Label.active IS NULL)
    3945UNION
    … …  
    5258    TRIM(TRAILING '.fits' FROM rawImfile.uri) AS path_base,
    5359    magicDSFile.backup_path_base AS alt_path_base,
    54     chipProcessedImfile.path_base AS chip_path_base,
     60    -- pass camera stage path base since we want the camera mask file. The script knows what to do
     61    camProcessedExp.path_base AS chip_path_base,
    5562    CAST('full' AS CHAR(255)) AS state,
    5663    CAST('full' AS CHAR(255)) AS data_state,
    … …  
    7784    AND distRun.clean = 0
    7885    AND distComponent.dist_id IS NULL
    79     AND chipProcessedImfile.data_state = 'full'
    80     AND chipProcessedImfile.magicked > 0
     86    AND chipProcessedImfile.magicked != 0
     87    AND camRun.magicked > 0
    8188    AND (Label.active OR Label.active IS NULL)
    8289UNION

  • branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_camera.sql

    r27945 r30675  
    1111    camProcessedExp.path_base,
    1212    camProcessedExp.path_base AS cam_path_base,
     13    recovery_path_base,
    1314    0 AS bothways,
    1415    0 AS bytes,

  • branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_chip.sql

    r29561 r30675  
    1111    chipProcessedImfile.path_base,
    1212    camProcessedExp.path_base AS cam_path_base,
     13    recovery_path_base,
    1314    0 AS bothways,
    1415    0 AS bytes,

  • branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_diff.sql

    r30256 r30675  
    1313    magicDSFile.recovery_path_base,
    1414    "NULL" AS cam_path_base,
     15    recovery_path_base,
    1516    CAST(diffRun.bothways AS SIGNED) AS bothways,
    1617    0 AS bytes,

  • branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_raw.sql

    r27945 r30675  
    1111    rawImfile.uri AS path_base,
    1212    CAST(NULL AS CHAR(255)) AS cam_path_base,
     13    recovery_path_base,
    1314    0 AS bothways,
    1415    bytes,

  • branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_warp.sql

    r27945 r30675  
    1111    warpSkyfile.path_base,
    1212    "NULL" AS cam_path_base,
     13    recovery_path_base,
    1314    0 AS bothways,
    1415    0 AS bytes,

  • branches/eam_branches/ipp-20110213/ippTools/share/pxadmin_create_tables.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20110213/ippTools/src

  • branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c

    r30548 r30675  
    709709
    710710
     711// update the magicked column for the underlying component
     712// Note: Must be called inside a transaction
     713
    711714static bool
    712 setMagicked(pxConfig *config, psS64 magic_ds_id, psString component)
     715setMagicked(pxConfig *config, psS64 magic_ds_id, psString component, bool clearMagicked)
    713716{
    714717    // first query the magicDSRun to find the stage and the stage_id
    … …  
    741744    psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
    742745    psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
     746    psFree(output);
     747
     748    psS64 newMagickedValue;
     749    if (clearMagicked) {
     750        newMagickedValue = 0;
     751    } else {
     752        newMagickedValue = magic_id;
     753    }
    743754
    744755    ippStage stageNum = ippStringToStage(stage);
    745756
    746757    // chose the appropriate query based on the stage
     758    char *clearRunQuery = NULL;
    747759    switch (stageNum) {
    748760    case IPP_STAGE_RAW:
    749761        query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'";
     762        clearRunQuery = "UPDATE rawExp SET magicked = 0 where exp_id = %" PRId64;
    750763        break;
    751764    case IPP_STAGE_CHIP:
    752765        query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'";
     766        clearRunQuery = "UPDATE chipRun set magicked = 0 where chip_id = %" PRId64;
    753767        break;
    754768    case IPP_STAGE_CHIP_BG:
    755769        query = "UPDATE chipBackgroundImfile SET magicked = %" PRId64 " where chip_bg_id = %" PRId64 " AND class_id = '%s'";
     770        clearRunQuery = "UPDATE chipBackgroundRun SET magicked = 0 where chip_bg_id = %" PRId64;
    756771        break;
    757772    case IPP_STAGE_CAMERA:
    758         // no there is no magicked column in camProcessedExp so we have nothing to do
    759         psFree(output);
    760         return true;
     773        psFree(query);
     774        query = NULL;
     775        clearRunQuery = "UPDATE chipBackgroundRun SET magicked = 0 where chip_bg_id = %" PRId64;
     776        break;
    761777    case IPP_STAGE_WARP:
    762778        query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'";
     779        clearRunQuery = "UPDATE warpRun SET magicked = 0 where warp_id = %" PRId64;
    763780        break;
    764781    case IPP_STAGE_WARP_BG:
    765782        query = "UPDATE warpBackgroundSkyfile SET magicked = %" PRId64 " where warp_bg_id = %" PRId64 " AND skycell_id = '%s'";
     783        clearRunQuery = "UPDATE warpBackgroundRun SET magicked = 0 where warp_bg_id = %" PRId64;
    766784        break;
    767785    case IPP_STAGE_DIFF:
    768786        query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'";
     787        clearRunQuery = "UPDATE diffRun SET magicked = 0 where diff_id = %" PRId64;
    769788        break;
    770789    default:
    771790        psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage);
    772         psFree(output);
    773         return false;
    774     }
    775 
    776     if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id, component)) {
    777         psError(PS_ERR_UNKNOWN, false, "database error");
    778         return false;
    779     }
    780     psFree(output);
    781 
    782     psU64 affected = psDBAffectedRows(config->dbh);
    783     if (affected != 1) {
    784         psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
    785         return false;
     791        return false;
     792    }
     793
     794    if (query) {
     795        if (!p_psDBRunQueryF(config->dbh, query, newMagickedValue, stage_id, component)) {
     796            psError(PS_ERR_UNKNOWN, false, "database error");
     797            return false;
     798        }
     799    }
     800
     801    if (clearMagicked && clearRunQuery) {
     802        if (!p_psDBRunQueryF(config->dbh, clearRunQuery, stage_id)) {
     803            psError(PS_ERR_UNKNOWN, false, "database error");
     804            return false;
     805        }
    786806    }
    787807
    … …  
    896916    if (setmagicked) {
    897917        // set the image file's magicked flag
    898         if (!setMagicked(config, magic_ds_id, component)) {
     918        if (!setMagicked(config, magic_ds_id, component, false)) {
    899919            psError(PS_ERR_UNKNOWN, false, "setMagicked failed");
    900920            if (!psDBRollback(config->dbh)) {
    … …  
    11111131    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
    11121132
     1133    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
     1134    PXOPT_LOOKUP_STR(component, config->args, "-component", true, false);
    11131135    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
    11141136    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
    … …  
    11161138    psString queryFile = NULL;
    11171139    bool stateIsUpdate = false;
     1140    bool toRestored = !strcmp(state, "goto_restored");
    11181141    if (state) {
    1119         if (! strcmp(state, "new") || !strcmp(state, "goto_restored")) {
     1142        if (! strcmp(state, "new") || toRestored) {
    11201143            queryFile = "magicdstool_revertdestreakedfile.sql";
    11211144        } else if (!strcmp(state, "update")) {
    … …  
    11531176    }
    11541177
     1178    if (!psDBTransaction(config->dbh)) {
     1179        psError(PS_ERR_UNKNOWN, false, "database error");
     1180        return false;
     1181    }
    11551182    if (!p_psDBRunQuery(config->dbh, query)) {
    11561183        psError(PS_ERR_UNKNOWN, false, "failed to revert");
    … …  
    11591186    }
    11601187    psFree(query);
     1188    if (toRestored) {
     1189        // clear the underlying component's magicked value
     1190        if (!setMagicked(config, magic_ds_id, component, true)) {
     1191            psError(PS_ERR_UNKNOWN, false, "setMagicked failed");
     1192            if (!psDBRollback(config->dbh)) {
     1193                psError(PS_ERR_UNKNOWN, false, "database error");
     1194            }
     1195            return false;
     1196        }
     1197    }
     1198    if (!psDBCommit(config->dbh)) {
     1199        psError(PS_ERR_UNKNOWN, false, "database error");
     1200        return false;
     1201    }
    11611202    return true;
    11621203}
    … …  
    16611702    }
    16621703    if (!strcmp(data_state, "full")) {
    1663         // if -tofullfile optoinally set the magicked value for the component
    1664         PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false);
    1665         // set the image file's magicked flag
    1666         if (!setMagicked(config, magic_ds_id, component)) {
     1704        // set the component's magicked flag
     1705       
     1706        if (!setMagicked(config, magic_ds_id, component, false)) {
    16671707            psError(PS_ERR_UNKNOWN, false, "setMagicked failed");
    16681708            if (!psDBRollback(config->dbh)) {
    … …  
    17161756    return change_file_data_state(config, "full");
    17171757}
    1718 /*
    1719 static bool topurgedimfileMode(pxConfig *config)
    1720 {
    1721     return change_imfile_data_state(config, "purged", "goto_purged");
    1722 }
    1723 static bool toscrubbedfileMode(pxConfig *config)
    1724 {
    1725   return change_file_data_state(config, "scrubbed", "goto_scrubbed");
    1726 }
    1727 */
    17281758
    17291759// a very specfic function to queue a cleaned magicDSFile to be updated

  • branches/eam_branches/ipp-20110213/ippTools/src/magictool.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20110213/ippTools/src/regtool.c

    r30594 r30675  
    259259      continue;
    260260    }
     261    char *is_ccim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'c');
     262    if (is_ccim) { // Is a camera commanded exposure
     263      //      fprintf(stderr,"IN camera commanded!\n");
     264      if ((strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"DOMEFLAT") == 0)||
     265          (strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"DARK") == 0) ||
     266          (strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"BIAS") == 0)) {
     267        continue;
     268      }
     269    }
    261270   
    262271    if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
    … …  
    367376    }
    368377
     378    char *is_ccim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'c');
     379    if (is_ccim) { // Is a camera commanded exposure
     380      //      fprintf(stderr,"IN camera commanded!\n");
     381      if ((strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"DOMEFLAT") == 0)||
     382          (strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"DARK") == 0) ||
     383          (strcasecmp(psMetadataLookupStr(NULL,row,"exp_type"),"BIAS") == 0)) {
     384        continue;
     385      }
     386    }
     387
    369388    bool status = false;
    370389    char *tmp_id = psMetadataLookupStr(&status,row,"summit_class_id");
    … …  
    389408    }
    390409
     410   
    391411    if (0 && !strcmp(this_class_id, "ota44")) {
    392412        printf("STAT 1: %s (%d %d) %d %d %d\n",
    … …  
    857877    PXOPT_LOOKUP_STR(set_md5sum, config->args, "-set_md5sum", false, false);
    858878    PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false);
    859 
     879    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     880   
    860881    if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state) {
    861882        psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected");
    … …  
    905926      psStringAppend(&setvalues,"rawImfile.data_state = '%s'",set_state);
    906927    }
    907 
     928    if (hostname) {
     929      if (setvalues) {
     930        psStringAppend(&setvalues,",");
     931      }
     932      psStringAppend(&setvalues,"rawImfile.hostname = '%s'", hostname);
     933    }
    908934    psString query = pxDataGet("regtool_updateprocessedimfile.sql");
    909935    if (!query) {

  • branches/eam_branches/ipp-20110213/ippTools/src/regtoolConfig.c

    r30352 r30675  
    189189    ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state",        "set burntool state", INT16_MAX);
    190190    ADD_OPT(S16, updateprocessedimfileArgs, "-fault",          "set fault code", INT16_MAX);
     191    ADD_OPT(Str, updateprocessedimfileArgs, "-hostname",       "set host name",                NULL);
    191192    ADD_OPT(S32, updateprocessedimfileArgs, "-set_bytes",      "set bytes", INT32_MAX);
    192193    ADD_OPT(Str, updateprocessedimfileArgs, "-set_md5sum",     "set md5sum", NULL);

  • branches/eam_branches/ipp-20110213/ppTranslate/src/ppMonet.c

    r25925 r30675  
    146146            psU32 flags = psMetadataLookupU32(NULL, row, "FLAGS");
    147147
    148             float width = 0.5 * (major + minor);
     148            float width = (float) (0.5 * (major + minor));
    149149
    150150            gzprintf(gz, "%s , %.2f , %.2f , %.8lf , %.8lf , %.3f , %.3f , %.2f , %#08x\n",

  • branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsDetections.c

    r29556 r30675  
    183183    det->apMagRadius = psVectorRealloc(det->apMagRadius, num);
    184184    det->apMagRaw = psVectorRealloc(det->apMagRadius, num);
    185     det->apFlux = psVectorRealloc(det->apMagRadius, num);
    186     det->apFluxSig = psVectorRealloc(det->apMagRadius, num);
     185    det->apFlux = psVectorRealloc(det->apFlux, num);
     186    det->apFluxSig = psVectorRealloc(det->apFluxSig, num);
    187187    det->peakFluxAsMag = psVectorRealloc(det->peakFluxAsMag, num);
    188188    det->calPsfMag = psVectorRealloc(det->calPsfMag, num);

  • branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsMerge.c

    r30532 r30675  
    2222    )
    2323{
    24     float dx = detections->x->data.F32[index] - detections->naxis1->data.S32[index] / 2.0;
    25     float dy = detections->y->data.F32[index] - detections->naxis2->data.S32[index] / 2.0;
    26     return PS_SQR(dx) + PS_SQR(dy);
     24  float dx = (float) (detections->x->data.F32[index] - detections->naxis1->data.S32[index] / 2.0);
     25  float dy = (float) (detections->y->data.F32[index] - detections->naxis2->data.S32[index] / 2.0);
     26  return PS_SQR(dx) + PS_SQR(dy);
    2727}
    2828
    … …  
    165165    if (num == 0) {
    166166        //All detections were NULL?!
    167         psTrace("ppMops.merge", 3, "All %d detections were NULL\n", detections->n);
     167        psTrace("ppMops.merge", 3, "All %ld detections were NULL\n", detections->n);
    168168        return NULL;
    169169    }
    170170    psTrace("ppMops.merge", 2, "%ld sources in merged detections list\n", merged->num);
    171171
    172     merged->seeing /= num;
     172    merged->seeing /= (float) num;
    173173
    174174    return merged;

  • branches/eam_branches/ipp-20110213/ppTranslate/src/ppMopsRead.c

    r29565 r30675  
    4545        if (args->version == 0) {
    4646          psTrace("ppMops.read", 1, "Changing args->version to %d\n", skyChipPsfVersion);
    47           args->version = skyChipPsfVersion;
     47          args->version = (unsigned short) skyChipPsfVersion;
    4848        }
    4949        if (skyChipPsfVersion == 0) {
    … …  
    8787        det->mjd = psMetadataLookupF64(NULL, header, "MJD-OBS") + det->exptime / 2.0 / 3600 / 24;
    8888
    89         det->seeing = 0.5 * (psMetadataLookupF32(NULL, header, "FWHM_MAJ") +
    90                              psMetadataLookupF32(NULL, header, "FWHM_MIN"));
     89        det->seeing = (float) 0.5 * (psMetadataLookupF32(NULL, header, "FWHM_MAJ") +
     90                                     psMetadataLookupF32(NULL, header, "FWHM_MIN"));
    9191
    9292        int naxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1"); // Number of columns
    … …  
    145145              //Values are set only if the version is 2
    146146              if (skyChipPsfVersion == 2) {
    147                 det->psfInstFlux->data.F32[numGood] = psMetadataLookupS32(NULL, row, "PSF_INST_FLUX");
    148                 det->psfInstFluxSig->data.F32[numGood] = psMetadataLookupS32(NULL, row, "PSF_INST_FLUX_SIG");
    149                 det->apMag->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_MAG");
    150                 det->apMagRaw->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_MAG_RAW");
    151                 det->apMagRadius->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_MAG_RADIUS");
    152                 det->apFlux->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_FLUX");
    153                 det->apFluxSig->data.F32[numGood] = psMetadataLookupS32(NULL, row, "AP_FLUX_SIG");
    154                 det->peakFluxAsMag->data.F32[numGood] = psMetadataLookupS32(NULL, row, "PEAK_FLUX_AS_MAG");
    155                 det->calPsfMag->data.F32[numGood] = psMetadataLookupS32(NULL, row, "CAL_PSF_MAG");
    156                 det->calPsfMagSig->data.F32[numGood] = psMetadataLookupS32(NULL, row, "CAL_PSF_MAG_SIG");
    157                 det->sky->data.F32[numGood] = psMetadataLookupS32(NULL, row, "SKY");
    158                 det->skySig->data.F32[numGood] = psMetadataLookupS32(NULL, row, "SKY_SIGMA");
     147                det->psfInstFlux->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_FLUX");
     148                det->psfInstFluxSig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_FLUX_SIG");
     149                det->apMag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_MAG");
     150                det->apMagRaw->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_MAG_RAW");
     151                det->apMagRadius->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_MAG_RADIUS");
     152                det->apFlux->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_FLUX");
     153                det->apFluxSig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "AP_FLUX_SIG");
     154                det->peakFluxAsMag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PEAK_FLUX_AS_MAG");
     155                det->calPsfMag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "CAL_PSF_MAG");
     156                det->calPsfMagSig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "CAL_PSF_MAG_SIG");
     157                det->sky->data.F32[numGood] = psMetadataLookupF32(NULL, row, "SKY");
     158                det->skySig->data.F32[numGood] = psMetadataLookupF32(NULL, row, "SKY_SIGMA");
    159159                det->qualityPerfect->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_QF_PERFECT");
    160160                det->momentsR1->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_R1");
    … …  
    236236            numGood++;
    237237        }
    238         det->seeing *= plateScale / numGood;
     238        det->seeing *= ((float) plateScale) / ((float) numGood);
    239239
    240240        det->x->n = numGood;

  • branches/eam_branches/ipp-20110213/tools/examine_burntool_pcontrol.pl

    r28886 r30675  
    145145    # Get the data.  The limit is there to keep the database happy.  I don't think you can observe that many
    146146    # images on a single night.
    147     $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,burntool_state,comment FROM rawImfile WHERE dateobs >= '${dmin}' AND dateobs <= '${dmax}' order by dateobs limit 600000";
     147    $sth = "SELECT exp_id,rawImfile.exp_name,rawImfile.obs_mode,rawImfile.dateobs,class_id,burntool_state,rawImfile.comment,rawExp.state FROM rawImfile JOIN rawExp USING(exp_id) WHERE rawImfile.dateobs >= '${dmin}' AND rawImfile.dateobs <= '${dmax}' order by rawImfile.dateobs limit 600000";
    148148
    149149    $data_ref = $db->selectall_arrayref( $sth );
    … …  
    168168    }
    169169    print "#                                                     0       1       2       3       4       5       6       7       \n";
    170     print "#exp_i exp_name       obs_mode dateobs             nB 0123456701234567012345670123456701234567012345670123456701234567 comment\n";
     170    print "#exp_i exp_name       obs_mode dateobs             nB 0123456701234567012345670123456701234567012345670123456701234567 rawExp.state    comment\n";
    171171   
    172172    # If the database returns no entries (because we have a mistake in pcontrol.pro perhaps), return a null entry.
    … …  
    180180   
    181181    foreach $row_ref (@{ $data_ref }) {
    182         ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment) = @{ $row_ref };
     182        ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment,$exp_state) = @{ $row_ref };
    183183
    184184        # Correct for nulls in the database.
    … …  
    202202           
    203203            if ($cur_exp_id != -99) {
    204                 printf("%6d %11s %11s %19s %2d %64s %s\n",
    205                        $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_comment);
     204                printf("%6d %11s %11s %19s %2d %64s %15s %s\n",
     205                       $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_exp_state,$cur_comment);
    206206
    207207                if (($burncount != 60)&&(exists($opt{A}))) {
    … …  
    228228            }
    229229            ($cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$cur_comment) = ($exp_id,$exp_name,$obs_mode,$dateobs,$comment);
     230            $cur_exp_state = $exp_state;
    230231            class_to_vector($class_id,$burntool_state);
    231232        }
    232233    }
    233234    # Final entry needs a manual print to clear it out. Let's tack on a footer as well.
    234     printf("%6d %11s %11s %19s %2d %64s %s\n",
    235            $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_comment);
     235    printf("%6d %11s %11s %19s %2d %64s %15s %s\n",
     236           $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_exp_state,$cur_comment);
    236237    print "#                                                     0       1       2       3       4       5       6       7       \n";
    237     print "#exp_i exp_name       obs_mode dateobs             nB 0123456701234567012345670123456701234567012345670123456701234567 comment\n";
     238    print "#exp_i exp_name       obs_mode dateobs             nB 0123456701234567012345670123456701234567012345670123456701234567 rawExp.state    comment\n";
    238239    print"\n";
    239240

  • branches/eam_branches/ipp-20110213/tools/log_parsing/parse_apache_log.py

    r30522 r30675  
    44import re
    55import datetime
     6import fileinput
    67
    78nebulousPattern = re.compile('POST /nebulous HTTP/1.1')
     9errorPattern = re.compile('exit signal Segmentation fault')
     10
    811# Resolution: 10 minutes of 60 seconds of 1000000 microseconds
    912datetime.resolution = 10*60*1000000
    1013
    1114def message(size):
     15    """
     16    Guess the message type according to the message size
     17    """
    1218    if size == 488:
    1319        # delete == 488
    … …  
    3440
    3541def show_results(counts, output=sys.stderr):
     42    """
     43    Display results
     44    """
    3645    output.write('#             Epoch | Create | FindIn | Delete | FailSt |   Stat |  Unkn. | NonNeb\n')
    3746    for count in sorted(counts.iterkeys()):
    … …  
    4554                                                                           counts[count][6]))
    4655
    47 if __name__ == '__main__':
     56def process(linesInFile = None):
     57    """
     58    Process Nebulous entries in apache2 log file
     59    """
    4860    counts = dict()
    49 
    5061    linesCount = 1
    51     for line in sys.stdin:
    52         if linesCount % 10000 == 0:
    53             sys.stderr.write('Analyzed lines: %d\n' % linesCount)
    54             if linesCount % 100000 == 0:
     62    if linesInFile == None:
     63        input = fileinput.input()
     64    else:
     65        input = linesInFile
     66    for line in input:
     67        if linesCount % 100000 == 0:
     68            sys.stderr.write('\tAnalyzed lines: %d\n' % linesCount)
     69            if linesCount % 500000 == 0:
    5570                show_results(counts)
    5671        linesCount += 1
    … …  
    85100                counts[dt.isoformat()] = [0, 0, 0, 0, 0, 0, 0]
    86101                counts[dt.isoformat()][6] += 1
     102    return counts
    87103
    88     show_results(counts, sys.stdout)
     104def show_errors(counts, output=sys.stderr):
     105    """
     106    Display error results
     107    """
     108    output.write('#             Epoch | Errors\n')
     109    for count in sorted(counts.iterkeys()):
     110        output.write('%19s | %6d\n' % (count, counts[count]))
     111
     112def processErrorLog(linesInFile = None):
     113    """
     114    Process errors in apache2 error log file
     115    """
     116    counts = dict()
     117    linesCount = 1
     118    if linesInFile == None:
     119        input = fileinput.input()
     120    else:
     121        input = linesInFile
     122    for line in input:
     123        if linesCount % 100000 == 0:
     124            sys.stderr.write('\tAnalyzed lines: %d\n' % linesCount)
     125            if linesCount % 500000 == 0:
     126                show_errors(counts)
     127        linesCount += 1
     128        if errorPattern.search(line):
     129            elements = line.split(']')
     130            date = elements[0]
     131            date = date[1:]
     132            # print '[%s]' % (date)
     133            dt = datetime.datetime.strptime(date,
     134                                            '%a %b %d %H:%M:%S %Y')
     135            dt = dt.replace(minute = int(dt.minute/10)*10,
     136                            second = 0)
     137            # print dt.isoformat()
     138            try:
     139                counts[dt.isoformat()] += 1
     140            except KeyError:
     141                counts[dt.isoformat()] = 1
     142    return counts
     143
     144def usage():
     145    sys.stderr.write(main.__doc__)
     146
     147def main(arguments):
     148    """
     149Usage: parse_apache_log [-a <nebulous apache access_log file>] [-e <nebulous apache error_log file>]
     150"""
     151    mainName = arguments.pop()
     152    if len(arguments) == 0:
     153        sys.stderr.write('Reading from /dev/stdin\n')
     154        counts = process()
     155        show_results(counts, sys.stdout)
     156    elif len(arguments) % 2 != 0:
     157        sys.stderr.write('Arguments length: %d\n' % (len(arguments)))
     158        usage()
     159        sys.exit(1)
     160    else:
     161        accessFilenames = []
     162        errorFilenames = []
     163        while len(arguments) != 0:
     164            flag = arguments.pop()
     165            filename = arguments.pop()
     166            if flag == '-a':
     167                accessFilenames.append(filename)
     168            elif flag == '-e':
     169                errorFilenames.append(filename)
     170            else:
     171                sys.stderr.write('Unknown flag: [%s]' % (flag))
     172                usage()
     173                sys.exit(1)
     174        for accessFilename in accessFilenames:
     175            sys.stderr.write('Processing: [%s]' % (accessFilename))
     176            inFile = open(accessFilename)
     177            counts = process(inFile)
     178            show_results(counts, sys.stdout)
     179        for errorFilename in errorFilenames:
     180            counts = processErrorLog(errorFilename)
     181            show_errors(counts, sys.stdout)
     182
     183if __name__ == '__main__':
     184    main(sys.argv)
Note: See TracChangeset for help on using the changeset viewer.