IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 33089


Ignore:
Timestamp:
Jan 11, 2012, 10:16:48 AM (14 years ago)
Author:
bills
Message:

Fix problems with IPP threading. In psThreadPoolWait wait until all threads have
finished before returning even if one of the threads has a fault.
Return false if any thread faults. Also add a new argument bool harvestOnFailure
which tells psThreadPoolWait to harvest the done jobs if there is a failure.
Many functions previously used psThreadPoolWait(harvest = false) because they
wanted to examine the jobs structs afterwards. However they weren't cleaning up the
jobs in the failure case.
These now call psThreadPoolWait(harvest = false, harvestOnFailure = true)
which causes the jobs to be cleaned up on fault. These lingering jobs were the
cause of the "Unknown task" failure in psImageConvolve*
Also changed psphotReadout to check for failure of psphotGuessModels and return
to the caller. This yields quality = 3007

Location:
trunk
Files:
27 edited

Legend:

Unmodified
Added
Removed

  • trunk/psLib/src/imageops/psImageConvolve.c

    r32745 r33089  
    10391039            }
    10401040        }
    1041         if (!psThreadPoolWait(true)) {
     1041        if (!psThreadPoolWait(true, true)) {
    10421042            psError(PS_ERR_UNKNOWN, false, "Error waiting for threads.");
    10431043            psFree(gaussNorm);
    … …  
    13901390              }
    13911391              // wait here for the threaded jobs to finish (NOP if threading is not active)
    1392               if (!psThreadPoolWait(true)) {
     1392              if (!psThreadPoolWait(true, true)) {
    13931393                  psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
    13941394                  psFree(calculation);
    … …  
    14371437
    14381438              // wait here for the threaded jobs to finish (NOP if threading is not active)
    1439               if (!psThreadPoolWait(true)) {
     1439              if (!psThreadPoolWait(true, true)) {
    14401440                  psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
    14411441                  psFree(calculation);
    … …  
    17521752            }
    17531753        }
    1754         if (!psThreadPoolWait(true)) {
     1754        if (!psThreadPoolWait(true, true)) {
    17551755            psError(PS_ERR_UNKNOWN, false, "Error waiting for threads.");
    17561756            psFree(conv);
    … …  
    17871787            }
    17881788        }
    1789         if (!psThreadPoolWait(true)) {
     1789        if (!psThreadPoolWait(true, true)) {
    17901790            psError(PS_ERR_UNKNOWN, false, "Error waiting for threads.");
    17911791            psFree(conv);

  • trunk/psLib/src/imageops/psImageCovariance.c

    r32714 r33089  
    193193    }
    194194
    195     if (threaded && !psThreadPoolWait(true)) {
     195    if (threaded && !psThreadPoolWait(true, true)) {
    196196        psError(PS_ERR_UNKNOWN, false, "Error waiting for threads.");
    197197        return false;
    … …  
    360360        }
    361361    }
    362     if (threaded && !psThreadPoolWait(true)) {
     362    if (threaded && !psThreadPoolWait(true, true)) {
    363363        psError(PS_ERR_UNKNOWN, false, "Error waiting for threads.");
    364364        return false;

  • trunk/psLib/src/sys/psThread.c

    r32715 r33089  
    179179    PS_ASSERT_THREAD_TASK_NON_NULL(task, false);
    180180
     181    // fprintf(stderr, "adding task %s\n", task->type);
     182
    181183    if (!tasks) {
    182184        tasks = psHashAlloc(TASK_BUCKETS);
    … …  
    189191{
    190192    PS_ASSERT_STRING_NON_EMPTY(type, false);
     193    // fprintf(stderr, "removing task %s\n", type);
    191194
    192195    return psHashRemove(tasks, type);
    … …  
    223226
    224227        psThreadTask *task = psHashLookup(tasks, job->type); // Task to execute job
     228        // fprintf(stderr, "launching job %s\n", job->type);
    225229#ifdef HAVE_BACKTRACE
    226230        if (!task && bt_buffer) {
    … …  
    246250                 "invalid number of arguments to %s (%ld supplied, expected %d)",
    247251                 task->type, job->args->n, task->nArgs);
     252        // fprintf(stderr, "    thread for %s %p launching on %p\n", job->type, task->function, self);
     253
     254        // Run the job's function
    248255        bool status = task->function(job); // Status of executing task
     256
     257        // fprintf(stderr, "    thread for %s %p finished on %p with status %d\n", job->type, task->function, self, status);
    249258
    250259        // Put the completed job on the 'done' queue
    … …  
    306315
    307316// call this function after you have added jobs to the queue and
    308 bool psThreadPoolWait(bool harvest)
    309 {
     317bool psThreadPoolWait(bool harvest, bool harvestOnFailure)
     318{
     319    // fprintf(stderr, "psThreadPoolWait called with harvest: %d\n", harvest);
    310320    if (!pool || pool->n == 0) {
    311321        // No threads initialised, so everything's done
    … …  
    326336#endif
    327337
     338    // accumulate the number of faulted jobs that we encounter
     339    int numFaults = 0;
    328340    while (1) {
    329341        // check for an error
    … …  
    331343            psThread *thread = pool->data[i];
    332344            if (thread->fault) {
    333                 return false;
     345                numFaults++;
    334346            }
    335347        }
    … …  
    354366            // Nothing in the queue and nothing more to add
    355367            // Ensure everything is harvested, if requested
    356             if (harvest) {
     368            if (harvest || (numFaults && harvestOnFailure)) {
    357369                psThreadJobHarvest();
    358370            }
    359371            psThreadUnlock();
    360             return true;
     372            return numFaults == 0;
    361373        }
    362374

  • trunk/psLib/src/sys/psThread.h

    r28405 r33089  
    116116/// Wait for the thread pool to finish
    117117///
    118 /// This function blocks (waits in usleep) until either an error is detected on one of the threads or until ll
    119 /// threads are idle and no jobs are left on the queue
    120 bool psThreadPoolWait(bool harvest      // Harvest the jobs from the queue?
     118/// This function blocks (waits in usleep) until all  threads are idle and no jobs
     119/// are left on the queue
     120/// returns success if all jobs return success, otherwise returns false
     121bool psThreadPoolWait(bool harvest,         // Harvest the jobs from the queue?
     122                      bool harvestOnFailure // If harvest is false, harvest the jobs if a failure is encountered
    121123    );
    122124

  • trunk/psModules/src/camera/pmReadoutFake.c

    r32347 r33089  
    314314                }
    315315            }
    316             if (!psThreadPoolWait(true)) {
     316            if (!psThreadPoolWait(true, true)) {
    317317                psError(PS_ERR_UNKNOWN, false, "Error waiting for threads.");
    318318                psFree(groups);

  • trunk/psModules/src/detrend/pmBias.c

    r29833 r33089  
    154154    if (threaded) {
    155155        // wait here for the threaded jobs to finish
    156         if (!psThreadPoolWait(true)) {
     156        if (!psThreadPoolWait(true, true)) {
    157157            psError(PS_ERR_UNKNOWN, false, "Unable to apply bias correction.");
    158158            return false;

  • trunk/psModules/src/detrend/pmDark.c

    r28405 r33089  
    601601    if (threaded) {
    602602        // wait here for the threaded jobs to finish
    603         if (!psThreadPoolWait(true)) {
     603        if (!psThreadPoolWait(true, true)) {
    604604            psError(PS_ERR_UNKNOWN, false, "Unable to apply dark.");
    605605            psFree(orders);

  • trunk/psModules/src/detrend/pmFlatField.c

    r28405 r33089  
    161161    if (threaded) {
    162162        // wait here for the threaded jobs to finish
    163         if (!psThreadPoolWait(true)) {
     163        if (!psThreadPoolWait(true, true)) {
    164164            psError(PS_ERR_UNKNOWN, false, "Unable to flat-field image.");
    165165            return false;

  • trunk/psModules/src/detrend/pmShutterCorrection.c

    r29004 r33089  
    805805        if (threaded) {
    806806            // wait here for the threaded jobs to finish
    807             if (!psThreadPoolWait(true)) {
     807            if (!psThreadPoolWait(true, true)) {
    808808                psError(PS_ERR_UNKNOWN, false, "Unable to apply shutter correction.");
    809809                psFree(shutterImage);

  • trunk/psModules/src/imcombine/pmStackReject.c

    r31435 r33089  
    313313    }
    314314
    315     if (!psThreadPoolWait(false)) {
     315    if (!psThreadPoolWait(false, true)) {
    316316        psError(psErrorCodeLast(), false, "Unable to grow bad pixels.");
    317317        psFree(source);

  • trunk/psModules/src/imcombine/pmSubtraction.c

    r32695 r33089  
    905905        }
    906906    }
    907     if (!psThreadPoolWait(true)) {
     907    if (!psThreadPoolWait(true, true)) {
    908908        psError(psErrorCodeLast(), false, "Error waiting for threads.");
    909909        return false;
    … …  
    14271427    }
    14281428
    1429     if (!psThreadPoolWait(false)) {
     1429    if (!psThreadPoolWait(false, true)) {
    14301430        psError(psErrorCodeLast(), false, "Error waiting for threads.");
    14311431        return false;

  • trunk/psModules/src/imcombine/pmSubtractionEquation.c

    r30622 r33089  
    958958    }
    959959
    960     if (!psThreadPoolWait(true)) {
     960    if (!psThreadPoolWait(true, true)) {
    961961        psError(psErrorCodeLast(), false, "Error waiting for threads.");
    962962        return false;

  • trunk/psModules/src/imcombine/pmSubtractionEquation.v0.c

    r30622 r33089  
    882882    }
    883883
    884     if (!psThreadPoolWait(true)) {
     884    if (!psThreadPoolWait(true, true)) {
    885885        psError(psErrorCodeLast(), false, "Error waiting for threads.");
    886886        return false;

  • trunk/psModules/src/imcombine/pmSubtractionMatch.c

    r31671 r33089  
    10911091    }
    10921092
    1093     if (!psThreadPoolWait(true)) {
     1093    if (!psThreadPoolWait(true, true)) {
    10941094        psError(psErrorCodeLast(), false, "Error waiting for threads.");
    10951095        psFree(models);

  • trunk/psphot/src/psphotApResid.c

    r32348 r33089  
    154154
    155155        // wait for the threads to finish and manage results
    156         if (!psThreadPoolWait (false)) {
     156        if (!psThreadPoolWait (false, true)) {
    157157            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
     158            psFree(cellGroups);
    158159            return false;
    159160        }

  • trunk/psphot/src/psphotBlendFit.c

    r32695 r33089  
    151151
    152152        // wait for the threads to finish and manage results
    153         if (!psThreadPoolWait (false)) {
     153        if (!psThreadPoolWait (false, true)) {
    154154            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
    155155            psFree (fitOptions);

  • trunk/psphot/src/psphotExtendedSourceAnalysis.c

    r32633 r33089  
    134134
    135135        // wait for the threads to finish and manage results
    136         if (!psThreadPoolWait (false)) {
     136        if (!psThreadPoolWait (false, true)) {
    137137            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
    138138            psFree(AnalysisRegion);

  • trunk/psphot/src/psphotExtendedSourceFits.c

    r32744 r33089  
    218218
    219219        // wait for the threads to finish and manage results
    220         if (!psThreadPoolWait (false)) {
     220        if (!psThreadPoolWait (false, true)) {
    221221            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
    222222            psFree(AnalysisRegion);

  • trunk/psphot/src/psphotFindFootprints.c

    r32348 r33089  
    3535        psArray *tmp = pmFootprintArrayGrow(footprints, growRadius);
    3636        psImageConvolveSetThreads(oldThreads);
    37         psLogMsg ("psphot", PS_LOG_MINUTIA, "grow footprint coverage by %d pixels, %ld footprints become %ld footprints: %f sec\n", growRadius, footprints->n, tmp->n, psTimerMark ("psphot.footprints"));
    38         psFree(footprints);
    39         footprints = tmp;
     37        psLogMsg ("psphot", PS_LOG_MINUTIA, "grow footprint coverage by %d pixels, %ld footprints become %ld footprints: %f sec\n", growRadius, footprints->n, tmp ? tmp->n : 0, psTimerMark ("psphot.footprints"));
     38        if (tmp) {
     39            psFree(footprints);
     40            footprints = tmp;
     41        } else {
     42            psLogMsg ("psphot", PS_LOG_WARN, "pmFootprintArray grow returned NULL\n");
     43        }
     44           
    4045    }
    4146

  • trunk/psphot/src/psphotGuessModels.c

    r32996 r33089  
    112112        // wait here for the threaded jobs to finish
    113113        // fprintf (stderr, "wait for threads (%d, %d)\n", jx, jy);
    114         if (!psThreadPoolWait (false)) {
     114        if (!psThreadPoolWait (false, true)) {
     115            // harvest our jobs
     116            psFree(cellGroups);
    115117            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
    116118            return false;

  • trunk/psphot/src/psphotKronIterate.c

    r32996 r33089  
    158158
    159159        // wait for the threads to finish and manage results
    160         if (!psThreadPoolWait (false)) {
     160        if (!psThreadPoolWait (false, true)) {
    161161            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
    162162            return false;

  • trunk/psphot/src/psphotMagnitudes.c

    r32348 r33089  
    129129
    130130        // wait for the threads to finish and manage results
    131         if (!psThreadPoolWait (false)) {
     131        if (!psThreadPoolWait (false, true)) {
    132132            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
    133133            return false;
    … …  
    309309
    310310        // wait for the threads to finish and manage results
    311         if (!psThreadPoolWait (false)) {
     311        if (!psThreadPoolWait (false, true)) {
    312312            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
    313313            return false;

  • trunk/psphot/src/psphotRadialApertures.c

    r32633 r33089  
    176176
    177177        // wait for the threads to finish and manage results
    178         if (!psThreadPoolWait (false)) {
     178        if (!psThreadPoolWait (false, true)) {
    179179            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
    180180            psFree(AnalysisRegion);

  • trunk/psphot/src/psphotRadialProfileWings.c

    r32996 r33089  
    147147
    148148        // wait for the threads to finish and manage results
    149         if (!psThreadPoolWait (false)) {
     149        if (!psThreadPoolWait (false, true)) {
    150150            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
    151151            return false;

  • trunk/psphot/src/psphotReadout.c

    r32996 r33089  
    133133    // Construct an initial model for each object, set the radius to fitRadius, set circular
    134134    // fit mask.  NOTE: only applied to sources without guess models
    135     psphotGuessModels (config, view, filerule); // pass 1
     135    // pass 1
     136    if (!psphotGuessModels (config, view, filerule)) {
     137        psLogMsg ("psphot", 3, "failure to Guess Model - pass 1");
     138        return psphotReadoutCleanup (config, view, filerule);
     139    }
    136140
    137141    // linear PSF fit to source peaks, subtract the models from the image (in PSF mask)
    … …  
    155159    // non-linear PSF and EXT fit to brighter sources
    156160    // replace model flux, adjust mask as needed, fit, subtract the models (full stamp)
     161    // XXX: can leave faulted job in done queue
    157162    psphotBlendFit (config, view, filerule); // pass 1 (detections->allSources)
    158163

  • trunk/psphot/src/psphotSourceStats.c

    r32699 r33089  
    216216
    217217        // wait for the threads to finish and manage results
    218         if (!psThreadPoolWait (false)) {
     218        if (!psThreadPoolWait (false, true)) {
    219219            psError(PS_ERR_UNKNOWN, false, "Failure in thread job PSPHOT_SOURCE_STATS");
    220220            psFree(detections->newSources);
    … …  
    347347
    348348        // wait for the threads to finish and manage results
    349         if (!psThreadPoolWait (false)) {
     349        if (!psThreadPoolWait (false, true)) {
    350350            psError(PS_ERR_UNKNOWN, false, "Failure in thread job PSPHOT_SOURCE_STATS");
    351351            return NULL;

  • trunk/psphot/src/psphotTest.c

    r29936 r33089  
    8585
    8686    // wait for the threads to finish and manage results
    87     if (!psThreadPoolWait (true)) {
     87    if (!psThreadPoolWait (true, true)) {
    8888        fprintf (stderr, "failure to run FillImage (2)");
    8989        exit (1);
Note: See TracChangeset for help on using the changeset viewer.