IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29790


Ignore:
Timestamp:
Nov 17, 2010, 3:12:10 PM (15 years ago)
Author:
eugene
Message:

new resort algorithm that uses a simple index rather than the linked-list thing; add a threaded resort option

Location:
branches/eam_branches/ipp-20101103/Ohana/src/addstar
Files:
2 added
6 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20101103/Ohana/src/addstar/Makefile

    r27582 r29790  
    7272$(SRC)/replace_match.$(ARCH).o \
    7373$(SRC)/resort_catalog.$(ARCH).o \
     74$(SRC)/resort_threaded.$(ARCH).o \
     75$(SRC)/resort_unthreaded.$(ARCH).o \
    7476$(SRC)/StarOps.$(ARCH).o \
    7577$(SRC)/ReadStarsFITS.$(ARCH).o \

  • branches/eam_branches/ipp-20101103/Ohana/src/addstar/include/addstar.h

    r29541 r29790  
    125125float   ZPT_ERR_PHU;
    126126
     127int     OLD_RESORT;
     128
    127129// carries the mosaic into gstars
    128130
    … …  
    135137double MIN_FWHM_X;
    136138double MIN_FWHM_Y;
     139int    NTHREADS;
    137140
    138141/* modify server behavior (make this an addstar cleanup mode?) */
    … …  
    207210Stars     *rd_gsc                 PROTO((char *filename, unsigned int *nstars));
    208211int        replace_match          PROTO((Average *average, Measure *measure, Stars *star));
     212int        resort_threaded        PROTO((AddstarClientOptions *options, SkyTable *sky));
     213int        resort_unthreaded      PROTO((AddstarClientOptions *options, SkyTable *sky));
    209214void       resort_catalog         PROTO((Catalog *catalog));
     215void       resort_catalog_old     PROTO((Catalog *catalog));
    210216Stars     *ReadStarsFITS          PROTO((FILE *f, Header *header, Header *in_theader, unsigned int *nstars));
    211217Stars     *ReadStarsTEXT          PROTO((FILE *f, unsigned int *nstars));

  • branches/eam_branches/ipp-20101103/Ohana/src/addstar/src/addstar.c

    r28241 r29790  
    3434  SkyTableSetFilenames (sky, CATDIR, "cpt");
    3535 
     36  if (options.mode == M_RESORT) {
     37    if (NTHREADS == 0) {
     38      resort_unthreaded (&options, sky);
     39    } else {
     40      resort_threaded (&options, sky);
     41    }
     42    exit (0);
     43  }
     44
    3645  stars = NULL;
    3746
    … …  
    157166        }
    158167
    159         resort_catalog (&catalog);
     168        if (OLD_RESORT) {
     169          resort_catalog_old (&catalog);
     170        } else {
     171          resort_catalog (&catalog);
     172        }
    160173        Nsubset = 1;
    161174        break;

  • branches/eam_branches/ipp-20101103/Ohana/src/addstar/src/args.c

    r27392 r29790  
    3434    remove_argument (N, &argc, argv);
    3535  }
     36  OLD_RESORT = FALSE;
     37  if ((N = get_argument (argc, argv, "-old-resort"))) {
     38    remove_argument (N, &argc, argv);
     39    OLD_RESORT = TRUE;
     40  }
    3641  if ((N = get_argument (argc, argv, "-fakeimage"))) {
    3742    options.mode = M_FAKEIMAGE;
    … …  
    6267    remove_argument (N, &argc, argv);
    6368    PMM_CCD_TABLE = strcreate (argv[N]);
     69    remove_argument (N, &argc, argv);
     70  }
     71
     72  // number of worker threads for resort (must have at least one worker thread)
     73  NTHREADS = 0;
     74  if ((N = get_argument (argc, argv, "-threads"))) {
     75    remove_argument (N, &argc, argv);
     76    NTHREADS = MAX(0, atoi (argv[N]));
    6477    remove_argument (N, &argc, argv);
    6578  }

  • branches/eam_branches/ipp-20101103/Ohana/src/addstar/src/resort_catalog.c

    r27435 r29790  
    11# include "addstar.h"
    22
    3 void resort_catalog (Catalog *catalog) {
     3void resort_catalog_old (Catalog *catalog) {
    44
    55  off_t *next_meas;
    … …  
    2828  return;
    2929}
     30
     31# define myAbort(MSG) { fprintf (stderr, "%s\n", MSG); abort(); }
     32# define myAssert(LOGIC,MSG) { if (!(LOGIC)) { fprintf (stderr, "%s\n", MSG); abort(); } }
     33
     34// sort the measure Sequence based on the average Sequence entries
     35void SortAveMeasMatch (off_t *MEAS, off_t *AVE, off_t N) {
     36
     37# define SWAPFUNC(A,B){ off_t tmp_meas; off_t tmp_ave;  \
     38    tmp_meas = MEAS[A]; MEAS[A] = MEAS[B]; MEAS[B] = tmp_meas;          \
     39    tmp_ave  = AVE[A];  AVE[A]  = AVE[B];  AVE[B]  = tmp_ave;           \
     40  }
     41# define COMPARE(A,B)(MEAS[A] < MEAS[B])
     42  OHANA_SORT (N, COMPARE, SWAPFUNC);
     43# undef SWAPFUNC
     44# undef COMPARE
     45}
     46
     47# define MARKTIME(MSG,...) { \
     48  float dtime; \
     49  gettimeofday (&stop, (void *) NULL); \
     50  dtime = DTIME (stop, start); start = stop; \
     51  fprintf (stderr, MSG, __VA_ARGS__); }
     52
     53// XXX : where is the time going?  perhaps the ALLOCATE?
     54// XXX : I don't thnk his is getting the right answer yet.
     55
     56void resort_catalog (Catalog *catalog) {
     57
     58  off_t Naverage, Nmeasure;
     59  Measure *measure;
     60  Average *average;
     61  off_t i, j, N, currentAve;
     62
     63  struct timeval start, stop;
     64
     65  off_t *measureSeq = NULL;
     66  off_t *averageSeq = NULL;
     67  Measure *measureTMP = NULL;
     68
     69  if (catalog[0].sorted == TRUE) return;
     70
     71  gettimeofday (&start, NULL);
     72
     73  /* internal counters */
     74  Nmeasure = catalog[0].Nmeasure;
     75  Naverage = catalog[0].Naverage;
     76
     77  measure = catalog[0].measure;
     78  average = catalog[0].average;
     79 
     80  // we have a table of average objects and an unsorted table of measurements.  each measurement
     81  // has a reference to the average object sequence (as well as an ID)
     82  // measure[i].averef -> average[averef]
     83  // measure[i].objID = average[averef].objID
     84  // measure[i].catID = average[averef].catID
     85
     86  // we want a sorted measure array with all averef entries in sequence
     87
     88  ALLOCATE (measureSeq, off_t,   Nmeasure);
     89  ALLOCATE (averageSeq, off_t,   Nmeasure);
     90
     91  for (i = 0; i < Nmeasure; i++) {
     92    measureSeq[i] = i;
     93    averageSeq[i] = measure[i].averef;
     94   
     95    myAssert(average[averageSeq[i]].objID == measure[measureSeq[i]].objID, "object / detection mismatch");
     96    myAssert(average[averageSeq[i]].catID == measure[measureSeq[i]].catID, "object / detection mismatch");
     97  }
     98 
     99  SortAveMeasMatch(measureSeq, averageSeq, Nmeasure);
     100  // MARKTIME("sort : %f sec\n", dtime);
     101
     102  // copy the measurements in the sorted order
     103  ALLOCATE (measureTMP, Measure, Nmeasure);
     104  for (i = 0; i < Nmeasure; i++) {
     105    j = measureSeq[i];
     106    measureTMP[i] = measure[j];
     107  }
     108  // MARKTIME("assign measure : %f sec\n", dtime);
     109
     110  // update the values of average.measureOffset and average.Nmeasure
     111  FREE(measure);
     112  catalog[0].measure = measureTMP;
     113
     114  N = 0;
     115  currentAve = averageSeq[0];
     116  average[currentAve].measureOffset = 0;
     117  for (i = 0; i < Nmeasure; i++) {
     118    if (averageSeq[i] != currentAve) {
     119      average[currentAve].Nmeasure = N;
     120      N = 0;
     121      currentAve = averageSeq[i];
     122      average[currentAve].measureOffset = i;
     123    }
     124    N++;
     125  }
     126  N++;
     127  average[currentAve].Nmeasure = N;
     128  // MARKTIME("update Nmeasure : %f sec\n", dtime);
     129
     130  MARKTIME("  match time %9.4f sec for %7lld measures, %6lld average\n", dtime, (long long) Nmeasure, (long long) Naverage);
     131
     132  FREE (measureSeq);
     133  FREE (averageSeq);
     134
     135  return;
     136}
     137

  • branches/eam_branches/ipp-20101103/Ohana/src/addstar/test/dvomerge.dvo

    r29001 r29790  
    44
    55# create 2 populated catdirs, each with a couple of cmf files
    6 macro test.dvomerge.update
     6macro test.dvomerge.continue
    77
    88  tapPLAN 51
    … …  
    1616
    1717  mkinput
     18  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec 10.0 20.0
     19  exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf
     20
     21  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 02:00:00 -radec 10.0 20.0
     22  exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf
     23
     24  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 03:00:00 -radec 9.9 20.0
     25  exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf
     26
     27  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 04:00:00 -radec 9.9 20.0
     28  exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf
     29
     30  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 05:00:00 -radec 10.0 19.9
     31  exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf
     32
     33  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 06:00:00 -radec 10.0 19.9
     34  exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf
     35
     36  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 07:00:00 -radec 9.9 19.9
     37  exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf
     38
     39  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 08:00:00 -radec 9.9 19.9
     40  exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf
     41
     42  break
     43
     44  exec rsync -auc catdir.test2/ catdir.test3/
     45
     46  date -var t1 -seconds -reftime 1276000000
     47  exec dvomerge catdir.test1 into catdir.test3
     48  date -var t2 -seconds -reftime 1276000000
     49  echo "merge time: {$t2 - $t1}"
     50
     51  catdir catdir.test3
     52  skyregion {$RA-1} {$RA+1} {$DEC-1} {$DEC+1}
     53  mextract ra dec mag
     54  create n 0 ra[]
     55  subset r0 = ra if (n % 4 == 0)
     56  subset r1 = ra if (n % 4 == 1)
     57  subset r2 = ra if (n % 4 == 2)
     58  subset r3 = ra if (n % 4 == 3)
     59
     60  catdir catdir.test1/
     61  mextract RA DEC MAG
     62  create N 0 RA[]
     63  subset R0 = RA if (N % 2 == 0)
     64  subset R1 = RA if (N % 2 == 1)
     65
     66  set dr0 = r0 - R0
     67  vstat -q dr0
     68  tapOK {abs($MEAN)  < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)"
     69  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
     70
     71  set dr1 = r1 - R1
     72  vstat -q dr1
     73  tapOK {abs($MEAN)  < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)"
     74  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
     75
     76  catdir catdir.test2/
     77  mextract RA DEC MAG
     78  create N 0 RA[]
     79  subset R2 = RA if (N % 2 == 0)
     80  subset R3 = RA if (N % 2 == 1)
     81
     82  set dr2 = r2 - R2
     83  vstat -q dr2
     84  tapOK {abs($MEAN)  < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)"
     85  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
     86
     87  set dr3 = r3 - R3
     88  vstat -q dr3
     89  tapOK {abs($MEAN)  < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)"
     90  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
     91
     92  # check on updates to imageID
     93  catdir catdir.test3
     94  imextract imageID
     95  sort imageID
     96  tapOK {imageID[]  == 4} "image IDs exist"
     97  tapOK {imageID[0] == 1} "updated image IDs"
     98  tapOK {imageID[1] == 2} "updated image IDs"
     99  tapOK {imageID[2] == 3} "updated image IDs"
     100  tapOK {imageID[3] == 4} "updated image IDs"
     101
     102  catdir catdir.test3
     103  mextract imageID, time
     104  set id = imageID
     105  set t = time
     106  imextract imageID, time
     107
     108  for i 0 time[]
     109    subset T = t if (id == imageID[$i])
     110    set dT = T - time[$i]
     111    vstat dT
     112    tapOK {abs($MEAN)  < 0.00001} "time for measure ID $i (MEAN)"
     113    tapOK {abs($SIGMA) < 0.00001} "time for measure ID $i (SIGMA)"
     114  end
     115
     116  # exec rm test.in.txt test.cmf
     117  # exec rm -rf catdir.test1
     118  # exec rm -rf catdir.test2
     119  # exec rm -rf catdir.test3
     120
     121  tapDONE
     122end
     123
     124# create 2 populated catdirs, each with a couple of cmf files
     125macro test.dvomerge.update
     126
     127  tapPLAN 51
     128
     129  exec rm -rf catdir.test1
     130  exec rm -rf catdir.test2
     131  exec rm -rf catdir.test3
     132
     133  $RA = 10.0
     134  $DEC = 20.0
     135
     136  mkinput
    18137  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC
    19138  exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf
    … …  
    27146  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 04:00:00 -radec $RA $DEC
    28147  exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf
     148
     149  break
    29150
    30151  exec rsync -auc catdir.test2/ catdir.test3/
Note: See TracChangeset for help on using the changeset viewer.