IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 28338


Ignore:
Timestamp:
Jun 15, 2010, 1:12:26 PM (16 years ago)
Author:
watersc1
Message:

Change to make stack summary code run at 9PM HST to attempt to dodge a race condition. Probablyl not a final solution, but will work for now.

Location:
branches/czw_branch/20100519
Files:
34 edited
1 copied

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20100519

  • branches/czw_branch/20100519/Ohana

  • branches/czw_branch/20100519/Ohana/src/addstar/test

    • Property svn:ignore

      •  

        old new  
        44test.in.txt
        55catdir.test
         6catdir.test1
         7catdir.test2
         8catdir.test3
        69.dvo
        710relphot.airmass.png

  • branches/czw_branch/20100519/Ohana/src/addstar/test/dvomerge.dvo

    r27435 r28338  
    105105end
    106106
    107 # create 2 populated catdirs, each with a couple of cmf files
    108 macro test.dvomerge.create
    109 
    110   tapPLAN 21
     107# create 2 populated catdirs, each with a couple of cmf files -- force some unmatched objects
     108macro test.dvomerge.update.extras
     109
     110  tapPLAN 51
    111111
    112112  exec rm -rf catdir.test1
    … …  
    124124  exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf
    125125
     126  # generate a few extra unmatched sources
     127  mkinput.extras
    126128  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 03:00:00 -radec $RA $DEC
    127129  exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf
    … …  
    130132  exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf
    131133
    132   exec dvomerge catdir.test1 and catdir.test2 to catdir.test3
     134  exec rsync -auc catdir.test2/ catdir.test3/
     135
     136  exec dvomerge catdir.test1 into catdir.test3
    133137
    134138  catdir catdir.test3
    … …  
    205209end
    206210
     211# create 2 populated catdirs, each with a couple of cmf files
     212macro test.dvomerge.create
     213
     214  tapPLAN 21
     215
     216  exec rm -rf catdir.test1
     217  exec rm -rf catdir.test2
     218  exec rm -rf catdir.test3
     219
     220  $RA = 10.0
     221  $DEC = 20.0
     222
     223  mkinput
     224  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC
     225  exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf
     226
     227  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 02:00:00 -radec $RA $DEC
     228  exec addstar -D CATDIR catdir.test1 -D CAMERA simtest test.cmf
     229
     230  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 03:00:00 -radec $RA $DEC
     231  exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf
     232
     233  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 04:00:00 -radec $RA $DEC
     234  exec addstar -D CATDIR catdir.test2 -D CAMERA simtest test.cmf
     235
     236  exec dvomerge catdir.test1 and catdir.test2 to catdir.test3
     237
     238  catdir catdir.test3
     239  skyregion {$RA-1} {$RA+1} {$DEC-1} {$DEC+1}
     240  mextract ra dec mag
     241  create n 0 ra[]
     242  subset r0 = ra if (n % 4 == 0)
     243  subset r1 = ra if (n % 4 == 1)
     244  subset r2 = ra if (n % 4 == 2)
     245  subset r3 = ra if (n % 4 == 3)
     246
     247  catdir catdir.test1/
     248  mextract RA DEC MAG
     249  create N 0 RA[]
     250  subset R0 = RA if (N % 2 == 0)
     251  subset R1 = RA if (N % 2 == 1)
     252
     253  set dr0 = r0 - R0
     254  vstat -q dr0
     255  tapOK {abs($MEAN)  < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)"
     256  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
     257
     258  set dr1 = r1 - R1
     259  vstat -q dr1
     260  tapOK {abs($MEAN)  < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)"
     261  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
     262
     263  catdir catdir.test2/
     264  mextract RA DEC MAG
     265  create N 0 RA[]
     266  subset R2 = RA if (N % 2 == 0)
     267  subset R3 = RA if (N % 2 == 1)
     268
     269  set dr2 = r2 - R2
     270  vstat -q dr2
     271  tapOK {abs($MEAN)  < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)"
     272  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
     273
     274  set dr3 = r3 - R3
     275  vstat -q dr3
     276  tapOK {abs($MEAN)  < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)"
     277  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
     278
     279  # check on updates to imageID
     280  catdir catdir.test3
     281  imextract imageID
     282  sort imageID
     283  tapOK {imageID[]  == 4} "image IDs exist"
     284  tapOK {imageID[0] == 1} "updated image IDs"
     285  tapOK {imageID[1] == 2} "updated image IDs"
     286  tapOK {imageID[2] == 3} "updated image IDs"
     287  tapOK {imageID[3] == 4} "updated image IDs"
     288
     289  catdir catdir.test3
     290  mextract imageID, time
     291  set id = imageID
     292  set t = time
     293  imextract imageID, time
     294
     295  for i 0 time[]
     296    subset T = t if (id == imageID[$i])
     297    set dT = T - time[$i]
     298    vstat -q dT
     299    tapOK {abs($MEAN)  < 0.00001} "time for measure ID $i (MEAN)"
     300    tapOK {abs($SIGMA) < 0.00001} "time for measure ID $i (SIGMA)"
     301  end
     302
     303  # exec rm test.in.txt test.cmf
     304  # exec rm -rf catdir.test1
     305  # exec rm -rf catdir.test2
     306  # exec rm -rf catdir.test3
     307
     308  tapDONE
     309end
     310
    207311# make a simple input file for mkcmf
    208312macro mkinput.alt
    … …  
    225329  for i 10 1024 100
    226330    for j 10 1024 100
     331      fprintf " %4d %4d  %6.2f" $i $j {-15.0 + 2.5*($i + $j)/1000.0}
     332    end
     333  end
     334  output stdout
     335end
     336
     337# make a simple input file for mkcmf
     338macro mkinput.extras
     339  exec rm -f test.in.txt
     340
     341  output test.in.txt
     342  for i 10 1024 100
     343    for j 10 1024 100
     344      fprintf " %4d %4d  %6.2f" $i $j {-15.0 + 2.5*($i + $j)/1000.0}
     345    end
     346  end
     347  for i 20 1024 500
     348    for j 20 1024 500
    227349      fprintf " %4d %4d  %6.2f" $i $j {-15.0 + 2.5*($i + $j)/1000.0}
    228350    end

  • branches/czw_branch/20100519/Ohana/src/dvomerge/src/LoadCatalog.c

    r24745 r28338  
    77    catalog[0].Nsecfilt  = GetPhotcodeNsecfilt ();
    88
    9     if (!strcmp (mode, "w")) {
    10         catalog[0].catflags = LOAD_NONE;
    11     } else {
    12         catalog[0].catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
    13     }
     9    // always load all of the data (if any exists)
     10    catalog[0].catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
    1411
    1512    catalog[0].catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data

  • branches/czw_branch/20100519/Ohana/src/dvomerge/src/dvomerge.c

    r27583 r28338  
    77  ConfigInit (&argc, argv);
    88  dvomerge_args (&argc, argv);
     9
     10  // XXX require both inputs to be sorted?
    911
    1012  if (argc == 6) dvomergeCreate (argc, argv);

  • branches/czw_branch/20100519/Ohana/src/dvomerge/src/dvomergeUpdate.c

    r27583 r28338  
    1616  output = argv[3];
    1717
    18   // the first input defines the photcode table & db layout
    19   sprintf (filename, "%s/Photcodes.dat", input);
     18  // since we are merging the input db into the output db, the output defines the photcode
     19  // table & db layout but, this requires the output to exist.  if it does not, instead use the
     20  // input.
     21  sprintf (filename, "%s/Photcodes.dat", output);
    2022  if (!LoadPhotcodes (filename, NULL, FALSE)) {
    21     fprintf (stderr, "error loading photcode table %s\n", filename);
    22     exit (1);
     23    sprintf (filename, "%s/Photcodes.dat", input);
     24    if (!LoadPhotcodes (filename, NULL, FALSE)) {
     25      fprintf (stderr, "error loading photcode table: tried %s/Photcodes.dat and %s/Photcodes.dat\n", output, input);
     26      exit (1);
     27    }
    2328  }
    2429
    … …  
    2732  // load the sky table for the existing database
    2833  insky = SkyTableLoadOptimal (input, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
     34  if (!insky) {
     35      Shutdown ("can't read SkyTable for %s", input);
     36  }
    2937  SkyTableSetFilenames (insky, input, "cpt");
    3038
    3139  // generate an output table populated at the desired depth
    32   outsky = SkyTableLoadOptimal (output, NULL, NULL, TRUE, SKY_DEPTH, VERBOSE);
     40  outsky = SkyTableLoadOptimal (output, NULL, GSCFILE, TRUE, SKY_DEPTH, VERBOSE);
     41  if (!outsky) {
     42      Shutdown ("can't read or create SkyTable for %s", output);
     43  }
    3344  SkyTableSetFilenames (outsky, output, "cpt");
    3445
    … …  
    4354    if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name);
    4455
    45     // load / create output catalog
    46     LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "a");
     56    // load / create output catalog (if catalog does not exist, it will be created)
     57    LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w");
    4758
    4859    // combine only tables at equal or larger depth
    … …  
    102113  // load the image table
    103114  if (inDB.dbstate == LCK_EMPTY) {
    104     Shutdown ("can't find input (1) image catalog %s", inDB.filename);
     115    Shutdown ("can't find input image catalog %s", inDB.filename);
    105116  }
    106117  if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
    107     Shutdown ("can't read input (1) image catalog %s", inDB.filename);
     118    Shutdown ("can't read input image catalog %s", inDB.filename);
    108119  }
    109120
    … …  
    117128  /* load the image table */
    118129  if (outDB.dbstate == LCK_EMPTY) {
    119     Shutdown ("can't find input (2) image catalog %s", outDB.filename);
    120   }
    121   if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {
    122     Shutdown ("can't read input (2) image catalog %s", outDB.filename);
     130    dvo_image_create (&outDB, GetZeroPoint());
     131  } else {
     132    if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {
     133      Shutdown ("can't read output image catalog %s", outDB.filename);
     134    }
    123135  }
    124136

  • branches/czw_branch/20100519/Ohana/src/dvomerge/src/merge_catalogs_old.c

    r28304 r28338  
    6363  tcoords.Npolyterms = 1;
    6464  strcpy (tcoords.ctype, "RA---ARC");
     65
     66  if (VERBOSE) fprintf (stderr, "merging %s into %s\n", input[0].filename, output[0].filename);
    6567
    6668  /* build spatial index (RA sort) referencing input array sequence */
    … …  
    161163      REALLOCATE (output[0].measure, Measure, NMEAS);
    162164    }
     165
     166    // 4) average properties from the input and the output db need to be properly merged.
    163167
    164168    /** add ALL measurements for this input average object **/
    … …  
    209213    }
    210214
     215    // update the average properties to reflect the incoming entries:
     216    // if the original value is NAN but the input value is not, accept the input:
     217    for (j = 0; j < Nsecfilt; j++) {
     218      if ( isfinite(output[0].secfilt[n*Nsecfilt+j].M)) continue;
     219      if (!isfinite( input[0].secfilt[N*Nsecfilt+j].M)) continue;
     220      output[0].secfilt[n*Nsecfilt+j].M     = input[0].secfilt[N*Nsecfilt+j].M;
     221      output[0].secfilt[n*Nsecfilt+j].dM    = input[0].secfilt[N*Nsecfilt+j].dM;
     222      output[0].secfilt[n*Nsecfilt+j].Xm    = input[0].secfilt[N*Nsecfilt+j].Xm;
     223      output[0].secfilt[n*Nsecfilt+j].M_20  = input[0].secfilt[N*Nsecfilt+j].M_20;
     224      output[0].secfilt[n*Nsecfilt+j].M_80  = input[0].secfilt[N*Nsecfilt+j].M_80;
     225      output[0].secfilt[n*Nsecfilt+j].Ncode = input[0].secfilt[N*Nsecfilt+j].Ncode;
     226      output[0].secfilt[n*Nsecfilt+j].Nused = input[0].secfilt[N*Nsecfilt+j].Nused;
     227    }
     228
    211229    /* Nm is updated, but not written out in -update mode (for existing entries)
    212230       Nm is recalculated in build_meas_links if loaded table is not sorted */
    … …  
    242260    if (!IN_REGION (input[0].average[N].R, input[0].average[N].D)) continue;
    243261
     262    // XXX should we accept the input measurements for the fields?
     263
    244264    output[0].average[Nave].R              = input[0].average[N].R;
    245265    output[0].average[Nave].D              = input[0].average[N].D;
    … …  
    247267    output[0].average[Nave].dD             = 0;
    248268
    249     output[0].average[Nave].Nmeasure       = 1;
     269    output[0].average[Nave].Nmeasure       = 0; // this value is update as the measurements are associated with this entry below
    250270    output[0].average[Nave].Nmissing       = 0;
    251271    output[0].average[Nave].Nextend        = 0;
    … …  
    282302
    283303    for (j = 0; j < Nsecfilt; j++) {
    284       output[0].secfilt[Nave*Nsecfilt+j].M  = NAN;
    285       output[0].secfilt[Nave*Nsecfilt+j].dM = NAN;
    286       output[0].secfilt[Nave*Nsecfilt+j].Xm = NAN_S_SHORT;
    287       output[0].secfilt[Nave*Nsecfilt+j].M_20   = NAN_S_SHORT;
    288       output[0].secfilt[Nave*Nsecfilt+j].M_80   = NAN_S_SHORT;
    289       output[0].secfilt[Nave*Nsecfilt+j].Ncode = 0;
    290       output[0].secfilt[Nave*Nsecfilt+j].Nused = 0;
     304      if (isfinite(input[0].secfilt[N*Nsecfilt+j].M)) {
     305        output[0].secfilt[Nave*Nsecfilt+j].M     = input[0].secfilt[N*Nsecfilt+j].M;
     306        output[0].secfilt[Nave*Nsecfilt+j].dM    = input[0].secfilt[N*Nsecfilt+j].dM;
     307        output[0].secfilt[Nave*Nsecfilt+j].Xm    = input[0].secfilt[N*Nsecfilt+j].Xm;
     308        output[0].secfilt[Nave*Nsecfilt+j].M_20  = input[0].secfilt[N*Nsecfilt+j].M_20;
     309        output[0].secfilt[Nave*Nsecfilt+j].M_80  = input[0].secfilt[N*Nsecfilt+j].M_80;
     310        output[0].secfilt[Nave*Nsecfilt+j].Ncode = input[0].secfilt[N*Nsecfilt+j].Ncode;
     311        output[0].secfilt[Nave*Nsecfilt+j].Nused = input[0].secfilt[N*Nsecfilt+j].Nused;
     312      } else {
     313        output[0].secfilt[Nave*Nsecfilt+j].M     = NAN;
     314        output[0].secfilt[Nave*Nsecfilt+j].dM    = NAN;
     315        output[0].secfilt[Nave*Nsecfilt+j].Xm    = NAN_S_SHORT;
     316        output[0].secfilt[Nave*Nsecfilt+j].M_20  = NAN_S_SHORT;
     317        output[0].secfilt[Nave*Nsecfilt+j].M_80  = NAN_S_SHORT;
     318        output[0].secfilt[Nave*Nsecfilt+j].Ncode = 0;
     319        output[0].secfilt[Nave*Nsecfilt+j].Nused = 0;
     320      }
    291321    }
    292322
    … …  
    304334      output[0].measure[Nmeas].catID    = output[0].catID;
    305335
    306       /* next[Nmeas] should always be -1 in this context (it is always the only
    307          measurement for the star) */
     336      // as we add measurements, update Nmeasure to match
     337      output[0].average[Nave].Nmeasure ++;
     338
     339      /* we set next[Nmeas] to -1 here, and update correctly below */
    308340      input[0].found[N] = Nmeas;
    309341      next_meas[Nmeas] = -1;
    … …  
    327359# endif
    328360
    329 # define NOSORT FALSE
     361# define NOSORT 0
    330362  if (NOSORT) {
    331363    output[0].sorted = FALSE;

  • branches/czw_branch/20100519/Ohana/src/libdvo/src/SavePhotcodesText.c

    r17190 r28338  
    6969             type,
    7070             table[0].code[i].C*SCALE,
    71              table[0].code[i].K*SCALE,
     71             table[0].code[i].K,
    7272             table[0].code[i].dC*SCALE);
    7373

  • branches/czw_branch/20100519/Ohana/src/opihi/cmd.data/create.c

    r27817 r28338  
    44 
    55  int i, N, INT;
    6   float start, end, delta;
     6  opihi_flt start, end, delta;
    77  Vector *vec;
    88 

  • branches/czw_branch/20100519/Ohana/src/opihi/cmd.data/help/histogram

    r4688 r28338  
    11
    2    histogram <buffer> <x> <y> [-region sx sy nx ny] [-range min max]
     2   histogram <invec> <outvec> <start> <end> [<delta>] [-range <dx_outvec>]
    33
    4    calculate a histogram of the image pixel values in the given
    5    buffer, optionally constrained to the given region, with optional
    6    max and min values.  the results are placed in the vectors x and y,
    7    which contain the pixel values and the number of occurences.
     4   calculate a histogram of the <invec> values and store the
     5   occurrences count in the <outvec> buffer. Optionally constrained to
     6   the given <start>-<end> region with <delta> step value (default
     7   step is 1). The optional '-range <dx_outvec>' parameter allows storing
     8   the range <start>-<end> values with <delta> increment
    89
     10   Sample code usage:
     11   
     12   # create a vector ('x') containing arbitrary values  [0.:1.] range
     13   create y 0 100 1; set x = sin(y)
     14
     15   # build histogram from x from 0. to 1. with 0.1 delta step
     16   histogram x xhist 0. 1. .1 -range dx
     17
     18   # plot corresponding histogram
     19   limits dx xhist; clear; box; plot dx xhist -x 1

  • branches/czw_branch/20100519/Ohana/src/opihi/dvo/gstar.c

    r28304 r28338  
    165165        gprint (GP_LOG, "%3d   ",  catalog.average[k].Nmeasure);
    166166        gprint (GP_LOG, "%4.1f ",  0.01*catalog.average[k].Xp);
    167         gprint (GP_LOG, "%5d ",     catalog.average[k].flags);
     167        gprint (GP_LOG, "%5x ",    catalog.average[k].flags);
     168        gprint (GP_LOG, "%x ",     catalog.average[k].objID);
     169        gprint (GP_LOG, "%x ",     catalog.average[k].catID);
    168170       
    169171        if (FULL_OUTPUT) {
    … …  
    176178            gprint (GP_LOG, "%f ",     catalog.average[k].P);
    177179            gprint (GP_LOG, "%f ",     catalog.average[k].dP);
    178             gprint (GP_LOG, "%x ",     catalog.average[k].objID);
    179             gprint (GP_LOG, "%x ",     catalog.average[k].catID);
    180180        }
    181181
    … …  
    244244                gprint (GP_LOG, "%f ", catalog.measure[m].crNsigma);
    245245                gprint (GP_LOG, "%f ", catalog.measure[m].extNsigma);
    246                 gprint (GP_LOG, "%f ", 0.01*catalog.measure[m].FWx);
    247                 gprint (GP_LOG, "%f ", 0.01*catalog.measure[m].FWy);
    248246                gprint (GP_LOG, "%f ", (360.0/(float)0xffff)*catalog.measure[m].theta);
    249247            }

  • branches/czw_branch/20100519/Ohana/src/opihi/pcontrol/StartJob.c

    r28304 r28338  
    3636  }
    3737
    38   fprintf (stderr, "command: %s\n", line);
     38  // fprintf (stderr, "command: %s\n", line);
    3939
    4040  status = PclientCommand (host, line, PCLIENT_PROMPT, PCONTROL_RESP_START_JOB);

  • branches/czw_branch/20100519/Ohana/src/photdbc/include/photdbc.h

    r27435 r28338  
    5252double CHISQ_MAX;
    5353double SIGMA_MAX;
     54double AVE_SIGMA_LIM;
    5455int    NMEAS_MIN;
    5556double ZERO_POINT;

  • branches/czw_branch/20100519/Ohana/src/photdbc/src/ConfigInit.c

    r25757 r28338  
    4242
    4343  ScanConfig (config, "SIGMA_MAX",              "%lf", 0, &SIGMA_MAX);
     44  ScanConfig (config, "AVE_SIGMA_LIM",          "%lf", 0, &AVE_SIGMA_LIM);
    4445  ScanConfig (config, "NMEAS_MIN",              "%d",  0, &NMEAS_MIN);
    4546

  • branches/czw_branch/20100519/Ohana/src/photdbc/src/copy_images.c

    r28334 r28338  
    66  off_t Nimage;
    77  char *ImageOut;
     8  unsigned int imageID;
    89  FITS_DB in;
    910  FITS_DB out;
    … …  
    4748  dvo_image_addrows (&out, image, Nimage);
    4849
     50  // note that imageID is unsigned int
     51  status = gfits_scan (&in.header, "IMAGEID", "%u", 1, &imageID);
     52  if (!status) {
     53    status = gfits_scan (&in.header, "NIMAGES", "%u", 1, &imageID);
     54    imageID++;
     55  }
     56  status = gfits_modify (&out.header, "IMAGEID", "%u", 1, imageID);
     57
    4958  dvo_image_update (&out, VERBOSE);
    5059  dvo_image_unlock (&out);

  • branches/czw_branch/20100519/Ohana/src/photdbc/src/make_subcatalog.c

    r28334 r28338  
    88  off_t NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm, Nsecfilt;
    99  double mag, minMag;
    10 
     10  int keep;
     11 
    1112  Nsecfilt = GetPhotcodeNsecfilt ();
    1213  assert (catalog[0].Nsecfilt == Nsecfilt);
    … …  
    2526    // exclude stars with too few measurements
    2627    if (NMEAS_MIN && (catalog[0].average[i].Nmeasure < NMEAS_MIN)) continue;
     28
     29    if (AVE_SIGMA_LIM) {
     30      // if all of the average magnitude errors are >AVE_SIGMA_LIM, drop the object
     31      keep = FALSE;
     32      for (j = 0; !keep && (j < Nsecfilt); j++) {
     33        if (catalog[0].secfilt[Nsecfilt*i+j].dM < AVE_SIGMA_LIM) {
     34          keep = TRUE;
     35        }
     36      }
     37      if (!keep) continue;
     38    }
    2739
    2840    /* assign average and secfilt values */

  • branches/czw_branch/20100519/Ohana/src/photdbc/src/photdbc.c

    r28334 r28338  
    2121  skylist = SkyListByPatch (sky, -1, &REGION);
    2222  for (i = 0; i < skylist[0].Nregions; i++) {
    23     if (i % 100 == 0) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
     23    if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
    2424
    2525    // set the parameters which guide catalog open/load/create
    … …  
    5656    }
    5757
     58    // the output catalog needs to have the same values for 'objID' and 'sorted' as the input
     59    outcatalog.objID = incatalog.objID;
     60    outcatalog.sorted = incatalog.sorted;
     61    if (!incatalog.sorted) {
     62      fprintf (stderr, "ERROR: input db must be sorted: %s\n", incatalog.filename);
     63      exit (2);
     64    }
     65
    5866    /* limit number of measures based on selections */
    5967    make_subcatalog (&outcatalog, &incatalog);

  • branches/czw_branch/20100519/ippScripts/scripts/automate_stacks.pl

    r28304 r28338  
    674674
    675675    my $cmd = "$dettool";
    676     $cmd .= " -pretend ";
     676#    $cmd .= " -pretend ";
    677677    $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type ";
    678678    $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter ";

  • branches/czw_branch/20100519/ippScripts/scripts/minidvodb_merge.pl

    r28304 r28338  
    171171            if ($this_is_the_first) {
    172172               
    173                 $merge_command = "cp -rp $minidvodb/* $mergedvodb";
     173                $merge_command = "rsync -rvat $minidvodb/* $mergedvodb";
    174174            } else {
    175175                $merge_command = "$dvomerge $minidvodb into $mergedvodb";

  • branches/czw_branch/20100519/ippTasks/nightly_stacks.pro

    r28304 r28338  
    214214  periods         -exec $LOADEXEC
    215215  periods         -timeout 30
    216   trange          23:00:00 24:00:00 -nmax 1
     216  trange          23:00:00 23:59:59 -nmax 1
    217217  npending        1
     218  active          false
    218219
    219220  task.exec

  • branches/czw_branch/20100519/ippTasks/stack.pro

    r28334 r28338  
    426426  periods      -exec $LOADEXEC
    427427  periods      -timeout 1200
     428  trange       07:00:00 08:00:00
    428429  npending     1
    429430

  • branches/czw_branch/20100519/ippTools/share/chiptool_processedimfile.sql

    r27880 r28338  
    2929    rawImfile.magicked AS raw_magicked,
    3030    rawImfile.burntool_state,
    31     magicDSRun.state AS dsRun_state
     31    magicDSRun.state AS dsRun_state,
     32    magicDSRun.magic_ds_id
    3233FROM chipRun
    3334JOIN chipImfile

  • branches/czw_branch/20100519/ippTools/share/pubtool_definerun.sql

    r27041 r28338  
    66FROM (
    77    -- Get diffs to publish
    8     SELECT
     8    SELECT DISTINCT
    99        client_id,
    1010        diff_id AS stage_id,
    11         label AS src_label
     11        diffRun.label AS src_label
    1212    FROM publishClient
    1313    JOIN diffRun
     14    JOIN diffInputSkyfile USING(diff_id)
     15    JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 -- Only JOINing input, not reference!
     16    JOIN fakeRun USING(fake_id)
     17    JOIN camRun USING(cam_id)
     18    JOIN chipRun USING(chip_id)
     19    JOIN rawExp USING(exp_id)
    1420    WHERE publishClient.stage = 'diff'
    1521        AND publishClient.active = 1
    16         AND diffRun.state = 'full'
     22        AND diffRun.state IN ('full', 'cleaned', 'goto_cleaned')
    1723        AND (diffRun.magicked > 0 OR diffRun.diff_mode = 4 OR publishClient.magicked = 0)
    1824    -- WHERE hook %s
    … …  
    2228        client_id,
    2329        cam_id AS stage_id,
    24         label AS src_label
     30        camRun.label AS src_label
    2531    FROM publishClient
    2632    JOIN camRun
     33    JOIN chipRun USING(chip_id)
     34    JOIN rawExp USING(exp_id)
    2735    WHERE publishClient.stage = 'camera'
    2836        AND publishClient.active = 1
    29         AND camRun.state = 'full'
     37        AND camRun.state IN ('full', 'cleaned', 'goto_cleaned')
    3038        AND (camRun.magicked > 0 OR publishClient.magicked = 0)
    3139    -- WHERE hook %s
    … …  
    3341-- Only get stuff that hasn't been published
    3442LEFT JOIN publishRun USING(client_id, stage_id)
    35 WHERE publishRun.client_id IS NULL

  • branches/czw_branch/20100519/ippTools/share/pubtool_pending.sql

    r27052 r28338  
    2626        AND publishClient.active = 1
    2727        AND publishRun.state = 'new'
    28         AND diffRun.state = 'full'
     28        AND diffRun.state IN ('full', 'cleaned', 'goto_cleaned')
    2929        AND (diffRun.magicked > 0 OR diffRun.diff_mode = 4 OR publishClient.magicked = 0)
    3030        -- WHERE hook %s
    … …  
    4747        AND publishClient.active = 1
    4848        AND publishRun.state ='new'
    49         AND camRun.state = 'full'
     49        AND camRun.state IN ('full', 'cleaned', 'goto_cleaned')
    5050        AND (camRun.magicked > 0 OR publishClient.magicked = 0)
    5151        -- WHERE hook %s

  • branches/czw_branch/20100519/ippTools/src/pstamptool.c

    r28334 r28338  
    870870
    871871    PXOPT_LOOKUP_S64(job_id,    config->args, "-job_id", false, false);
     872    PXOPT_LOOKUP_S64(req_id,    config->args, "-req_id", false, false);
    872873    PXOPT_LOOKUP_S64(dep_id,    config->args, "-dep_id", false, false);
    873874
    874     if (!job_id && !dep_id) {
    875         psError(PS_ERR_UNKNOWN, true, "at least -job_id or -dep_id is required");
     875    if (!job_id && !req_id && !dep_id) {
     876        psError(PS_ERR_UNKNOWN, true, "at least one of -job_id -req_id or -dep_id is required");
    876877        return false;
    877878    }
    … …  
    888889
    889890    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
     891    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
    890892    PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
     893    PXOPT_COPY_STR(config->args, where, "-state",  "pstampJob.state", "==");
    891894
    892895    psString query = pxDataGet("pstamptool_updatejob.sql");

  • branches/czw_branch/20100519/ippTools/src/pstamptoolConfig.c

    r28334 r28338  
    160160    // -updatejob
    161161    psMetadata *updatejobArgs = psMetadataAlloc();
    162     psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0,            "req_id for which to change state", 0);
    163     psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0,            "dep_id for which to change state", 0);
     162    psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-req_id", 0,            "req_id of jobs to update", 0);
     163    psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0,            "job_id of jobs to update", 0);
     164    psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0,            "dep_id of jobs to update", 0);
     165    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0,            "current state of jobs to update", 0);
    164166    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-set_state", 0,            "new state", NULL);
    165167    psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-set_fault", 0,            "new result", 0);

  • branches/czw_branch/20100519/ippTools/src/pubtool.c

    r27071 r28338  
    151151    PS_ASSERT_PTR_NON_NULL(config, false);
    152152
    153     psMetadata *where = psMetadataAlloc(); // WHERE conditions
     153    psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffs
     154    psMetadata *camWhere = psMetadataAlloc(); // WHERE conditions for cams
    154155
    155156    // required
    156157
    157158    // optional
    158     PXOPT_COPY_S64(config->args, where, "-client_id", "client_id", "==");
    159     pxAddLabelSearchArgs(config, where, "-label", "label", "=="); // define using newExp label
     159    PXOPT_COPY_S64(config->args, diffWhere, "-client_id", "client_id", "==");
     160    pxAddLabelSearchArgs(config, diffWhere, "-label", "diffRun.label", "==");
     161    pxAddLabelSearchArgs(config, diffWhere, "-data_group", "diffRun.data_group", "LIKE");
     162    PXOPT_COPY_TIME(config->args, diffWhere, "-dateobs_begin", "rawExp.dateobs", ">=");
     163    PXOPT_COPY_TIME(config->args, diffWhere, "-dateobs_end", "rawExp.dateobs", "<=");
     164    PXOPT_COPY_STR(config->args, diffWhere, "-filter", "rawExp.filter", "LIKE");
     165    PXOPT_COPY_STR(config->args, diffWhere, "-obs_mode", "rawExp.obs_mode", "LIKE");
     166
     167    PXOPT_COPY_S64(config->args, camWhere, "-client_id", "client_id", "==");
     168    pxAddLabelSearchArgs(config, camWhere, "-label", "camRun.label", "==");
     169    pxAddLabelSearchArgs(config, camWhere, "-data_group", "camRun.data_group", "LIKE");
     170    PXOPT_COPY_TIME(config->args, camWhere, "-dateobs_begin", "rawExp.dateobs", ">=");
     171    PXOPT_COPY_TIME(config->args, camWhere, "-dateobs_end", "rawExp.dateobs", "<=");
     172    PXOPT_COPY_STR(config->args, camWhere, "-filter", "rawExp.filter", "LIKE");
     173    PXOPT_COPY_STR(config->args, camWhere, "-obs_mode", "rawExp.obs_mode", "LIKE");
     174
     175    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
     176    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
     177    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     178    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    160179
    161180    psString query = pxDataGet("pubtool_definerun.sql"); // Query to run
    162181    if (!query) {
    163182        psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement");
    164         psFree(where);
    165         return false;
     183        psFree(diffWhere);
     184        psFree(camWhere);
     185        return false;
     186    }
     187
     188    if (!rerun) {
     189        psStringAppend(&query, "\nWHERE publishRun.client_id IS NULL");
     190    }
     191
     192    if (limit) {
     193        psString limitString = psDBGenerateLimitSQL(limit);
     194        psStringAppend(&query, "\n%s", limitString);
     195        psFree(limitString);
    166196    }
    167197
    168198    if (!psDBTransaction(config->dbh)) {
    169199        psError(PS_ERR_UNKNOWN, false, "Database error");
    170         psFree(where);
    171         return false;
    172     }
    173 
    174     psString whereClause = psStringCopy(""); // Additional constraints to add to query
    175     if (psListLength(where->list)) {
    176         psString clause = psDBGenerateWhereConditionSQL(where, NULL);
    177         psStringAppend(&whereClause, "\n AND %s", clause);
     200        psFree(diffWhere);
     201        psFree(camWhere);
     202        return false;
     203    }
     204
     205    psString whereDiff = psStringCopy(""); // Additional constraints to add to query
     206    if (psListLength(diffWhere->list)) {
     207        psString clause = psDBGenerateWhereConditionSQL(diffWhere, NULL);
     208        psStringAppend(&whereDiff, "\n AND %s", clause);
    178209        psFree(clause);
    179210    }
    180     psFree(where);
    181 
    182     if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) {
     211    psFree(diffWhere);
     212
     213    psString whereCam = psStringCopy(""); // Additional constraints to add to query
     214    if (psListLength(camWhere->list)) {
     215        psString clause = psDBGenerateWhereConditionSQL(camWhere, NULL);
     216        psStringAppend(&whereCam, "\n AND %s", clause);
     217        psFree(clause);
     218    }
     219    psFree(camWhere);
     220
     221    if (!p_psDBRunQueryF(config->dbh, query, whereDiff, whereCam)) {
    183222        psError(PS_ERR_UNKNOWN, false, "Database error");
    184223        psFree(query);
    185         psFree(whereClause);
     224        psFree(whereDiff);
     225        psFree(whereCam);
    186226        if (!psDBRollback(config->dbh)) {
    187227            psError(PS_ERR_UNKNOWN, false, "Database error");
    … …  
    190230    }
    191231    psFree(query);
    192     psFree(whereClause);
     232    psFree(whereDiff);
     233    psFree(whereCam);
    193234
    194235    psArray *output = p_psDBFetchResult(config->dbh); // Output of SELECT statement
    … …  
    202243    if (!psArrayLength(output)) {
    203244        psTrace("pubtool", PS_LOG_INFO, "No rows found");
     245        psFree(output);
     246        return true;
     247    }
     248
     249    if (pretend) {
     250        if (!ippdbPrintMetadatas(stdout, output, "publishRun", !simple)) {
     251            psError(psErrorCodeLast(), false, "Failed to print array");
     252            psFree(output);
     253            return false;
     254        }
    204255        psFree(output);
    205256        return true;
    … …  
    241292    PXOPT_COPY_STR(config->args, where, "-stage", "publishClient.stage", "==");
    242293    PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE");
    243     PXOPT_COPY_STR(config->args, where, "-label", "publishRun.label", "==");
     294    pxAddLabelSearchArgs(config, where, "-label", "label", "==");
    244295
    245296    // optional

  • branches/czw_branch/20100519/ippTools/src/pubtoolConfig.c

    r27311 r28338  
    6464    psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-client_id", 0, "search by client_id", 0);
    6565    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "set and search by label", NULL);
     66    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", NULL);
     67    psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL);
     68    psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL);
     69    psMetadataAddStr(definerunArgs,  PS_LIST_TAIL, "-filter", 0, "search for filter", NULL);
     70    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL);
     71    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-rerun", 0, "Re-run publish?", false);
     72    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-pretend", 0, "Pretend to define?", false);
     73    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple",  0, "use simple output format?", false);
     74    psMetadataAddU64(definerunArgs, PS_LIST_TAIL, "-limit",  0, "limit result set", 0);
    6675
    6776    // -pending
    … …  
    7079    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-stage", 0, "search on source", NULL);
    7180    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL);
    72     psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL);
     81    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on label", NULL);
    7382    psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple",  0, "use simple output format?", false);
    7483    psMetadataAddU64(pendingArgs, PS_LIST_TAIL, "-limit",  0, "limit result set", 0);

  • branches/czw_branch/20100519/ippTools/src/pxtools.c

    r28304 r28338  
    250250        psMetadataItem *item = NULL;
    251251        while ((item = psListGetAndIncrement(iter))) {
     252            psMetadataItem *new = psMetadataItemCopy(item);
    252253            // need to change the name and comment
    253             psFree (item->name);
    254             item->name = psStringCopy (field);
    255             psFree (item->comment);
    256             item->comment = psStringCopy (op);
    257             if (!psMetadataAddItem(where, item, PS_LIST_TAIL, PS_META_DUPLICATE_OK)) {
     254            psFree (new->name);
     255            new->name = psStringCopy (field);
     256            psFree (new->comment);
     257            new->comment = psStringCopy (op);
     258            if (!psMetadataAddItem(where, new, PS_LIST_TAIL, PS_META_DUPLICATE_OK)) {
    258259                psError(psErrorCodeLast(), false, "failed to add item %s", field);
    259260                psFree(where);
    … …  
    261262            }
    262263        }
    263         psFree (iter);
     264        psFree(iter);
    264265    }
    265266    return true;

  • branches/czw_branch/20100519/ippconfig/recipes/nightly_science.config

    r28304 r28338  
    121121#  COMMENT STR Stellar Transit%
    122122  STACKABLE BOOL FALSE
     123  NOCLEAN BOOL TRUE
    123124END
    124125TARGETS METADATA

  • branches/czw_branch/20100519/magic/remove/src/streaksio.c

    r27474 r28338  
    12941294        }
    12951295    }
    1296     sfiles->maskMask = ~convPoor;
     1296    // preserve pixels that are only suspect
     1297    psU32 suspect = (double) psMetadataLookupU32(&status, masks, "SUSPECT");
     1298    if (!status) {
     1299        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for SUSPECT in recipes\n");
     1300        streaksExit("", PS_EXIT_CONFIG_ERROR);
     1301    }
     1302
     1303    sfiles->maskMask = ~(convPoor | suspect);
    12971304}
    12981305

  • branches/czw_branch/20100519/psModules/src/imcombine/pmPSFEnvelope.c

    r27331 r28338  
    3333
    3434
    35 // #define TESTING                         // Enable test output
     35//#define TESTING                         // Enable test output
    3636// #define PEAK_NORM                       // Normalise peaks?
    3737#define PEAK_FLUX 1.0e4                 // Peak flux for each source
    … …  
    235235
    236236            // Get the radius
    237             pmModel *model = pmModelFromPSFforXY(psf, x, y, PEAK_FLUX); // Model for source
     237            pmModel *model = pmModelFromPSFforXY(psf, source->peak->xf, source->peak->yf, PEAK_FLUX); // Model for source
    238238            if (!model || (model->flags & MODEL_MASK)) {
    239239                continue;
    … …  
    321321    numFakes = fakes->n;
    322322
    323     if (numFakes == 0.0) {
     323    if (numFakes == 0) {
    324324        psError(PS_ERR_UNKNOWN, false, "No fake sources are suitable for PSF fitting.");
    325325        psFree(fakes);

  • branches/czw_branch/20100519/pstamp/src/ppstampMakeStamp.c

    r27989 r28338  
    688688        }
    689689    }
    690     psU32 maskMask = ~convPoor;
     690    psU32 suspect = psMetadataLookupU32(&status, masks, "SUSPECT");
     691    if (!status) {
     692        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for SUSPECT in recipes\n");
     693        return false;
     694    }
     695    psU32 maskMask = ~(convPoor | suspect);
    691696
    692697    double exciseValue;
Note: See TracChangeset for help on using the changeset viewer.