Index: /branches/eam_branch_20080421/Ohana/configure =================================================================== --- /branches/eam_branch_20080421/Ohana/configure (revision 18423) +++ /branches/eam_branch_20080421/Ohana/configure (revision 18424) @@ -2,10 +2,11 @@ # strip out CC, CFLAGS, CPPFLAGS, LDFLAGS and set env vars -while (( $# > 0 )); do +while ( test $# -gt 0 ); do + skip=0 # strip out CC, set as env variable echo $1 | grep "^CC=" > /dev/null - if (( $? == 0 )) ; then + if ( test $? -eq 0 ) ; then val=`echo $1 | sed "s|^CC=||"` export CC=$val @@ -14,5 +15,5 @@ # strip out CFLAGS, set as env variable echo $1 | grep "^CFLAGS=" > /dev/null - if (( $? == 0 )) ; then + if ( test $? -eq 0 ) ; then val=`echo $1 | sed "s|^CFLAGS=||"` export CFLAGS=$val @@ -21,5 +22,5 @@ # strip out CPPFLAGS, set as env variable echo $1 | grep "^CPPFLAGS=" > /dev/null - if (( $? == 0 )) ; then + if ( test $? -eq 0 ) ; then val=`echo $1 | sed "s|^CPPFLAGS=||"` export CPPFLAGS=$val @@ -28,10 +29,10 @@ # strip out LDFLAGS, set as env variable echo $1 | grep "^LDFLAGS=" > /dev/null - if (( $? == 0 )) ; then + if ( test $? -eq 0 ) ; then val=`echo $1 | sed "s|^LDFLAGS=||"` export LDFLAGS=$val skip=1 fi - if (( $skip == 0 )) ; then + if ( test $skip -eq 0 ) ; then args="$args $1" fi Index: /branches/eam_branch_20080421/Ohana/configure.tcsh =================================================================== --- /branches/eam_branch_20080421/Ohana/configure.tcsh (revision 18423) +++ /branches/eam_branch_20080421/Ohana/configure.tcsh (revision 18424) @@ -350,7 +350,19 @@ foreach f ( ncurses curses termcap ) foreach g ( $libpath $libdir $syslibpath ) + # echo "trying $g" set name = $g/lib$f.a + # echo "trying $name" if (-e $name[1]) goto got_curses; set name = $g/lib$f.$dlltype + # echo "trying $name" + if (-e $name[1]) goto got_curses; + end + # try versioned libraries such as .so.N + foreach g ( $libpath $libdir $syslibpath ) + # echo "trying $g" + set name = $g/lib$f.$dlltype.* + # echo "$#name : $name[1]" + if ($#name < 2) continue + # echo "trying $name[1]" if (-e $name[1]) goto got_curses; end @@ -359,5 +371,5 @@ echo "missing a valid curses library" echo "missing: $faillibs" -echo "please find one of them and install them in $lib" +echo "please find one of them and install them in $libpath" exit 1 Index: /branches/eam_branch_20080421/Ohana/src/addstar/include/addstar.h =================================================================== --- /branches/eam_branch_20080421/Ohana/src/addstar/include/addstar.h (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/addstar/include/addstar.h (revision 18424) @@ -129,4 +129,5 @@ SkyRegion UserPatch; // used by MODE CAT char *SELECT_2MASS_QUALITY; // used only by get2mass_as +int NREFSTAR_GROUP; /*** addstar prototypes ***/ Index: /branches/eam_branch_20080421/Ohana/src/addstar/src/args.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/addstar/src/args.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/addstar/src/args.c (revision 18424) @@ -6,4 +6,7 @@ int i, N; int QUALITY_AIRMASS; + + // a global used by find_matches_refstars.c (value is 1 except for load2mass) + NREFSTAR_GROUP = 1; /* check for help request */ Index: /branches/eam_branch_20080421/Ohana/src/addstar/src/args_client.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/addstar/src/args_client.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/addstar/src/args_client.c (revision 18424) @@ -11,4 +11,7 @@ help (); } + + // a global used by find_matches_refstars.c (value is 1 except for load2mass) + NREFSTAR_GROUP = 1; /*** check for command line options ***/ Index: /branches/eam_branch_20080421/Ohana/src/addstar/src/args_load2mass.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/addstar/src/args_load2mass.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/addstar/src/args_load2mass.c (revision 18424) @@ -11,4 +11,7 @@ help (); } + + // a global used by find_matches_refstars.c (value is 1 except for load2mass) + NREFSTAR_GROUP = 3; /*** check for command line options ***/ Index: /branches/eam_branch_20080421/Ohana/src/addstar/src/args_sedstar.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/addstar/src/args_sedstar.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/addstar/src/args_sedstar.c (revision 18424) @@ -11,4 +11,7 @@ help (); } + + // a global used by find_matches_refstars.c (value is 1 except for load2mass) + NREFSTAR_GROUP = 1; /*** check for command line options ***/ Index: /branches/eam_branch_20080421/Ohana/src/addstar/src/args_server.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/addstar/src/args_server.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/addstar/src/args_server.c (revision 18424) @@ -11,4 +11,7 @@ help (); } + + // a global used by find_matches_refstars.c (value is 1 except for load2mass) + NREFSTAR_GROUP = 1; /* restrict to a portion of the sky? (REFCAT only) */ Index: /branches/eam_branch_20080421/Ohana/src/addstar/src/find_matches_refstars.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/addstar/src/find_matches_refstars.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/addstar/src/find_matches_refstars.c (revision 18424) @@ -14,4 +14,9 @@ Coords tcoords; int Nsecfilt; + + if ((NREFSTAR_GROUP != 1) && (NREFSTAR_GROUP != 3)) { + fprintf (stderr, "ERROR: NREFSTAR_GROUP NOT SET!\n"); + exit (1); + } /* photcode data -- should not have to modify secfilt / average */ @@ -98,4 +103,5 @@ /** find matched stars **/ + // XXX could use NREFSTAR_GROUP to do this match more quicky for (i = j = 0; (i < Nstars) && (j < Nave); ) { @@ -225,5 +231,5 @@ the reference up-to-date with known stars only */ - for (i = 0; (i < Nstars) && !options.only_match; i++) { + for (i = 0; (i < Nstars) && !options.only_match; i+=NREFSTAR_GROUP) { N = N1[i]; if (stars[N][0].found >= 0) continue; @@ -232,5 +238,5 @@ catalog[0].average[Nave].D = stars[N][0].D; catalog[0].average[Nave].Xp = 0; - catalog[0].average[Nave].Nmeasure = 1; + catalog[0].average[Nave].Nmeasure = NREFSTAR_GROUP; catalog[0].average[Nave].Nmissing = 0; catalog[0].average[Nave].measureOffset = Nmeas; @@ -268,54 +274,58 @@ } - catalog[0].measure[Nmeas].dR = 0.0; - catalog[0].measure[Nmeas].dD = 0.0; - catalog[0].measure[Nmeas].M = stars[N][0].M; - catalog[0].measure[Nmeas].dM = stars[N][0].dM; - catalog[0].measure[Nmeas].Mcal = 0; - catalog[0].measure[Nmeas].t = (stars[N][0].t == 0) ? TIMEREF : stars[N][0].t; /** careful : time_t vs e_time **/ - catalog[0].measure[Nmeas].averef = Nave; - catalog[0].measure[Nmeas].photcode = stars[N][0].code; - catalog[0].measure[Nmeas].dophot = 0; - catalog[0].measure[Nmeas].dbFlags = 0; - catalog[0].measure[Nmeas].dt = 0xffff; - - catalog[0].measure[Nmeas].photFlags = stars[N][0].flags; // XXX make sure these are zero'ed as needed - catalog[0].measure[Nmeas].qPSF = 0; - catalog[0].measure[Nmeas].psfChisq = 0; - catalog[0].measure[Nmeas].crNsigma = 0; - catalog[0].measure[Nmeas].extNsigma = 0; - catalog[0].measure[Nmeas].Sky = 0; - catalog[0].measure[Nmeas].dSky = 0; - - catalog[0].measure[Nmeas].stargal = 0; // XXX not yet set - - catalog[0].measure[Nmeas].detID = 0; - catalog[0].measure[Nmeas].imageID = 0; - - catalog[0].measure[Nmeas].dXccd = 0; - catalog[0].measure[Nmeas].dYccd = 0; - - catalog[0].measure[Nmeas].Xccd = stars[N][0].X; // XXX make sure these are zero'ed as needed - catalog[0].measure[Nmeas].Yccd = stars[N][0].Y; // XXX make sure these are zero'ed as needed - - catalog[0].measure[Nmeas].airmass = 0; - catalog[0].measure[Nmeas].Map = NAN; - catalog[0].measure[Nmeas].FWx = stars[N][0].fx; // XXX make sure these are zero'ed as needed - catalog[0].measure[Nmeas].FWy = stars[N][0].fy; // XXX make sure these are zero'ed as needed - catalog[0].measure[Nmeas].theta = stars[N][0].df; // XXX make sure these are zero'ed as needed - - catalog[0].measure[Nmeas].Xccd = 0.0; - catalog[0].measure[Nmeas].Yccd = 0.0; - - stars[N][0].found = Nmeas; - next[last] = Nmeas; - next[Nmeas] = -1; - last = Nmeas; - Nmeas ++; - if (Nmeas == NMEAS) { - NMEAS = Nmeas + 1000; - REALLOCATE (next, int, NMEAS); - REALLOCATE (catalog[0].measure, Measure, NMEAS); - } + for (j = 0; j < NREFSTAR_GROUP; j++) { + N = N1[i + j]; + catalog[0].measure[Nmeas].dR = 0.0; + catalog[0].measure[Nmeas].dD = 0.0; + catalog[0].measure[Nmeas].M = stars[N][0].M; + catalog[0].measure[Nmeas].dM = stars[N][0].dM; + catalog[0].measure[Nmeas].Mcal = 0; + catalog[0].measure[Nmeas].t = (stars[N][0].t == 0) ? TIMEREF : stars[N][0].t; /** careful : time_t vs e_time **/ + catalog[0].measure[Nmeas].averef = Nave; + catalog[0].measure[Nmeas].photcode = stars[N][0].code; + catalog[0].measure[Nmeas].dophot = 0; + catalog[0].measure[Nmeas].dbFlags = 0; + catalog[0].measure[Nmeas].dt = 0xffff; + + catalog[0].measure[Nmeas].photFlags = stars[N][0].flags; // XXX make sure these are zero'ed as needed + catalog[0].measure[Nmeas].qPSF = 0; + catalog[0].measure[Nmeas].psfChisq = 0; + catalog[0].measure[Nmeas].crNsigma = 0; + catalog[0].measure[Nmeas].extNsigma = 0; + catalog[0].measure[Nmeas].Sky = 0; + catalog[0].measure[Nmeas].dSky = 0; + + catalog[0].measure[Nmeas].stargal = 0; // XXX not yet set + + catalog[0].measure[Nmeas].detID = 0; + catalog[0].measure[Nmeas].imageID = 0; + + catalog[0].measure[Nmeas].dXccd = 0; + catalog[0].measure[Nmeas].dYccd = 0; + + catalog[0].measure[Nmeas].Xccd = stars[N][0].X; // XXX make sure these are zero'ed as needed + catalog[0].measure[Nmeas].Yccd = stars[N][0].Y; // XXX make sure these are zero'ed as needed + + catalog[0].measure[Nmeas].airmass = 0; + catalog[0].measure[Nmeas].Map = NAN; + catalog[0].measure[Nmeas].FWx = stars[N][0].fx; // XXX make sure these are zero'ed as needed + catalog[0].measure[Nmeas].FWy = stars[N][0].fy; // XXX make sure these are zero'ed as needed + catalog[0].measure[Nmeas].theta = stars[N][0].df; // XXX make sure these are zero'ed as needed + + catalog[0].measure[Nmeas].Xccd = 0.0; + catalog[0].measure[Nmeas].Yccd = 0.0; + + stars[N][0].found = Nmeas; + next[last] = Nmeas; + next[Nmeas] = -1; + last = Nmeas; + Nmeas ++; + if (Nmeas == NMEAS) { + NMEAS = Nmeas + 1000; + REALLOCATE (next, int, NMEAS); + REALLOCATE (catalog[0].measure, Measure, NMEAS); + } + } + Nave ++; if (Nave == NAVE) { Index: /branches/eam_branch_20080421/Ohana/src/kapa2/include/structures.h =================================================================== --- /branches/eam_branch_20080421/Ohana/src/kapa2/include/structures.h (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/kapa2/include/structures.h (revision 18424) @@ -138,4 +138,6 @@ char isaxis, areticks, islabel, islog; double fx, dfx, fy, dfy; /* axis location on graphic */ + double lweight; + int color; } Axis; Index: /branches/eam_branch_20080421/Ohana/src/kapa2/src/DrawFrame.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/kapa2/src/DrawFrame.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/kapa2/src/DrawFrame.c (revision 18424) @@ -8,10 +8,7 @@ int i, fx, fy, dfx, dfy, P, IsLabel, IsMajor; - double range, major, minor, first, next; + double range, major, minor, first, next, lweight; graphic = GetGraphic(); - - // XXX why is this called for every redraw? - DrawRotTextInit (graphic->display, graphic->window, graphic->gc, graphic->fore, graphic->back); /* each axis is drawn independently, but ticks and labels are placed according to @@ -23,4 +20,9 @@ dfy = graph[0].axis[i].dfy; P = hypot ((double)graph[0].axis[(i+1)%2].dfx, (double)graph[0].axis[(i+1)%2].dfy); + + lweight = MAX (0, MIN (10, graph[0].axis[i].lweight)); + XSetLineAttributes (graphic->display, graphic->gc, lweight, LineSolid, CapNotLast, JoinMiter); + XSetForeground (graphic->display, graphic->gc, graphic->color[graph[0].axis[i].color]); + DrawRotTextInit (graphic->display, graphic->window, graphic->gc, graphic->color[graph[0].axis[i].color], graphic->back); if (graph[0].axis[i].isaxis) { Index: /branches/eam_branch_20080421/Ohana/src/kapa2/src/InterpretKeys.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/kapa2/src/InterpretKeys.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/kapa2/src/InterpretKeys.c (revision 18424) @@ -152,9 +152,23 @@ case XK_KP_Add: - image[0].image[0].zero += 0.1*image[0].image[0].range; + if (modstate & ControlMask) { + image[0].image[0].zero -= 0.05*image[0].image[0].range; + image[0].image[0].range *= 1.1; + } else { + image[0].image[0].zero += 0.1*image[0].image[0].range; + } + SetColorScale (graphic, image); + Remap (graphic, image); Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, 0); break; case XK_KP_Subtract: - image[0].image[0].zero -= 0.1*image[0].image[0].range; + if (modstate & ControlMask) { + image[0].image[0].zero += 0.05*image[0].image[0].range; + image[0].image[0].range *= 0.90; + } else { + image[0].image[0].zero -= 0.1*image[0].image[0].range; + } + SetColorScale (graphic, image); + Remap (graphic, image); Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, 0); break; Index: /branches/eam_branch_20080421/Ohana/src/kapa2/src/LoadFrame.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/kapa2/src/LoadFrame.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/kapa2/src/LoadFrame.c (revision 18424) @@ -3,6 +3,7 @@ int LoadFrame (int sock) { - int i; + int i, color; char Axis[16], Labels[16], Ticks[16]; + double lweight; Section *section; KapaGraphWidget *graph; @@ -24,7 +25,12 @@ graph[0].axis[2].max = graph[0].axis[0].max; + KiiScanMessage (sock, "%lf %d", &lweight, &color); + KiiScanMessage (sock, "%s %s %s", Axis, Labels, Ticks); for (i = 0; i < 4; i++) { + graph[0].axis[i].lweight = lweight; + graph[0].axis[i].color = color; + switch (Axis[i]) { case '0': Index: /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap16.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap16.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap16.c (revision 18424) @@ -86,5 +86,5 @@ /**** fill in area to the left of the picture ****/ - for (jj = 0; (i_start > 0) && (jj < expand_out); jj++) { + for (jj = 0; (i_start > 0) && (jj < expand_out) && (j + jj < dy); jj++) { out_pix2 = out_pix + jj*dx; for (i = 0; i < i_start; i++, out_pix2++) { @@ -104,5 +104,5 @@ pixvalue = pixel[*in_pix2]; out_pix2 = out_pix; - for (jj = 0; jj < expand_out; jj++, out_pix2+=(dx-expand_out)) { + for (jj = 0; (jj < expand_out) && (j + jj < dy); jj++, out_pix2+=(dx-expand_out)) { for (ii = 0; ii < expand_out; ii++, out_pix2++) { *out_pix2 = pixvalue; @@ -114,5 +114,5 @@ /**** fill in area to the right of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { + for (jj = 0; (jj < expand_out) && (j + jj < dy); jj++) { out_pix2 = out_pix + jj*dx; for (i = i_end; i < dx; i++, out_pix2++) { Index: /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap24.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap24.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap24.c (revision 18424) @@ -84,5 +84,5 @@ /**** fill in area to the left of the picture ****/ - for (jj = 0; (i_start > 0) && (jj < expand_out); jj++) { + for (jj = 0; (i_start > 0) && (jj < expand_out) && (j + jj < dy); jj++) { out_pix2 = out_pix + jj*(3*dx + extra); for (i = 0; i < i_start; i++, out_pix2+=3) { @@ -108,5 +108,5 @@ pixvalue3 = pixel3[*in_pix2]; out_pix2 = out_pix; - for (jj = 0; jj < expand_out; jj++, out_pix2+=3*(dx-expand_out)+extra) { + for (jj = 0; (jj < expand_out) && (j + jj < dy); jj++, out_pix2+=3*(dx-expand_out)+extra) { for (ii = 0; ii < expand_out; ii++, out_pix2+=3) { out_pix2[0] = pixvalue1; @@ -120,5 +120,6 @@ /**** fill in area to the right of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { + for (jj = 0; (jj < expand_out) && (j + jj < dy); jj++) { + // for (jj = 0; jj < expand_out; jj++) { out_pix2 = out_pix + jj*(3*dx+extra); for (i = i_end; i < dx; i++, out_pix2+=3) { @@ -142,5 +143,5 @@ picture[0].pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - picture[0].data, picture[0].dx, picture[0].dy, 32, 0); + picture[0].data, picture[0].dx, picture[0].dy, 32, 0); free (pixel1); Index: /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap32.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap32.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap32.c (revision 18424) @@ -100,5 +100,5 @@ /**** fill in area to the left of the picture ****/ - for (jj = 0; (i_start > 0) && (jj < expand_out); jj++) { + for (jj = 0; (i_start > 0) && (jj < expand_out) && (j + jj < dy); jj++) { out_pix2 = out_pix + jj*dx; for (i = 0; i < i_start; i++, out_pix2++) { @@ -118,5 +118,5 @@ pixvalue = pixel[*in_pix2]; out_pix2 = out_pix; - for (jj = 0; jj < expand_out; jj++, out_pix2+=(dx-expand_out)) { + for (jj = 0; (jj < expand_out) && (j + jj < dy); jj++, out_pix2+=(dx-expand_out)) { for (ii = 0; ii < expand_out; ii++, out_pix2++) { *out_pix2 = pixvalue; @@ -133,5 +133,5 @@ /**** fill in area to the right of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { + for (jj = 0; (jj < expand_out) && (j + jj < dy); jj++) { out_pix2 = out_pix + jj*dx; for (i = i_end; i < dx; i++, out_pix2++) { Index: /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap8.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap8.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/kapa2/src/Remap8.c (revision 18424) @@ -69,5 +69,5 @@ /**** fill in area to the left of the picture ****/ - for (jj = 0; (i_start > 0) && (jj < expand_out); jj++) { + for (jj = 0; (i_start > 0) && (jj < expand_out) && (j + jj < dy); jj++) { out_pix2 = out_pix + jj*dx; for (i = 0; i < i_start; i++, out_pix2++) { @@ -87,5 +87,5 @@ pixvalue = pixel[*in_pix2]; out_pix2 = out_pix; - for (jj = 0; jj < expand_out; jj++, out_pix2+=(dx-expand_out)) { + for (jj = 0; (jj < expand_out) & (j + jj < dy); jj++, out_pix2+=(dx-expand_out)) { for (ii = 0; ii < expand_out; ii++, out_pix2++) { *out_pix2 = pixvalue; @@ -97,5 +97,5 @@ /**** fill in area to the right of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { + for (jj = 0; (jj < expand_out) && (j + jj < dy); jj++) { out_pix2 = out_pix + jj*dx; for (i = i_end; i < dx; i++, out_pix2++) { @@ -115,5 +115,5 @@ } picture[0].pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - picture[0].data, picture[0].dx, picture[0].dy, 8, 0); + picture[0].data, picture[0].dx, picture[0].dy, 8, 0); free (pixel); } Index: /branches/eam_branch_20080421/Ohana/src/libfits/extern/ricecomp.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libfits/extern/ricecomp.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libfits/extern/ricecomp.c (revision 18424) @@ -1,2 +1,8 @@ + +// 2008.06.22 EAM : This code was taken from CFITSIO and included in Ohana for rice +// decompression. In order to include this .c file (or future upgrades), we include our own +// version of ricecomp.h (not the one available in the CFITSIO tree) and include it instead of +// fitsio2.h. This file defines our own version of ffpmsg. + /* The following code was written by Richard White at STScI and made @@ -23,7 +29,19 @@ #include #include -#include "ricecomp.h" /* originally included in rcomp.c file (WDP) */ + +typedef unsigned char Buffer_t; + +typedef struct { + int bitbuffer; /* bit buffer */ + int bits_to_go; /* bits to go in buffer */ + Buffer_t *start; /* start of buffer */ + Buffer_t *current; /* current position in buffer */ + Buffer_t *end; /* end of buffer */ +} Buffer; + +#define putcbuf(c,mf) ((*(mf->current)++ = c), 0) + // #include "fitsio2.h" - +# include "ricecomp.h" static void start_outputing_bits(Buffer *buffer); @@ -32,4 +50,5 @@ /* this routine used to be called 'rcomp' (WDP) */ +/*---------------------------------------------------------------------------*/ int fits_rcomp(int a[], /* input array */ @@ -55,5 +74,6 @@ */ bsize = 4; -/* nblock = 32; */ + +/* nblock = 32; now an input parameter*/ /* * From bsize derive: @@ -62,4 +82,6 @@ * BBITS = bits/pixel for direct coding */ + +/* switch (bsize) { case 1: @@ -76,7 +98,12 @@ break; default: - fprintf (stderr, "rdecomp: bsize must be 1, 2, or 4 bytes"); + ffpmsg("rdecomp: bsize must be 1, 2, or 4 bytes"); return(-1); } +*/ + + /* move out of switch block, to tweak performance */ + fsbits = 5; + fsmax = 25; bbits = 1<> 1; for (fs = 0; psum>0; fs++) psum >>= 1; + /* * write the codes @@ -149,5 +178,5 @@ */ if (output_nbits(buffer, fsmax+1, fsbits) == EOF) { - fprintf (stderr, "rice_encode: end of buffer"); + ffpmsg("rice_encode: end of buffer"); free(diff); return(-1); @@ -155,5 +184,5 @@ for (j=0; j=8; top -= 8) { - if (putcbuf(0, buffer) == EOF) { - fprintf (stderr, "rice_encode: end of buffer"); - free(diff); - return(-1); - } + putcbuf(0, buffer); } lbitbuffer = 1; @@ -222,12 +244,463 @@ lbits_to_go -= fs; while (lbits_to_go <= 0) { - if (putcbuf((lbitbuffer>>(-lbits_to_go)) & 0xff,buffer)==EOF) { - fprintf (stderr, "rice_encode: end of buffer"); - free(diff); - return(-1); - } + putcbuf((lbitbuffer>>(-lbits_to_go)) & 0xff,buffer); lbits_to_go += 8; } } + } + + /* check if overflowed output buffer */ + if (buffer->current > buffer->end) { + ffpmsg("rice_encode: end of buffer"); + free(diff); + return(-1); + } + buffer->bitbuffer = lbitbuffer; + buffer->bits_to_go = lbits_to_go; + } + } + done_outputing_bits(buffer); + free(diff); + /* + * return number of bytes used + */ + return(buffer->current - buffer->start); +} +/*---------------------------------------------------------------------------*/ + +int fits_rcomp_short( + short a[], /* input array */ + int nx, /* number of input pixels */ + unsigned char *c, /* output buffer */ + int clen, /* max length of output */ + int nblock) /* coding block size */ +{ +Buffer bufmem, *buffer = &bufmem; +int bsize, i, j, thisblock; + +/* +NOTE: in principle, the following 2 variable could be declared as 'short' +but in fact the code runs faster (on 32-bit Linux at least) as 'int' +*/ +int lastpix, nextpix; +/* int pdiff; */ +short pdiff; +int v, fs, fsmask, top, fsmax, fsbits, bbits; +int lbitbuffer, lbits_to_go; +/* unsigned int psum; */ +unsigned short psum; +double pixelsum, dpsum; +unsigned int *diff; + + /* + * Original size of each pixel (bsize, bytes) and coding block + * size (nblock, pixels) + * Could make bsize a parameter to allow more efficient + * compression of short & byte images. + */ + bsize = 2; + +/* nblock = 32; now an input parameter */ + /* + * From bsize derive: + * FSBITS = # bits required to store FS + * FSMAX = maximum value for FS + * BBITS = bits/pixel for direct coding + */ + +/* + switch (bsize) { + case 1: + fsbits = 3; + fsmax = 6; + break; + case 2: + fsbits = 4; + fsmax = 14; + break; + case 4: + fsbits = 5; + fsmax = 25; + break; + default: + ffpmsg("rdecomp: bsize must be 1, 2, or 4 bytes"); + return(-1); + } +*/ + + /* move these out of switch block to further tweak performance */ + fsbits = 4; + fsmax = 14; + bbits = 1<start = c; + buffer->current = c; + buffer->end = c+clen; + buffer->bits_to_go = 8; + /* + * array for differences mapped to non-negative values + */ + diff = (unsigned int *) malloc(nblock*sizeof(unsigned int)); + if (diff == (unsigned int *) NULL) { + ffpmsg("fits_rcomp: insufficient memory"); + return(-1); + } + /* + * Code in blocks of nblock pixels + */ + start_outputing_bits(buffer); + + /* write out first short value to the first 2 bytes of the buffer */ + if (output_nbits(buffer, a[0], 16) == EOF) { + ffpmsg("rice_encode: end of buffer"); + free(diff); + return(-1); + } + + lastpix = a[0]; /* the first difference will always be zero */ + + thisblock = nblock; + for (i=0; i> 1; */ + psum = ((unsigned short) dpsum ) >> 1; + for (fs = 0; psum>0; fs++) psum >>= 1; + + /* + * write the codes + * fsbits ID bits used to indicate split level + */ + if (fs >= fsmax) { + /* Special high entropy case when FS >= fsmax + * Just write pixel difference values directly, no Rice coding at all. + */ + if (output_nbits(buffer, fsmax+1, fsbits) == EOF) { + ffpmsg("rice_encode: end of buffer"); + free(diff); + return(-1); + } + for (j=0; jbitbuffer; + lbits_to_go = buffer->bits_to_go; + for (j=0; j> fs; + /* + * top is coded by top zeros + 1 + */ + if (lbits_to_go >= top+1) { + lbitbuffer <<= top+1; + lbitbuffer |= 1; + lbits_to_go -= top+1; + } else { + lbitbuffer <<= lbits_to_go; + putcbuf(lbitbuffer & 0xff,buffer); + for (top -= lbits_to_go; top>=8; top -= 8) { + putcbuf(0, buffer); + } + lbitbuffer = 1; + lbits_to_go = 7-top; + } + /* + * bottom FS bits are written without coding + * code is output_nbits, moved into this routine to reduce overheads + * This code potentially breaks if FS>24, so I am limiting + * FS to 24 by choice of FSMAX above. + */ + if (fs > 0) { + lbitbuffer <<= fs; + lbitbuffer |= v & fsmask; + lbits_to_go -= fs; + while (lbits_to_go <= 0) { + putcbuf((lbitbuffer>>(-lbits_to_go)) & 0xff,buffer); + lbits_to_go += 8; + } + } + } + /* check if overflowed output buffer */ + if (buffer->current > buffer->end) { + ffpmsg("rice_encode: end of buffer"); + free(diff); + return(-1); + } + buffer->bitbuffer = lbitbuffer; + buffer->bits_to_go = lbits_to_go; + } + } + done_outputing_bits(buffer); + free(diff); + /* + * return number of bytes used + */ + return(buffer->current - buffer->start); +} +/*---------------------------------------------------------------------------*/ + +int fits_rcomp_byte( + signed char a[], /* input array */ + int nx, /* number of input pixels */ + unsigned char *c, /* output buffer */ + int clen, /* max length of output */ + int nblock) /* coding block size */ +{ +Buffer bufmem, *buffer = &bufmem; +int bsize, i, j, thisblock; + +/* +NOTE: in principle, the following 2 variable could be declared as 'short' +but in fact the code runs faster (on 32-bit Linux at least) as 'int' +*/ +int lastpix, nextpix; +/* int pdiff; */ +signed char pdiff; +int v, fs, fsmask, top, fsmax, fsbits, bbits; +int lbitbuffer, lbits_to_go; +/* unsigned int psum; */ +unsigned char psum; +double pixelsum, dpsum; +unsigned int *diff; + + /* + * Original size of each pixel (bsize, bytes) and coding block + * size (nblock, pixels) + * Could make bsize a parameter to allow more efficient + * compression of short & byte images. + */ + bsize = 1; + +/* nblock = 32; now an input parameter */ + /* + * From bsize derive: + * FSBITS = # bits required to store FS + * FSMAX = maximum value for FS + * BBITS = bits/pixel for direct coding + */ + +/* + switch (bsize) { + case 1: + fsbits = 3; + fsmax = 6; + break; + case 2: + fsbits = 4; + fsmax = 14; + break; + case 4: + fsbits = 5; + fsmax = 25; + break; + default: + ffpmsg("rdecomp: bsize must be 1, 2, or 4 bytes"); + return(-1); + } +*/ + + /* move these out of switch block to further tweak performance */ + fsbits = 3; + fsmax = 6; + bbits = 1<start = c; + buffer->current = c; + buffer->end = c+clen; + buffer->bits_to_go = 8; + /* + * array for differences mapped to non-negative values + */ + diff = (unsigned int *) malloc(nblock*sizeof(unsigned int)); + if (diff == (unsigned int *) NULL) { + ffpmsg("fits_rcomp: insufficient memory"); + return(-1); + } + /* + * Code in blocks of nblock pixels + */ + start_outputing_bits(buffer); + + /* write out first byte value to the first byte of the buffer */ + if (output_nbits(buffer, a[0], 8) == EOF) { + ffpmsg("rice_encode: end of buffer"); + free(diff); + return(-1); + } + + lastpix = a[0]; /* the first difference will always be zero */ + + thisblock = nblock; + for (i=0; i> 1; */ + psum = ((unsigned char) dpsum ) >> 1; + for (fs = 0; psum>0; fs++) psum >>= 1; + + /* + * write the codes + * fsbits ID bits used to indicate split level + */ + if (fs >= fsmax) { + /* Special high entropy case when FS >= fsmax + * Just write pixel difference values directly, no Rice coding at all. + */ + if (output_nbits(buffer, fsmax+1, fsbits) == EOF) { + ffpmsg("rice_encode: end of buffer"); + free(diff); + return(-1); + } + for (j=0; jbitbuffer; + lbits_to_go = buffer->bits_to_go; + for (j=0; j> fs; + /* + * top is coded by top zeros + 1 + */ + if (lbits_to_go >= top+1) { + lbitbuffer <<= top+1; + lbitbuffer |= 1; + lbits_to_go -= top+1; + } else { + lbitbuffer <<= lbits_to_go; + putcbuf(lbitbuffer & 0xff,buffer); + for (top -= lbits_to_go; top>=8; top -= 8) { + putcbuf(0, buffer); + } + lbitbuffer = 1; + lbits_to_go = 7-top; + } + /* + * bottom FS bits are written without coding + * code is output_nbits, moved into this routine to reduce overheads + * This code potentially breaks if FS>24, so I am limiting + * FS to 24 by choice of FSMAX above. + */ + if (fs > 0) { + lbitbuffer <<= fs; + lbitbuffer |= v & fsmask; + lbits_to_go -= fs; + while (lbits_to_go <= 0) { + putcbuf((lbitbuffer>>(-lbits_to_go)) & 0xff,buffer); + lbits_to_go += 8; + } + } + } + /* check if overflowed output buffer */ + if (buffer->current > buffer->end) { + ffpmsg("rice_encode: end of buffer"); + free(diff); + return(-1); } buffer->bitbuffer = lbitbuffer; @@ -270,4 +743,11 @@ int lbitbuffer; int lbits_to_go; + /* AND mask for the right-most n bits */ + static unsigned int mask[33] = + {0, + 0x1, 0x3, 0x7, 0xf, 0x1f, 0x3f, 0x7f, 0xff, + 0x1ff, 0x3ff, 0x7ff, 0xfff, 0x1fff, 0x3fff, 0x7fff, 0xffff, + 0x1ffff, 0x3ffff, 0x7ffff, 0xfffff, 0x1fffff, 0x3fffff, 0x7fffff, 0xffffff, + 0x1ffffff, 0x3ffffff, 0x7ffffff, 0xfffffff, 0x1fffffff, 0x3fffffff, 0x7fffffff, 0xffffffff}; /* @@ -282,11 +762,13 @@ */ lbitbuffer <<= lbits_to_go; - lbitbuffer |= (bits>>(n-lbits_to_go)) & ((1<>(n-lbits_to_go)) & ((1<>(n-lbits_to_go)) & *(mask+lbits_to_go); + putcbuf(lbitbuffer & 0xff,buffer); n -= lbits_to_go; lbits_to_go = 8; } lbitbuffer <<= n; - lbitbuffer |= ( bits & ((1<>(-lbits_to_go)) & 0xff,buffer) == EOF) - return(EOF); + putcbuf((lbitbuffer>>(-lbits_to_go)) & 0xff,buffer); lbits_to_go += 8; } @@ -302,5 +783,4 @@ return(0); } - /*---------------------------------------------------------------------------*/ /* Flush out the last bits */ @@ -309,6 +789,9 @@ { if(buffer->bits_to_go < 8) { - if (putcbuf(buffer->bitbuffer<bits_to_go,buffer) == EOF) + putcbuf(buffer->bitbuffer<bits_to_go,buffer); + +/* if (putcbuf(buffer->bitbuffer<bits_to_go,buffer) == EOF) return(EOF); +*/ } return(0); @@ -334,4 +817,5 @@ */ +/*---------------------------------------------------------------------------*/ /* this routine used to be called 'rdecomp' (WDP) */ @@ -356,5 +840,6 @@ */ bsize = 4; -/* nblock = 32; */ + +/* nblock = 32; now an input parameter */ /* * From bsize derive: @@ -363,4 +848,6 @@ * BBITS = bits/pixel for direct coding */ + +/* switch (bsize) { case 1: @@ -377,12 +864,15 @@ break; default: - fprintf (stderr, "rdecomp: bsize must be 1, 2, or 4 bytes"); + ffpmsg("rdecomp: bsize must be 1, 2, or 4 bytes"); return 1; } +*/ + + /* move out of switch block, to tweak performance */ + fsbits = 5; + fsmax = 25; + bbits = 1<> nbits) - 1; + b &= (1<>nbits); b &= (1< cend) { - fprintf (stderr, "decompression error: hit end of compressed byte stream"); + ffpmsg("decompression error: hit end of compressed byte stream"); return 1; } } if (c < cend) { - fprintf (stderr, "decompression warning: unused bytes at end of compressed buffer"); + ffpmsg("decompression warning: unused bytes at end of compressed buffer"); } return 0; } +/*---------------------------------------------------------------------------*/ +/* this routine used to be called 'rdecomp' (WDP) */ + +int fits_rdecomp_short (unsigned char *c, /* input buffer */ + int clen, /* length of input */ + unsigned short array[], /* output array */ + int nx, /* number of output pixels */ + int nblock) /* coding block size */ +{ +int i, imax; +int bsize, k; +int nbits, nzero, fs; +unsigned char *cend, bytevalue; +unsigned int b, diff, lastpix; +int fsmax, fsbits, bbits; +static int *nonzero_count = (int *)NULL; + + /* + * Original size of each pixel (bsize, bytes) and coding block + * size (nblock, pixels) + * Could make bsize a parameter to allow more efficient + * compression of short & byte images. + */ + + bsize = 2; + +/* nblock = 32; now an input parameter */ + /* + * From bsize derive: + * FSBITS = # bits required to store FS + * FSMAX = maximum value for FS + * BBITS = bits/pixel for direct coding + */ + +/* + switch (bsize) { + case 1: + fsbits = 3; + fsmax = 6; + break; + case 2: + fsbits = 4; + fsmax = 14; + break; + case 4: + fsbits = 5; + fsmax = 25; + break; + default: + ffpmsg("rdecomp: bsize must be 1, 2, or 4 bytes"); + return 1; + } +*/ + + /* move out of switch block, to tweak performance */ + fsbits = 4; + fsmax = 14; + + bbits = 1<=0; ) { + for ( ; i>=k; i--) nonzero_count[i] = nzero; + k = k/2; + nzero--; + } + } + /* + * Decode in blocks of nblock pixels + */ + + /* first 2 bytes of input buffer contain the value of the first */ + /* 2 byte integer value, without any encoding */ + + lastpix = 0; + bytevalue = c[0]; + lastpix = lastpix | (bytevalue<<8); + bytevalue = c[1]; + lastpix = lastpix | bytevalue; + + c += 2; + cend = c + clen - 2; + + b = *c++; /* bit buffer */ + nbits = 8; /* number of bits remaining in b */ + for (i = 0; i> nbits) - 1; + + b &= (1< nx) imax = nx; + if (fs<0) { + /* low-entropy case, all zero differences */ + for ( ; i= 0; k -= 8) { + b = *c++; + diff |= b<0) { + b = *c++; + diff |= b>>(-k); + b &= (1<>1; + } else { + diff = ~(diff>>1); + } + array[i] = diff+lastpix; + lastpix = array[i]; + } + } else { + /* normal case, Rice coding */ + for ( ; i>nbits); + b &= (1<>1; + } else { + diff = ~(diff>>1); + } + array[i] = diff+lastpix; + lastpix = array[i]; + } + } + if (c > cend) { + ffpmsg("decompression error: hit end of compressed byte stream"); + return 1; + } + } + if (c < cend) { + ffpmsg("decompression warning: unused bytes at end of compressed buffer"); + } + return 0; +} +/*---------------------------------------------------------------------------*/ +/* this routine used to be called 'rdecomp' (WDP) */ + +int fits_rdecomp_byte (unsigned char *c, /* input buffer */ + int clen, /* length of input */ + unsigned char array[], /* output array */ + int nx, /* number of output pixels */ + int nblock) /* coding block size */ +{ +int i, imax; +int bsize, k; +int nbits, nzero, fs; +unsigned char *cend; +unsigned int b, diff, lastpix; +int fsmax, fsbits, bbits; +static int *nonzero_count = (int *)NULL; + + /* + * Original size of each pixel (bsize, bytes) and coding block + * size (nblock, pixels) + * Could make bsize a parameter to allow more efficient + * compression of short & byte images. + */ + + bsize = 1; + +/* nblock = 32; now an input parameter */ + /* + * From bsize derive: + * FSBITS = # bits required to store FS + * FSMAX = maximum value for FS + * BBITS = bits/pixel for direct coding + */ + +/* + switch (bsize) { + case 1: + fsbits = 3; + fsmax = 6; + break; + case 2: + fsbits = 4; + fsmax = 14; + break; + case 4: + fsbits = 5; + fsmax = 25; + break; + default: + ffpmsg("rdecomp: bsize must be 1, 2, or 4 bytes"); + return 1; + } +*/ + + /* move out of switch block, to tweak performance */ + fsbits = 3; + fsmax = 6; + + bbits = 1<=0; ) { + for ( ; i>=k; i--) nonzero_count[i] = nzero; + k = k/2; + nzero--; + } + } + /* + * Decode in blocks of nblock pixels + */ + + /* first byte of input buffer contain the value of the first */ + /* byte integer value, without any encoding */ + + lastpix = c[0]; + c += 1; + cend = c + clen - 1; + + b = *c++; /* bit buffer */ + nbits = 8; /* number of bits remaining in b */ + for (i = 0; i> nbits) - 1; + + b &= (1< nx) imax = nx; + if (fs<0) { + /* low-entropy case, all zero differences */ + for ( ; i= 0; k -= 8) { + b = *c++; + diff |= b<0) { + b = *c++; + diff |= b>>(-k); + b &= (1<>1; + } else { + diff = ~(diff>>1); + } + array[i] = diff+lastpix; + lastpix = array[i]; + } + } else { + /* normal case, Rice coding */ + for ( ; i>nbits); + b &= (1<>1; + } else { + diff = ~(diff>>1); + } + array[i] = diff+lastpix; + lastpix = array[i]; + } + } + if (c > cend) { + ffpmsg("decompression error: hit end of compressed byte stream"); + return 1; + } + } + if (c < cend) { + ffpmsg("decompression warning: unused bytes at end of compressed buffer"); + } + return 0; +} Index: /branches/eam_branch_20080421/Ohana/src/libfits/extern/ricecomp.h =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libfits/extern/ricecomp.h (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libfits/extern/ricecomp.h (revision 18424) @@ -1,107 +1,8 @@ -/* @(#) buffer.h 1.1 98/07/21 12:34:27 */ -/* buffer.h: structure for compression to buffer rather than to a file, including - * bit I/O buffer - * - * R. White, 19 June 1998 - */ -typedef unsigned char Buffer_t; +// 2008.06.22 EAM : This code was taken from CFITSIO and included in Ohana for rice +// decompression. In order to include this .c file (or future upgrades), we include our own +// version of ricecomp.h (not the one available in the CFITSIO tree) and include it instead of +// fitsio2.h. This file defines our own version of ffpmsg. -typedef struct { - int bitbuffer; /* bit buffer */ - int bits_to_go; /* bits to go in buffer */ - Buffer_t *start; /* start of buffer */ - Buffer_t *current; /* current position in buffer */ - Buffer_t *end; /* end of buffer */ -} Buffer; - -#define buffree(mf) (free(mf->start), free(mf)) -#define bufused(mf) (mf->current - mf->start) -#define bufreset(mf) (mf->current = mf->start) - -/* - * getcbuf, putcbuf macros for character IO to buffer - * putcbuf returns EOF on end of buffer, else returns 0 - */ -#define getcbuf(mf) ((mf->current >= mf->end) ? EOF : *(mf->current)++) -#define putcbuf(c,mf) \ - ((mf->current >= mf->end) ? \ - EOF :\ - ((*(mf->current)++ = c), 0)) - -/* - * bufalloc sets up buffer of length n - */ - -/* not needed by CFITSIO - -static Buffer *bufalloc(int n) -{ -Buffer *mf; - - mf = (Buffer *) malloc(sizeof(Buffer)); - if (mf == (Buffer *)NULL) return((Buffer *)NULL); - - mf->start = (Buffer_t *) malloc(n*sizeof(Buffer_t)); - if (mf->start == (Buffer_t *)NULL) { - free(mf); - return((Buffer *)NULL); - } - mf->bits_to_go = 8; - mf->end = mf->start + n; - mf->current = mf->start; - return(mf); -} -*/ - -/* - * bufrealloc extends buffer (or truncates it) by - * reallocating memory - */ - -/* not needed by CFITSIO -static int bufrealloc(Buffer *mf, int n) -{ -int len; - - len = mf->current - mf->start; - - * silently throw away data if buffer is already longer than n * - if (len>n) len = n; - if (len<0) len = 0; - - mf->start = (Buffer_t *) realloc(mf->start, n*sizeof(Buffer_t)); - if (mf->start == (Buffer_t *)NULL) return(0); - - mf->end = mf->start + n; - mf->current = mf->start + len; - return(n); -} -*/ - -/* - * bufdump dumps contents of buffer to outfile and resets - * it to be empty. Returns number of bytes written. - * - * Note we don't write out the bit buffer -- you must call - * done_outputing_bits() first to ensure that the bit buffer - * is written out. I do it this way to allow incremental - * buffer dumps while bit IO is still going on. - */ - -/* not needed by CFITSIO - -static int bufdump(FILE *outfile, Buffer *buffer) -{ -int ndump; - - ndump = bufused(buffer); - if (fwrite(buffer->start, 1, ndump, outfile) != ndump) { - fprintf(stderr, "bufdump: error in write\n"); - exit(1); - } - bufreset(buffer); - return(ndump); -} -*/ +#define ffpmsg(MSG) fprintf(stderr, "%s\n", MSG) Index: /branches/eam_branch_20080421/Ohana/src/libfits/header/F_copy_H.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libfits/header/F_copy_H.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libfits/header/F_copy_H.c (revision 18424) @@ -23,4 +23,5 @@ out[0].bscale = in[0].bscale; + if (out[0].buffer != NULL) free (out[0].buffer); ALLOCATE (out[0].buffer, char, out[0].size); Index: /branches/eam_branch_20080421/Ohana/src/libfits/include/gfitsio.h =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libfits/include/gfitsio.h (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libfits/include/gfitsio.h (revision 18424) @@ -161,5 +161,5 @@ int gfits_extension_is_compressed PROTO((Header *header)); int gfits_tile_size PROTO((Matrix *matrix, int *otile, int *ztile)); -int gfits_uncompressed_data_pixsize PROTO((char *cmptype, int out_bitpix)); +int gfits_uncompressed_data_pixsize PROTO((char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions)); int gfits_vartable_heap_pixsize PROTO((char format)); Index: /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_compress_M.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_compress_M.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_compress_M.c (revision 18424) @@ -18,13 +18,14 @@ return (FALSE); } -# define MOD_KEYWORD(NAME,ZNAME,TYPE,IN,OUT) { \ - gfits_scan (header, ZNAME, TYPE, 1, IN); \ - gfits_delete (header, ZNAME, 1); \ - gfits_print (header, NAME, TYPE, 1, OUT); } +# define MOD_KEYWORD(ZNAME,NAME,TYPE,IN,OUT) { \ + if (gfits_scan (header, ZNAME, TYPE, 1, IN)) { \ + gfits_modify (header, NAME, TYPE, 1, OUT); \ + } \ + gfits_delete (header, ZNAME, 1); } # define MOD_KEYWORD_REQUIRED(ZNAME,NAME,TYPE,IN,OUT) { \ if (!gfits_scan (header, ZNAME, TYPE, 1, IN)) ESCAPE; \ gfits_delete (header, ZNAME, 1); \ - gfits_print (header, NAME, TYPE, 1, OUT); } + gfits_modify (header, NAME, TYPE, 1, OUT); } int gfits_uncompress_image (Header *header, Matrix *matrix, FTable *ftable) { @@ -55,4 +56,7 @@ // copy original header to output header gfits_copy_header (ftable->header, header); + + // delete ZIMAGE from output header + gfits_delete (header, "ZIMAGE", 1); // extract compression-specific keywords, update header as needed. @@ -79,4 +83,5 @@ } } else { + gfits_delete (header, "ZTILE1", 1); for (i = 1; i < header->Naxes; i++) { snprintf (key, 10, "ZTILE%d", i + 1); @@ -143,11 +148,28 @@ gfits_delete (header, "ZTENSION", 1); gfits_modify_extended (header, exttype, "Image extension"); - // XXX validate that exttype == 'IMAGE'? - - MOD_KEYWORD ("ZPCOUNT", "PCOUNT", "%d", &header->pcount, header->pcount); - MOD_KEYWORD ("ZGCOUNT", "GCOUNT", "%d", &header->gcount, header->gcount); - } - - MOD_KEYWORD ("ZHECKSUM", "ZHECKSUM", "%s", checksum, checksum); + + // we may have an uncompressed PCOUNT / GCOUNT value, otherwise set to 0,1 + if (gfits_scan (header, "ZPCOUNT", "%d", 1, &header->pcount)) { + gfits_delete (header, "ZPCOUNT", 1); + gfits_modify (header, "PCOUNT", "%d", 1, header->pcount); + } else { + header->pcount = 0; + gfits_modify (header, "PCOUNT", "%d", 1, header->pcount); + } + if (gfits_scan (header, "ZGCOUNT", "%d", 1, &header->gcount)) { + gfits_delete (header, "ZGCOUNT", 1); + gfits_modify (header, "GCOUNT", "%d", 1, header->gcount); + } else { + header->pcount = 1; + gfits_modify (header, "GCOUNT", "%d", 1, header->gcount); + } + } else { + header->pcount = 0; + header->gcount = 1; + gfits_modify (header, "PCOUNT", "%d", 1, header->pcount); + gfits_modify (header, "GCOUNT", "%d", 1, header->gcount); + } + + MOD_KEYWORD ("ZHECKSUM", "CHECKSUM", "%s", checksum, checksum); MOD_KEYWORD ("ZDATASUM", "DATASUM", "%s", datasum, datasum); @@ -167,4 +189,9 @@ if (!gfits_varlength_column_define (ftable, &zdef, zcol)) ESCAPE; + gfits_delete (header, "TFIELDS", 1); + snprintf (key, 10, "TTYPE%d", zcol); + gfits_delete (header, key, 1); + snprintf (key, 10, "TFORM%d", zcol); + gfits_delete (header, key, 1); // create the output image @@ -200,5 +227,5 @@ // size of a pixel in the output from the decompression routine - odata_pixsize = gfits_uncompressed_data_pixsize (cmptype, header[0].bitpix); + odata_pixsize = gfits_uncompressed_data_pixsize (cmptype, header[0].bitpix, optname, optvalue, Noptions); ALLOCATE (out, char, odata_pixsize*max_tile_size); @@ -385,5 +412,5 @@ case 2: - for (i = 0; i < Nzdata; i+=2) { + for (i = 0; i < 2*Nzdata; i+=2) { tmp = zdata[i]; zdata[i] = zdata[i+1]; @@ -393,5 +420,5 @@ case 4: - for (i = 0; i < Nzdata; i+=4) { + for (i = 0; i < 4*Nzdata; i+=4) { tmp = zdata[i+1]; zdata[i+1] = zdata[i+2]; @@ -404,5 +431,5 @@ case 8: - for (i = 0; i < Nzdata; i+=8) { + for (i = 0; i < 8*Nzdata; i+=8) { tmp = zdata[i+0]; zdata[i+0] = zdata[i+7]; @@ -426,10 +453,12 @@ int gfits_extension_is_compressed (Header *header) { - int has_extname, has_zimage, zimage; - char extname[80]; - + int has_extension, has_extname, has_zimage, zimage; + char extname[80], extension[80]; + + has_extension = gfits_scan (header, "XTENSION", "%s", 1, extension); has_extname = gfits_scan (header, "EXTNAME", "%s", 1, extname); has_zimage = gfits_scan (header, "ZIMAGE", "%t", 1, &zimage); + if (has_extension && !strcmp (extension, "IMAGE")) return (FALSE); if (has_zimage && zimage) return (TRUE); if (has_extname) { @@ -453,5 +482,7 @@ } -int gfits_uncompressed_data_pixsize (char *cmptype, int out_bitpix) { +int gfits_uncompressed_data_pixsize (char *cmptype, int out_bitpix, char **optname, char **optvalue, int Noptions) { + + int i, Nbyte; if (!strcasecmp(cmptype, "GZIP_1")) { @@ -459,8 +490,17 @@ } if (!strcasecmp(cmptype, "RICE_1")) { + + // if BYTEPIX option is specified, use that for Nbyte + for (i = 0; i < Noptions; i++) { + if (!strcmp(optname[i], "BYTEPIX")) { + Nbyte = atoi (optvalue[i]); + return (Nbyte); + } + } + return (4); } if (!strcasecmp(cmptype, "PLIO_1")) { - return (1); + return (4); } if (!strcasecmp(cmptype, "HCOMPRESS_1")) { Index: /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_copy_M.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_copy_M.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_copy_M.c (revision 18424) @@ -16,4 +16,5 @@ matrix2[0].Naxis[i] = matrix1[0].Naxis[i]; + if (matrix2[0].buffer != NULL) free (matrix2[0].buffer); ALLOCATE (matrix2[0].buffer, char, matrix2[0].size); Index: /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_create_M.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_create_M.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_create_M.c (revision 18424) @@ -27,2 +27,3 @@ } +// XXX free buffer if non-null: need to double check for existing frees Index: /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_uncompress_data.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_uncompress_data.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libfits/matrix/F_uncompress_data.c (revision 18424) @@ -6,4 +6,6 @@ int fits_rcomp(int a[], int nx, unsigned char *c, int clen, int nblock); int fits_rdecomp (unsigned char *c, int clen, unsigned int array[], int nx, int nblock); +int fits_rdecomp_short (unsigned char *c, int clen, unsigned short array[], int nx, int nblock); +int fits_rdecomp_byte (unsigned char *c, int clen, unsigned char array[], int nx, int nblock); /* functions defined in fits_hcompress.c */ @@ -60,11 +62,33 @@ // fprintf (stderr, "%d comp bytes; %d uncomp 'pixels', totals: %d %d\n", Nzdata, Npix, Ninsum, Noutsum); - // rice decompression from the CFITSIO source tree : we need to tell it the expected number of pixels - // is also REQUIRES 4byte output, which is fairly stupid. - if (fits_rdecomp ((unsigned char *) zdata, Nzdata, (unsigned int *) outdata, Npix, blocksize)) { - fprintf (stderr, "error in rice decompression\n"); - return (FALSE); + switch (out_pixsize) { + case 4: + // rice decompression from the CFITSIO source tree : we need to tell it the expected number of pixels + // is also REQUIRES 4byte output, which is fairly stupid. + if (fits_rdecomp ((unsigned char *) zdata, Nzdata, (unsigned int *) outdata, Npix, blocksize)) { + fprintf (stderr, "error in rice decompression\n"); + return (FALSE); + } + return (TRUE); + + case 2: + if (fits_rdecomp_short ((unsigned char *) zdata, Nzdata, (unsigned short *) outdata, Npix, blocksize)) { + fprintf (stderr, "error in rice decompression\n"); + return (FALSE); + } + return (TRUE); + + case 1: + if (fits_rdecomp_byte ((unsigned char *) zdata, Nzdata, (unsigned char *) outdata, Npix, blocksize)) { + fprintf (stderr, "error in rice decompression\n"); + return (FALSE); + } + return (TRUE); + + default: + fprintf (stderr, "invalid output pixel size %d\n", out_pixsize); + return (FALSE); } - return (TRUE); + } Index: /branches/eam_branch_20080421/Ohana/src/libfits/table/F_table_varlength.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libfits/table/F_table_varlength.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libfits/table/F_table_varlength.c (revision 18424) @@ -70,4 +70,5 @@ void *gfits_varlength_column_pointer (FTable *ftable, VarLengthColumn *column, int row, int *length) { + void *result; int offset, Nx, *ptr; @@ -92,40 +93,5 @@ offset = ptr[1]; - switch (column->format) { - case 'X': - case 'L': - case 'A': - case 'B': { - char *result; - result = (char *) ftable->buffer + column->heap_start + offset; - return result; - } - case 'I': { - short *result; - result = (short *) ftable->buffer + column->heap_start + offset; - return result; - } - case 'J': { - int *result; - result = (int *) ftable->buffer + column->heap_start + offset; - return result; - } - case 'E': - case 'C': { - float *result; - result = (float *) ftable->buffer + column->heap_start + offset; - return result; - } - case 'D': - case 'M': { - double *result; - result = (double *) ftable->buffer + column->heap_start + offset; - return result; - } - case 'P': - default: - abort(); - } - abort(); + result = (void *) (ftable->buffer + column->heap_start + offset); + return result; } - Index: /branches/eam_branch_20080421/Ohana/src/libkapa/src/IOfuncs.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libkapa/src/IOfuncs.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libkapa/src/IOfuncs.c (revision 18424) @@ -1,3 +1,4 @@ # include +# define DEBUG 0 /** these function expect to operate with a BLOCKing socket **/ @@ -63,5 +64,8 @@ status = read (device, buffer, 16); buffer[16] = 0; - if (status != 16) fprintf (stderr, "dropped message length\n"); + if (status != 16) { + fprintf (stderr, "dropped message length\n"); + } + if (DEBUG) fprintf (stderr, "recv buffer: %s\n", buffer); /* find the message length, allocate space */ @@ -71,9 +75,11 @@ /* read Nbytes from the device */ status = read (device, message, Nbytes); - if (status != Nbytes) fprintf (stderr, "Kii/Kapa comm error\n"); + if (status != Nbytes) { + fprintf (stderr, "Kii/Kapa comm error\n"); + } message[status] = 0; /* make the string easy to parse */ - // fprintf (stderr, "recv: %s\n", message); + if (DEBUG) fprintf (stderr, "recv: %s\n", message); /* scan the incoming message */ @@ -112,5 +118,5 @@ write (device, string, length); - // fprintf (stderr, "send: %s\n", string); + if (DEBUG) fprintf (stderr, "send: %s\n", string); free (string); @@ -129,4 +135,5 @@ /* read Nbytes from the device */ status = read (device, message, length); + if (DEBUG) fprintf (stderr, "recv message: %s\n", message); if (status != length) { Index: /branches/eam_branch_20080421/Ohana/src/libkapa/src/KapaWindow.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libkapa/src/KapaWindow.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libkapa/src/KapaWindow.c (revision 18424) @@ -23,4 +23,6 @@ KiiSendMessage (fd, "%12.6g %12.6g %12.6g %12.6g", graphdata[0].xmin, graphdata[0].xmax, graphdata[0].ymin, graphdata[0].ymax); + + KiiSendMessage (fd, "%lf %d", graphdata[0].lweight, graphdata[0].color); KiiSendMessage (fd, "%s %s %s", graphdata[0].axis, graphdata[0].labels, graphdata[0].ticks); Index: /branches/eam_branch_20080421/Ohana/src/libohana/src/IOBufferOps.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libohana/src/IOBufferOps.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libohana/src/IOBufferOps.c (revision 18424) @@ -57,9 +57,9 @@ switch (errno) { case EAGAIN: - case EIO: - /** no data available in pipe **/ + case EINTR: + /** data not available in pipe or read interrupted : just try again **/ return (-1); default: - /** error reading from pipe **/ + /** serious error (buffer overflow, invalid fd, etc **/ perror ("ReadtoIOBuffer read error"); return (-2); Index: /branches/eam_branch_20080421/Ohana/src/libohana/src/findexec.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/libohana/src/findexec.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/libohana/src/findexec.c (revision 18424) @@ -213,6 +213,6 @@ ALLOCATE (file, char, strlen(name) + 1); - strcpy (file, name); - c = strrchr (file, '/'); + + c = strrchr (name, '/'); if (c == (char *) NULL) { strcpy (file, name); Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.astro/coord_systems.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.astro/coord_systems.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.astro/coord_systems.c (revision 18424) @@ -59,4 +59,5 @@ abort(); } + break; case COORD_GALACTIC: switch (output) { @@ -77,4 +78,5 @@ abort(); } + break; default: abort(); Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.astro/coords.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.astro/coords.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.astro/coords.c (revision 18424) @@ -104,4 +104,6 @@ gprint (GP_ERR, "USAGE: coords [buffer] (-c R D) | (-p X Y)\n"); gprint (GP_ERR, "only one of -p or -c can be used\n"); + gprint (GP_ERR, " -p : from pixels to ra/dec\n"); + gprint (GP_ERR, " -c : from ra/dec to pixels\n"); escape: if (MOSAIC != NULL) free (MOSAIC); Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.basic/Makefile =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.basic/Makefile (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.basic/Makefile (revision 18424) @@ -22,4 +22,5 @@ $(SRC)/config.$(ARCH).o \ $(SRC)/continue.$(ARCH).o \ +$(SRC)/basename.$(ARCH).o \ $(SRC)/dirname.$(ARCH).o \ $(SRC)/date.$(ARCH).o \ Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.basic/basename.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.basic/basename.c (revision 18424) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.basic/basename.c (revision 18424) @@ -0,0 +1,51 @@ +# include "basic.h" + +int basename_opihi (int argc, char **argv) { + + int N; + char *baseName, *varName, *suffixName; + + varName = NULL; + if ((N = get_argument (argc, argv, "-var"))) { + remove_argument (N, &argc, argv); + varName = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + suffixName = NULL; + if ((N = get_argument (argc, argv, "-suffix"))) { + remove_argument (N, &argc, argv); + suffixName = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc < 2) { + gprint (GP_ERR, "USAGE: dirname (path) [-var name] [-suffix suffix]\n"); + return (FALSE); + } + + baseName = filebasename (argv[1]); + + // strip suffix, if supplied + if (suffixName != NULL) { + if (strlen(baseName) > strlen(suffixName)) { + char *ptr = baseName + strlen(baseName) - strlen(suffixName); + if (!strcmp (ptr, suffixName)) { + *ptr = 0; + } + } + } + + if (varName == NULL) { + gprint (GP_LOG, "%s\n", baseName); + } else { + set_str_variable (varName, baseName); + free (varName); + } + + free (baseName); + return (TRUE); +} + +// XXX need to add mode option +// XXX need to respect umask (need umask command?) Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.basic/init.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.basic/init.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.basic/init.c (revision 18424) @@ -1,4 +1,5 @@ # include "basic.h" +int basename_opihi PROTO((int, char **)); int config PROTO((int, char **)); int exec_sleep PROTO((int, char **)); @@ -47,4 +48,5 @@ {1, "cd", cd, "change directory"}, {1, "date", date, "get current date"}, + {1, "basename", basename_opihi, "built-in basename function"}, {1, "dirname", dirname_opihi, "built-in dirname function"}, {1, "echo", echo, "type this line *"}, Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/Makefile =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/Makefile (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/Makefile (revision 18424) @@ -89,4 +89,5 @@ $(SRC)/queuepop.$(ARCH).o \ $(SRC)/queueprint.$(ARCH).o \ +$(SRC)/queuesubstr.$(ARCH).o \ $(SRC)/queueinit.$(ARCH).o \ $(SRC)/radial.$(ARCH).o \ Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/box.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/box.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/box.c (revision 18424) @@ -15,4 +15,19 @@ if (!GetGraph (&graphmode, &kapa, name)) return (FALSE); FREE (name); + + graphmode.lweight = 1; + if ((N = get_argument (argc, argv, "-lw"))) { + remove_argument (N, &argc, argv); + graphmode.lweight = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + graphmode.color = KapaColorByName ("black"); + if ((N = get_argument (argc, argv, "-c"))) { + remove_argument (N, &argc, argv); + graphmode.color = KapaColorByName (argv[N]); + if (graphmode.color == -1) return (FALSE); + remove_argument (N, &argc, argv); + } strcpy (graphmode.ticks, "2222"); Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/dbselect.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/dbselect.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/dbselect.c (revision 18424) @@ -5,4 +5,5 @@ int dbselect (int argc, char **argv) { + time_t seconds; int i, j, Nbytes, Ncols, Nrows; char *query; @@ -80,8 +81,30 @@ } for (i = 0; i < Ncols; i++) { - vec[i][0].elements[j] = atof (row[i]); + if (row[i]) { + switch (fields[i].type) { + case FIELD_TYPE_TINY: + case FIELD_TYPE_SHORT: + case FIELD_TYPE_LONG: + case FIELD_TYPE_INT24: + case FIELD_TYPE_LONGLONG: + case FIELD_TYPE_DECIMAL: + case FIELD_TYPE_FLOAT: + case FIELD_TYPE_DOUBLE: + vec[i][0].elements[j] = atof (row[i]); + break; + case FIELD_TYPE_TIME: + case FIELD_TYPE_DATE: + case FIELD_TYPE_DATETIME: + seconds = ohana_date_to_sec (row[i]); + vec[i][0].elements[j] = ohana_sec_to_mjd (seconds); + break; + default: + vec[i][0].elements[j] = NAN; + } + } else { + vec[i][0].elements[j] = NAN; + } } } - free (query); free (vec); @@ -89,5 +112,4 @@ return (TRUE); } - # else Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/histogram.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/histogram.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/histogram.c (revision 18424) @@ -34,4 +34,7 @@ REALLOCATE (yvec[0].elements, float, yvec[0].Nelements); bzero (yvec[0].elements, sizeof(float)*yvec[0].Nelements); + if (Nbins < 1) { + return (TRUE); + } V = xvec[0].elements; Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/init.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/init.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/init.c (revision 18424) @@ -74,4 +74,5 @@ int queueprint PROTO((int, char **)); int queuepush PROTO((int, char **)); +int queuesubstr PROTO((int, char **)); int queuesize PROTO((int, char **)); int rd PROTO((int, char **)); @@ -196,4 +197,5 @@ {1, "queuelist", queuelist, "list defined queues"}, {1, "queueload", queueload, "load queue from command"}, + {1, "queuesubstr", queuesubstr, "bulk replace strings in queue"}, {1, "queuesize", queuesize, "show queue size"}, {1, "rd", rd, "load fits image"}, Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/queuesubstr.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/queuesubstr.c (revision 18424) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/queuesubstr.c (revision 18424) @@ -0,0 +1,42 @@ +# include "data.h" + +int queuesubstr (int argc, char **argv) { + + int i; + char *p, *q; + char *match, *replace; + Queue *queue; + + if (argc != 4) { + gprint (GP_ERR, "USAGE: queueprint (queue) (match) (replace)\n"); + return (FALSE); + } + + match = argv[2]; + replace = argv[3]; + + assert (match); + assert (replace); + + queue = FindQueue (argv[1]); + if (queue == NULL) { + gprint (GP_ERR, "ERROR: queue %s not found\n", argv[1]); + return (FALSE); + } + + for (i = 0; i < queue[0].Nlines; i++) { + + p = queue[0].lines[i]; + if (p == NULL) continue; + + while (strstr (p, match) != NULL) { + q = strsubs (p, match, replace); + free (p); + p = q; + } + queue[0].lines[i] = p; + } + + return (TRUE); +} + Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/rd.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/rd.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/rd.c (revision 18424) @@ -182,7 +182,10 @@ Header theader; ftable.header = &theader; + ftable.header[0].buffer = NULL; gfits_copy_header (&buf[0].header, ftable.header); status = gfits_fread_ftable_data (f, &ftable); // this just reads the bytes (not even a SWAP) status = gfits_uncompress_image (&buf[0].header, &buf[0].matrix, &ftable); + // uncompressing the image leaves the format as an extension + gfits_extended_to_primary (&buf[0].header, TRUE, "Standard FITS"); gfits_free_table (&ftable); // XXX this currently does not work for a cube (we get a cube back, not a specific plane) @@ -190,6 +193,6 @@ sprintf (region, "-1 -1 -1 -1 %d %d", (plane - 1), plane); status = gfits_fread_matrix_segment (f, &buf[0].matrix, &buf[0].header, region); - fclose (f); - } + } + fclose (f); if (!status) { Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/spline_construct.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/spline_construct.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/spline_construct.c (revision 18424) @@ -16,6 +16,6 @@ if ((out = SelectBuffer (argv[2], ANYBUFFER, TRUE)) == NULL) return (FALSE); + // XXX move this to gfits_create_matrix free (out[0].matrix.buffer); - free (out[0].header.buffer); if ((in[0].file[0] != '*') && (in[0].file[0] != '(')) { sprintf (out[0].file, "*%s", in[0].file); Index: /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/wd.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/wd.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/cmd.data/wd.c (revision 18424) @@ -81,5 +81,12 @@ memcpy (temp_header.buffer, buf[0].header.buffer, temp_header.size); - if (temp_header.Naxes) gfits_convert_format (&temp_header, &temp_matrix, outBitpix, outScale, outZero, outUnsign); + if (temp_header.Naxes) { + gfits_convert_format (&temp_header, &temp_matrix, outBitpix, outScale, outZero, outUnsign); + } else { + gfits_modify (&temp_header, "BITPIX", "%d", 1, outBitpix); + gfits_modify (&temp_header, "BSCALE", "%lf", 1, outScale); + gfits_modify (&temp_header, "BZERO", "%lf", 1, outZero); + gfits_modify (&temp_header, "UNSIGN", "%t", 1, outUnsign); + } if (Extend) { Index: /branches/eam_branch_20080421/Ohana/src/opihi/dvo/ImageOps.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/dvo/ImageOps.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/dvo/ImageOps.c (revision 18424) @@ -50,4 +50,8 @@ if (selection->useDisplay) { if (!FindMosaicForImage (image, Nimage, i)) continue; + // first check if region center is in image + status = RD_to_XY (&X, &Y, Rmid, graph.coords.crval2, &image[i].coords); + if (status && (X >= 0) && (X < image[i].NX) && (Y >= 0) && (Y < image[i].NY)) goto in_region; + /* project this image to screen display coords */ x[0] = 0; y[0] = 0; Index: /branches/eam_branch_20080421/Ohana/src/opihi/dvo/dbExtractMeasures.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/dvo/dbExtractMeasures.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/dvo/dbExtractMeasures.c (revision 18424) @@ -291,5 +291,5 @@ break; case MEAS_FWHM: /* OK */ - value = 0.01*(measure[0].FWx + measure[0].FWy); + value = 0.01*(measure[0].FWx + measure[0].FWy) / 2.0; break; case MEAS_FWHM_MAJ: /* OK */ Index: /branches/eam_branch_20080421/Ohana/src/opihi/include/pcontrol.h =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/include/pcontrol.h (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/include/pcontrol.h (revision 18424) @@ -8,6 +8,8 @@ /** job status values **/ typedef enum { + PCONTROL_JOB_ALLJOBS, PCONTROL_JOB_PENDING, PCONTROL_JOB_BUSY, + PCONTROL_JOB_RESP, PCONTROL_JOB_HUNG, PCONTROL_JOB_DONE, @@ -26,10 +28,23 @@ /** host status values **/ typedef enum { + PCONTROL_HOST_ALLHOSTS, PCONTROL_HOST_IDLE, PCONTROL_HOST_BUSY, + PCONTROL_HOST_RESP, PCONTROL_HOST_DOWN, PCONTROL_HOST_DONE, PCONTROL_HOST_OFF, } HostStat; + +/** host response options **/ +typedef enum { + PCONTROL_RESP_NONE, + PCONTROL_RESP_START_JOB, + PCONTROL_RESP_CHECK_BUSY_JOB, + PCONTROL_RESP_CHECK_DONE_HOST, + PCONTROL_RESP_CHECK_HOST, + PCONTROL_RESP_KILL_JOB, + PCONTROL_RESP_STOP_HOST, +} HostResp; typedef enum { @@ -97,4 +112,7 @@ Ptime nexttry; IDtype HostID; + IOBuffer comms_buffer; + char *response; + HostResp response_state; struct Job *job; } Host; @@ -156,16 +174,30 @@ /*** own files ***/ +int StartJob (Job *job, Host *host); +int StartJobResponse (Host *host); + int CheckHost (Host *host); +int CheckHostResponse (Host *host); + +int CheckDoneHost (Host *host); +int CheckDoneHostResponse (Host *host); + +int CheckBusyJob (Job *job, Host *host); +int CheckBusyJobResponse (Host *host); + +int KillJob (Job *job, Host *host); +int KillJobResponse (Host *host); + int StartHost (Host *host); int CheckIdleHost (Host *host); -int CheckDoneHost (Host *host); -int CheckBusyJob (Job *job, Host *host); int CheckDoneJob (Job *job, Host *host); -int KillJob (Job *job, Host *host); -int StartJob (Job *job, Host *host); -int ResetJob (Job *job); int GetJobOutput (char *command, Host *host, IOBuffer *buffer, int Nbytes); -int PclientCommand (Host *host, char *command, char *response, IOBuffer *buffer); int rconnect (char *command, char *hostname, char *shell, int *stdio); + +int PclientCommand (Host *host, char *command, char *response, HostResp response_state); +int PclientResponse (Host *host, char *response, IOBuffer *buffer); + +int CheckRespHosts (float MaxDelay); +int CheckRespHost (Host *host); /*** misc files ***/ @@ -178,4 +210,5 @@ IDtype NextHostID (); void PrintID (gpDest dest, IDtype ID); +IDtype GetID (char *IDword); /*** CheckPoint.c ***/ @@ -203,4 +236,5 @@ int DownHosts (); int StopHost (Host *host); +int StopHostResponse (Host *host); int HarvestHost (int pid); Index: /branches/eam_branch_20080421/Ohana/src/opihi/lib.data/fft.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/lib.data/fft.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/lib.data/fft.c (revision 18424) @@ -127,4 +127,5 @@ Nminor *= Nsize[iDim]; } + free (Nbit); free (tmpX); free (tmpY); Index: /branches/eam_branch_20080421/Ohana/src/opihi/lib.data/open_kapa.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/lib.data/open_kapa.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/lib.data/open_kapa.c (revision 18424) @@ -108,5 +108,5 @@ if (!strncmp (kapa_exec, "unix://", 7)) { - fd = KapaOpenNamedSocket (&kapa_exec[7], "psphot"); + fd = KapaOpenNamedSocket (&kapa_exec[7], kapa_name); } else { fd = KapaOpen (kapa_exec, kapa_name); Index: /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/BufferOps.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/BufferOps.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/BufferOps.c (revision 18424) @@ -122,6 +122,4 @@ int CopyBuffer (Buffer *out, Buffer *in) { - free (out[0].matrix.buffer); - free (out[0].header.buffer); out[0].bitpix = in[0].bitpix; out[0].unsign = in[0].unsign; Index: /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/evaluate_stack.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/evaluate_stack.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/evaluate_stack.c (revision 18424) @@ -13,5 +13,5 @@ int i, j, Nvar, Nout, status; - char line[512]; + char line[512]; // this is only used to report an error StackVar tmp_stack; Nout = Nvar = 0; Index: /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/multicommand.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/multicommand.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/multicommand.c (revision 18424) @@ -93,5 +93,5 @@ // receive the command exit status - if (ExpectMessage (server, 2.0, &message)) { + if (ExpectMessage (server, 10.0, &message)) { switch (errno) { case EPIPE: @@ -108,5 +108,5 @@ // receive the resulting stderr - if (ExpectMessage (server, 2.0, &message)) { + if (ExpectMessage (server, 10.0, &message)) { switch (errno) { case EPIPE: @@ -123,5 +123,5 @@ // receive the resulting stdout - if (ExpectMessage (server, 2.0, &message)) { + if (ExpectMessage (server, 10.0, &message)) { switch (errno) { case EPIPE: Index: /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/stack_math.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/stack_math.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/lib.shell/stack_math.c (revision 18424) @@ -10,5 +10,5 @@ int i, Nx; float *out, *M1, *M2; - char line[512]; + char line[512]; // this is only used to report an error Nx = V1[0].vector[0].Nelements; @@ -83,6 +83,5 @@ switch (op[0]) { case '+': - VV_FUN - + VV_FUNC if ((M1->type == OPIHI_FLOAT) && (M2->type == OPIHI_FLOAT)) { @@ -222,5 +221,5 @@ int i, Nx; float *out, *M1, *M2; - char line[512]; + char line[512]; // this is only used to report an error Nx = V2[0].vector[0].Nelements; @@ -340,5 +339,5 @@ int i, Nx; float *out, *M1, *M2; - char line[512]; + char line[512]; // this is only used to report an error Nx = V1[0].vector[0].Nelements; @@ -459,5 +458,5 @@ int i, j, Nx, Ny; float *out, *M1, *M2; - char line[512]; + char line[512]; // this is only used to report an error Nx = V1[0].buffer[0].matrix.Naxis[0]; @@ -629,5 +628,5 @@ int i, j, Nx, Ny; float *out, *M1, *M2; - char line[512]; + char line[512]; // this is only used to report an error Nx = V2[0].buffer[0].matrix.Naxis[0]; @@ -815,5 +814,5 @@ int i, Nx, Ny; float *out, *M1, *M2; - char line[512]; + char line[512]; // this is only used to report an error Nx = V1[0].buffer[0].matrix.Naxis[0]; @@ -947,5 +946,5 @@ int i, Nx, Ny; float *out, *M1, *M2; - char line[512]; + char line[512]; // this is only used to report an error Nx = V1[0].buffer[0].matrix.Naxis[0]; @@ -1066,5 +1065,5 @@ int i, Nx, Ny; float *out, *M1, *M2; - char line[512]; + char line[512]; // this is only used to report an error Nx = V2[0].buffer[0].matrix.Naxis[0]; @@ -1183,5 +1182,5 @@ float *M1, *M2, *out; - char line[512]; + char line[512]; // this is only used to report an error M1 = V1[0].ptr; @@ -1267,5 +1266,5 @@ int value; - char line[512]; + char line[512]; // this is only used to report an error /* only 'N' and 'E' are allowed for WW_binary operations. anything else is either a Index: /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/ControllerOps.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/ControllerOps.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/ControllerOps.c (revision 18424) @@ -1,6 +1,6 @@ # include "pantasks.h" /* adding a new host can delay controller up to a second or so */ -# define CONTROLLER_TIMEOUT 1000 -# define CONNECT_TIMEOUT 300 +# define CONTROLLER_TIMEOUT 5000 +# define CONNECT_TIMEOUT 1000 /* local static variables to hold the connection to the controller */ @@ -88,14 +88,20 @@ /** parse status message **/ p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer); + if (p == NULL) goto escape; sscanf (p, "%*s %s", status_string); p = memstr (buffer.buffer, "EXITST", buffer.Nbuffer); + if (p == NULL) goto escape; sscanf (p, "%*s %d", &job[0].exit_status); p = memstr (buffer.buffer, "STDOUT", buffer.Nbuffer); + if (p == NULL) goto escape; sscanf (p, "%*s %d", &job[0].stdout_size); p = memstr (buffer.buffer, "STDERR", buffer.Nbuffer); + if (p == NULL) goto escape; sscanf (p, "%*s %d", &job[0].stderr_size); p = memstr (buffer.buffer, "DTIME", buffer.Nbuffer); + if (p == NULL) goto escape; sscanf (p, "%*s %lf", &job[0].dtime); p = memstr (buffer.buffer, "HOSTNAME", buffer.Nbuffer); + if (p == NULL) goto escape; sscanf (p, "%*s %s", string); job[0].realhost = strcreate (string); @@ -109,10 +115,13 @@ if (!strcmp(status_string, "EXIT")) outstate = JOB_EXIT; if (!strcmp(status_string, "CRASH")) outstate = JOB_CRASH; - if (outstate == -1) { - gprint (GP_ERR, "programming error?\n"); - exit (1); - } + if (outstate == -1) goto escape; + job[0].state = outstate; return (TRUE); + + escape: + gprint (GP_ERR, "garbage in pcontrol reponse\n"); + FreeIOBuffer (&buffer); + return (FALSE); } @@ -297,5 +306,5 @@ status = ReadtoIOBuffer (&buffer, stdout_fd[0]); p = memstr (buffer.buffer, "CONNECTED", buffer.Nbuffer); - usleep (10000); // wait for controller to start up + usleep (50000); // wait for controller to start up } if (status == 0) goto pipe_error; @@ -436,4 +445,11 @@ } +int FlushControllerOutput () { + + FlushIOBuffer (&stdout_buffer); + FlushIOBuffer (&stderr_buffer); + + return (TRUE); +} int KillControllerJob (Job *job) { Index: /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/JobOps.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/JobOps.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/JobOps.c (revision 18424) @@ -16,4 +16,5 @@ } +/* free all jobs, only used on shutdown */ void FreeJobs () { int i; @@ -159,4 +160,7 @@ if (task[0].stderr_dump != NULL) job[0].stderr_dump = strcreate (task[0].stderr_dump); + job[0].stdout_fd = -1; + job[0].stderr_fd = -1; + jobs[Njobs] = job; Njobs ++; @@ -185,4 +189,7 @@ } free (job[0].optv); + + if (job[0].stdout_fd >= 0) close (job[0].stdout_fd); + if (job[0].stderr_fd >= 0) close (job[0].stderr_fd); if (job[0].stdout_dump != NULL) free (job[0].stdout_dump); Index: /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/LocalJob.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/LocalJob.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/LocalJob.c (revision 18424) @@ -21,4 +21,6 @@ close (job[0].stdout_fd); close (job[0].stderr_fd); + job[0].stdout_fd = -1; // prevent FreeJob from trying to close again + job[0].stderr_fd = -1; // prevent FreeJob from trying to close again } else { /* read stdout buffer */ Index: /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/Makefile =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/Makefile (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/Makefile (revision 18424) @@ -79,4 +79,5 @@ $(SRC)/controller_status.$(ARCH).o \ $(SRC)/controller_jobstack.$(ARCH).o \ +$(SRC)/controller_verbose.$(ARCH).o \ $(SRC)/controller_run.$(ARCH).o \ $(SRC)/controller_output.$(ARCH).o \ Index: /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller.c (revision 18424) @@ -5,4 +5,5 @@ int controller_status PROTO((int, char **)); int controller_jobstack PROTO((int, char **)); +int controller_verbose PROTO((int, char **)); int controller_run PROTO((int, char **)); int controller_stop PROTO((int, char **)); @@ -19,4 +20,5 @@ {1, "status", controller_status, "check controller status"}, {1, "jobstack", controller_jobstack, "check controller status"}, + {1, "verbose", controller_verbose, "set controller verbosity"}, {1, "output", controller_output, "print controller output"}, {1, "pulse", controller_pulse, "set controller pulse"}, Index: /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller_jobstack.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller_jobstack.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller_jobstack.c (revision 18424) @@ -21,5 +21,5 @@ // XXX this has an error? test this out... - sprintf (command, "jobstack %s", argv[2]); + sprintf (command, "jobstack %s", argv[1]); InitIOBuffer (&buffer, 0x100); Index: /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller_output.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller_output.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller_output.c (revision 18424) @@ -3,6 +3,6 @@ int controller_output (int argc, char **argv) { - if (argc != 1) { - gprint (GP_ERR, "USAGE: controller status\n"); + if ((argc != 1) || ((argc == 2) && (strcmp(argv[1], "flush")))) { + gprint (GP_ERR, "USAGE: controller output\n"); return (FALSE); } @@ -10,4 +10,9 @@ CheckControllerOutput (); PrintControllerOutput (); + + if (argc == 2) { + FlushControllerOutput (); + } + return (TRUE); } Index: /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller_verbose.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller_verbose.c (revision 18424) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/controller_verbose.c (revision 18424) @@ -0,0 +1,37 @@ +# include "pantasks.h" + +int controller_verbose (int argc, char **argv) { + + int status; + char command[1024]; + IOBuffer buffer; + + if (argc != 2) { + gprint (GP_ERR, "USAGE: controller verbose (level)\n"); + gprint (GP_ERR, " (level) : off, on, toggle\n"); + return (FALSE); + } + + /* check if controller is running */ + status = CheckControllerStatus (); + if (!status) { + gprint (GP_LOG, "controller is not active\n"); + return (TRUE); + } + + // XXX this has an error? test this out... + sprintf (command, "verbose %s", argv[1]); + InitIOBuffer (&buffer, 0x100); + + SerialThreadLock (); + status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer); + SerialThreadUnlock (); + + if (status) { + gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG); + } else { + gprint (GP_LOG, "controller is not communicating\n"); + } + FreeIOBuffer (&buffer); + return (TRUE); +} Index: /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/ipptool2book.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/ipptool2book.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pantasks/ipptool2book.c (revision 18424) @@ -11,5 +11,6 @@ int i, N, onPage, found, Unique, Nkeys, NKEYS; char *line, *tmpword, *tmpvalue; - char pagename[512], *bookName, **keys, *p, *q; + char pagename[512]; // XXX this should be made dynamic, though it is an unlikey problem + char *bookName, **keys, *p, *q; char **setWordList; char **setWordValue; @@ -102,5 +103,5 @@ tmpword = nextword (line); - if (strcmp(tmpword, "MULTI")) { + if (!tmpword || strcmp(tmpword, "MULTI")) { gprint (GP_ERR, "ERROR: missing metadata output name on first line\n"); free (bookName); Index: /branches/eam_branch_20080421/Ohana/src/opihi/pclient/pclient.c.in =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pclient/pclient.c.in (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pclient/pclient.c.in (revision 18424) @@ -23,5 +23,5 @@ rl_attempted_completion_function = command_completer; rl_event_hook = CheckChild; - rl_set_keyboard_input_timeout (1000); + rl_set_keyboard_input_timeout (20000); /* 1 ms seems to be the minimum valid number */ @@ -63,4 +63,5 @@ FreeBasic (); FreePclient (); + gprint (GP_LOG, "Goodbye\n"); return; } Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckBusyJob.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckBusyJob.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckBusyJob.c (revision 18424) @@ -4,9 +4,5 @@ int CheckBusyJob (Job *job, Host *host) { - int status; - int outstate; - char *p; - char string[64]; - IOBuffer buffer; + int status; /* we are checking a job which is currently busy. it has been pulled from the @@ -16,50 +12,57 @@ ASSERT (job, "job not set"); ASSERT (host, "host not set"); - ASSERT (host == (Host *) job[0].host, "invalid host"); ASSERT (job == (Job *) host[0].job, "invalid job"); - InitIOBuffer (&buffer, 0x100); - - status = PclientCommand (host, "status", PCLIENT_PROMPT, &buffer); + status = PclientCommand (host, "status", PCLIENT_PROMPT, PCONTROL_RESP_CHECK_BUSY_JOB); /* check on success of pclient command */ switch (status) { case PCLIENT_DOWN: - HarvestHost (host[0].pid); + // free the realhost name + if (job[0].realhost) free (job[0].realhost); + job[0].realhost = NULL; + // unlink host & job + if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname); job[0].host = NULL; host[0].job = NULL; - if (job[0].realhost) free (job[0].realhost); - job[0].realhost = NULL; + HarvestHost (host[0].pid); PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM); - FreeIOBuffer (&buffer); return (FALSE); - - case PCLIENT_HUNG: - // don't do anything drastic, just keep trying - if (DEBUG || VerboseMode()) gprint (GP_ERR, "client is busy, not responding"); - PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM); - PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM); - FreeIOBuffer (&buffer); - return (TRUE); case PCLIENT_GOOD: if (DEBUG || VerboseMode()) gprint (GP_ERR, "message received (CheckBusyJob)"); - break; + PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM); + PutJob (job, PCONTROL_JOB_RESP, STACK_BOTTOM); + return (TRUE); default: ABORT ("unknown status for pclient command"); } +} + +int CheckBusyJobResponse (Host *host) { + + int outstate; + char *p; + char string[64]; + IOBuffer *buffer; + Job *job; + + /* job must have assigned host */ + ASSERT (host, "missing host"); + ASSERT (host[0].job, "missing job"); + buffer = &host[0].comms_buffer; + job = (Job *) host[0].job; /** host is up, need to parse message **/ - p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer); - // this condition means the message is garbage. toss it and try again + p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer); if (p == NULL) { - PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM); - PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM); - FreeIOBuffer (&buffer); - return (FALSE); + if (DEBUG || VerboseMode()) gprint (GP_ERR, "missing STATUS in response; try again\n"); + PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM); + PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM); + return (TRUE); } @@ -71,5 +74,4 @@ PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM); PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM); - FreeIOBuffer (&buffer); return (TRUE); } @@ -79,12 +81,12 @@ if (!strcmp(string, "EXIT")) outstate = PCONTROL_JOB_EXIT; if (!strcmp(string, "CRASH")) outstate = PCONTROL_JOB_CRASH; - ASSERT (outstate != PCONTROL_JOB_BUSY, "should not reach here (CheckJob)"); + ASSERT (outstate != PCONTROL_JOB_BUSY, "invalid status response (CheckBusyJobResponse)"); /* parse the exit status and sizes of output buffers */ - p = memstr (buffer.buffer, "EXITST", buffer.Nbuffer); + p = memstr (buffer[0].buffer, "EXITST", buffer[0].Nbuffer); sscanf (p, "%*s %d", &job[0].exit_status); - p = memstr (buffer.buffer, "STDOUT", buffer.Nbuffer); + p = memstr (buffer[0].buffer, "STDOUT", buffer[0].Nbuffer); sscanf (p, "%*s %d", &job[0].stdout_size); - p = memstr (buffer.buffer, "STDERR", buffer.Nbuffer); + p = memstr (buffer[0].buffer, "STDERR", buffer[0].Nbuffer); sscanf (p, "%*s %d", &job[0].stderr_size); @@ -98,7 +100,6 @@ PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM); PutJobSetState (job, PCONTROL_JOB_DONE, STACK_BOTTOM, outstate); - gettimeofday (&job[0].stop, (void *) NULL); + gettimeofday (&job[0].stop, NULL); job[0].dtime = DTIME(job[0].stop, job[0].start); - FreeIOBuffer (&buffer); return (TRUE); } Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckDoneHost.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckDoneHost.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckDoneHost.c (revision 18424) @@ -5,12 +5,8 @@ int status; - char *p; - IOBuffer buffer; ASSERT (host, "host not set"); - InitIOBuffer (&buffer, 0x100); - - status = PclientCommand (host, "reset", PCLIENT_PROMPT, &buffer); + status = PclientCommand (host, "reset", PCLIENT_PROMPT, PCONTROL_RESP_CHECK_DONE_HOST); /* check on success of pclient command */ @@ -18,29 +14,38 @@ case PCLIENT_DOWN: if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname); - /* DONE host does not have an incomplete job */ HarvestHost (host[0].pid); PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); - FreeIOBuffer (&buffer); return (FALSE); + /* DONE host does not have an incomplete job */ // XXX do we need to close the connection? - case PCLIENT_HUNG: - // don't do anything drastic, just try again later - PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM); - if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is not responding\n", host[0].hostname); - FreeIOBuffer (&buffer); - return (FALSE); - case PCLIENT_GOOD: - if (VerboseMode()) gprint (GP_ERR, "message received (CheckDoneHost)\n"); - break; + if (VerboseMode()) gprint (GP_ERR, "checking done host %s\n", host[0].hostname); + PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM); + return (TRUE); default: ABORT ("unknown status for pclient command"); } + ABORT ("should not reach here (CheckDoneHost)"); +} + +int CheckDoneHostResponse (Host *host) { + + int status; + char *p; + IOBuffer *buffer; + + /* job must have assigned host */ + ASSERT (host, "missing host"); + buffer = &host[0].comms_buffer; /** successful command, examine result **/ - p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer); - ASSERT (p != NULL, "missing STATUS in pclient message (CheckDoneHost)"); + p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer); + if (p == NULL) { + if (DEBUG || VerboseMode()) gprint (GP_ERR, "missing STATUS in response; try again\n"); + PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM); + return (FALSE); + } sscanf (p, "%*s %d", &status); @@ -52,5 +57,4 @@ if (DEBUG || VerboseMode()) gprint (GP_ERR, "reset failed\n"); PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM); - FreeIOBuffer (&buffer); return (FALSE); @@ -59,5 +63,4 @@ if (DEBUG || VerboseMode()) gprint (GP_ERR, "successful reset\n"); PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM); - FreeIOBuffer (&buffer); return (FALSE); @@ -67,5 +70,2 @@ ABORT ("should not reach here (CheckDoneHost)"); } - -/** probably need to flush the buffer before the command **/ -/** need to add timeout check here **/ Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckHost.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckHost.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckHost.c (revision 18424) @@ -1,8 +1,9 @@ # include "pcontrol.h" +// if the host has a job, we skip it (down or crash state will be caught elsewhere) +// in fact, just touch the IDLE hosts, not the BUSY hosts? int CheckHost (Host *host) { int status; - IOBuffer buffer; ASSERT (host, "host not set"); @@ -14,34 +15,38 @@ host[0].markoff = FALSE; StopHost (host); - OffHost (host); return (TRUE); } - InitIOBuffer (&buffer, 0x100); + // the argument to echo (OK) is the expected response below in CheckHostResponse + status = PclientCommand (host, "echo OK", PCLIENT_PROMPT, PCONTROL_RESP_CHECK_HOST); - status = PclientCommand (host, "echo OK", PCLIENT_PROMPT, &buffer); switch (status) { - case 0: + case PCLIENT_DOWN: if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname); HarvestHost (host[0].pid); PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); - FreeIOBuffer (&buffer); return (FALSE); - case -1: - if (VerboseMode()) gprint (GP_ERR, "host %s is not responding\n", host[0].hostname); - /*** do we mark this in some way (HUNG) ? ***/ - PutHost (host, host[0].stack, STACK_BOTTOM); - FreeIOBuffer (&buffer); - return (FALSE); + case PCLIENT_GOOD: + PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM); + return (TRUE); default: - PutHost (host, host[0].stack, STACK_BOTTOM); - FreeIOBuffer (&buffer); - return (TRUE); + ABORT ("unknown status for pclient command"); } - ABORT ("should not reach here (Check Host)"); + ABORT ("should not reach here (CheckHost)"); } -// if the host has a job, we skip it (down or crash state will be caught elsewhere) -// in fact, just touch the IDLE hosts, not the BUSY hosts? +int CheckHostResponse (Host *host) { + + IOBuffer *buffer; + + /* we only check IDLE hosts without jobs */ + ASSERT (host, "missing host"); + buffer = &host[0].comms_buffer; + + // XXX check on the value of the response? (OK) + + PutHost (host, PCONTROL_HOST_IDLE, STACK_BOTTOM); + return (TRUE); +} Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckIdleHost.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckIdleHost.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckIdleHost.c (revision 18424) @@ -14,5 +14,4 @@ host[0].markoff = FALSE; StopHost (host); - OffHost (host); return (TRUE); } Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckRespHost.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckRespHost.c (revision 18424) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckRespHost.c (revision 18424) @@ -0,0 +1,96 @@ +# include "pcontrol.h" +# define DEBUG 0 + +// this function operates on hosts waiting for a response. we simply check if the message is +// complete, and if so, send it to the correct parsing function +int CheckRespHost (Host *host) { + + int status; + Job *job; + + ASSERT (host, "host not set"); + job = (Job *) host[0].job; + + status = PclientResponse (host, host[0].response, &host[0].comms_buffer); + + /* check on output from pclient command */ + switch (status) { + case PCLIENT_DOWN: + if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname); + + // not all hosts here have a job; if it does, return it to PENDING + if (job) { + // unlink host & job + if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname); + job[0].host = NULL; + host[0].job = NULL; + PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM); + } + + // clear the response data + host[0].response_state = PCONTROL_RESP_NONE; + host[0].response = NULL; + + // host has shutdown; harvest the defunct process + HarvestHost (host[0].pid); + PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); + return (FALSE); + + case PCLIENT_HUNG: + // not done yet; try again later + PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM); + if (job) { + PutJob (job, PCONTROL_JOB_RESP, STACK_BOTTOM); + } + if (DEBUG || VerboseMode()) gprint (GP_ERR, "host %s is not responding\n", host[0].hostname); + return (FALSE); + + case PCLIENT_GOOD: + if (VerboseMode()) gprint (GP_ERR, "message received (CheckRespHost)\n"); + break; + + default: + ABORT ("unknown status for pclient command"); + } + + switch (host[0].response_state) { + case PCONTROL_RESP_START_JOB: + if (DEBUG) fprintf (stderr, "PCONTROL_RESP_START_JOB\n"); + status = StartJobResponse (host); + break; + + case PCONTROL_RESP_CHECK_HOST: + if (DEBUG) fprintf (stderr, "PCONTROL_RESP_CHECK_HOST\n"); + status = CheckHostResponse (host); + break; + + case PCONTROL_RESP_CHECK_DONE_HOST: + if (DEBUG) fprintf (stderr, "PCONTROL_RESP_CHECK_DONE_HOST\n"); + status = CheckDoneHostResponse (host); + break; + + case PCONTROL_RESP_CHECK_BUSY_JOB: + if (DEBUG) fprintf (stderr, "PCONTROL_RESP_BUSY_JOB\n"); + status = CheckBusyJobResponse (host); + break; + + case PCONTROL_RESP_KILL_JOB: + if (DEBUG) fprintf (stderr, "PCONTROL_RESP_KILL_JOB\n"); + status = KillJobResponse (host); + break; + + case PCONTROL_RESP_STOP_HOST: + if (DEBUG) fprintf (stderr, "PCONTROL_RESP_STOP_HOST\n"); + status = StopHostResponse (host); + break; + + default: + ABORT ("undefined response state"); + } + + // we have detected a valid response, clear the response data + host[0].response_state = PCONTROL_RESP_NONE; + host[0].response = NULL; + return (status); +} + Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckSystem.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckSystem.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/CheckSystem.c (revision 18424) @@ -99,4 +99,6 @@ if (RunLevel != PCONTROL_RUN_NONE) { + Nhostchecks += CheckRespHosts(0.020); /* check for incoming messages */ + TestCheckPoint (); Nhostchecks += CheckDoneHosts(0.020); /* reset the host */ TestCheckPoint (); @@ -258,4 +260,45 @@ } +int CheckRespHosts (float MaxDelay) { + + struct timeval start, stop; + int i, Nobject; + Stack *hoststack; + Stack *jobstack; + Host *host; + Job *job; + float dtime; + + /* Loop through objects on the stack, no more than once. see note above */ + hoststack = GetHostStack (PCONTROL_HOST_RESP); + jobstack = GetJobStack (PCONTROL_JOB_RESP); + Nobject = hoststack[0].Nobject; + + /* always allow at least one test */ + gettimeofday (&start, (void *) NULL); + dtime = 0.0; + for (i = 0; (i < Nobject) && (dtime < MaxDelay); i++) { + LockStack (jobstack); + host = PullStackByLocation (hoststack, STACK_TOP); + if (host == NULL) { + UnlockStack (jobstack); + break; + } + + // if the host has a job, we need to pull the job from its stack + job = (Job *) host[0].job; + if (job != NULL) { + RemoveStackByID (jobstack, job[0].JobID); + } + UnlockStack (jobstack); + + CheckRespHost (host); + gettimeofday (&stop, (void *) NULL); + dtime = DTIME (stop, start); + } + if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i); + return (i); +} + int CheckDoneHosts (float MaxDelay) { @@ -303,4 +346,5 @@ if (host == NULL) break; if (host[0].markoff) { + // DOWN -> OFF host[0].markoff = FALSE; OffHost (host); @@ -311,4 +355,7 @@ PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); } else { + // DOWN -> IDLE (maybe) + // this is a race condition with "host retry", but the only + // consequence is that both StartHost and reset set the times to 0.0 StartHost (host); } Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/HostOps.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/HostOps.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/HostOps.c (revision 18424) @@ -1,13 +1,19 @@ # include "pcontrol.h" -Stack *HostPool_Idle; -Stack *HostPool_Busy; -Stack *HostPool_Done; -Stack *HostPool_Down; -Stack *HostPool_Off; +Stack *HostPool_AllHosts; // virtual pool for user status queries + +Stack *HostPool_Idle; // these hosts are waiting for something to do +Stack *HostPool_Busy; // these hosts are working +Stack *HostPool_Resp; // these hosts are trying to respond +Stack *HostPool_Done; // these hosts have finished a job +Stack *HostPool_Down; // these hosts are not responding +Stack *HostPool_Off; // these hosts are off void InitHostStacks () { + HostPool_AllHosts = InitStack (); + HostPool_Idle = InitStack (); HostPool_Busy = InitStack (); + HostPool_Resp = InitStack (); HostPool_Done = InitStack (); HostPool_Down = InitStack (); @@ -26,13 +32,19 @@ FreeHostStack (HostPool_Idle); FreeHostStack (HostPool_Busy); + FreeHostStack (HostPool_Resp); FreeHostStack (HostPool_Done); FreeHostStack (HostPool_Down); FreeHostStack (HostPool_Off ); + + // AllHosts is a virtual stack : all hosts are references + FreeStack (HostPool_AllHosts); } char *GetHostStackName (int StackID) { switch (StackID) { + case PCONTROL_HOST_ALLHOSTS: return ("ALLHOSTS"); case PCONTROL_HOST_IDLE: return ("IDLE"); case PCONTROL_HOST_DOWN: return ("DOWN"); + case PCONTROL_HOST_RESP: return ("RESP"); case PCONTROL_HOST_DONE: return ("DONE"); case PCONTROL_HOST_BUSY: return ("BUSY"); @@ -46,6 +58,8 @@ Stack *GetHostStack (int StackID) { switch (StackID) { + case PCONTROL_HOST_ALLHOSTS: return (HostPool_AllHosts); case PCONTROL_HOST_IDLE: return (HostPool_Idle); case PCONTROL_HOST_DOWN: return (HostPool_Down); + case PCONTROL_HOST_RESP: return (HostPool_Resp); case PCONTROL_HOST_DONE: return (HostPool_Done); case PCONTROL_HOST_BUSY: return (HostPool_Busy); @@ -58,6 +72,8 @@ Stack *GetHostStackByName (char *name) { + if (!strcasecmp (name, "all")) return (HostPool_AllHosts); if (!strcasecmp (name, "idle")) return (HostPool_Idle); if (!strcasecmp (name, "down")) return (HostPool_Down); + if (!strcasecmp (name, "resp")) return (HostPool_Resp); if (!strcasecmp (name, "done")) return (HostPool_Done); if (!strcasecmp (name, "busy")) return (HostPool_Busy); @@ -71,4 +87,6 @@ int stat; Stack *stack; + + // fprintf (stderr, "move host %s to %s\n", host[0].hostname, GetHostStackName(StackID)); stack = GetHostStack (StackID); @@ -94,4 +112,8 @@ if (host != NULL) return (host); + *StackID = PCONTROL_HOST_RESP; + host = PullHostFromStackByID (*StackID, HostID); + if (host != NULL) return (host); + *StackID = PCONTROL_HOST_DONE; host = PullHostFromStackByID (*StackID, HostID); @@ -120,4 +142,8 @@ *StackID = PCONTROL_HOST_DOWN; + host = PullHostFromStackByName (*StackID, name); + if (host != NULL) return (host); + + *StackID = PCONTROL_HOST_RESP; host = PullHostFromStackByName (*StackID, name); if (host != NULL) return (host); @@ -180,6 +206,12 @@ host[0].nexttry.tv_usec = 0; + InitIOBuffer (&host[0].comms_buffer, 0x100); + host[0].response_state = PCONTROL_RESP_NONE; + host[0].response = NULL; + host[0].markoff = FALSE; host[0].job = NULL; + + PutHost (host, PCONTROL_HOST_ALLHOSTS, STACK_BOTTOM); PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); return (host[0].HostID); @@ -187,4 +219,11 @@ void DelHost (Host *host) { + + Host *copy; + + copy = PullStackByID (HostPool_AllHosts, host[0].HostID); + ASSERT (copy == host, "programming error: ALLHOSTS entry does not match"); + + FreeIOBuffer (&host[0].comms_buffer); FREE (host[0].hostname); FREE (host[0].job); Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/IDops.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/IDops.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/IDops.c (revision 18424) @@ -1,11 +1,12 @@ # include "pcontrol.h" -static IDtype CurrentJobID = 0; -static IDtype CurrentHostID = 0; +static IDtype CurrentJobID = 1; +static IDtype CurrentHostID = 1; -/* for now, no persistence : we could use the date/time to seed the upper byte(s) if needed */ +/* for now, no persistence between sessions : we could use the date/time to seed the upper + * byte(s) if needed */ void InitIDs () { - CurrentJobID = 0; - CurrentHostID = 0; + CurrentJobID = 1; + CurrentHostID = 1; } @@ -41,2 +42,29 @@ } +IDtype GetID (char *IDword) { + + int Nargs; + IDtype ID; + unsigned int word0, word1, word2, word3; + char *endptr; + + Nargs = sscanf (IDword, "%x.%x.%x.%x", &word3, &word2, &word1, &word0); + if (Nargs == 4) { + IDtype tmp; + ID = 0; + ID |= (word0 << 0); + ID |= (word1 << 16); + tmp = word2; + ID |= (tmp << 32); + tmp = word3; + ID |= (tmp << 48); + return ID; + } + + ID = strtoll (IDword, &endptr, 10); + if (*endptr == 0) { + return ID; + } + + return 0; +} Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/JobID.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/JobID.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/JobID.c (revision 18424) @@ -40,2 +40,25 @@ } +IDtype GetID (char *IDword) { + + int Nargs; + IDtype ID; + unsigned short int word0, word1, word2, word3; + + Nargs = sscanf (IDword, "%x.%x.%x.%x", &word0, &word1, &word2, &word3); + if (Nargs == 4) { + ID = 0; + ID |= (word0 << 0); + ID |= (word1 << 16); + ID |= (word2 << 32); + ID |= (word3 << 48); + return ID; + } + + ID = strtoll (IDword, &endptr, 10); + if (*endptr == 0) { + return ID; + } + + return 0; +} Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/JobOps.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/JobOps.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/JobOps.c (revision 18424) @@ -1,6 +1,9 @@ # include "pcontrol.h" + +Stack *JobPool_AllJobs; // virtual pool for user status queries Stack *JobPool_Pending; Stack *JobPool_Busy; +Stack *JobPool_Resp; Stack *JobPool_Done; Stack *JobPool_Kill; @@ -9,6 +12,9 @@ void InitJobStacks () { + JobPool_AllJobs = InitStack (); + JobPool_Pending = InitStack (); JobPool_Busy = InitStack (); + JobPool_Resp = InitStack (); JobPool_Done = InitStack (); JobPool_Kill = InitStack (); @@ -28,14 +34,21 @@ FreeJobStack (JobPool_Pending); FreeJobStack (JobPool_Busy ); + FreeJobStack (JobPool_Resp ); FreeJobStack (JobPool_Done ); FreeJobStack (JobPool_Kill ); FreeJobStack (JobPool_Exit ); FreeJobStack (JobPool_Crash ); + + // AllJobs is a virtual stack : all jobs are references + FreeStack (JobPool_AllJobs); } char *GetJobStackName (int StackID) { switch (StackID) { + case PCONTROL_JOB_ALLJOBS: return ("ALLJOBS"); + case PCONTROL_JOB_PENDING: return ("PENDING"); case PCONTROL_JOB_BUSY: return ("BUSY"); + case PCONTROL_JOB_RESP: return ("RESP"); case PCONTROL_JOB_DONE: return ("DONE"); case PCONTROL_JOB_KILL: return ("KILL"); @@ -50,6 +63,9 @@ Stack *GetJobStack (int StackID) { switch (StackID) { + case PCONTROL_JOB_ALLJOBS: return (JobPool_AllJobs); + case PCONTROL_JOB_PENDING: return (JobPool_Pending); case PCONTROL_JOB_BUSY: return (JobPool_Busy); + case PCONTROL_JOB_RESP: return (JobPool_Resp); case PCONTROL_JOB_DONE: return (JobPool_Done); case PCONTROL_JOB_KILL: return (JobPool_Kill); @@ -64,6 +80,9 @@ Stack *GetJobStackByName (char *name) { + if (!strcasecmp (name, "all")) return (JobPool_AllJobs); + if (!strcasecmp (name, "pending")) return (JobPool_Pending); if (!strcasecmp (name, "busy")) return (JobPool_Busy); + if (!strcasecmp (name, "resp")) return (JobPool_Resp); if (!strcasecmp (name, "done")) return (JobPool_Done); if (!strcasecmp (name, "exit")) return (JobPool_Exit); @@ -77,4 +96,6 @@ int stat; Stack *stack; + + // fprintf (stderr, "move job %s to %s\n", job[0].argv[0], GetJobStackName(StackID)); stack = GetJobStack (StackID); @@ -118,4 +139,8 @@ if (job != NULL) return (job); + *StackID = PCONTROL_JOB_RESP; + job = PullJobFromStackByID (*StackID, JobID); + if (job != NULL) return (job); + *StackID = PCONTROL_JOB_EXIT; job = PullJobFromStackByID (*StackID, JobID); @@ -160,9 +185,15 @@ job[0].argv = argv; job[0].hostname = hostname; + job[0].realhost = NULL; + + job[0].exit_status = 0; + job[0].Reset = FALSE; + job[0].stdout_size = 0; + job[0].stderr_size = 0; + job[0].mode = mode; - job[0].host = NULL; - job[0].JobID = NextJobID(); - job[0].Reset = FALSE; - job[0].realhost = NULL; + + job[0].state = 0; + job[0].stack = 0; /* do this step on start? */ @@ -170,6 +201,17 @@ InitIOBuffer (&job[0].stderr_buff, 0x1000); + job[0].dtime = 0.0; + job[0].pid = 0; + + job[0].JobID = NextJobID(); + job[0].host = NULL; + JobID = job[0].JobID; + + // Put a copy of all created jobs on the ALLJOBS stack + // This is a virtual stack: do not free the job from this stack + PutJob (job, PCONTROL_JOB_ALLJOBS, STACK_BOTTOM); PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM); + if (VerboseMode()) gprint (GP_ERR, "added new job\n"); return (JobID); @@ -179,4 +221,9 @@ int i; + + Job *copy; + + copy = PullStackByID (JobPool_AllJobs, job[0].JobID); + ASSERT (copy == job, "programming error: ALLJOBS entry does not match"); FREE (job[0].hostname); Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/KillJob.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/KillJob.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/KillJob.c (revision 18424) @@ -3,52 +3,60 @@ int KillJob (Job *job, Host *host) { - IOBuffer buffer; int status; - char *p; ASSERT (host != NULL, "host missing"); ASSERT (job != NULL, "job missing"); - ASSERT (host == (Host *) job[0].host, "invalid host"); ASSERT (job == (Job *) host[0].job, "invalid job"); - InitIOBuffer (&buffer, 0x100); - - status = PclientCommand (host, "reset", PCLIENT_PROMPT, &buffer); + status = PclientCommand (host, "reset", PCLIENT_PROMPT, PCONTROL_RESP_KILL_JOB); /* check on success of pclient command */ switch (status) { case PCLIENT_DOWN: - HarvestHost (host[0].pid); // unlink host & job + if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname); job[0].host = NULL; host[0].job = NULL; + HarvestHost (host[0].pid); PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); PutJob (job, PCONTROL_JOB_CRASH, STACK_BOTTOM); - FreeIOBuffer (&buffer); return (FALSE); - case PCLIENT_HUNG: - // don't do anything drastic, just keep trying - // XXX move to which stack?? - gprint (GP_ERR, "client is busy, not responding (KillJob)"); - FreeIOBuffer (&buffer); + case PCLIENT_GOOD: + if (VerboseMode()) gprint (GP_ERR, "kill job on host %s\n", host[0].hostname); + FlushIOBuffer (&host[0].comms_buffer); + PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM); + PutJob (job, PCONTROL_JOB_RESP, STACK_BOTTOM); return (TRUE); - - case PCLIENT_GOOD: - if (VerboseMode()) gprint (GP_ERR, "message received (KillJob)\n"); - break; default: ABORT ("unknown status for pclient command"); } +} - /** host is up, need to parse message **/ - p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer); - ASSERT (p != NULL, "missing STATUS in pclient message"); - if (VerboseMode()) gprint (GP_ERR, "client message: %s\n", buffer.buffer); +int KillJobResponse (Host *host) { + + int status; + char *p; + IOBuffer *buffer; + Job *job; + + ASSERT (host != NULL, "host missing"); + ASSERT (host[0].job, "missing job"); + buffer = &host[0].comms_buffer; + job = (Job *) host[0].job; + + /** check on response to pclient command **/ + p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer); + if (p == NULL) { + if (VerboseMode()) gprint (GP_ERR, "missing STATUS in response; try again\n"); + PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM); + PutJob (job, PCONTROL_JOB_KILL, STACK_BOTTOM); + return (FALSE); + } + if (VerboseMode()) gprint (GP_ERR, "client message: %s\n", buffer[0].buffer); sscanf (p, "%*s %d", &status); - FreeIOBuffer (&buffer); gprint (GP_ERR, "client status: %d\n", status); @@ -62,5 +70,5 @@ return (FALSE); case 1: - gprint (GP_ERR, "killing job %s on %s\n", job[0].argv[0], host[0].hostname); + gprint (GP_ERR, "killed job %s on %s\n", job[0].argv[0], host[0].hostname); // unlink host & job job[0].host = NULL; Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/Makefile =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/Makefile (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/Makefile (revision 18424) @@ -13,5 +13,5 @@ LIBS1 = -lkapa -lFITS -lohana LIBS2 = -lbasiccmd -lshell -ldata -FULL_CFLAGS = $(BASE_CFLAGS) +FULL_CFLAGS = $(BASE_CFLAGS) -Wall -Werror FULL_CPPFLAGS = $(BASE_CPPFLAGS) FULL_LDFLAGS = $(LIBS1) $(LIBS2) $(BASE_LDFLAGS) @@ -27,4 +27,5 @@ $(SRC)/CheckBusyJob.$(ARCH).o \ $(SRC)/CheckDoneHost.$(ARCH).o \ +$(SRC)/CheckRespHost.$(ARCH).o \ $(SRC)/CheckDoneJob.$(ARCH).o \ $(SRC)/CheckHost.$(ARCH).o \ @@ -38,5 +39,4 @@ $(SRC)/StackOps.$(ARCH).o \ $(SRC)/PclientCommand.$(ARCH).o \ -$(SRC)/ResetJob.$(ARCH).o \ $(SRC)/StartHost.$(ARCH).o \ $(SRC)/StopHosts.$(ARCH).o \ Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/PclientCommand.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/PclientCommand.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/PclientCommand.c (revision 18424) @@ -1,26 +1,19 @@ # include "pcontrol.h" -# define PCLIENT_TIMEOUT 5000 +# define PCLIENT_TIMEOUT 100 -int PclientCommand (Host *host, char *command, char *response, IOBuffer *buffer) { +// send a command and check for errors; ignore output +int PclientCommand (Host *host, char *command, char *response, HostResp response_state) { - int i; int status; - char *line; - struct timespec request, remain; ASSERT (host != NULL, "host missing"); - ASSERT (buffer != NULL, "buffer missing"); ASSERT (command != NULL, "command missing"); - ASSERT (response != NULL, "response missing"); - - /* avoid blocking on read, test every 100 usec, up to 50 msec */ - request.tv_sec = 0; - request.tv_nsec = 100000; // flush the stdout and stderr buffers here - ReadtoIOBuffer (buffer, host[0].stdout_fd); - FlushIOBuffer (buffer); - ReadtoIOBuffer (buffer, host[0].stderr_fd); - FlushIOBuffer (buffer); + // recycle comms_buffer to minimize page thrashing + ReadtoIOBuffer (&host[0].comms_buffer, host[0].stdout_fd); + FlushIOBuffer (&host[0].comms_buffer); + ReadtoIOBuffer (&host[0].comms_buffer, host[0].stderr_fd); + FlushIOBuffer (&host[0].comms_buffer); /* send command to client (adding on \n) */ @@ -33,7 +26,35 @@ } + // prepare host to accept response + host[0].response_state = response_state; + host[0].response = response; + FlushIOBuffer (&host[0].comms_buffer); + + // fprintf (stderr, "command: %s\n", command); + + return (PCLIENT_GOOD); +} + +// check for response; message must end with specified string. +// accumulate the response in the buffer +int PclientResponse (Host *host, char *response, IOBuffer *buffer) { + + int i; + int status; + char *line; + struct timespec request, remain; + + ASSERT (response != NULL, "response missing"); + ASSERT (buffer != NULL, "buffer missing"); + + /* avoid blocking very long on read, test every 100 usec, up to 0.1 sec */ + request.tv_sec = 0; + request.tv_nsec = 100000; + /* watch for response - wait up to 1 second */ line = NULL; status = -1; + + // how long does each cycle really take? for (i = 0; (i < PCLIENT_TIMEOUT) && (status != 0) && (line == NULL); i++) { status = ReadtoIOBuffer (buffer, host[0].stdout_fd); @@ -45,8 +66,17 @@ return (PCLIENT_DOWN); } + if (line == NULL) return (PCLIENT_HUNG); if (status == -1) return (PCLIENT_HUNG); - /* gprint (GP_ERR, "buffer.buffer: %s\n", buffer[0].buffer); */ + + // fprintf (stderr, "response: %s\n", buffer[0].buffer); + return (PCLIENT_GOOD); } /* memstr returns a view, not an allocated string : don't free */ +/* ReadtoIOBuffer returns : + 0 - pipe closed + -1 - no more data in pipe, data not ready + -2 - serious error reading from pipe + >0 - data read from pipe +*/ Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/ResetJob.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/ResetJob.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/ResetJob.c (revision 18424) @@ -1,8 +1,9 @@ # include "pcontrol.h" + +// XXX deprecated int ResetJob (Job *job) { int status; - IOBuffer buffer; Host *host; @@ -13,41 +14,42 @@ ASSERT (job != NULL, "host missing"); - InitIOBuffer (&buffer, 0x100); - /* we have tried to reset the job; may not get status */ job[0].Reset = TRUE; - status = PclientCommand (host, "reset", PCLIENT_PROMPT, &buffer); + status = PclientCommand (host, "reset"); /* check on success of pclient command */ switch (status) { case PCLIENT_DOWN: - /*** different behavior for ANYHOST, WANTHOST, NEEDHOST? ***/ - gprint (GP_ERR, "host %s is down\n", host[0].hostname); + if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname); HarvestHost (host[0].pid); PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); - FreeIOBuffer (&buffer); - return (FALSE); - - case PCLIENT_HUNG: - /*** should we consider a HUNG host DOWN? ***/ - gprint (GP_ERR, "host %s is not responding (ResetJob)\n", host[0].hostname); - FreeIOBuffer (&buffer); return (FALSE); case PCLIENT_GOOD: - gprint (GP_ERR, "message received (ResetJob)\n"); - FreeIOBuffer (&buffer); + host[0].response_state = PCONTROL_RESP_RESET_JOB; + host[0].response = PCLIENT_PROMPT; + FlushIOBuffer (&host[0].comms_buffer, 0x100); + PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM); return (TRUE); default: - gprint (GP_ERR, "unknown status for pclient command: programming error\n"); - pcontrol_exit (55); + ABORT ("unknown status for pclient command"); } + ABORT ("should not reach here (ResetJob)"); +} - gprint (GP_ERR, "programming error in ResetJob (should not reach here)\n"); - FreeIOBuffer (&buffer); - pcontrol_exit (56); - return (FALSE); +int ResetJobResponse (Host *host) { + + int status; + IOBuffer *buffer; + + /* job must have assigned host */ + ASSERT (host, "missing host"); + ASSERT (host[0].job, "missing job"); + buffer = host[0].comms_buffer; + + gprint (GP_ERR, "message received (ResetJob)\n"); + return (TRUE); } Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/StartJob.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/StartJob.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/StartJob.c (revision 18424) @@ -1,11 +1,9 @@ # include "pcontrol.h" +// job and host are bound together (why pass in both?) int StartJob (Job *job, Host *host) { int i, Nline, status; - char *line, *p; - IOBuffer buffer; - - InitIOBuffer (&buffer, 0x100); + char *line; /* job must have assigned host */ @@ -28,5 +26,5 @@ } - status = PclientCommand (host, line, PCLIENT_PROMPT, &buffer); + status = PclientCommand (host, line, PCLIENT_PROMPT, PCONTROL_RESP_START_JOB); free (line); @@ -34,36 +32,53 @@ switch (status) { case PCLIENT_DOWN: + // unlink host & job if (VerboseMode()) gprint (GP_ERR, "host %s is down\n", host[0].hostname); - goto failure; - - case PCLIENT_HUNG: - // we need the job start to return a valid Job ID, - // give up on jobs which don't get started. - // XXX we are sensitive here to the time it takes pclient - // to fork the job. if this is slow, the client may appear to hang. - gprint (GP_ERR, "host %s is not responding (StartJob)\n", host[0].hostname); - if (VerboseMode()) gprint (GP_ERR, "host %s is not responding\n", host[0].hostname); - - // unlink host & job job[0].host = NULL; host[0].job = NULL; - if (job[0].realhost) free (job[0].realhost); - job[0].realhost = NULL; - PutHost (host, PCONTROL_HOST_DONE, STACK_BOTTOM); + HarvestHost (host[0].pid); + PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM); - FreeIOBuffer (&buffer); return (FALSE); case PCLIENT_GOOD: - if (VerboseMode()) gprint (GP_ERR, "message received (StartJob)\n"); - break; + job[0].realhost = strcreate (host[0].hostname); + job[0].pid = -1; + gettimeofday (&job[0].start, (void *) NULL); + + if (VerboseMode()) gprint (GP_ERR, "started job on host %s\n", host[0].hostname); + PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM); + PutJob (job, PCONTROL_JOB_RESP, STACK_BOTTOM); + return (TRUE); default: ABORT ("unknown status for pclient command"); } +} + +// message has been received from the host, interpret results +int StartJobResponse (Host *host) { + + int status; + char *p; + IOBuffer *buffer; + Job *job; + + /* job must have assigned host */ + ASSERT (host, "missing host"); + ASSERT (host[0].job, "missing job"); + buffer = &host[0].comms_buffer; + job = (Job *) host[0].job; /* check on result of pclient command */ - p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer); - ASSERT (p != NULL, "missing STATUS in pclient message"); + p = memstr (buffer[0].buffer, "STATUS", buffer[0].Nbuffer); + if (p == NULL) { + // failed to get a valid response. kill the job and try again, + // or accept a running process without a PID? + if (VerboseMode()) gprint (GP_ERR, "failed to get a valid PID, trying to continue without\n"); + PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM); + PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM); + gettimeofday (&job[0].start, NULL); + return (TRUE); + } sscanf (p, "%*s %d", &status); @@ -71,5 +86,11 @@ case -1: if (VerboseMode()) gprint (GP_ERR, "error in pclient child\n"); - goto failure; + // unlink host & job + job[0].host = NULL; + host[0].job = NULL; + HarvestHost (host[0].pid); + PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); + PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM); + return (FALSE); case -2: @@ -80,23 +101,13 @@ default: - job[0].realhost = strcreate (host[0].hostname); + if (VerboseMode()) gprint (GP_ERR, "message received (StartJobResponse)\n"); job[0].pid = status; PutHost (host, PCONTROL_HOST_BUSY, STACK_BOTTOM); PutJob (job, PCONTROL_JOB_BUSY, STACK_BOTTOM); - FreeIOBuffer (&buffer); - gettimeofday (&job[0].start, (void *) NULL); + gettimeofday (&job[0].start, NULL); return (TRUE); } + /* we should never reach here */ ABORT ("should not reach here (StartJob)"); - -failure: - // unlink host & job - job[0].host = NULL; - host[0].job = NULL; - HarvestHost (host[0].pid); - PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); - PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM); - FreeIOBuffer (&buffer); - return (FALSE); } Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/StopHosts.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/StopHosts.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/StopHosts.c (revision 18424) @@ -72,26 +72,28 @@ int status; - IOBuffer buffer; - InitIOBuffer (&buffer, 0x100); - status = PclientCommand (host, "exit", "Goodbye", &buffer); - FreeIOBuffer (&buffer); + status = PclientCommand (host, "exit", "Goodbye", PCONTROL_RESP_STOP_HOST); /* check on success of pclient command */ switch (status) { case PCLIENT_DOWN: - break; - - case PCLIENT_HUNG: - gprint (GP_ERR, "host %s is not responding\n", host[0].hostname); + // XXX this is the desired result in any case, so ignore it break; case PCLIENT_GOOD: - break; + if (VerboseMode()) gprint (GP_ERR, "stop host %s\n", host[0].hostname); + FlushIOBuffer (&host[0].comms_buffer); + PutHost (host, PCONTROL_HOST_RESP, STACK_BOTTOM); + return (TRUE); default: - gprint (GP_ERR, "unknown status for pclient command: programming error\n"); - pcontrol_exit (57); + ABORT ("unknown status for pclient command"); } + ABORT ("should not reach here"); +} + +int StopHostResponse (Host *host) { + + OffHost (host); HarvestHost (host[0].pid); return (TRUE); @@ -103,6 +105,6 @@ int i, result, waitstatus; - gprint (GP_ERR, "harvesting within thread %d\n", pthread_self()); - gprint (GP_ERR, "child process %d is down, wait for exit status\n", pid); + if (VerboseMode()) gprint (GP_ERR, "harvesting within thread %p\n", pthread_self()); + if (VerboseMode()) gprint (GP_ERR, "child process %d is down, wait for exit status\n", pid); // Loop a few times waiting for child to exit Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/check.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/check.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/check.c (revision 18424) @@ -3,7 +3,9 @@ int check (int argc, char **argv) { - Job *job; - Host *host; - int JobID, HostID, StackID; + int JobID, HostID; + + Stack *stack = NULL; + Job *job = NULL; + Host *host = NULL; if (argc != 3) { @@ -14,14 +16,18 @@ if (!strcasecmp (argv[1], "JOB")) { - JobID = atoi (argv[2]); + JobID = GetID (argv[2]); + if (!JobID) { + gprint (GP_ERR, "invalid job id %s\n", argv[2]); + return (FALSE); + } - SetCheckPoint (); // ensure the JOB is on one of the stacks - job = PullJobByID (JobID, &StackID); + stack = GetJobStack (PCONTROL_JOB_ALLJOBS); + job = PullStackByID (stack, JobID); if (job == NULL) { gprint (GP_LOG, "job not found\n"); - ClearCheckPoint (); return (FALSE); } - gprint (GP_LOG, "STATUS %s\n", GetJobStackName(StackID)); + + gprint (GP_LOG, "STATUS %s\n", GetJobStackName(job[0].stack)); gprint (GP_LOG, "EXITST %d\n", job[0].exit_status); gprint (GP_LOG, "STDOUT %d\n", job[0].stdout_size); @@ -33,6 +39,5 @@ gprint (GP_LOG, "HOSTNAME NONE\n"); } - PutJob (job, StackID, STACK_BOTTOM); - ClearCheckPoint (); + PushStack (stack, STACK_BOTTOM, job, job[0].JobID, job[0].argv[0]); return (TRUE); } @@ -41,14 +46,12 @@ HostID = atoi (argv[2]); - SetCheckPoint (); // ensure the HOST is on one of the stacks - host = PullHostByID (HostID, &StackID); + stack = GetHostStack (PCONTROL_HOST_ALLHOSTS); + host = PullStackByID (stack, HostID); if (host == NULL) { gprint (GP_LOG, "host not found\n"); - ClearCheckPoint (); return (FALSE); } - gprint (GP_LOG, "host %s\n", GetHostStackName(StackID)); - PutHost (host, StackID, STACK_BOTTOM); - ClearCheckPoint (); + gprint (GP_LOG, "host %s\n", GetHostStackName(host[0].stack)); + PushStack (stack, STACK_BOTTOM, host, host[0].HostID, host[0].hostname); return (TRUE); } Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/delete.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/delete.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/delete.c (revision 18424) @@ -10,5 +10,10 @@ return (FALSE); } - JobID = atoi (argv[1]); + JobID = GetID (argv[1]); + if (!JobID) { + gprint (GP_ERR, "invalid job id %s\n", argv[1]); + return (FALSE); + } + /* use a string interp to convert JobIDs to ints ? */ Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/host.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/host.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/host.c (revision 18424) @@ -1,12 +1,13 @@ # include "pcontrol.h" -// we use CheckPoints in this function to prevent objects in flight from being missing. int host (int argc, char **argv) { - int StackID; IDtype HostID; Host *host; + Stack *AllHosts; if (argc != 3) goto usage; + + AllHosts = GetHostStack (PCONTROL_HOST_ALLHOSTS); if (!strcasecmp (argv[1], "ADD")) { @@ -15,4 +16,6 @@ return (TRUE); } + + // this one is safe from in-flight entries: no one else pulls from OFF if (!strcasecmp (argv[1], "ON")) { host = PullHostFromStackByName (PCONTROL_HOST_OFF, argv[2]); @@ -25,49 +28,51 @@ return (TRUE); } + + // this is a race condition with "CheckDownHosts", but the only + // consequence is that both StartHost and reset set the times to 0.0 if (!strcasecmp (argv[1], "RETRY")) { // no need to use a check point [thief: CheckDownHost (DOWN->IDLE)] - host = PullHostFromStackByName (PCONTROL_HOST_DOWN, argv[2]); + host = PullHostFromStackByName (PCONTROL_HOST_ALLHOSTS, argv[2]); if (!host) { + gprint (GP_LOG, "host %s not found\n", argv[2]); + return (FALSE); + } + if (host[0].stack != PCONTROL_HOST_DOWN) { gprint (GP_LOG, "host %s is not DOWN\n", argv[2]); return (FALSE); } - /* reset time, place back on DOWN stack */ + /* reset time, place back on ALLHOSTS stack */ host[0].nexttry.tv_sec = 0; host[0].nexttry.tv_usec = 0; host[0].lasttry.tv_sec = 0; host[0].lasttry.tv_usec = 0; - PutHost (host, PCONTROL_HOST_DOWN, STACK_BOTTOM); + PushStack (AllHosts, STACK_BOTTOM, host, host[0].HostID, host[0].hostname); return (TRUE); } + if (!strcasecmp (argv[1], "CHECK")) { - SetCheckPoint (); // ensure the host is on one of the stacks - host = PullHostByName (argv[2], &StackID); + host = PullHostFromStackByName (PCONTROL_HOST_ALLHOSTS, argv[2]); if (host == NULL) { gprint (GP_LOG, "host %s not found\n", argv[2]); - ClearCheckPoint (); return (FALSE); } - PutHost (host, StackID, STACK_BOTTOM); - ClearCheckPoint (); - - gprint (GP_LOG, "host %s is %s\n", argv[2], GetHostStackName (StackID)); + gprint (GP_LOG, "host %s is %s\n", argv[2], GetHostStackName (host[0].stack)); + PushStack (AllHosts, STACK_BOTTOM, host, host[0].HostID, host[0].hostname); return (TRUE); } + if (!strcasecmp (argv[1], "OFF")) { - SetCheckPoint (); // ensure we can find the specified host - host = PullHostByName (argv[2], &StackID); + host = PullHostFromStackByName (PCONTROL_HOST_ALLHOSTS, argv[2]); if (host == NULL) { gprint (GP_LOG, "host %s not found\n", argv[2]); - ClearCheckPoint (); return (FALSE); } host[0].markoff = TRUE; - PutHost (host, StackID, STACK_BOTTOM); - ClearCheckPoint (); + PushStack (AllHosts, STACK_BOTTOM, host, host[0].HostID, host[0].hostname); return (TRUE); } + // this one is safe from in-flight entries: no one else pulls from OFF if (!strcasecmp (argv[1], "DELETE")) { - // a check point is not required: no possible thief host = PullHostFromStackByName (PCONTROL_HOST_OFF, argv[2]); if (!host) { Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/notes.txt =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/notes.txt (revision 18424) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/notes.txt (revision 18424) @@ -0,0 +1,69 @@ + +2008.06.11 + + Upgrade: add a virtual stack of the jobs and hosts so that the user + status queries do not need to block until the processing thread is + caught up. + + + +2008.04.21 + + I am having trouble with pcontrol managing the clients. The + functions which call PclientCommand expect to see a complete + response in the same call. However, latency in the client means, if + the client is super busy, that this interaction will either result + in dropped messages (which look like HUNG machines) or in long + blocks against the pantasks/pcontrol interface. + + A possible solution may be to break the functions which call + PclientCommand into a PclientCommand and a PclientResponse call. + The PclientCommand would send the command and report any comms + errors in that interaction (pipe closed, interrupts, etc). It would + return immediately, pushing the + +-- pcontrol / pclient interactions + +StartJob: +xmt: job argv[0] argv[1] ... +rcv: + STATUS (PID) + pclient: + +CheckBusyJob: +xmt: status +rcv: + EXITST N + STDOUT N + STDERR N + STATUS (value) + pclient: + +CheckDoneHost: +xmt: reset +rcv: + STATUS (status) + pclient: + +CheckHost: +xmt: echo OK +rcv: + OK + pclient: + +KillJob: +xmt: reset +rcv: + STATUS (status) + pclient: + +ResetJob: +xmt: reset +rcv: + STATUS (status) + pclient: + +StopHost: +xmt: ext +rcv: + Goodbye Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/pcontrol.c.in =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/pcontrol.c.in (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/pcontrol.c.in (revision 18424) @@ -68,4 +68,6 @@ /* add program-dependent exit functions here */ void cleanup () { + // stop checking on the jobs + SetRunLevel (PCONTROL_RUN_HOSTS); DownHosts (); ConfigFree (); Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/rconnect.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/rconnect.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/rconnect.c (revision 18424) @@ -2,5 +2,5 @@ /* connection can take a while, allow up to 2 sec */ -# define CONNECT_TIMEOUT 500 +# define CONNECT_TIMEOUT 5000 /* connect to host, start the shell: ssh hostname pclient -> command hostname shell @@ -106,5 +106,5 @@ connect_error: - if (VerboseMode()) gprint (GP_ERR, "error while connecting\n"); + if (VerboseMode()) gprint (GP_ERR, "error while connecting, status: %d, ncycles: %d\n", status, i); /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */ Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/status.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/status.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/status.c (revision 18424) @@ -6,18 +6,7 @@ int status (int argc, char **argv) { - SetCheckPoint (); - PrintJobStack (PCONTROL_JOB_PENDING); - PrintJobStack (PCONTROL_JOB_BUSY); - PrintJobStack (PCONTROL_JOB_DONE); - PrintJobStack (PCONTROL_JOB_KILL); - PrintJobStack (PCONTROL_JOB_EXIT); - PrintJobStack (PCONTROL_JOB_CRASH); + PrintJobStack (PCONTROL_JOB_ALLJOBS); + PrintHostStack (PCONTROL_HOST_ALLHOSTS); - PrintHostStack (PCONTROL_HOST_OFF); - PrintHostStack (PCONTROL_HOST_DOWN); - PrintHostStack (PCONTROL_HOST_IDLE); - PrintHostStack (PCONTROL_HOST_BUSY); - PrintHostStack (PCONTROL_HOST_DONE); - ClearCheckPoint (); return (TRUE); } @@ -39,5 +28,10 @@ job = stack[0].object[i]; ASSERT (job != NULL, "programming error"); - gprint (GP_LOG, "%d %s %d ", i, job[0].hostname, job[0].argc); + if (job[0].realhost == NULL) { + gprint (GP_LOG, "%3d %9s ", i, job[0].hostname); + } else { + gprint (GP_LOG, "%3d %9s ", i, job[0].realhost); + } + gprint (GP_LOG, "%7s ", GetJobStackName (job[0].state)); for (j = 0; j < job[0].argc; j++) { gprint (GP_LOG, "%s ", job[0].argv[j]); @@ -66,4 +60,5 @@ host = stack[0].object[i]; gprint (GP_LOG, "%d %s ", i, host[0].hostname); + gprint (GP_LOG, "%5s ", GetHostStackName (host[0].stack)); PrintID (GP_LOG, host[0].HostID); gprint (GP_LOG, "\n"); Index: /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/stdout.c =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/stdout.c (revision 18423) +++ /branches/eam_branch_20080421/Ohana/src/opihi/pcontrol/stdout.c (revision 18424) @@ -1,12 +1,21 @@ # include "pcontrol.h" +// XXX unify by testing for value of argv[0] int stdout_pc (int argc, char **argv) { - int JobID, StackID; + int N, JobID, StackID; Job *job; IOBuffer *buffer; + char *varName; + + varName = NULL; + if ((N = get_argument (argc, argv, "-var"))) { + remove_argument (N, &argc, argv); + varName = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } if (argc != 2) { - gprint (GP_ERR, "USAGE: stdout (JobID)\n"); + gprint (GP_ERR, "USAGE: stdout (JobID) [-var name]\n"); gprint (GP_LOG, "STATUS %d\n", -1); return (FALSE); @@ -25,11 +34,22 @@ gprint (GP_ERR, "job not found in EXIT or CRASH\n"); - gprint (GP_LOG, "STATUS %d\n", -2); + if (varName == NULL) { + gprint (GP_LOG, "STATUS %d\n", -2); + } else { + set_str_variable (varName, "NULL"); + free (varName); + } return (FALSE); found_stdout: buffer = &job[0].stdout_buff; - fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout); - gprint (GP_LOG, "STATUS %d\n", 0); + if (varName == NULL) { + fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout); + gprint (GP_LOG, "STATUS %d\n", 0); + } else { + // XXX this can drop '0' values + set_str_variable (varName, buffer[0].buffer); + free (varName); + } PutJob (job, StackID, STACK_BOTTOM); return (TRUE); @@ -38,10 +58,18 @@ int stderr_pc (int argc, char **argv) { - int JobID, StackID; + int N, JobID, StackID; Job *job; IOBuffer *buffer; + char *varName; + + varName = NULL; + if ((N = get_argument (argc, argv, "-var"))) { + remove_argument (N, &argc, argv); + varName = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } if (argc != 2) { - gprint (GP_ERR, "USAGE: stderr (JobID)\n"); + gprint (GP_ERR, "USAGE: stderr (JobID) [-var name]\n"); gprint (GP_LOG, "STATUS %d\n", -1); return (FALSE); @@ -60,11 +88,22 @@ gprint (GP_ERR, "job not found in EXIT or CRASH\n"); - gprint (GP_LOG, "STATUS %d\n", -2); + if (varName == NULL) { + gprint (GP_LOG, "STATUS %d\n", -2); + } else { + set_str_variable (varName, "NULL"); + free (varName); + } return (FALSE); found_stderr: buffer = &job[0].stderr_buff; - fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout); - gprint (GP_LOG, "STATUS %d\n", 0); + if (varName == NULL) { + fwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, stdout); + gprint (GP_LOG, "STATUS %d\n", 0); + } else { + // XXX this can drop '0' values + set_str_variable (varName, buffer[0].buffer); + free (varName); + } PutJob (job, StackID, STACK_BOTTOM); return (TRUE); Index: /branches/eam_branch_20080421/Ohana/src/opihi/test/strings.pro =================================================================== --- /branches/eam_branch_20080421/Ohana/src/opihi/test/strings.pro (revision 18424) +++ /branches/eam_branch_20080421/Ohana/src/opihi/test/strings.pro (revision 18424) @@ -0,0 +1,32 @@ + +macro test1 + $test1 = 01234567890.01234567890.1234567890 + $test2 = $test1 + $test1 + $test3 = $test2 and $test2 + $test4 = $test3 or $test3 + $test5 = $test4 with $test4 + + echo strlen test1; strlen "$test1" + echo strlen test2; strlen "$test2" + echo strlen test3; strlen "$test3" + echo strlen test4; strlen "$test4" + echo strlen test5; strlen "$test5" +end + +macro test2 + $test1 = 01234567890.01234567890.1234567890 + appendup test1 test2 + appendup test2 test3 + appendup test3 test4 + appendup test4 test5 + + echo strlen test1; strlen "$test1" + echo strlen test2; strlen "$test2" + echo strlen test3; strlen "$test3" + echo strlen test4; strlen "$test4" + echo strlen test5; strlen "$test5" +end + +macro appendup + $$2 = $$1 and $$1 +end