# include "data.h"

FILE *f = (FILE *) NULL;
char filename[2048];

void read_vectors_cleanup ();
int read_table_sizes (Header *header);

int datafile (int argc, char **argv) {

  if (argc < 2) {
    gprint (GP_ERR, "USAGE: datafile (filename)\n");
    return (FALSE);
  }
  
  strcpy (filename, argv[1]);
  if (f != (FILE *) NULL) { fclose (f); }
  f = fopen (filename, "r");
  if (f == (FILE *) NULL) {
    gprint (GP_ERR, "failed to open file %s\n", argv[1]);
    return (FALSE);
  }
  return (TRUE);
}

// vector types
enum {COLTYPE_NONE, COLTYPE_FLT, COLTYPE_INT, COLTYPE_TIME, COLTYPE_CHAR};

static int      Nvec     = 0;
static Vector **vec      = NULL;
static char   **listname = NULL;
static int     *col      = NULL;
static int     *coltype  = NULL;
static char    *buffer   = NULL;

int read_vectors (int argc, char **argv) {
  
  int TimeFormat;
  time_t TimeReference;
  int i, j, Nskip, Narg, IsCSV, VERBOSE;
  int Nbytes, Nstart, NELEM, Nelem, nread;
  char *colstr, *c0, *c1, *extname;
  char varname[1024];  // used as a buffer for the names of string fields

  /* auto-sense table type */
  if ((Narg = get_argument (argc, argv, "-fits"))) {
    remove_argument (Narg, &argc, argv);
    extname = strcreate (argv[Narg]);
    if (extname == (char *) NULL) return (FALSE);
    remove_argument (Narg, &argc, argv);
    int status = read_table_vectors (argc, argv, extname);
    free (extname);
    return (status);
  }

  Nskip = 0;
  if ((Narg = get_argument (argc, argv, "-skip"))) {
    remove_argument (Narg, &argc, argv);
    Nskip = atof (argv[Narg]);
    remove_argument (Narg, &argc, argv);
  }

  IsCSV = FALSE;
  if ((Narg = get_argument (argc, argv, "-csv"))) {
    remove_argument (Narg, &argc, argv);
    IsCSV = TRUE;
  }

  VERBOSE = FALSE;
  if ((Narg = get_argument (argc, argv, "-v"))) {
    remove_argument (Narg, &argc, argv);
    VERBOSE = TRUE;
  }

  if ((argc < 3) || !(argc % 2)) {
    gprint (GP_ERR, "USAGE: read name N name N ...\n");
    return (FALSE);
  }
  /* read name N name N  */

  // do this only optionally?
  GetTimeFormat (&TimeReference, &TimeFormat);

  if (f == (FILE *) NULL) {
    gprint (GP_ERR, "no open file for read\n");
    return (FALSE);
  }
  fseeko (f, 0LL, SEEK_SET);

  Nvec = (argc - 1) / 2;
  ALLOCATE (listname, char *, Nvec);
  ALLOCATE (vec, Vector *, Nvec);
  ALLOCATE (col, int, Nvec);
  ALLOCATE (coltype, int, Nvec);
  for (i = 0; i < Nvec; i++) {
    listname[i] = NULL;
    vec[i] = NULL;
  }

  for (i = 0; i < Nvec; i++) {

    // interpret the column names including type flags
    // XXX review the grammar before releasing this : is foo:type best, or is something else needed?
    // Note the conflict wrt list entries
    // the name may be of the form foo:type, where type may be one of : int, float, time
    
    coltype[i] = COLTYPE_FLT;
    char *colname = argv[2*i + 1];
    char *ptr = strchr (colname, ':');
    if (ptr) {
      // split out colname and type
      *ptr = 0;
      ptr ++;
      if (!ptr) goto bad_colname;
      coltype[i] = COLTYPE_NONE;
      if (!strcasecmp(ptr, "float")) { coltype[i] = COLTYPE_FLT; }
      if (!strcasecmp(ptr, "int"))   { coltype[i] = COLTYPE_INT; }
      if (!strcasecmp(ptr, "char"))  { coltype[i] = COLTYPE_CHAR; }
      if (!strcasecmp(ptr, "time"))  { coltype[i] = COLTYPE_TIME; }
      if (!coltype[i]) goto bad_colname;
    }

    if (coltype[i] == COLTYPE_CHAR) {
      listname[i] = strcreate (argv[2*i + 1]);
    } else {
      if ((vec[i] = SelectVector (argv[2*i + 1], ANYVECTOR, TRUE)) == NULL) {
	gprint (GP_ERR, "USAGE: read name N name N ...\n");
	read_vectors_cleanup();
	return (FALSE);    
      }
    }

    colstr = argv[2*i+2];
    for (j = 0; j < strlen (colstr); j++) {
      if (isdigit(colstr[j])) {
	col[i] = atof (colstr);
	continue;
      }
      // allow 'excel' columns names of the form A-Z, AA-AZ, BA-BZ, .. ZA-ZZ
      if (strlen(colstr) >= 3) goto bad_colname;
      if (colstr[0] < 'A') goto bad_colname;
      if (colstr[0] > 'Z') goto bad_colname;
      if (colstr[1] && colstr[1] < 'A') goto bad_colname;
      if (colstr[1] && colstr[1] > 'Z') goto bad_colname;

      col[i] = colstr[0] - 'A' + 1;
      if (colstr[1]) {
	col[i] *= 26;
	col[i] += colstr[1] - 'A' + 1;
      }
      continue;

    bad_colname:
      gprint (GP_ERR, "USAGE: read name N name N ...\n");
      read_vectors_cleanup();
      return (FALSE);    
    }
  }

  // currently, all read vectors are forced to be type FLT
  NELEM = 1000;
  for (i = 0; i < Nvec; i++) {
    switch (coltype[i]) {
      case COLTYPE_INT:
	ResetVector (vec[i], OPIHI_INT, NELEM);
	break;
      case COLTYPE_FLT:
      case COLTYPE_TIME:
	// note that COLTYPE_TIME is a type of float
	ResetVector (vec[i], OPIHI_FLT, NELEM);
	break;
      case COLTYPE_CHAR:
      default:
	break;
    }
  }
  
  ALLOCATE (buffer, char, 0x10001);
  bzero (buffer, 0x10001);
  for (i = 0; i < Nskip; i++) {
    if (scan_line_maxlen (f, buffer, 0x10000) == EOF) {
      gprint (GP_ERR, "problem reading file %s\n", filename);
      read_vectors_cleanup();
      return FALSE;
    }
  }

  // we have a working buffer read from the file. we parse the lines in the working buffer
  // until we reach the last chunk without an EOL char.  at that point, we shift the start
  // of the last (partial) line to the start of the buffer and re-fill.

  Nstart = 0; // location of the last valid byte in the buffer (start filling here)
  Nelem = 0; // number of valid rows read (vector elements)
  while (TRUE) {
    Nbytes = 0x10000 - Nstart;
    bzero (&buffer[Nstart], Nbytes);
    nread = fread (&buffer[Nstart], 1, Nbytes, f);
    if (ferror (f)) {
      perror ("error reading data file");
      break;
    }
    if (nread == 0) break; // end of the file
    // nbytes = nread + Nstart;
    
    int bufferStatus = TRUE; 
    c0 = buffer; // c0 always marks the start of a line
    while (bufferStatus) {
      c1 = strchr (c0, '\n'); // find the end of this current line
      if (!c1) {
	c1 = strchr (c0, '\r'); // try \r for non-UNIX files (what do we do about a Mac? \n\r?)
      }
      if (c1 == (char *) NULL) {
	Nstart = strlen (c0);
	memmove (buffer, c0, Nstart);
	bufferStatus = FALSE;
	continue;
      }
      *c1 = 0; // mark the end of the line 

      if (*c0 == '#') {	c0 = c1 + 1; continue; }
      if (*c0 == '!') {	c0 = c1 + 1; continue; }

      // parse the vectors in this line.  this code is a bit inefficient: each column
      // requires a separate pass through the line.

      int lineStatus = TRUE;
      for (i = 0; i < Nvec; i++) {
	int ivalue;
	double dvalue;
	time_t tvalue;
	int readStatus = FALSE;
	// need to make the if cases for coltype[i]
	switch (coltype[i]) {
	  case COLTYPE_INT:
	    readStatus = IsCSV ? iparse_csv (&ivalue, col[i], c0) : iparse (&ivalue, col[i], c0);
	    vec[i][0].elements.Int[Nelem] = readStatus ? ivalue : 0;
	    break;
	  case COLTYPE_CHAR:
	    {
	      // I need to get an isolated word in 'value' with the string value of this field 
	      char *ptr = IsCSV ? ptrparse_csv (col[i], c0) : ptrparse (col[i], c0);
	      char *value = NULL;
	      if (IsCSV) {
		char *end = parse_nextword_csv (ptr);
		if (end) {
		  value = end ? strncreate (ptr, end - ptr) : strcreate (ptr);
		}
	      } else {
		value = thisword(ptr);
	      }
	      set_list_varname (varname, listname[i], Nelem, FALSE);
	      set_str_variable (varname, value);
	      free (value);
	      break;
	    }
	  case COLTYPE_FLT:
	    readStatus = IsCSV ? dparse_csv (&dvalue, col[i], c0) : dparse (&dvalue, col[i], c0);
	    vec[i][0].elements.Flt[Nelem] = readStatus ? dvalue : NAN;
	    break;
	  case COLTYPE_TIME:
	    readStatus = IsCSV ? tparse_csv (&tvalue, col[i], c0) : tparse (&tvalue, col[i], c0);
	    dvalue = TimeValue (tvalue, TimeReference, TimeFormat);
	    vec[i][0].elements.Flt[Nelem] = readStatus ? dvalue : NAN;
	    break;
	}
	if (!readStatus && VERBOSE) {
	  if (IsCSV) {
	    gprint (GP_ERR, "suspect field: %d (%s) in %s\n", col[i], argv[2*i+2], c0);
	  } else {
	    gprint (GP_ERR, "suspect field: %d in %s\n", col[i], c0);
	  }
	}
	lineStatus &= readStatus;
      }
      if (!lineStatus && VERBOSE) {
	char temp[32];
	strncpy (temp, c0, 32);
	temp[31] = 0;
	gprint (GP_ERR, "skip line %s\n\n", temp);
      }
      Nelem ++;
      if (Nelem == NELEM) {
	NELEM += 1000;
	for (i = 0; i < Nvec; i++) {
	  switch (coltype[i]) {
	    case COLTYPE_INT:
	      ResetVector (vec[i], OPIHI_INT, NELEM);
	      break;
	    case COLTYPE_FLT:
	    case COLTYPE_TIME:
	      ResetVector (vec[i], OPIHI_FLT, NELEM);
	      break;
	    case COLTYPE_CHAR:
	    default:
	      break;
	  }
	}
      }
      c0 = c1 + 1;
    }
  }
  // set the final vector / list length
  for (i = 0; i < Nvec; i++) {
    switch (coltype[i]) {
      case COLTYPE_INT:
	ResetVector (vec[i], OPIHI_INT, Nelem);
	break;
      case COLTYPE_FLT:
      case COLTYPE_TIME:
	ResetVector (vec[i], OPIHI_FLT, Nelem);
	break;
      case COLTYPE_CHAR:
	sprintf (varname, "%s:n", listname[i]);
	set_int_variable (varname, Nelem);
	break;
      default:
	break;
    }
  }
  read_vectors_cleanup();
  return (TRUE);
}

# define ESCAPE(...) {		\
  gprint (GP_ERR, __VA_ARGS__); \
  if (CCDKeyword != NULL) free (CCDKeyword); \
  gfits_free_table  (&table); \
  gfits_free_header (&header); \
  return (FALSE);  }

int read_table_vectors (int argc, char **argv, char *extname) {

  off_t Nbytes;
  int i, j, N, Ny, Binary, vecType;
  char type[16], ID[80];
  FTable table;
  Header header;
  Vector **vec;

  table.buffer = NULL;
  header.buffer = NULL;

  int FITS_TRANSPOSE = FALSE;
  if ((N = get_argument (argc, argv, "-transpose"))) {
    remove_argument (N, &argc, argv);
    FITS_TRANSPOSE = TRUE;
  }

  char *CCDKeyword = NULL;
  if ((N = get_argument (argc, argv, "-keyword"))) {
    remove_argument (N, &argc, argv);
    CCDKeyword = strcreate (argv[N]);
    remove_argument (N, &argc, argv);
  }

  int padIfShort = FALSE;
  if ((N = get_argument (argc, argv, "-pad-if-short"))) {
    remove_argument (N, &argc, argv);
    padIfShort = TRUE;
  }

  int getSizes = FALSE;
  if ((N = get_argument (argc, argv, "-sizes"))) {
    remove_argument (N, &argc, argv);
    getSizes = TRUE;
  }

  int Nextend = -1;
  if ((N = get_argument (argc, argv, "-extnum"))) {
    remove_argument (N, &argc, argv);
    Nextend = atoi (extname);
  }

  int start = 0;
  int Nrows = -1; // -1 : read entire table
  if ((N = get_argument (argc, argv, "-range"))) {
    remove_argument (N, &argc, argv);
    start = atoi (argv[N]);
    remove_argument (N, &argc, argv);
    Nrows = atoi (argv[N]);
    remove_argument (N, &argc, argv);
  }

  // if CharAsVectors, char fields will be saved as vectors NAME:0 -- NAME:n for n characters
  // else char fields will be saved as $NAME:0 - $NAME:m for m rows
  // if (Ny > 10000), force CharAsVectors
  int CharAsVectors = FALSE;
  if ((N = get_argument (argc, argv, "-char-vectors"))) {
    remove_argument (N, &argc, argv);
    CharAsVectors = TRUE;
  }

  // XXX ReadAll needs: deal with Extnum vs Extname, save vectors, etc
  // ReadAll = FALSE;
  // if ((N = get_argument (argc, argv, "-all"))) {
  //   remove_argument (N, &argc, argv);
  //   ReadAll = atoi (extname);
  //   if (argc != 1) ESCAPE ("-all option cannot be mixed with selected field");
  // }

  if (argc < 2) ESCAPE ("USAGE: read -fits extension [-extnum] [-keyword key] name name ...\n");

  if (f == NULL) ESCAPE ("file not found\n");
  fseeko (f, 0LL, SEEK_SET);
  table.header = &header;

  /* load appropriate extension (if extname is a number, use count) */
  if (Nextend > -1) {
    // first extension is PHU, cannot be a table. 
    // Nextend counts from 0 for first extension
    if (!gfits_load_header (f, &header)) ESCAPE ("error reading primary header for file\n");
    Nbytes = gfits_data_size (&header);
    fseeko (f, Nbytes, SEEK_CUR);
    gfits_free_header (&header);

    for (i = 0; i < Nextend; i++) {
      if (!gfits_load_header (f, &header)) ESCAPE ("extension %d not found\n", i);
      Nbytes = gfits_data_size (&header);
      /* skip the prior data buffers */
      fseeko (f, Nbytes, SEEK_CUR);
      gfits_free_header (&header);
    }
    if (!gfits_load_header (f, &header)) ESCAPE ("error reading header for extension %d\n", Nextend);

    if (getSizes) {
      read_table_sizes (&header);
      if (CCDKeyword != NULL) free (CCDKeyword); 
      gfits_free_header (&header); 
      return TRUE;
    }

    if (Nrows == -1) {
      Nrows = header.Naxis[1] - start;
    }
    if (start < 0) ESCAPE ("invalid range: start < 0\n");
    if (start >= header.Naxis[1]) ESCAPE ("invalid range: start >= Ny (%d)\n", header.Naxis[1]);
    if (Nrows < 0) ESCAPE ("invalid range: Nrows < 0\n");
    if (start + Nrows > header.Naxis[1]) ESCAPE ("invalid range: start + Nrows > Ny (%d)\n", header.Naxis[1]);

    // Ny = 100, start = 0, Nrows = -1 -> Nrows => 100
    // Ny = 100, start = 10, Nrows = 90

    if (!gfits_fread_ftable_range (f, padIfShort, &table, start, Nrows)) ESCAPE ("error reading table for extension %d\n", Nextend);
  } else {
    if (CCDKeyword == NULL) {
      CCDKeyword = get_variable ("CCDKEYWORD");
    }
    if (CCDKeyword == NULL) {
      CCDKeyword = strcreate ("EXTNAME");
    }

    while (1) {
      if (!gfits_load_header (f, &header)) {
	gprint (GP_ERR, "extension %s not found in file\n", extname);
	if (CCDKeyword != NULL) free (CCDKeyword); 
	gfits_free_table  (&table); 
	gfits_free_header (&header); 
	return (TRUE);  
      }

      Nbytes = gfits_data_size (&header);

      if (!gfits_scan (&header, CCDKeyword, "%s", 1, ID)) {
	fseeko (f, Nbytes, SEEK_CUR);
	gfits_free_header (&header);
	continue;
      }
      if (strcmp (ID, extname)) {
	fseeko (f, Nbytes, SEEK_CUR);
	gfits_free_header (&header);
	continue;
      }

      if (Nrows == -1) {
	Nrows = header.Naxis[1] - start;
      }
      if (start < 0) ESCAPE ("invalid range: start < 0\n");
      if (start >= header.Naxis[1]) ESCAPE ("invalid range: start >= Ny (%d)\n", header.Naxis[1]);
      if (Nrows < 0) ESCAPE ("invalid range: Nrows < 0\n");
      if (start + Nrows > header.Naxis[1]) ESCAPE ("invalid range: start + Nrows > Ny (%d)\n", header.Naxis[1]);

      if (!gfits_fread_ftable_range (f, padIfShort, &table, start, Nrows)) ESCAPE ("error reading table for extension %d\n", Nextend);

      // if (!gfits_fread_ftable_data (f, &table, padIfShort)) ESCAPE ("error reading table for extension\n");

      break;
    }
  }

  /* identify table type (ascii / binary) */
  Binary = FALSE;
  gfits_scan (&header, "XTENSION", "%s", 1, type);
  if (strcmp (type, "BINTABLE") && strcmp (type, "TABLE")) {
    ESCAPE ("specified extension %s is not a table\n", type);
  }
  Binary = !strcmp (type, "BINTABLE");
  Ny = header.Naxis[1];

  /* find columns which match requested vectors */
  for (i = 1; i < argc; i++) {
    void   *data;
    int Nval;
    char name[80];
      
    Nval = 0;
    if (Binary) {
      if (!gfits_get_bintable_column_type (&header, argv[i], type, &Nval)) ESCAPE ("requested field not found");
      if (!gfits_get_bintable_column (&header, &table, argv[i], &data)) ESCAPE ("error reading data from specified field");
    } else {
      if (!gfits_get_table_column_type (&header, argv[i], type)) ESCAPE ("requested field not found");
      if (!gfits_get_table_column (&header, &table, argv[i], &data)) ESCAPE ("error reading data from specified field");
    }
    if (Nval == 0) ESCAPE ("no data for field in table");
    
    vecType = OPIHI_INT;
    if (!strcmp (type, "double") || !strcmp (type, "float")) {
      vecType = OPIHI_FLT;
    }
	
    if (!FITS_TRANSPOSE) {
      // read string column into a list rather than a vector
      if (!strcmp (type, "char") && !CharAsVectors && (Ny < 3000)) {
	char *fieldName = argv[i];
	char *Ptr = data;
	char varname[1024];  // used as a buffer for the names of string fields
	for (j = 0; j < Ny; j++) {
	  set_list_varname (varname, fieldName, j, FALSE);
	  char *value = strncreate (&Ptr[j*Nval], Nval);
	  // replace instances of $ with _
	  char *p = strchr (value, '$');
	  while (p) { 
	    *p = '_';
	    p = strchr (p, '$');
	  }
	  set_str_variable (varname, value);
	  free (value);
	}
	sprintf (varname, "%s:n", fieldName);
	set_int_variable (varname, Ny);
	continue;
      }

      // define the multifield vector names (Nval vectors x Ny elements)
      ALLOCATE (vec, Vector *, Nval);
      for (j = 0; j < Nval; j++) {
	if (Nval == 1) 
	  sprintf (name, "%s", argv[i]);
	else
	  sprintf (name, "%s:%d", argv[i], j);
	if ((vec[j] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) ESCAPE ("bad vector name");
	ResetVector (vec[j], vecType, Ny);
      }

      if (!VectorAssignData (vec, type, data, Ny, Nval)) ESCAPE ("bad column type %s", type);

    } else {
      // define the multifield vector names (Ny vectors x Nval elements)
      ALLOCATE (vec, Vector *, Ny);
      for (j = 0; j < Ny; j++) {
	if (Ny == 1) 
	  sprintf (name, "%s", argv[i]);
	else
	  sprintf (name, "%s:%d", argv[i], j);
	if ((vec[j] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) ESCAPE ("bad vector name");
	ResetVector (vec[j], vecType, Nval);
      }

      if (!VectorAssignDataTranspose (vec, type, data, Ny, Nval)) ESCAPE ("bad column type %s", type);
    }
    free (data);
    free (vec);
  }
  if (CCDKeyword != NULL) free (CCDKeyword);
  gfits_free_table (&table);
  gfits_free_header (&header);
  return (TRUE);
}

void read_vectors_cleanup () {

  int i;

  if (col) free (col);
  if (coltype) free (coltype);
  if (buffer) free (buffer);
  if (listname) {
    for (i = 0; i < Nvec; i++) {
      if (listname[i]) free (listname[i]);
    }
    free (listname);
  }
  if (vec) free (vec);
}

// read -fits foo -sizes -- Nx, Ny, Nfields -> $table:Nx, $table:Ny, $table:$Nfields
// read -fits foo -fields 

int read_table_sizes (Header *header) {
  
  int Nfields;

  gfits_scan (header, "TFIELDS", "%d", 1, &Nfields);

  set_int_variable ("table:Nx", header->Naxis[0]);
  set_int_variable ("table:Nx", header->Naxis[0]);
  set_int_variable ("table:Nfields", Nfields);

  return TRUE;
}

