Merge commit '23c7930d27fe11c4655e1291a07a821dbbaba78d' as 'funtools'

1 files changed, 1412 insertions, 0 deletions

diff --git a/funtools/funtab.c b/funtools/funtab.c
new file mode 100644
index 0000000..5f0a3f3
--- /dev/null
+++ b/funtools/funtab.c
@@ -0,0 +1,1412 @@
+/*
+ *	Copyright (c) 1999-2003 Smithsonian Astrophysical Observatory
+ */
+
+#include <ctype.h>
+#include <funtoolsP.h>
+
+/*
+ *
+ * private routines
+ *
+ */
+
+#ifdef ANSI_FUNC
+static void *
+swap(void *obuf, void *ibuf, int width, int type)
+#else
+static void *swap(obuf, ibuf, width, type)
+     void *obuf;
+     void *ibuf;
+     int width;
+     int type;
+#endif
+{
+  switch ( type ) {
+  case 'A':
+  case 'B':
+  case 'X':
+  case 'L':
+    memcpy(obuf, ibuf, width);
+    break;
+  case 'I':
+  case 'U':
+    swap2(obuf, ibuf, width);
+    break;
+  case 'J':
+  case 'V':
+  case 'E':
+    swap4(obuf, ibuf, width);
+    break;
+  case 'K':
+  case 'D':
+    swap8(obuf, ibuf, width);
+    break;
+  default:
+    memcpy(obuf, ibuf, width);
+    break;
+  }
+  return obuf;
+}
+
+#ifdef ANSI_FUNC
+static void
+_FunTableSave(Fun fun, char *rows, char *ebuf, int nrow)
+#else
+static void _FunTableSave(fun, rows, ebuf, nrow)
+     Fun fun;
+     char *rows;
+     char *ebuf;
+     int nrow;
+#endif
+{
+  SaveBuf new, cur;
+  int n;
+
+  /* allocate new data space */
+  new = (SaveBuf)xcalloc(1, sizeof(SaveBufRec));
+  /* this is how many rows we are saving */
+  new->nrow = nrow;
+  /* see if we can store in memory or if we have to shunt to a file */
+  n = nrow * (fun->rowsize + fun->rawsize);
+  if( (fun->head->lmem < 0) || ((fun->head->lmem-n)>0) ){
+    new->type = 1;
+    /* save alloc'ed raw buf null out buf so funtools does not free it */
+    new->rbuf = fun->rawbuf;
+    fun->rawbuf = NULL;
+    /* if user passed in a row buffer, we save a copy */
+    if( rows ){
+      new->ebuf = xmalloc(nrow*fun->rawsize);
+      memcpy(new->ebuf, ebuf, new->nrow*fun->rowsize);
+    }
+    /* otherwise just save funtools-allocated row buffer */
+    else{
+      new->ebuf  = ebuf;
+    }
+    /* decrease amount of memory we have left for saved data */
+    if( fun->head->lmem > 0 )
+      fun->head->lmem -= n;
+  }
+  else{
+    new->type = 2;
+    /* write user rows */
+    fwrite(ebuf, fun->rowsize, nrow, fun->lefp);
+    /* free up user row buffer unless user allocated it */
+    if( !rows && ebuf ) xfree(ebuf);
+    /* write raw rows */
+    fwrite(fun->rawbuf, fun->rawsize, nrow, fun->lrfp);
+    /* free up raw buffer */
+    if( fun->rawbuf ) xfree(fun->rawbuf);
+    fun->rawbuf = NULL;
+  }
+  /* add to end of list */
+  if( fun->save == NULL ){
+    fun->save = new;
+  }
+  else{
+    for(cur=fun->save; cur->next!=NULL; cur=cur->next)
+      ;
+    cur->next = new;
+  }
+}
+
+#ifdef ANSI_FUNC
+static char *
+_FunTableRestore(Fun fun, char *rows, int *nrow)
+#else
+static char *_FunTableRestore(fun, rows, nrow)
+     Fun fun;
+     char *rows;
+     int *nrow;
+#endif
+{
+  SaveBuf cur;
+  char *ebuf=NULL;
+  unsigned char *rbuf;
+
+  /* start of pessimistically */
+  *nrow = 0;
+  /* make sure we have something to restore */
+  if( !fun->save )
+    return NULL;
+  /* bump list to next */
+  cur = fun->save;
+  fun->save = cur->next;
+  /* fill in the blanks */
+  *nrow = cur->nrow;
+  switch(cur->type){
+  case 1:
+    /* we might have to fill the passed row buffer */
+    if( rows ){
+      /* copy into user-supplied buffer */
+      memcpy(rows, cur->ebuf, cur->nrow*fun->rowsize);
+      /* we will return user-supplied buffer */
+      ebuf = rows;
+      /* free up temp buffer */
+      if(cur->ebuf ) xfree(cur->ebuf);
+    }
+    /* otherwise, we just pass back the saved row buffer */
+    else{
+      ebuf = cur->ebuf;
+    }
+    /* move raw rows buf back into funtools struct */
+    if( fun->rawbuf ) xfree(fun->rawbuf);
+    fun->rawbuf = cur->rbuf;
+    break;
+  case 2:
+    /* we might have to fill the passed row buffer */
+    if( rows )
+      ebuf = rows;
+    else
+      ebuf = xmalloc(fun->rowsize*cur->nrow);
+    /* read into user-supplied buffer */
+    fread(ebuf, fun->rowsize, cur->nrow, fun->lefp);
+    /* allocate space for the raw buffer */
+    rbuf = xmalloc(fun->rawsize*cur->nrow);
+    /* read into raw buffer */
+    fread(rbuf, fun->rawsize, cur->nrow, fun->lrfp);
+    /* move raw rows buf back into funtools struct */
+    if( fun->rawbuf ) xfree(fun->rawbuf);
+    fun->rawbuf = rbuf;
+    break;
+  }
+  /* this is the last file we "read" in the list */
+  if( fun->head )
+    fun->head->current = fun;
+  else
+    fun->current = fun;
+  if( cur ) xfree(cur);
+  /* return results */
+  return ebuf;
+}
+
+#ifdef ANSI_FUNC
+static void
+_FunBlank(Fun fun, int type, char *buf)
+#else
+static void _FunBlank(fun, type, buf)
+     Fun fun;
+     int type;
+     char *buf;
+#endif
+{
+  unsigned char bval;
+  short sval;
+  int ival;
+  longlong lval;
+  float fval;
+  double dval;
+
+  switch(type){
+  case 'B':
+    bval = fun->blank;
+    memcpy(buf, &bval, sizeof(char));
+    break;
+  case 'I':
+  case 'U':
+    sval = fun->blank;
+    memcpy(buf, &sval, sizeof(short));
+    break;
+  case 'J':
+  case 'V':
+    ival = fun->blank;
+    memcpy(buf, &ival, sizeof(int));
+    break;
+  case 'K':
+#if HAVE_LONG_LONG == 0
+    gerror(stderr,
+	   "long long support was not built into this program\n");
+    exit(1);
+#endif
+    lval = fun->blank;
+    memcpy(buf, &lval, sizeof(longlong));
+    break;
+  case 'E':
+    fval = getnanf();
+    memcpy(buf, &fval, sizeof(float));
+    break;
+  case 'D':
+    dval = getnand();
+    memcpy(buf, &dval, sizeof(double));
+    break;
+  default:
+    memset(buf, 0, ft_sizeof(type));
+    break;
+  }
+}
+  
+#ifdef ANSI_FUNC
+static void *
+_FunTextRead(Fun fun, char *buf, size_t UNUSED(size), size_t get,  size_t *got)
+#else
+static void *_FunTextRead(fun, buf, size, get,  got)
+     Fun fun;
+     char *buf;
+     size_t size;
+     size_t get;
+     size_t *got;
+#endif
+{
+  int p, t;
+  int len;
+  int dodata1;
+  int xtype;
+  int ntoken;
+  int nline=0;
+  char lbuf[SZ_LINE];
+  char *bp;
+  Parse parser=NULL;
+  ParsedLine line;
+
+  /* sanity checks
+     NB: the order of ascii columns should be identical to the fitsy columns
+     (since the latter was generated by the former) */
+  if( !fun->header || !fun->header->table || !fun->header->table->col )
+    return 0;
+
+  /* if we have not read any bytes, use data1 */
+  dodata1 = (fun->bytes==0);
+
+  /* convert ASCII columns to binary columns */
+  bp = buf;
+  while( (size_t)nline < get ){
+    /* first line should be in data1 */
+    if( dodata1 ){
+      /* look for the parser with data1 that has the most tokens */
+      for(parser=NULL, ntoken=0, p=0; p<fun->nparser; p++){
+	if( !(fun->parsers[p]->state & PARSE_STATE_BAD) && 
+	    fun->parsers[p]->data1 ){
+	  if( fun->parsers[p]->ntoken > ntoken ){
+	    parser = fun->parsers[p];
+	    ntoken = parser->ntoken;
+	  }
+	}
+      }
+      /* use the parser with the max number of tokens */
+      if( parser ){
+	line = parser->data1;
+	if( PARSE_ISCOMMENT(line) ) continue;
+	for(t=0; t<line->ntoken; t++){
+	  switch(line->tokens[t].type){
+	  case PARSE_FLOAT:
+	  case PARSE_HEXINT:
+	  case PARSE_INTEGER:
+	    /* handle string type specially */
+	    if( fun->header->table->col[t].type == 'A' ){
+	      len = strlen(line->tokens[t].sval);
+	      if( len >= fun->header->table->col[t].width){
+		memmove(bp, line->tokens[t].sval, 
+			fun->header->table->col[t].width);
+	      }
+	      else{
+		memmove(bp, line->tokens[t].sval, len);
+		memset(bp+len, 0, fun->header->table->col[t].width-len);
+	      }
+	    }
+	    /* handle bit-field processing specially */
+	    else if( fun->header->table->col[t].type == 'X' ){
+	      switch(fun->header->table->col[t].width){
+	      case 0:
+	      case 1:
+		xtype = 'B'; break;
+	      case 2:
+		xtype = 'U'; break;
+	      case 4:
+		xtype = 'V'; break;
+	      default:
+		xtype = 'B'; break;
+	      }
+	      if( (line->tokens[t].type == PARSE_INTEGER) ||
+		  (line->tokens[t].type == PARSE_HEXINT)   )
+		ft_acht2(xtype, bp, 'K', &line->tokens[t].lval, 1, 0, 0);
+	      else
+		ft_acht2(xtype, bp, 'D', &line->tokens[t].dval, 1, 0, 0);
+	    }
+	    else{
+	      if( (line->tokens[t].type == PARSE_INTEGER) ||
+		  (line->tokens[t].type == PARSE_HEXINT)   )
+		ft_acht2(fun->header->table->col[t].type, bp, 
+			 'K', &line->tokens[t].lval, 1, 0, 0);
+	      else
+		ft_acht2(fun->header->table->col[t].type, bp, 
+			 'D', &line->tokens[t].dval, 1, 0, 0);
+	    }
+	    bp += fun->header->table->col[t].width;
+	    break;
+	  case PARSE_NULL:
+	    _FunBlank(fun, fun->header->table->col[t].type, bp);
+	    bp += fun->header->table->col[t].width;
+	    break;
+	  case PARSE_STRING:
+	    len = strlen(line->tokens[t].sval);
+	    if( len >= fun->header->table->col[t].width){
+	      memmove(bp, line->tokens[t].sval, 
+		      fun->header->table->col[t].width);
+	    }
+	    else{
+	      memmove(bp, line->tokens[t].sval, len);
+	      memset(bp+len, 0, fun->header->table->col[t].width-len);
+	    }
+	    bp += fun->header->table->col[t].width;
+	    break;
+	  default:
+	    break;
+	  }
+	}
+	/* processed data1 line */
+	nline++;
+      }
+      /* went through all parsers looking for data1 */
+      dodata1 = 0;
+    }
+    /* read and process the next line */
+    else{
+      /* get next line or be done */
+      if( !ggets(fun->gio, lbuf, SZ_LINE) )
+	break;
+      /* analyze line and make sure one parser succeeded (even if its EOT) */
+      if( !ParseAnalyze(fun->parsers, fun->nparser, lbuf) ){
+	gerror(stderr, "text parser failure analyzing line: %s\n", lbuf);
+	break;
+      }
+      /* look for valid parser with the most tokens */
+      for(parser=NULL, ntoken=0, parser=NULL, p=0; p<fun->nparser; p++){
+	if( fun->parsers[p]->state & PARSE_STATE_BAD) 
+	  continue;
+	else if( fun->parsers[p]->ntoken > ntoken ){
+	  parser = fun->parsers[p];
+	  ntoken = parser->ntoken;
+	}
+      }
+      /* stop if no valid parser or valid parser found EOT */
+      if( !parser ) break;
+      if( parser->state & PARSE_STATE_EOT) break;
+      /* use selected parser to fill up row buffer */
+      line = parser->cur;
+      if( PARSE_ISCOMMENT(line) ) continue;
+      for(t=0; t<line->ntoken; t++){
+	switch(line->tokens[t].type){
+	case PARSE_FLOAT:
+	case PARSE_HEXINT:
+	case PARSE_INTEGER:
+	  /* handle string type specially */
+	  if( fun->header->table->col[t].type == 'A' ){
+	    len = strlen(line->tokens[t].sval);
+	    if( len >= fun->header->table->col[t].width){
+	      memmove(bp, line->tokens[t].sval, 
+		      fun->header->table->col[t].width);
+	    }
+	    else{
+	      memmove(bp, line->tokens[t].sval, len);
+	      memset(bp+len, 0, fun->header->table->col[t].width-len);
+	    }
+	  }
+	  /* handle bit-field processing specially */
+	  else if( fun->header->table->col[t].type == 'X' ){
+	    switch(fun->header->table->col[t].width){
+	    case 0:
+	    case 1:
+	      xtype = 'B'; break;
+	    case 2:
+	      xtype = 'U'; break;
+	    case 4:
+	      xtype = 'V'; break;
+	    default:
+	      xtype = 'B'; break;
+	    }
+	    if( (line->tokens[t].type == PARSE_INTEGER) ||
+		(line->tokens[t].type == PARSE_HEXINT)   )
+	      ft_acht2(xtype, bp, 'K', &line->tokens[t].lval, 1, 0, 0);
+	    else
+	      ft_acht2(xtype, bp, 'D', &line->tokens[t].dval, 1, 0, 0);
+	  }
+	  else{
+	    if( (line->tokens[t].type == PARSE_INTEGER) ||
+		(line->tokens[t].type == PARSE_HEXINT)   )
+	      ft_acht2(fun->header->table->col[t].type, bp, 
+		       'K', &line->tokens[t].lval, 1, 0, 0);
+	    else
+	      ft_acht2(fun->header->table->col[t].type, bp, 
+		       'D', &line->tokens[t].dval, 1, 0, 0);
+	  }
+	  bp += fun->header->table->col[t].width;
+	  break;
+	case PARSE_STRING:
+	  /* handle string type specially */
+	  if( fun->header->table->col[t].type == 'A' ){
+	    len = strlen(line->tokens[t].sval);
+	    if( len >= fun->header->table->col[t].width){
+	      memmove(bp, line->tokens[t].sval, 
+		      fun->header->table->col[t].width);
+	    }
+	    else{
+	      memmove(bp, line->tokens[t].sval, len);
+	      memset(bp+len, 0, fun->header->table->col[t].width-len);
+	    }
+	  }
+	  bp += fun->header->table->col[t].width;
+	  break;
+	case PARSE_NULL:
+	  _FunBlank(fun, fun->header->table->col[t].type, bp);
+	  bp += fun->header->table->col[t].width;
+	  break;
+	default:
+	  break;
+	}
+      }
+      /* processed another line */
+      nline++;
+    }
+  }
+  *got = nline;
+  return buf;
+}
+
+/*
+ *
+ * semi-public routines
+ *
+ */
+
+#ifdef ANSI_FUNC
+void *
+_FunRead(Fun fun, char *buf, size_t size, size_t get,  size_t *got)
+#else
+void *_FunRead(fun, buf, size, get,  got)
+     Fun fun;
+     char *buf;
+     size_t size;
+     size_t get;
+     size_t *got;
+#endif
+{
+  /* initialize */
+  *got = 0;
+  
+  /* sanity check */
+  if( !fun || !buf ) return NULL;
+
+  /* text parsers are handled one line at a time */
+  if( fun->parsers ){
+    return _FunTextRead(fun, buf, size, get,  got);
+  }
+  /* binary reads */
+  else{
+    /* ordinary binary read */
+    if( (fun->filt == NOFILTER) || (fun->filt->doidx != 1) ){
+      return _gread(fun->gio, (void *)buf, size, get, got);
+    }
+    /* indexed read */
+    else{
+      return idxread(fun->filt->idx, fun->gio, fun->header,
+		     (void *)buf, size, get, got, &fun->dofilt);
+    }
+  }
+}
+
+/*
+ *
+ * _FunFixNaxis2 -- change the value of an NAXIS2 card
+ * used to correct a copied binary table after we filter
+ *
+ */
+#ifdef ANSI_FUNC
+int
+_FunFixNaxis2(Fun fun)
+#else
+int _FunFixNaxis2(fun)
+     Fun fun;
+#endif
+{
+  if( (fun->ops & OP_WRHEAD) && (fun->headpos >=0) ){
+    return FunParamPuti(fun,
+			"NAXIS", 2, fun->io, "Number of entries in table", 0);
+  }
+  else{
+    return(0);
+  }
+}
+
+/*
+ *
+ * _FunTablePutHeader -- writes binary table header to file
+ * used by FunTablePut and sometimes FunFlush (if no Put was done)
+ *
+ */
+#ifdef ANSI_FUNC
+int
+_FunTablePutHeader(Fun fun)
+#else
+int _FunTablePutHeader(fun)
+	Fun fun;
+#endif
+{
+  /* write table header, if necessary */
+  if( !fun->ops ){
+    /* if we have no columns yet, try to get some from the assoc. input file */
+    if( !fun->ncol ){
+      /* if we have a header to copy from, that is */
+      if( fun->ifun && fun->ifun->header && fun->ifun->header->table ){
+	FunColumnSelect(fun, 0, "copy=select", NULL);
+	/* grab the copied merge value  */
+	fun->merge = fun->ifun->merge;
+	/* and the organization value */
+	fun->org = fun->ifun->org;
+      }
+      /* else we lose */
+      else
+	return 0;
+    }
+
+    /* write the dummy primary header, if necessary */
+    _FunPrimaryExtension(fun);
+
+    /* generate the fitsy header for specified columns */
+    _FunColumnHeader(fun, fun->cols, fun->ncol);
+
+    /* save file position for later updating */
+    fun->headpos = gtell(fun->gio);
+
+    /* write the binary table header */
+    ft_headwrite(fun->gio, fun->header);
+
+    /* we just wrote the header */
+    fun->ops |= OP_WRHEAD;
+  }
+  return 1;
+}
+
+/*
+ *
+ * _FunTableRowGet -- get rows from a table
+ *
+*/
+#ifdef ANSI_FUNC
+void *
+_FunTableRowGet(Fun fun, void *rows, int maxrow, char *plist, int *nrow)
+#else
+void *_FunTableRowGet(fun, rows, maxrow, plist, nrow)
+     Fun fun;
+     void *rows;
+     int maxrow;
+     char *plist;
+     int *nrow;
+#endif
+{
+  int i, j, k, l;		/* counters */
+  int tival;			/* temp int value */
+  int dofilt;			/* true if we can filter */
+  int skip;			/* bytes to skip to end of extension */
+  int *rbuf;			/* valid row flags */
+  int rowbufsize;		/* alloc'ed output buffer size */
+  int maxbufsize;		/* max alloc'ed output buffer size */
+  int smax;			/* max elements in sbuf */
+  int pad=sizeof(double);	/* double boundary pad bytes */
+  size_t get;			/* number of rows to read */
+  size_t got;			/* number of rows read */
+  double *sbuf=NULL;		/* temp buffer for scaling */
+  unsigned char *vheap;		/* vheap space for output vector allocation */
+  unsigned char *rawbase=NULL;	/* base pointer into rawbuf */
+  unsigned char *rawptr;	/* current pointer into rawbuf */
+  unsigned char *rowbase=NULL;	/* base pointer to output row buf */
+  unsigned char *rowptr;	/* current pointer into output row buf */
+  char *rawoff;			/* temp pointer to input */
+  char *rowoff;			/* temp pointer to output */
+  char *pbuf;			/* temp plist buffer */
+  char tbuf[SZ_LINE];		/* ever-present temp buffer */
+  static char paint[SZ_LINE];	/* paint mode (like zhtools) */
+  static char debug[SZ_LINE];
+  static int rconvert;
+
+  /* init result */
+  *nrow = 0;
+  rbuf = NULL;
+  
+  /* just in case ... */
+  if( !_FunValid(fun) ){
+    gerror(stderr, "invalid funtools handle\n");
+    return NULL;
+  }
+
+  /* if we delayed the open before, we have to open now */
+  if( !fun->header && strchr(fun->mode, 'r') )
+    _FunFITSOpen(fun, fun->fname, "r");
+
+  /* make sure something good happened */
+  if( !fun->header || !fun->header->table ){
+    gerror(stderr, "not a binary table\n");
+    return NULL;
+  }
+
+  /* check plist values: convert, mask ... */
+  if( fun->iconvert < 0 ){
+    rconvert = 0;
+    fun->transparent = 0;
+    pbuf = xstrdup(plist);
+    if( _FunKeyword(pbuf, "convert", NULL, tbuf, SZ_LINE) ){
+      fun->iconvert = istrue(tbuf);
+      if( fun->iconvert != is_bigendian() ){
+	rconvert = 1;
+      }
+    }
+    else{
+      fun->iconvert = (fun->endian != is_bigendian());
+    }
+    if( _FunKeyword(pbuf, "mask", "FUN_MASK", tbuf, SZ_LINE) ){
+      if( !strcasecmp(tbuf, "transparent") )
+	fun->transparent = 1;
+    }
+    if( _FunKeyword(pbuf, "paint", "FILTER_PAINT", tbuf, SZ_LINE) ){
+      strncpy(paint, tbuf, SZ_LINE);
+    }
+    else{
+      *paint = '\0';
+    }
+    if( _FunKeyword(pbuf, "debug", "FILTER_DEBUG", tbuf, SZ_LINE) ){
+      strncpy(debug, tbuf, SZ_LINE);
+    }
+    else{
+      *debug = '\0';
+    }
+    if( pbuf ) xfree(pbuf);
+  }
+
+  /* check for no more row to read in this file */
+  if( !fun->left ){
+    /* if possible, set up to read next file in list */
+    if((fun->head) && (fun->head->ltype != LIST_NONE) && fun->head->current){
+      fun->head->current = fun->next;
+      /* if there is a next file, set it up */
+      if(  fun->head->current ){
+	/* rewind save file pointers */
+	rewind(fun->head->current->lefp);
+	rewind(fun->head->current->lrfp);
+	/* and if we read no rows but we have set up a next file,
+	   try again immediately */
+	return FunTableRowGet(fun->head, rows, maxrow, plist, nrow);
+      }
+    }
+    return NULL;
+  }
+
+  /* open the filter, if its not already been done */
+  if( !fun->filt ){
+    if( fun->filter && *fun->filter ){
+      /* create the mode string */
+      if( fun->endian != is_bigendian() )
+	strncpy(tbuf, "convert=true", SZ_LINE-1);
+      else
+	strncpy(tbuf, "convert=false", SZ_LINE-1);
+      /* add the binning key */
+      if( fun->bincols ){
+	strncat(tbuf, ",", SZ_LINE-1);
+	strncat(tbuf, fun->bincols, SZ_LINE-1);
+      }
+      /* add the index flag */
+      if( fun->idx == 1 ){
+	strncat(tbuf, ",idx=true", SZ_LINE-1);
+      }
+      else if( fun->idx == -1 ){
+	strncat(tbuf, ",idx=false", SZ_LINE-1);
+      }
+      /* add paint mode */
+      if( *paint ){
+	strncat(tbuf, ",", SZ_LINE-1);
+	strncat(tbuf, "paint=", SZ_LINE-1);
+	strncat(tbuf, paint, SZ_LINE-1);
+      }
+      /* add debug flag */
+      if( *debug ){
+	strcat(tbuf, ",");
+	strcat(tbuf, "debug=");
+	strcat(tbuf, debug);
+      }
+      fun->filt = FilterOpen(fun->header, fun->filter, tbuf);
+      if( fun->filt == NULL ){
+	gerror(stderr, "can't open filter\n");
+	return NULL;
+      }
+      else if( fun->filt == NOFILTER ){
+	fun->dofilt = 0;
+      }
+    }
+    else{
+      fun->filt = NOFILTER;
+      fun->dofilt = 0;
+    }
+  }
+
+  /* sanity check (after setup, to allow setup without actually reading) */
+  if( maxrow <=0 ) return NULL;
+
+  /* allocate output buffer, if necessary */
+  if( rows ){
+    rowbase = rows;
+  }
+  else{
+    maxbufsize = fun->maxbufsize;
+    rowbufsize = fun->rowsize + pad;
+    /* we might have to allocate space for pointers as well */
+    for(j=0; j<fun->ncol; j++){
+      if( fun->cols[j]->mode & COL_PTR ){
+	if( fun->cols[j]->poff && (fun->cols[j]->mode & COL_READ) ){
+	  gerror(stderr,
+	  "FunTableRowGet: [poff] not yet supported for allocated pointers\n");
+	}
+	rowbufsize += (fun->cols[j]->width + pad);
+      }
+    }
+    /* ensure that we don't try to allocate too large a buffer, or we might
+       swap our brains out and then die anyway */
+    if( maxrow * rowbufsize > maxbufsize ){
+      if( rowbufsize > maxbufsize )
+	maxrow = 1;
+      else
+	maxrow = maxbufsize / rowbufsize;
+    }
+    if( maxrow <= 0 ) maxrow = 1;
+    /* don't call xcalloc (which has internal error checking) but instead
+       call system calloc and handle failure by halving the row number */
+    while( !(rowbase = (unsigned char *)malloc(maxrow*rowbufsize+pad)) ){
+      maxrow /= 2;
+      if( maxrow <= 0 ){
+	gerror(stderr, "can't allocate row buffer\n");
+	return NULL;
+      }
+    }
+    /* clear buffer */
+    memset(rowbase, 0, maxrow*rowbufsize+pad);
+    /* the vheap for pointers starts after row buffer */
+    vheap = rowbase + (maxrow * fun->rowsize);
+    /* make sure its on a double boundary */
+    vheap += (pad - ((long)vheap % pad));
+    /* set up the allocated pointers in each record */
+    for(j=0; j<fun->ncol; j++){
+      if( fun->cols[j]->mode & COL_PTR ){
+	for(i=0; i<maxrow; i++){
+	  rowptr = rowbase + (i*fun->rowsize);
+	  *((unsigned char **)(rowptr+fun->cols[j]->offset)) = vheap;
+	  vheap += fun->cols[j]->width + (pad - (fun->cols[j]->width % pad));
+	  if( vheap >= (rowbase + (maxrow*rowbufsize+pad)) ){
+	    gerror(stderr, "internal funtools error: vheap too small\n");
+	    return NULL;
+	  }
+	}
+      }
+    }
+  }
+
+  /* free previously saved buffer of raw input data */
+  if( fun->rawbuf ) xfree(fun->rawbuf);
+  /* allocate space for a pile of row records */
+  if( fun->left > 0 )
+    fun->rawbufsize = MIN(maxrow, fun->left);
+  /* this is the case where we read til EOF */
+  else
+    fun->rawbufsize = maxrow;
+  if( !(fun->rawbuf=(unsigned char *)xmalloc(fun->rawsize*fun->rawbufsize)) ||
+      !(rbuf = (int *)xmalloc(sizeof(int)*fun->rawbufsize)) ){
+    gerror(stderr, "can't allocate row buffer\n");
+    return NULL;
+  }
+
+  /* allocate a scaling buffer for the largest input or output column */
+  smax = -1;
+  for(j=0; j<fun->ncol; j++){
+    smax = MAX(fun->cols[j]->n, smax);
+  }
+  if( smax >0 )
+    sbuf = (double *)malloc(smax*sizeof(double));
+
+  /* seed output pointer to start of output buffer */
+  rowptr = rowbase;
+
+  /* while there are still rows to read and we have not read the max ... */ 
+  while( (fun->left != 0) && (*nrow < maxrow) ){
+    /* figure out how many to read this time */
+    if( fun->left > 0 )
+      get = MIN(maxrow - *nrow, fun->left);
+    /* read to EOF */
+    else
+      get = maxrow - *nrow;
+    /* read in a pile of rows AFTER the currently saved raw rows */
+    rawbase = fun->rawbuf + (*nrow * fun->rawsize);
+    _FunRead(fun, (void *)rawbase, fun->rawsize, get, &got);
+    /* if we got what we asked for ... */
+    if( got == get ){
+      /* if we are reading a set number of rows ... */
+      if( fun->left > 0 ){
+	/* subtract the number we just got */
+	fun->left -= got;
+	/* just in case! */
+	if( fun->left < 0 )
+	  fun->left = 0;
+      }
+    }
+    /* didn't get what we asked for ... */
+    else{
+      /* if we are reading a set number of rows ... */
+      if( fun->left > 0 ){
+	/* using an index => already filtering, so fewer rows are acceptable */
+	if( (fun->filt == NOFILTER) || (fun->filt->doidx != 1) ){
+	  /* otherwise its unexpected */
+	  gerror(stderr, "unexpected EOF reading rows\n");
+	}
+	/* ensure we stop after this iteration */
+	fun->left = 0;
+      }
+      /* if we are reading til EOF, we just got there */
+      else{
+	fun->left = 0;
+      }
+    }
+
+    /* filter these rows through the co-process */
+    if( fun->dofilt ){
+      dofilt = FilterEvents(fun->filt, (void *)rawbase, fun->rawsize, (int)got,
+			    (void *)rbuf);
+      /* look for error on filter call */
+      if( dofilt < 0 ){
+	gerror(stderr, "event filter failed\n");
+      }
+    }
+    else{
+      dofilt = 0;
+    }
+
+    /* process each row in this batch */
+    for(rawptr=rawbase, i=0; (size_t)i<got; i++, rawptr += fun->rawsize){
+      /* if its not a valid row, skip it */
+      if( dofilt && !fun->transparent && !rbuf[i] ){
+	continue;
+      }
+      /* get specified columns, convert to type, and store in output */
+      for(j=0; j<fun->ncol; j++){
+	/* if this column is not active, or is not to be read, or if its
+	   to be taken from the input data, skip it */
+	if(!(fun->cols[j]->mode & COL_ACTIVE)	||
+	   !(fun->cols[j]->mode & COL_READ)  	||
+	    (fun->cols[j]->mode & COL_IBUF)  	)
+	  continue;
+	/* this is the (input) fitsy column number */
+	k = fun->cols[j]->tcol;
+	if( fun->cols[j]->mode & COL_PTR )
+	  rowoff = *((char **)(((char *)rowptr)+fun->cols[j]->offset));
+	else
+	  rowoff = (char *)rowptr+fun->cols[j]->offset;
+	rowoff += fun->cols[j]->poff;
+	/* for soa, rowbuf (i.e. rowbase) is indexed by nrows */
+	if( fun->org == FUN_ORG_SOA )
+	  rowoff += *nrow * fun->cols[j]->width;
+	/* process special values, or data values */
+	switch(k){
+        /* unknown means mode is "rw" but we don't actually have the col
+	   in the input data -- do nothing */
+	case COL_UNKNOWN_ID:
+	  break;
+        /* $region: the special region id */
+	case COL_REGION_ID:
+	case COL_NUM_ID:
+	  if( k == COL_REGION_ID ){
+	    /* we either use the return value or -1 (no region filtering) */
+	    if( dofilt )
+	      tival = rbuf[i];
+	    else
+	      tival = COL_REGION_ID;
+	  }
+	  if( k == COL_NUM_ID ){
+	    tival = fun->io + i + 1;
+	  }
+	  /* common code */
+	  /* if source and dest type diff, we must do full type convert */
+	  if( fun->cols[j]->type != 'J' ){
+	    ft_acht2(fun->cols[j]->type, rowoff, 'J', &tival, 1, rconvert, 0);
+	  }
+	  /* for same types, we might have to swap bytes */
+	  else if( rconvert ){
+	    swap(rowoff, &tival, ft_sizeof('J'), 'J');
+	  }
+	  /* otherwise its an easy copy */
+	  else{
+	    memcpy(rowoff, &tival, ft_sizeof('J'));
+	  }
+	  break;
+        /* data column */
+	default:
+	  /* this is the offset into the input record */
+	  rawoff = (char *)rawptr + fun->header->table->col[k].offset;
+	  /* if source and dest type diff or we are scaling the column,
+	     we must do full type convert */
+	  if( (fun->cols[j]->type != fun->header->table->col[k].type)	||
+	       fun->header->table->col[k].scaled			){
+	    /* handle bit-field processing specially */
+	    if( fun->header->table->col[k].type == 'X' ){
+	      int xtype;
+	      switch(fun->header->table->col[k].width/fun->cols[j]->n){
+	      case 0:
+	      case 1:
+		xtype = 'B'; break;
+	      case 2:
+		xtype = 'U'; break;
+	      case 4:
+		xtype = 'V'; break;
+	      default:
+		xtype = 'B'; break;
+	      }
+	      ft_acht2(fun->cols[j]->type, rowoff,
+		       xtype, rawoff,
+		       fun->cols[j]->n, fun->iconvert, 0);
+	    }
+	    /* handle scaled input data */
+	    else if( fun->header->table->col[k].scaled ){
+	      /* convert raw to double */
+	      ft_acht2('D', sbuf,
+		       fun->header->table->col[k].type, rawoff,
+		       fun->cols[j]->n, fun->iconvert, 0);
+	      /* calculate scaled value */
+	      for(l=0; l<fun->cols[j]->n; l++){
+		sbuf[l] = sbuf[l] * fun->header->table->col[k].scale +
+		          fun->header->table->col[k].zero;
+	      }
+	      /* convert double to type */
+	      ft_acht2(fun->cols[j]->type, rowoff,
+		       'D', sbuf, fun->cols[j]->n, 0, 0);
+	      /* mark column as having been scaled */
+	      fun->cols[j]->scaled |= FUN_SCALE_APPLIED;
+	    }
+	    /* handle straight-forward change of type */
+	    else{
+	      ft_acht2(fun->cols[j]->type, rowoff,
+		       fun->header->table->col[k].type, rawoff,
+		       fun->cols[j]->n, fun->iconvert, 0);
+	    }
+	  }
+	  /* for same types, we might have to swap bytes */
+	  else if( fun->iconvert ){
+	    if( fun->header->table->col[k].width >= fun->cols[j]->width ){
+	      swap(rowoff, rawoff, fun->cols[j]->width, fun->cols[j]->type);
+	    }
+	    else{
+	      swap(rowoff, rawoff, fun->header->table->col[k].width,
+		   fun->cols[j]->type);
+	      memset(rowoff+fun->header->table->col[k].width, 0,
+		     fun->cols[j]->width-fun->header->table->col[k].width);
+	    }
+	  }
+	  /* otherwise its an easy copy */
+	  else{
+	    if( fun->header->table->col[k].width >= fun->cols[j]->width ){
+	      memcpy(rowoff, rawoff, fun->cols[j]->width);
+	    }
+	    else{
+	      memcpy(rowoff, rawoff, fun->header->table->col[k].width);
+	      memset(rowoff+fun->header->table->col[k].width, 0,
+		     fun->cols[j]->width-fun->header->table->col[k].width);
+	    }
+	  }
+	  break;
+	}
+      }
+      /* save the input data in rawbuf */
+      if( (fun->rawbuf + (*nrow * fun->rawsize)) != rawptr)
+	memcpy((fun->rawbuf + (*nrow * fun->rawsize)), rawptr, fun->rawsize);
+      /* got another row */
+      *nrow += 1;
+      /* bump to next output pointer */
+      if( fun->org == FUN_ORG_AOS )
+	rowptr += fun->rowsize;
+    }
+    /* update how many input rows we have read */
+    fun->io += got;
+    fun->bytes += (got*fun->rawsize);
+  }
+
+  /* free up space */
+  if( rbuf ) xfree(rbuf);
+  if( sbuf ) xfree(sbuf); 
+
+  /* we just read a table */
+  fun->ops |= OP_RDTABLE;
+  /* this is the last file we read in the list */
+  if( fun->head )
+    fun->head->current = fun;
+  else
+    fun->current = fun;
+
+  /* if we have no more rows left, clean up */
+  if( !fun->left ){
+    /* skip to end of extension */
+    if( fun->bytes ){
+      /* skip any unread rows */
+      skip = (fun->rawsize*fun->total) - fun->bytes;
+      gskip(fun->gio, (off_t)skip);
+      fun->bytes += skip;
+      /* skip padding */
+      if( (skip = FT_BLOCK - (fun->bytes % FT_BLOCK)) == FT_BLOCK )
+	skip = 0;
+      gskip(fun->gio, (off_t)skip);
+      /* reset io flag */
+      /* fun->io = 0; */
+    }
+    /* close filter if done with the filter process */
+    if( fun->filt && (fun->filt != NOFILTER) ){
+      FilterClose(fun->filt);
+      fun->filt = NULL;
+    }
+  }
+
+  /* return buffer */
+  return rowbase;
+}
+
+/*
+ *
+ * public routines
+ *
+ */
+
+/*
+ *
+ * FunTableRowGet -- get rows from a table
+ *
+*/
+#ifdef ANSI_FUNC
+void *
+FunTableRowGet(Fun fun, void *rows, int maxrow, char *plist, int *nrow)
+#else
+void *FunTableRowGet(fun, rows, maxrow, plist, nrow)
+     Fun fun;
+     void *rows;
+     int maxrow;
+     char *plist;
+     int *nrow;
+#endif
+{
+  int got;
+  fd_set readfds;		/* read fds for select() */
+  char *obuf;
+  Fun cur, wrap, mark=NULL;
+
+  /* just in case ... */
+  *nrow = 0;
+  if( !_FunValid(fun) ){
+    gerror(stderr, "invalid data handle\n");
+    return NULL;
+  }
+  
+  /* process list type */
+  switch(fun->ltype){
+  case LIST_NONE:
+    return _FunTableRowGet(fun, rows, maxrow, plist, nrow);
+  case LIST_FILEORDER:
+    return _FunTableRowGet(fun->current, rows, maxrow, plist, nrow);
+  case LIST_SORT:
+  case LIST_UNSORT:
+  case LIST_TRYSORT:
+again:
+    /* if we are sorting -- look for first valid file before we read */
+    if( fun->ltype == LIST_SORT ){
+      for(mark=fun->head; mark; mark=mark->next ){
+	if( mark->left )
+	  break;
+      }
+      /* if we have no valid files left, we are done */
+      if( !mark ){
+	return NULL;
+      }
+      /* if we are sorting and we have current buffers, use one of them */
+      else if( mark->save ){
+	return _FunTableRestore(mark, rows, nrow);
+      }
+    }
+    /* if we have no more active fds to process, select() for more */
+    FD_ZERO(&readfds);
+    for(got=0, cur=fun; cur; cur=cur->next){
+      if( (cur->ifd >=0) && cur->left ){
+	FD_SET(cur->ifd, &readfds);
+	got++;
+      }
+    }
+    /* if we have no more files, we are done */
+    if( got == 0 ) return NULL;
+    /* find out who is ready to be read */
+    got = select(FD_SETSIZE, &readfds, NULL, NULL, NULL);
+    if( got >0 ){
+      /* get current starting place in list */
+      if( fun->current && fun->current->next )
+	cur = fun->current->next;
+      else
+	cur = fun;
+      wrap = cur;
+      while( !FD_ISSET(cur->ifd, &readfds) ){
+	/* next file in list (with wrap around) */
+	if( cur->next )
+	  cur = cur->next;
+	else
+	  cur = fun;
+	/* should not happen -- we went through the whole list unsucessfully */
+	if( cur == wrap ){
+	  return NULL;
+	}
+      }
+      /* read rows */
+      obuf = _FunTableRowGet(cur, rows, maxrow, plist, nrow);
+      /* return if not sorting, or if we are and this file is current */
+      if( fun->ltype != LIST_SORT ){
+	return obuf;
+      }
+      else{
+	/* if the currently accessed file is the one we want, return it */
+	if( fun->current == mark ){
+	  return obuf;
+	}
+	/* else we must save these rows for when this file is current */
+	else{
+	  /* store these rows */
+	  _FunTableSave(fun->current, rows, obuf, *nrow);
+	  /* reset, go back for current */
+	  *nrow = 0;
+	  goto again;
+	}
+      }
+    }
+    else
+      return NULL;
+  default:
+    return NULL;
+  }
+}
+
+/*
+ *
+ * FunTableRowPut -- put rows to a table
+ *
+*/
+#ifdef ANSI_FUNC
+int
+FunTableRowPut(Fun fun, void *rows, int nrow, int idx, char *plist)
+#else
+int FunTableRowPut(fun, rows, nrow, idx, plist)
+     Fun fun;
+     void *rows;
+     int nrow;
+     int idx;
+     char *plist;
+#endif
+{
+  int i;
+  int ieoff1, ieoff2;
+  int to, from;
+  int width;
+  int convert;
+  int shortcut;
+  char *rawptr;
+  char *oebuf;
+  char *rowptr;
+  char *pbuf;
+  char tbuf[SZ_LINE];
+  Fun ifun;
+
+  /* gotta have it */
+  if( !_FunValid(fun) )
+    return 0;
+
+  /* this extension is a table */
+  fun->type = FUN_TABLE;
+
+  /* temp switch this fun's ifun to be the last fun we read in the ifun list */
+  /* we need to do this to pick up the correct raw rawbuf */
+  if( (ifun = fun->ifun) ){
+    if( ifun->current )
+      ifun = ifun->current;
+  }
+  /* no reference fun handle -- rely on the output fun */
+  else
+    ifun = fun;
+
+  /* we have to convert from native if the data is the big-endian */
+  if( fun->oconvert < 0 ){
+    pbuf = xstrdup(plist);
+    if( _FunKeyword(pbuf, "convert", NULL, tbuf, SZ_LINE) )
+      fun->oconvert = istrue(tbuf);
+    else
+      fun->oconvert = !is_bigendian();
+    if( pbuf ) xfree(pbuf);
+  }
+
+  /* initialize for writing */
+  if( !_FunTablePutHeader(fun) )
+    return(0);
+
+  /* exit if we are not actually writing any events */
+  if( !nrow )
+    return(0);
+
+  /* allocate temp buffer for rows */
+  shortcut = 1;
+  for(width=0, i=0; i<fun->ncol; i++){
+    if( (fun->cols[i]->mode  & COL_ACTIVE)     &&
+	(fun->cols[i]->mode  & COL_WRITE)      &&
+        !(fun->cols[i]->mode & COL_REPLACEME) ){
+      /* if we are writing anything from the user buffer, no shortcuts */
+      if( !(fun->cols[i]->mode & COL_IBUF) || (i != fun->cols[i]->from) )
+	shortcut = 0;
+      /* add to the size of the output row record */
+      width += fun->cols[i]->width;
+    }
+    /* if we are not writing anything from the input buffer, no shortcuts */
+    else if( (fun->cols[i]->mode & COL_IBUF) )
+      shortcut = 0;
+  }
+
+  /* make sure we have something to write */
+  if( !width ){
+    gerror(stderr, "No columns defined for FunTableRowPut()\n");
+  }
+
+  /* if we are just writing the input rows, we can do it much faster */
+  if( shortcut ){
+    oebuf = (char *)ifun->rawbuf + (idx * ifun->rawsize); 
+  }
+  /* normally, we have to move all columns to a temp buffer and then write */
+  else{
+    /* write rows (might have to convert to big_endian) */
+    oebuf = xmalloc(width*nrow);
+    rowptr = oebuf;
+    ieoff1 = idx * ifun->rawsize;
+    ieoff2 = 0;
+    /* process rows */
+    for(i=0; i<nrow; i++){
+      /* process columns in a row */
+      for(to=0; to<fun->ncol; to++){
+	/* skip if necessary */
+	if( !(fun->cols[to]->mode & COL_ACTIVE)    ||
+	    !(fun->cols[to]->mode & COL_WRITE)     ||
+	     (fun->cols[to]->mode & COL_REPLACEME) )
+	  continue;
+	/* if from value is -1, that means "use value from this column" */
+	if( fun->cols[to]->from <0 )
+	  from = to;
+	/* otherwise we use the value from some other column */
+	else
+	  from = fun->cols[to]->from;
+	/* raw input data */
+	if( (fun->cols[from]->mode & COL_IBUF) ){
+	  rawptr = (char *)ifun->rawbuf+ieoff1;
+	  convert = !ifun->endian;
+	}
+	/* user data */
+	else{
+	  rawptr = (char *)rows+ieoff2;
+	  convert = fun->oconvert;
+	}
+	if( fun->cols[from]->mode & COL_PTR ){
+	  rawptr = *((char **)(((char *)rawptr)+fun->cols[from]->offset));
+	}
+	else{
+	  rawptr = rawptr + fun->cols[from]->offset;
+	}
+	rawptr += fun->cols[from]->poff;
+	/* for soa, rowbuf (i.e. rowbase) is indexed by nrows */
+	if( fun->org == FUN_ORG_SOA )
+	  rawptr += i * fun->cols[from]->width;
+	/* if source and dest type are different, we do full type convert */
+	if( fun->cols[to]->type != fun->cols[from]->type ){
+	  /* but try to handle writing int to bit-field specially */
+	  if( fun->cols[to]->type == 'X' ){
+	    int xtype;
+	    switch(fun->cols[to]->width/fun->cols[from]->n){
+	    case 0:
+	    case 1:
+	      xtype = 'B'; break;
+	    case 2:
+	      xtype = 'U'; break;
+	    case 4:
+	      xtype = 'V'; break;
+	    default:
+	      xtype = 'B'; break;
+	    }
+	    ft_acht2(xtype, rowptr,
+		     fun->cols[from]->type, rawptr,
+		     fun->cols[from]->n, convert, 1);
+	  }
+	  else{
+	    ft_acht2(fun->cols[to]->type, rowptr, 
+		     fun->cols[from]->type, rawptr,
+		     fun->cols[from]->n, convert, 1);
+	  }
+	}
+	/* for same types, we might have to swap bytes */
+	else if( convert ){
+	  swap(rowptr, rawptr, fun->cols[to]->width, fun->cols[to]->type);
+	}
+	/* otherwise its an easy copy */
+	else{
+	  memcpy(rowptr, rawptr, fun->cols[to]->width);
+	}
+	/* bump output pointer */
+	rowptr += fun->cols[to]->width;
+      }
+      /* bump raw and user row pointers */
+      ieoff1 += ifun->rawsize;
+      if( fun->org == FUN_ORG_AOS )
+	ieoff2 += fun->rowsize;
+    }
+  }
+
+  /* write rows */
+  if( gwrite(fun->gio, oebuf, width, nrow) != (size_t)nrow ){
+    gerror(stderr, "unexpected error writing rows\n");
+    return 0;
+  }
+
+  /* update bookkeeping */
+  fun->io += nrow;
+  fun->bytes += (width*nrow);
+
+  /* free up allocated space */
+  if( !shortcut && oebuf ) xfree(oebuf);
+
+  /* we just wrote a table */
+  fun->ops |= OP_WRTABLE;
+
+  /* return the number of rows output */
+  return nrow;
+}
+
+/*
+ *
+ * FunTableRowSeek -- seek to a specified rows in a table
+ *
+*/
+#ifdef ANSI_FUNC
+off_t
+FunTableRowSeek(Fun fun, int nrow, char *UNUSED(plist))
+#else
+off_t FunTableRowSeek(fun, nrow, plist)
+     Fun fun;
+     int nrow;
+     char *plist;
+#endif
+{
+  off_t ipos, opos;
+
+  /* gotta have it */
+  if( !_FunValid(fun) )
+    return (off_t)-1;
+  /* get new position */
+  ipos = fun->datastart + (fun->rawsize*(nrow-1));
+  /* try to set position */
+  opos = gseek(fun->gio, ipos, 0);
+  /* if successful, reset the byte positon */
+  if( opos >= 0 ){
+    fun->bytes = opos;
+    fun->io = nrow-1;
+    fun->left = fun->total - fun->io;
+    opos = ((opos - fun->datastart) / fun->rawsize) + 1;
+    /* a negative record number only means we started from an odd place */
+    if( opos < 0 ) opos = 0;
+  }
+  /* return results */
+  return opos;
+}