diff options
Diffstat (limited to 'funtools/funtab.c')
-rw-r--r-- | funtools/funtab.c | 1412 |
1 files changed, 1412 insertions, 0 deletions
diff --git a/funtools/funtab.c b/funtools/funtab.c new file mode 100644 index 0000000..5f0a3f3 --- /dev/null +++ b/funtools/funtab.c @@ -0,0 +1,1412 @@ +/* + * Copyright (c) 1999-2003 Smithsonian Astrophysical Observatory + */ + +#include <ctype.h> +#include <funtoolsP.h> + +/* + * + * private routines + * + */ + +#ifdef ANSI_FUNC +static void * +swap(void *obuf, void *ibuf, int width, int type) +#else +static void *swap(obuf, ibuf, width, type) + void *obuf; + void *ibuf; + int width; + int type; +#endif +{ + switch ( type ) { + case 'A': + case 'B': + case 'X': + case 'L': + memcpy(obuf, ibuf, width); + break; + case 'I': + case 'U': + swap2(obuf, ibuf, width); + break; + case 'J': + case 'V': + case 'E': + swap4(obuf, ibuf, width); + break; + case 'K': + case 'D': + swap8(obuf, ibuf, width); + break; + default: + memcpy(obuf, ibuf, width); + break; + } + return obuf; +} + +#ifdef ANSI_FUNC +static void +_FunTableSave(Fun fun, char *rows, char *ebuf, int nrow) +#else +static void _FunTableSave(fun, rows, ebuf, nrow) + Fun fun; + char *rows; + char *ebuf; + int nrow; +#endif +{ + SaveBuf new, cur; + int n; + + /* allocate new data space */ + new = (SaveBuf)xcalloc(1, sizeof(SaveBufRec)); + /* this is how many rows we are saving */ + new->nrow = nrow; + /* see if we can store in memory or if we have to shunt to a file */ + n = nrow * (fun->rowsize + fun->rawsize); + if( (fun->head->lmem < 0) || ((fun->head->lmem-n)>0) ){ + new->type = 1; + /* save alloc'ed raw buf null out buf so funtools does not free it */ + new->rbuf = fun->rawbuf; + fun->rawbuf = NULL; + /* if user passed in a row buffer, we save a copy */ + if( rows ){ + new->ebuf = xmalloc(nrow*fun->rawsize); + memcpy(new->ebuf, ebuf, new->nrow*fun->rowsize); + } + /* otherwise just save funtools-allocated row buffer */ + else{ + new->ebuf = ebuf; + } + /* decrease amount of memory we have left for saved data */ + if( fun->head->lmem > 0 ) + fun->head->lmem -= n; + } + else{ + new->type = 2; + /* write user rows */ + fwrite(ebuf, fun->rowsize, nrow, fun->lefp); + /* free up user row buffer unless user allocated it */ + if( !rows && ebuf ) xfree(ebuf); + /* write raw rows */ + fwrite(fun->rawbuf, fun->rawsize, nrow, fun->lrfp); + /* free up raw buffer */ + if( fun->rawbuf ) xfree(fun->rawbuf); + fun->rawbuf = NULL; + } + /* add to end of list */ + if( fun->save == NULL ){ + fun->save = new; + } + else{ + for(cur=fun->save; cur->next!=NULL; cur=cur->next) + ; + cur->next = new; + } +} + +#ifdef ANSI_FUNC +static char * +_FunTableRestore(Fun fun, char *rows, int *nrow) +#else +static char *_FunTableRestore(fun, rows, nrow) + Fun fun; + char *rows; + int *nrow; +#endif +{ + SaveBuf cur; + char *ebuf=NULL; + unsigned char *rbuf; + + /* start of pessimistically */ + *nrow = 0; + /* make sure we have something to restore */ + if( !fun->save ) + return NULL; + /* bump list to next */ + cur = fun->save; + fun->save = cur->next; + /* fill in the blanks */ + *nrow = cur->nrow; + switch(cur->type){ + case 1: + /* we might have to fill the passed row buffer */ + if( rows ){ + /* copy into user-supplied buffer */ + memcpy(rows, cur->ebuf, cur->nrow*fun->rowsize); + /* we will return user-supplied buffer */ + ebuf = rows; + /* free up temp buffer */ + if(cur->ebuf ) xfree(cur->ebuf); + } + /* otherwise, we just pass back the saved row buffer */ + else{ + ebuf = cur->ebuf; + } + /* move raw rows buf back into funtools struct */ + if( fun->rawbuf ) xfree(fun->rawbuf); + fun->rawbuf = cur->rbuf; + break; + case 2: + /* we might have to fill the passed row buffer */ + if( rows ) + ebuf = rows; + else + ebuf = xmalloc(fun->rowsize*cur->nrow); + /* read into user-supplied buffer */ + fread(ebuf, fun->rowsize, cur->nrow, fun->lefp); + /* allocate space for the raw buffer */ + rbuf = xmalloc(fun->rawsize*cur->nrow); + /* read into raw buffer */ + fread(rbuf, fun->rawsize, cur->nrow, fun->lrfp); + /* move raw rows buf back into funtools struct */ + if( fun->rawbuf ) xfree(fun->rawbuf); + fun->rawbuf = rbuf; + break; + } + /* this is the last file we "read" in the list */ + if( fun->head ) + fun->head->current = fun; + else + fun->current = fun; + if( cur ) xfree(cur); + /* return results */ + return ebuf; +} + +#ifdef ANSI_FUNC +static void +_FunBlank(Fun fun, int type, char *buf) +#else +static void _FunBlank(fun, type, buf) + Fun fun; + int type; + char *buf; +#endif +{ + unsigned char bval; + short sval; + int ival; + longlong lval; + float fval; + double dval; + + switch(type){ + case 'B': + bval = fun->blank; + memcpy(buf, &bval, sizeof(char)); + break; + case 'I': + case 'U': + sval = fun->blank; + memcpy(buf, &sval, sizeof(short)); + break; + case 'J': + case 'V': + ival = fun->blank; + memcpy(buf, &ival, sizeof(int)); + break; + case 'K': +#if HAVE_LONG_LONG == 0 + gerror(stderr, + "long long support was not built into this program\n"); + exit(1); +#endif + lval = fun->blank; + memcpy(buf, &lval, sizeof(longlong)); + break; + case 'E': + fval = getnanf(); + memcpy(buf, &fval, sizeof(float)); + break; + case 'D': + dval = getnand(); + memcpy(buf, &dval, sizeof(double)); + break; + default: + memset(buf, 0, ft_sizeof(type)); + break; + } +} + +#ifdef ANSI_FUNC +static void * +_FunTextRead(Fun fun, char *buf, size_t UNUSED(size), size_t get, size_t *got) +#else +static void *_FunTextRead(fun, buf, size, get, got) + Fun fun; + char *buf; + size_t size; + size_t get; + size_t *got; +#endif +{ + int p, t; + int len; + int dodata1; + int xtype; + int ntoken; + int nline=0; + char lbuf[SZ_LINE]; + char *bp; + Parse parser=NULL; + ParsedLine line; + + /* sanity checks + NB: the order of ascii columns should be identical to the fitsy columns + (since the latter was generated by the former) */ + if( !fun->header || !fun->header->table || !fun->header->table->col ) + return 0; + + /* if we have not read any bytes, use data1 */ + dodata1 = (fun->bytes==0); + + /* convert ASCII columns to binary columns */ + bp = buf; + while( (size_t)nline < get ){ + /* first line should be in data1 */ + if( dodata1 ){ + /* look for the parser with data1 that has the most tokens */ + for(parser=NULL, ntoken=0, p=0; p<fun->nparser; p++){ + if( !(fun->parsers[p]->state & PARSE_STATE_BAD) && + fun->parsers[p]->data1 ){ + if( fun->parsers[p]->ntoken > ntoken ){ + parser = fun->parsers[p]; + ntoken = parser->ntoken; + } + } + } + /* use the parser with the max number of tokens */ + if( parser ){ + line = parser->data1; + if( PARSE_ISCOMMENT(line) ) continue; + for(t=0; t<line->ntoken; t++){ + switch(line->tokens[t].type){ + case PARSE_FLOAT: + case PARSE_HEXINT: + case PARSE_INTEGER: + /* handle string type specially */ + if( fun->header->table->col[t].type == 'A' ){ + len = strlen(line->tokens[t].sval); + if( len >= fun->header->table->col[t].width){ + memmove(bp, line->tokens[t].sval, + fun->header->table->col[t].width); + } + else{ + memmove(bp, line->tokens[t].sval, len); + memset(bp+len, 0, fun->header->table->col[t].width-len); + } + } + /* handle bit-field processing specially */ + else if( fun->header->table->col[t].type == 'X' ){ + switch(fun->header->table->col[t].width){ + case 0: + case 1: + xtype = 'B'; break; + case 2: + xtype = 'U'; break; + case 4: + xtype = 'V'; break; + default: + xtype = 'B'; break; + } + if( (line->tokens[t].type == PARSE_INTEGER) || + (line->tokens[t].type == PARSE_HEXINT) ) + ft_acht2(xtype, bp, 'K', &line->tokens[t].lval, 1, 0, 0); + else + ft_acht2(xtype, bp, 'D', &line->tokens[t].dval, 1, 0, 0); + } + else{ + if( (line->tokens[t].type == PARSE_INTEGER) || + (line->tokens[t].type == PARSE_HEXINT) ) + ft_acht2(fun->header->table->col[t].type, bp, + 'K', &line->tokens[t].lval, 1, 0, 0); + else + ft_acht2(fun->header->table->col[t].type, bp, + 'D', &line->tokens[t].dval, 1, 0, 0); + } + bp += fun->header->table->col[t].width; + break; + case PARSE_NULL: + _FunBlank(fun, fun->header->table->col[t].type, bp); + bp += fun->header->table->col[t].width; + break; + case PARSE_STRING: + len = strlen(line->tokens[t].sval); + if( len >= fun->header->table->col[t].width){ + memmove(bp, line->tokens[t].sval, + fun->header->table->col[t].width); + } + else{ + memmove(bp, line->tokens[t].sval, len); + memset(bp+len, 0, fun->header->table->col[t].width-len); + } + bp += fun->header->table->col[t].width; + break; + default: + break; + } + } + /* processed data1 line */ + nline++; + } + /* went through all parsers looking for data1 */ + dodata1 = 0; + } + /* read and process the next line */ + else{ + /* get next line or be done */ + if( !ggets(fun->gio, lbuf, SZ_LINE) ) + break; + /* analyze line and make sure one parser succeeded (even if its EOT) */ + if( !ParseAnalyze(fun->parsers, fun->nparser, lbuf) ){ + gerror(stderr, "text parser failure analyzing line: %s\n", lbuf); + break; + } + /* look for valid parser with the most tokens */ + for(parser=NULL, ntoken=0, parser=NULL, p=0; p<fun->nparser; p++){ + if( fun->parsers[p]->state & PARSE_STATE_BAD) + continue; + else if( fun->parsers[p]->ntoken > ntoken ){ + parser = fun->parsers[p]; + ntoken = parser->ntoken; + } + } + /* stop if no valid parser or valid parser found EOT */ + if( !parser ) break; + if( parser->state & PARSE_STATE_EOT) break; + /* use selected parser to fill up row buffer */ + line = parser->cur; + if( PARSE_ISCOMMENT(line) ) continue; + for(t=0; t<line->ntoken; t++){ + switch(line->tokens[t].type){ + case PARSE_FLOAT: + case PARSE_HEXINT: + case PARSE_INTEGER: + /* handle string type specially */ + if( fun->header->table->col[t].type == 'A' ){ + len = strlen(line->tokens[t].sval); + if( len >= fun->header->table->col[t].width){ + memmove(bp, line->tokens[t].sval, + fun->header->table->col[t].width); + } + else{ + memmove(bp, line->tokens[t].sval, len); + memset(bp+len, 0, fun->header->table->col[t].width-len); + } + } + /* handle bit-field processing specially */ + else if( fun->header->table->col[t].type == 'X' ){ + switch(fun->header->table->col[t].width){ + case 0: + case 1: + xtype = 'B'; break; + case 2: + xtype = 'U'; break; + case 4: + xtype = 'V'; break; + default: + xtype = 'B'; break; + } + if( (line->tokens[t].type == PARSE_INTEGER) || + (line->tokens[t].type == PARSE_HEXINT) ) + ft_acht2(xtype, bp, 'K', &line->tokens[t].lval, 1, 0, 0); + else + ft_acht2(xtype, bp, 'D', &line->tokens[t].dval, 1, 0, 0); + } + else{ + if( (line->tokens[t].type == PARSE_INTEGER) || + (line->tokens[t].type == PARSE_HEXINT) ) + ft_acht2(fun->header->table->col[t].type, bp, + 'K', &line->tokens[t].lval, 1, 0, 0); + else + ft_acht2(fun->header->table->col[t].type, bp, + 'D', &line->tokens[t].dval, 1, 0, 0); + } + bp += fun->header->table->col[t].width; + break; + case PARSE_STRING: + /* handle string type specially */ + if( fun->header->table->col[t].type == 'A' ){ + len = strlen(line->tokens[t].sval); + if( len >= fun->header->table->col[t].width){ + memmove(bp, line->tokens[t].sval, + fun->header->table->col[t].width); + } + else{ + memmove(bp, line->tokens[t].sval, len); + memset(bp+len, 0, fun->header->table->col[t].width-len); + } + } + bp += fun->header->table->col[t].width; + break; + case PARSE_NULL: + _FunBlank(fun, fun->header->table->col[t].type, bp); + bp += fun->header->table->col[t].width; + break; + default: + break; + } + } + /* processed another line */ + nline++; + } + } + *got = nline; + return buf; +} + +/* + * + * semi-public routines + * + */ + +#ifdef ANSI_FUNC +void * +_FunRead(Fun fun, char *buf, size_t size, size_t get, size_t *got) +#else +void *_FunRead(fun, buf, size, get, got) + Fun fun; + char *buf; + size_t size; + size_t get; + size_t *got; +#endif +{ + /* initialize */ + *got = 0; + + /* sanity check */ + if( !fun || !buf ) return NULL; + + /* text parsers are handled one line at a time */ + if( fun->parsers ){ + return _FunTextRead(fun, buf, size, get, got); + } + /* binary reads */ + else{ + /* ordinary binary read */ + if( (fun->filt == NOFILTER) || (fun->filt->doidx != 1) ){ + return _gread(fun->gio, (void *)buf, size, get, got); + } + /* indexed read */ + else{ + return idxread(fun->filt->idx, fun->gio, fun->header, + (void *)buf, size, get, got, &fun->dofilt); + } + } +} + +/* + * + * _FunFixNaxis2 -- change the value of an NAXIS2 card + * used to correct a copied binary table after we filter + * + */ +#ifdef ANSI_FUNC +int +_FunFixNaxis2(Fun fun) +#else +int _FunFixNaxis2(fun) + Fun fun; +#endif +{ + if( (fun->ops & OP_WRHEAD) && (fun->headpos >=0) ){ + return FunParamPuti(fun, + "NAXIS", 2, fun->io, "Number of entries in table", 0); + } + else{ + return(0); + } +} + +/* + * + * _FunTablePutHeader -- writes binary table header to file + * used by FunTablePut and sometimes FunFlush (if no Put was done) + * + */ +#ifdef ANSI_FUNC +int +_FunTablePutHeader(Fun fun) +#else +int _FunTablePutHeader(fun) + Fun fun; +#endif +{ + /* write table header, if necessary */ + if( !fun->ops ){ + /* if we have no columns yet, try to get some from the assoc. input file */ + if( !fun->ncol ){ + /* if we have a header to copy from, that is */ + if( fun->ifun && fun->ifun->header && fun->ifun->header->table ){ + FunColumnSelect(fun, 0, "copy=select", NULL); + /* grab the copied merge value */ + fun->merge = fun->ifun->merge; + /* and the organization value */ + fun->org = fun->ifun->org; + } + /* else we lose */ + else + return 0; + } + + /* write the dummy primary header, if necessary */ + _FunPrimaryExtension(fun); + + /* generate the fitsy header for specified columns */ + _FunColumnHeader(fun, fun->cols, fun->ncol); + + /* save file position for later updating */ + fun->headpos = gtell(fun->gio); + + /* write the binary table header */ + ft_headwrite(fun->gio, fun->header); + + /* we just wrote the header */ + fun->ops |= OP_WRHEAD; + } + return 1; +} + +/* + * + * _FunTableRowGet -- get rows from a table + * +*/ +#ifdef ANSI_FUNC +void * +_FunTableRowGet(Fun fun, void *rows, int maxrow, char *plist, int *nrow) +#else +void *_FunTableRowGet(fun, rows, maxrow, plist, nrow) + Fun fun; + void *rows; + int maxrow; + char *plist; + int *nrow; +#endif +{ + int i, j, k, l; /* counters */ + int tival; /* temp int value */ + int dofilt; /* true if we can filter */ + int skip; /* bytes to skip to end of extension */ + int *rbuf; /* valid row flags */ + int rowbufsize; /* alloc'ed output buffer size */ + int maxbufsize; /* max alloc'ed output buffer size */ + int smax; /* max elements in sbuf */ + int pad=sizeof(double); /* double boundary pad bytes */ + size_t get; /* number of rows to read */ + size_t got; /* number of rows read */ + double *sbuf=NULL; /* temp buffer for scaling */ + unsigned char *vheap; /* vheap space for output vector allocation */ + unsigned char *rawbase=NULL; /* base pointer into rawbuf */ + unsigned char *rawptr; /* current pointer into rawbuf */ + unsigned char *rowbase=NULL; /* base pointer to output row buf */ + unsigned char *rowptr; /* current pointer into output row buf */ + char *rawoff; /* temp pointer to input */ + char *rowoff; /* temp pointer to output */ + char *pbuf; /* temp plist buffer */ + char tbuf[SZ_LINE]; /* ever-present temp buffer */ + static char paint[SZ_LINE]; /* paint mode (like zhtools) */ + static char debug[SZ_LINE]; + static int rconvert; + + /* init result */ + *nrow = 0; + rbuf = NULL; + + /* just in case ... */ + if( !_FunValid(fun) ){ + gerror(stderr, "invalid funtools handle\n"); + return NULL; + } + + /* if we delayed the open before, we have to open now */ + if( !fun->header && strchr(fun->mode, 'r') ) + _FunFITSOpen(fun, fun->fname, "r"); + + /* make sure something good happened */ + if( !fun->header || !fun->header->table ){ + gerror(stderr, "not a binary table\n"); + return NULL; + } + + /* check plist values: convert, mask ... */ + if( fun->iconvert < 0 ){ + rconvert = 0; + fun->transparent = 0; + pbuf = xstrdup(plist); + if( _FunKeyword(pbuf, "convert", NULL, tbuf, SZ_LINE) ){ + fun->iconvert = istrue(tbuf); + if( fun->iconvert != is_bigendian() ){ + rconvert = 1; + } + } + else{ + fun->iconvert = (fun->endian != is_bigendian()); + } + if( _FunKeyword(pbuf, "mask", "FUN_MASK", tbuf, SZ_LINE) ){ + if( !strcasecmp(tbuf, "transparent") ) + fun->transparent = 1; + } + if( _FunKeyword(pbuf, "paint", "FILTER_PAINT", tbuf, SZ_LINE) ){ + strncpy(paint, tbuf, SZ_LINE); + } + else{ + *paint = '\0'; + } + if( _FunKeyword(pbuf, "debug", "FILTER_DEBUG", tbuf, SZ_LINE) ){ + strncpy(debug, tbuf, SZ_LINE); + } + else{ + *debug = '\0'; + } + if( pbuf ) xfree(pbuf); + } + + /* check for no more row to read in this file */ + if( !fun->left ){ + /* if possible, set up to read next file in list */ + if((fun->head) && (fun->head->ltype != LIST_NONE) && fun->head->current){ + fun->head->current = fun->next; + /* if there is a next file, set it up */ + if( fun->head->current ){ + /* rewind save file pointers */ + rewind(fun->head->current->lefp); + rewind(fun->head->current->lrfp); + /* and if we read no rows but we have set up a next file, + try again immediately */ + return FunTableRowGet(fun->head, rows, maxrow, plist, nrow); + } + } + return NULL; + } + + /* open the filter, if its not already been done */ + if( !fun->filt ){ + if( fun->filter && *fun->filter ){ + /* create the mode string */ + if( fun->endian != is_bigendian() ) + strncpy(tbuf, "convert=true", SZ_LINE-1); + else + strncpy(tbuf, "convert=false", SZ_LINE-1); + /* add the binning key */ + if( fun->bincols ){ + strncat(tbuf, ",", SZ_LINE-1); + strncat(tbuf, fun->bincols, SZ_LINE-1); + } + /* add the index flag */ + if( fun->idx == 1 ){ + strncat(tbuf, ",idx=true", SZ_LINE-1); + } + else if( fun->idx == -1 ){ + strncat(tbuf, ",idx=false", SZ_LINE-1); + } + /* add paint mode */ + if( *paint ){ + strncat(tbuf, ",", SZ_LINE-1); + strncat(tbuf, "paint=", SZ_LINE-1); + strncat(tbuf, paint, SZ_LINE-1); + } + /* add debug flag */ + if( *debug ){ + strcat(tbuf, ","); + strcat(tbuf, "debug="); + strcat(tbuf, debug); + } + fun->filt = FilterOpen(fun->header, fun->filter, tbuf); + if( fun->filt == NULL ){ + gerror(stderr, "can't open filter\n"); + return NULL; + } + else if( fun->filt == NOFILTER ){ + fun->dofilt = 0; + } + } + else{ + fun->filt = NOFILTER; + fun->dofilt = 0; + } + } + + /* sanity check (after setup, to allow setup without actually reading) */ + if( maxrow <=0 ) return NULL; + + /* allocate output buffer, if necessary */ + if( rows ){ + rowbase = rows; + } + else{ + maxbufsize = fun->maxbufsize; + rowbufsize = fun->rowsize + pad; + /* we might have to allocate space for pointers as well */ + for(j=0; j<fun->ncol; j++){ + if( fun->cols[j]->mode & COL_PTR ){ + if( fun->cols[j]->poff && (fun->cols[j]->mode & COL_READ) ){ + gerror(stderr, + "FunTableRowGet: [poff] not yet supported for allocated pointers\n"); + } + rowbufsize += (fun->cols[j]->width + pad); + } + } + /* ensure that we don't try to allocate too large a buffer, or we might + swap our brains out and then die anyway */ + if( maxrow * rowbufsize > maxbufsize ){ + if( rowbufsize > maxbufsize ) + maxrow = 1; + else + maxrow = maxbufsize / rowbufsize; + } + if( maxrow <= 0 ) maxrow = 1; + /* don't call xcalloc (which has internal error checking) but instead + call system calloc and handle failure by halving the row number */ + while( !(rowbase = (unsigned char *)malloc(maxrow*rowbufsize+pad)) ){ + maxrow /= 2; + if( maxrow <= 0 ){ + gerror(stderr, "can't allocate row buffer\n"); + return NULL; + } + } + /* clear buffer */ + memset(rowbase, 0, maxrow*rowbufsize+pad); + /* the vheap for pointers starts after row buffer */ + vheap = rowbase + (maxrow * fun->rowsize); + /* make sure its on a double boundary */ + vheap += (pad - ((long)vheap % pad)); + /* set up the allocated pointers in each record */ + for(j=0; j<fun->ncol; j++){ + if( fun->cols[j]->mode & COL_PTR ){ + for(i=0; i<maxrow; i++){ + rowptr = rowbase + (i*fun->rowsize); + *((unsigned char **)(rowptr+fun->cols[j]->offset)) = vheap; + vheap += fun->cols[j]->width + (pad - (fun->cols[j]->width % pad)); + if( vheap >= (rowbase + (maxrow*rowbufsize+pad)) ){ + gerror(stderr, "internal funtools error: vheap too small\n"); + return NULL; + } + } + } + } + } + + /* free previously saved buffer of raw input data */ + if( fun->rawbuf ) xfree(fun->rawbuf); + /* allocate space for a pile of row records */ + if( fun->left > 0 ) + fun->rawbufsize = MIN(maxrow, fun->left); + /* this is the case where we read til EOF */ + else + fun->rawbufsize = maxrow; + if( !(fun->rawbuf=(unsigned char *)xmalloc(fun->rawsize*fun->rawbufsize)) || + !(rbuf = (int *)xmalloc(sizeof(int)*fun->rawbufsize)) ){ + gerror(stderr, "can't allocate row buffer\n"); + return NULL; + } + + /* allocate a scaling buffer for the largest input or output column */ + smax = -1; + for(j=0; j<fun->ncol; j++){ + smax = MAX(fun->cols[j]->n, smax); + } + if( smax >0 ) + sbuf = (double *)malloc(smax*sizeof(double)); + + /* seed output pointer to start of output buffer */ + rowptr = rowbase; + + /* while there are still rows to read and we have not read the max ... */ + while( (fun->left != 0) && (*nrow < maxrow) ){ + /* figure out how many to read this time */ + if( fun->left > 0 ) + get = MIN(maxrow - *nrow, fun->left); + /* read to EOF */ + else + get = maxrow - *nrow; + /* read in a pile of rows AFTER the currently saved raw rows */ + rawbase = fun->rawbuf + (*nrow * fun->rawsize); + _FunRead(fun, (void *)rawbase, fun->rawsize, get, &got); + /* if we got what we asked for ... */ + if( got == get ){ + /* if we are reading a set number of rows ... */ + if( fun->left > 0 ){ + /* subtract the number we just got */ + fun->left -= got; + /* just in case! */ + if( fun->left < 0 ) + fun->left = 0; + } + } + /* didn't get what we asked for ... */ + else{ + /* if we are reading a set number of rows ... */ + if( fun->left > 0 ){ + /* using an index => already filtering, so fewer rows are acceptable */ + if( (fun->filt == NOFILTER) || (fun->filt->doidx != 1) ){ + /* otherwise its unexpected */ + gerror(stderr, "unexpected EOF reading rows\n"); + } + /* ensure we stop after this iteration */ + fun->left = 0; + } + /* if we are reading til EOF, we just got there */ + else{ + fun->left = 0; + } + } + + /* filter these rows through the co-process */ + if( fun->dofilt ){ + dofilt = FilterEvents(fun->filt, (void *)rawbase, fun->rawsize, (int)got, + (void *)rbuf); + /* look for error on filter call */ + if( dofilt < 0 ){ + gerror(stderr, "event filter failed\n"); + } + } + else{ + dofilt = 0; + } + + /* process each row in this batch */ + for(rawptr=rawbase, i=0; (size_t)i<got; i++, rawptr += fun->rawsize){ + /* if its not a valid row, skip it */ + if( dofilt && !fun->transparent && !rbuf[i] ){ + continue; + } + /* get specified columns, convert to type, and store in output */ + for(j=0; j<fun->ncol; j++){ + /* if this column is not active, or is not to be read, or if its + to be taken from the input data, skip it */ + if(!(fun->cols[j]->mode & COL_ACTIVE) || + !(fun->cols[j]->mode & COL_READ) || + (fun->cols[j]->mode & COL_IBUF) ) + continue; + /* this is the (input) fitsy column number */ + k = fun->cols[j]->tcol; + if( fun->cols[j]->mode & COL_PTR ) + rowoff = *((char **)(((char *)rowptr)+fun->cols[j]->offset)); + else + rowoff = (char *)rowptr+fun->cols[j]->offset; + rowoff += fun->cols[j]->poff; + /* for soa, rowbuf (i.e. rowbase) is indexed by nrows */ + if( fun->org == FUN_ORG_SOA ) + rowoff += *nrow * fun->cols[j]->width; + /* process special values, or data values */ + switch(k){ + /* unknown means mode is "rw" but we don't actually have the col + in the input data -- do nothing */ + case COL_UNKNOWN_ID: + break; + /* $region: the special region id */ + case COL_REGION_ID: + case COL_NUM_ID: + if( k == COL_REGION_ID ){ + /* we either use the return value or -1 (no region filtering) */ + if( dofilt ) + tival = rbuf[i]; + else + tival = COL_REGION_ID; + } + if( k == COL_NUM_ID ){ + tival = fun->io + i + 1; + } + /* common code */ + /* if source and dest type diff, we must do full type convert */ + if( fun->cols[j]->type != 'J' ){ + ft_acht2(fun->cols[j]->type, rowoff, 'J', &tival, 1, rconvert, 0); + } + /* for same types, we might have to swap bytes */ + else if( rconvert ){ + swap(rowoff, &tival, ft_sizeof('J'), 'J'); + } + /* otherwise its an easy copy */ + else{ + memcpy(rowoff, &tival, ft_sizeof('J')); + } + break; + /* data column */ + default: + /* this is the offset into the input record */ + rawoff = (char *)rawptr + fun->header->table->col[k].offset; + /* if source and dest type diff or we are scaling the column, + we must do full type convert */ + if( (fun->cols[j]->type != fun->header->table->col[k].type) || + fun->header->table->col[k].scaled ){ + /* handle bit-field processing specially */ + if( fun->header->table->col[k].type == 'X' ){ + int xtype; + switch(fun->header->table->col[k].width/fun->cols[j]->n){ + case 0: + case 1: + xtype = 'B'; break; + case 2: + xtype = 'U'; break; + case 4: + xtype = 'V'; break; + default: + xtype = 'B'; break; + } + ft_acht2(fun->cols[j]->type, rowoff, + xtype, rawoff, + fun->cols[j]->n, fun->iconvert, 0); + } + /* handle scaled input data */ + else if( fun->header->table->col[k].scaled ){ + /* convert raw to double */ + ft_acht2('D', sbuf, + fun->header->table->col[k].type, rawoff, + fun->cols[j]->n, fun->iconvert, 0); + /* calculate scaled value */ + for(l=0; l<fun->cols[j]->n; l++){ + sbuf[l] = sbuf[l] * fun->header->table->col[k].scale + + fun->header->table->col[k].zero; + } + /* convert double to type */ + ft_acht2(fun->cols[j]->type, rowoff, + 'D', sbuf, fun->cols[j]->n, 0, 0); + /* mark column as having been scaled */ + fun->cols[j]->scaled |= FUN_SCALE_APPLIED; + } + /* handle straight-forward change of type */ + else{ + ft_acht2(fun->cols[j]->type, rowoff, + fun->header->table->col[k].type, rawoff, + fun->cols[j]->n, fun->iconvert, 0); + } + } + /* for same types, we might have to swap bytes */ + else if( fun->iconvert ){ + if( fun->header->table->col[k].width >= fun->cols[j]->width ){ + swap(rowoff, rawoff, fun->cols[j]->width, fun->cols[j]->type); + } + else{ + swap(rowoff, rawoff, fun->header->table->col[k].width, + fun->cols[j]->type); + memset(rowoff+fun->header->table->col[k].width, 0, + fun->cols[j]->width-fun->header->table->col[k].width); + } + } + /* otherwise its an easy copy */ + else{ + if( fun->header->table->col[k].width >= fun->cols[j]->width ){ + memcpy(rowoff, rawoff, fun->cols[j]->width); + } + else{ + memcpy(rowoff, rawoff, fun->header->table->col[k].width); + memset(rowoff+fun->header->table->col[k].width, 0, + fun->cols[j]->width-fun->header->table->col[k].width); + } + } + break; + } + } + /* save the input data in rawbuf */ + if( (fun->rawbuf + (*nrow * fun->rawsize)) != rawptr) + memcpy((fun->rawbuf + (*nrow * fun->rawsize)), rawptr, fun->rawsize); + /* got another row */ + *nrow += 1; + /* bump to next output pointer */ + if( fun->org == FUN_ORG_AOS ) + rowptr += fun->rowsize; + } + /* update how many input rows we have read */ + fun->io += got; + fun->bytes += (got*fun->rawsize); + } + + /* free up space */ + if( rbuf ) xfree(rbuf); + if( sbuf ) xfree(sbuf); + + /* we just read a table */ + fun->ops |= OP_RDTABLE; + /* this is the last file we read in the list */ + if( fun->head ) + fun->head->current = fun; + else + fun->current = fun; + + /* if we have no more rows left, clean up */ + if( !fun->left ){ + /* skip to end of extension */ + if( fun->bytes ){ + /* skip any unread rows */ + skip = (fun->rawsize*fun->total) - fun->bytes; + gskip(fun->gio, (off_t)skip); + fun->bytes += skip; + /* skip padding */ + if( (skip = FT_BLOCK - (fun->bytes % FT_BLOCK)) == FT_BLOCK ) + skip = 0; + gskip(fun->gio, (off_t)skip); + /* reset io flag */ + /* fun->io = 0; */ + } + /* close filter if done with the filter process */ + if( fun->filt && (fun->filt != NOFILTER) ){ + FilterClose(fun->filt); + fun->filt = NULL; + } + } + + /* return buffer */ + return rowbase; +} + +/* + * + * public routines + * + */ + +/* + * + * FunTableRowGet -- get rows from a table + * +*/ +#ifdef ANSI_FUNC +void * +FunTableRowGet(Fun fun, void *rows, int maxrow, char *plist, int *nrow) +#else +void *FunTableRowGet(fun, rows, maxrow, plist, nrow) + Fun fun; + void *rows; + int maxrow; + char *plist; + int *nrow; +#endif +{ + int got; + fd_set readfds; /* read fds for select() */ + char *obuf; + Fun cur, wrap, mark=NULL; + + /* just in case ... */ + *nrow = 0; + if( !_FunValid(fun) ){ + gerror(stderr, "invalid data handle\n"); + return NULL; + } + + /* process list type */ + switch(fun->ltype){ + case LIST_NONE: + return _FunTableRowGet(fun, rows, maxrow, plist, nrow); + case LIST_FILEORDER: + return _FunTableRowGet(fun->current, rows, maxrow, plist, nrow); + case LIST_SORT: + case LIST_UNSORT: + case LIST_TRYSORT: +again: + /* if we are sorting -- look for first valid file before we read */ + if( fun->ltype == LIST_SORT ){ + for(mark=fun->head; mark; mark=mark->next ){ + if( mark->left ) + break; + } + /* if we have no valid files left, we are done */ + if( !mark ){ + return NULL; + } + /* if we are sorting and we have current buffers, use one of them */ + else if( mark->save ){ + return _FunTableRestore(mark, rows, nrow); + } + } + /* if we have no more active fds to process, select() for more */ + FD_ZERO(&readfds); + for(got=0, cur=fun; cur; cur=cur->next){ + if( (cur->ifd >=0) && cur->left ){ + FD_SET(cur->ifd, &readfds); + got++; + } + } + /* if we have no more files, we are done */ + if( got == 0 ) return NULL; + /* find out who is ready to be read */ + got = select(FD_SETSIZE, &readfds, NULL, NULL, NULL); + if( got >0 ){ + /* get current starting place in list */ + if( fun->current && fun->current->next ) + cur = fun->current->next; + else + cur = fun; + wrap = cur; + while( !FD_ISSET(cur->ifd, &readfds) ){ + /* next file in list (with wrap around) */ + if( cur->next ) + cur = cur->next; + else + cur = fun; + /* should not happen -- we went through the whole list unsucessfully */ + if( cur == wrap ){ + return NULL; + } + } + /* read rows */ + obuf = _FunTableRowGet(cur, rows, maxrow, plist, nrow); + /* return if not sorting, or if we are and this file is current */ + if( fun->ltype != LIST_SORT ){ + return obuf; + } + else{ + /* if the currently accessed file is the one we want, return it */ + if( fun->current == mark ){ + return obuf; + } + /* else we must save these rows for when this file is current */ + else{ + /* store these rows */ + _FunTableSave(fun->current, rows, obuf, *nrow); + /* reset, go back for current */ + *nrow = 0; + goto again; + } + } + } + else + return NULL; + default: + return NULL; + } +} + +/* + * + * FunTableRowPut -- put rows to a table + * +*/ +#ifdef ANSI_FUNC +int +FunTableRowPut(Fun fun, void *rows, int nrow, int idx, char *plist) +#else +int FunTableRowPut(fun, rows, nrow, idx, plist) + Fun fun; + void *rows; + int nrow; + int idx; + char *plist; +#endif +{ + int i; + int ieoff1, ieoff2; + int to, from; + int width; + int convert; + int shortcut; + char *rawptr; + char *oebuf; + char *rowptr; + char *pbuf; + char tbuf[SZ_LINE]; + Fun ifun; + + /* gotta have it */ + if( !_FunValid(fun) ) + return 0; + + /* this extension is a table */ + fun->type = FUN_TABLE; + + /* temp switch this fun's ifun to be the last fun we read in the ifun list */ + /* we need to do this to pick up the correct raw rawbuf */ + if( (ifun = fun->ifun) ){ + if( ifun->current ) + ifun = ifun->current; + } + /* no reference fun handle -- rely on the output fun */ + else + ifun = fun; + + /* we have to convert from native if the data is the big-endian */ + if( fun->oconvert < 0 ){ + pbuf = xstrdup(plist); + if( _FunKeyword(pbuf, "convert", NULL, tbuf, SZ_LINE) ) + fun->oconvert = istrue(tbuf); + else + fun->oconvert = !is_bigendian(); + if( pbuf ) xfree(pbuf); + } + + /* initialize for writing */ + if( !_FunTablePutHeader(fun) ) + return(0); + + /* exit if we are not actually writing any events */ + if( !nrow ) + return(0); + + /* allocate temp buffer for rows */ + shortcut = 1; + for(width=0, i=0; i<fun->ncol; i++){ + if( (fun->cols[i]->mode & COL_ACTIVE) && + (fun->cols[i]->mode & COL_WRITE) && + !(fun->cols[i]->mode & COL_REPLACEME) ){ + /* if we are writing anything from the user buffer, no shortcuts */ + if( !(fun->cols[i]->mode & COL_IBUF) || (i != fun->cols[i]->from) ) + shortcut = 0; + /* add to the size of the output row record */ + width += fun->cols[i]->width; + } + /* if we are not writing anything from the input buffer, no shortcuts */ + else if( (fun->cols[i]->mode & COL_IBUF) ) + shortcut = 0; + } + + /* make sure we have something to write */ + if( !width ){ + gerror(stderr, "No columns defined for FunTableRowPut()\n"); + } + + /* if we are just writing the input rows, we can do it much faster */ + if( shortcut ){ + oebuf = (char *)ifun->rawbuf + (idx * ifun->rawsize); + } + /* normally, we have to move all columns to a temp buffer and then write */ + else{ + /* write rows (might have to convert to big_endian) */ + oebuf = xmalloc(width*nrow); + rowptr = oebuf; + ieoff1 = idx * ifun->rawsize; + ieoff2 = 0; + /* process rows */ + for(i=0; i<nrow; i++){ + /* process columns in a row */ + for(to=0; to<fun->ncol; to++){ + /* skip if necessary */ + if( !(fun->cols[to]->mode & COL_ACTIVE) || + !(fun->cols[to]->mode & COL_WRITE) || + (fun->cols[to]->mode & COL_REPLACEME) ) + continue; + /* if from value is -1, that means "use value from this column" */ + if( fun->cols[to]->from <0 ) + from = to; + /* otherwise we use the value from some other column */ + else + from = fun->cols[to]->from; + /* raw input data */ + if( (fun->cols[from]->mode & COL_IBUF) ){ + rawptr = (char *)ifun->rawbuf+ieoff1; + convert = !ifun->endian; + } + /* user data */ + else{ + rawptr = (char *)rows+ieoff2; + convert = fun->oconvert; + } + if( fun->cols[from]->mode & COL_PTR ){ + rawptr = *((char **)(((char *)rawptr)+fun->cols[from]->offset)); + } + else{ + rawptr = rawptr + fun->cols[from]->offset; + } + rawptr += fun->cols[from]->poff; + /* for soa, rowbuf (i.e. rowbase) is indexed by nrows */ + if( fun->org == FUN_ORG_SOA ) + rawptr += i * fun->cols[from]->width; + /* if source and dest type are different, we do full type convert */ + if( fun->cols[to]->type != fun->cols[from]->type ){ + /* but try to handle writing int to bit-field specially */ + if( fun->cols[to]->type == 'X' ){ + int xtype; + switch(fun->cols[to]->width/fun->cols[from]->n){ + case 0: + case 1: + xtype = 'B'; break; + case 2: + xtype = 'U'; break; + case 4: + xtype = 'V'; break; + default: + xtype = 'B'; break; + } + ft_acht2(xtype, rowptr, + fun->cols[from]->type, rawptr, + fun->cols[from]->n, convert, 1); + } + else{ + ft_acht2(fun->cols[to]->type, rowptr, + fun->cols[from]->type, rawptr, + fun->cols[from]->n, convert, 1); + } + } + /* for same types, we might have to swap bytes */ + else if( convert ){ + swap(rowptr, rawptr, fun->cols[to]->width, fun->cols[to]->type); + } + /* otherwise its an easy copy */ + else{ + memcpy(rowptr, rawptr, fun->cols[to]->width); + } + /* bump output pointer */ + rowptr += fun->cols[to]->width; + } + /* bump raw and user row pointers */ + ieoff1 += ifun->rawsize; + if( fun->org == FUN_ORG_AOS ) + ieoff2 += fun->rowsize; + } + } + + /* write rows */ + if( gwrite(fun->gio, oebuf, width, nrow) != (size_t)nrow ){ + gerror(stderr, "unexpected error writing rows\n"); + return 0; + } + + /* update bookkeeping */ + fun->io += nrow; + fun->bytes += (width*nrow); + + /* free up allocated space */ + if( !shortcut && oebuf ) xfree(oebuf); + + /* we just wrote a table */ + fun->ops |= OP_WRTABLE; + + /* return the number of rows output */ + return nrow; +} + +/* + * + * FunTableRowSeek -- seek to a specified rows in a table + * +*/ +#ifdef ANSI_FUNC +off_t +FunTableRowSeek(Fun fun, int nrow, char *UNUSED(plist)) +#else +off_t FunTableRowSeek(fun, nrow, plist) + Fun fun; + int nrow; + char *plist; +#endif +{ + off_t ipos, opos; + + /* gotta have it */ + if( !_FunValid(fun) ) + return (off_t)-1; + /* get new position */ + ipos = fun->datastart + (fun->rawsize*(nrow-1)); + /* try to set position */ + opos = gseek(fun->gio, ipos, 0); + /* if successful, reset the byte positon */ + if( opos >= 0 ){ + fun->bytes = opos; + fun->io = nrow-1; + fun->left = fun->total - fun->io; + opos = ((opos - fun->datastart) / fun->rawsize) + 1; + /* a negative record number only means we started from an odd place */ + if( opos < 0 ) opos = 0; + } + /* return results */ + return opos; +} |