Merge commit '23c7930d27fe11c4655e1291a07a821dbbaba78d' as 'funtools'

7 files changed, 500 insertions, 0 deletions

diff --git a/funtools/notes/ds9.notes b/funtools/notes/ds9.notes
new file mode 100644
index 0000000..2d947d5
--- /dev/null
+++ b/funtools/notes/ds9.notes
@@ -0,0 +1,90 @@
+		Extensions to Analysis specification syntax
+		   to support mainlib (internal) analysis
+
+# add user-specified name/value pair for analysis macro expansion
+set LIB $HOME/lib
+
+# xtype: for backward compatibility, default has type=external,
+# i.e. execute current sh -c "cmd"
+menu
+menu xtype=external
+
+#  internal uses mainlib, requires package name and (maybe) shared object name
+menu xtype=internal package=funtools source=static
+menu xtype=internal package=funtools source=$LIB/libfuntoolsMainLib.dylib
+
+bind x xtype=internal package=funtools source=static
+bind x xtype=internal package=funtools source=$LIB/libfuntoolsMainLib.dylib
+
+Notes:
+
+1. set name val
+
+Add user-specified name and value to the macro expansion.  Not
+specific to internal processing but will help with shared object pathnames.
+
+2. xtype=internal
+
+Execute a mainlib action by performing macro subsititution, generating
+an in-memory FITS file if necessary, and then calling:
+
+	set result [mainlib "command"]
+
+Question: what triggers generation of in-memory FITS? Is it $file or a new
+keyword?
+
+3. source=static package=<name>
+
+Valid for xtype=internal only.
+
+Assumes that a routine named <name>MainLibInit() is linked into
+ds9. The name and entry point will have to be pre-loaded into a table
+so that the entry point for the routine can be looked up and
+executed. Calling sequence:
+
+	int nameMainLibInit(void)
+
+returns 1 on success, 0 on failure.
+
+4. source=<shared object> package=<name>
+
+Valid for xtype=internal only.
+
+ds9 must call:
+
+      int MainLibLoad(char *name, char *source,  void **dl, char **ermsg) 
+
+or the Tcl equivalent:
+
+      set dl [mainlibload name source]
+
+to load shared object and call the init routine. This can be done either
+when processing the analysis specification or just before executing
+the command for the first time.  The returned dl is the handle from
+dlopen and can be used with dlclose(), if desired.
+
+5. ds9 will link libfuntools.a into the executable, as usual. 
+To access the mainlib support, we can either:
+
+a. link libfuntoolsMainLib.a into the executable and use source=static
+in the analysis menu. In this case, the static library is created using:
+
+	make mainlib
+
+and requires --enable-mainlib=yes on the configure command line.
+
+b. we can load libfuntoolsMainLib.[so,dylib] and use source=<shobj>
+in the analysis menu. In this case, the dynamic library is created using:
+
+	make shmainlib
+
+and requires --enable-mainlib=shared on the configure command line.
+
+Probably the static method is preferable so that funtools is part of a
+single execuable?
+
+6. The LD_LIBRARY_PATH is critical for loading shared libraries. How
+will we deal with this? One possibility is to add an argument to MainLibLoad()
+to specify a directory which gets added to LD_LIBRARY_PATH before the call
+to dlopen()???
+
diff --git a/funtools/notes/index.notes b/funtools/notes/index.notes
new file mode 100644
index 0000000..838633d
--- /dev/null
+++ b/funtools/notes/index.notes
@@ -0,0 +1,30 @@
+bynars: setenv B /pool7/eric/big.fits 
+bynars: setenv BX /pool7/eric/big_chipx.ind 
+
+# funtable filters 1 column
+bynars: time funtable big.fits'[chipx=78]' foo.fits
+9.74u 6.71s 1:05.59 25.0%
+bynars: time funtable big.fits'[chipx=78]' foo.fits
+8.99u 9.82s 1:02.83 29.9%
+bynars: time funtable big.fits'[chipx=78]' foo.fits
+9.09u 7.40s 1:06.21 24.9%
+
+# experimental index program filters 1 column
+bynars: time ./foo $B /pool7/eric/foo2.fits $BX chipx 78
+1.21u 1.57s 0:04.80 57.9%
+bynars: time ./foo $B /pool7/eric/foo2.fits $BX chipx 78
+1.24u 1.34s 0:02.71 95.2%
+bynars: time ./foo $B /pool7/eric/foo2.fits $BX chipx 78
+1.20u 1.42s 0:02.73 95.9%
+
+# funtable filters 2 columns
+bynars: time funtable big.fits'[chipx=78&&chipy=152]' foo.fits
+11.04u 6.96s 1:08.03 26.4%
+bynars: time funtable big.fits'[chipx=78&&chipy=152]' foo.fits
+11.18u 6.11s 1:07.29 25.6%
+
+# experimental index program filters 2 columns
+bynars: time ./foo $B /pool7/eric/foo2.fits $BX chipx 78 $BY chipy 152
+1.07u 1.00s 0:02.73 75.8%
+bynars: time ./foo $B /pool7/eric/foo2.fits $BX chipx 78 $BY chipy 152
+1.07u 0.89s 0:02.01 97.5%
diff --git a/funtools/notes/mainlib.notes b/funtools/notes/mainlib.notes
new file mode 100644
index 0000000..a1e6c2b
--- /dev/null
+++ b/funtools/notes/mainlib.notes
@@ -0,0 +1,68 @@
+As see on Linux when running:
+
+  saoconfig shmainlib
+  make
+  make shmainlib
+  setenv LD_LIBRARY_PATH .
+  gcc -g -o tmain -I. -I./util tmain.c libfuntoolsMainLib.a libfuntools.a -ldl -lm
+or
+  gcc -g -o tmain -I. -I./util tmain.c libfuntools.a -ldl -lm
+  ./tmain -s libfuntoolsMainLib.so $S "fundisp foo[cir(512,512,.1)]"
+
+gives:
+...
+0xb7ef80f4 0xb7ef80f4
+0xb7ef80f4
+0xb7ef80f4 0xb7ef80f4
+executing: fundisp "buf:0xb7f3e008:648000[cir(512,512,.1)]" 
+0x8062650
+0x8062650 0x8062650
+
+The hex numbers are the address of the mainlibhead list header for
+mainlib records. How can this address change?
+
+We link tmain against either libutil.a or libfuntools.a to get the
+MainLibLoad() routine. When this routine is called, it loads the
+libfuntoolsMainLib.so library, and, using that library calls
+funtoolsMainLibInit(). This routine makes a bunch of MainLibNew()
+and uses mainlibhead in the static executable as the list head.
+
+Apparently, because we add -L. -lfuntools to the mklib command line:
+
+	./mklib -o $(PACKAGE)MainLib -L. -lfuntools $(PACKAGE)tmp/*.o; \
+
+libfuntools.so is loaded as well. This has the mainlib.o module in it,
+and of course, a new copy of mainlibhead.
+
+So, when MainLibProcess() is subsequently called, it probably comes
+from the shared object, not the statically linked executable. At any
+rate, it uses the mainllibhead from the shared object, which is NULL,
+and therefore does not see that fundisp has been defined.
+
+If we don't add -L. -lfuntools to the mklib command line:
+
+    ./mklib -o $(PACKAGE)MainLib $(PACKAGE)tmp/*.o; \
+
+and remake the libfuntoolsMainLib.so file, we get an error when we try to run:
+
+  make shmainlib
+  mklib: Making Linux shared library:  libfuntoolsMainLib.so.1.0
+  ./tmain -s libfuntoolsMainLib.so $S "fundisp foo[cir(512,512,.1)]"
+  can't load shared object libfuntoolsMainLib.so (./libfuntoolsMainLib.so: undefined symbol: SAOdtype)
+
+This is because the statically linked program does not call SAOdtype
+and therefore does not have it linked in. It must come from a loaded
+shared object.
+
+So we have to load a shared object to get funtools entry points, but we
+don't want a second copy of mainlibhead ...
+
+Even if we separate MainLibLoad from mainlib.c and don't explicitly
+load mainlib.c, we still get the wrong mainlibhead:
+
+	nm tmain | egrep mainlib
+	08062650 b mainlibhead
+	08062654 b mainlibinit
+	08062658 b mainlib_path
+	080625f0 d mainlibundef
+
diff --git a/funtools/notes/mingw.notes b/funtools/notes/mingw.notes
new file mode 100644
index 0000000..75ec185
--- /dev/null
+++ b/funtools/notes/mingw.notes
@@ -0,0 +1,12 @@
+Since the gcc.exe install directory, C:\mingw\bin, is not in the
+Windows PATH, the FILTER_CC variable must be set. You have to use a
+Windows pathname, not a Posix pathname, because the Windows system()
+routine passes the string to the DOS shell.
+
+	export FILTER_CC="C:\mingw\bin\gcc.exe"
+
+
+
+
+
+
diff --git a/funtools/notes/tcl.notes b/funtools/notes/tcl.notes
new file mode 100644
index 0000000..5662f2e
--- /dev/null
+++ b/funtools/notes/tcl.notes
@@ -0,0 +1,10 @@
+
+http://www.equi4.com/starkit.html
+
+"A Starkit is a wrapping mechanism for delivering an application in a
+self-contained, installation-free, and portable way. The name comes
+from being based on a StandAlone Runtime, called Tclkit."
+
+http://www.equi4.com/tkstarted.html
+
+http://www.equi4.com/papers/skpaper1.html
diff --git a/funtools/notes/text.notes b/funtools/notes/text.notes
new file mode 100644
index 0000000..48960ef
--- /dev/null
+++ b/funtools/notes/text.notes
@@ -0,0 +1,29 @@
+# keywords:
+foo	100
+foo :	100
+foo:	100
+foo =	100
+foo=	100
+
+# strings:
+foo	/usr/local/bin
+foo:	'/usr/local/bin	/opt/local/bin'
+foo =	"/usr/local/bin	/opt/local/bin"
+
+# comments:
+foo	foo1				/comment
+foo	'foo1 foo2 foo3'		/ comment
+foo	/usr/local/bin			/ path dir
+foo	"/usr/local/bin	/opt/local/bin"	/ path dir
+foo	"/usr/local/bin	/opt/local/bin"/path dir
+
+# put it all together:
+http_ ref	http://hea-www.cfa.harvard.edu	/ base URL
+http_ ref:	http://hea-www.cfa.harvard.edu	/ base URL
+mypaths =	"/usr/local/bin	/opt/local/bin"	/ paths to use
+
+text support for funtools
+
+1. add "allowtypechange" option and die if data type for a column
+goes from int to float.
+
diff --git a/funtools/notes/view.notes b/funtools/notes/view.notes
new file mode 100644
index 0000000..bd9db20
--- /dev/null
+++ b/funtools/notes/view.notes
@@ -0,0 +1,261 @@
+Views:
+
+We can implement views as pseudo files that actually refer to a row in a
+data base containing column, filter, and format specifiers for that view.
+
+To create a view:
+
+	funview -p "format" -r "filter" -c "columns" view file
+
+This will make an entry into a data base having the form:
+
+	view	file		columns		filter		format
+	----	-----------	-----------	-----------	--------------
+
+Data base name will be the first one found:
+
+1. getenv("FUN_VIEWBASE")
+2. ./.funtools.vu
+3. ~/.funtools.vu
+
+For example:
+
+	funview -p "I=%3d" -r "cir 512 512 .1" -c "x y pha pi" myview snr.ev
+
+adds to database:
+
+	view	file		columns		filter		format
+	----	-----------	-----------	-----------	--------------
+	myview	snr.ev		x y pha pi	cir 512 512 .1	I=%.3d
+
+Also:
+
+	funview -d view		# delete view from database
+	funview -l		# list views
+
+Of course, you can edit the database file directly (but does this avoid
+some needed checking?) ...  and we might also have a GUI ...
+
+You then use the view id instead of a file in any funtools program,
+prefixed by v:, i.e.
+
+	fundisp v:myview
+	funtable v:myview foo.fits
+
+The first command will run fundisp on the associated file, but using
+the specified filter, column, and format spec. The second runs funtable,
+which has no need of the print format but still utilizes the filter
+and column specifiers.
+
+You can override any of the three view specifiers explicitly:
+
+	fundisp v:myview'[cir 400 400 1]'
+
+uses the view format and column spec, but overrides the filter, while:
+
+	fundisp -f "I=%5d" v:myview'[cir 400 400 1]' "x y time"
+
+overrides everything but the file name ... probably useless but at
+least consistent.
+
+The syntax of a view should support templates, so that the view filter,
+columns, and format parameters can be used with multiple files.
+
+For example:
+
+	funview -p "I=%3d" -r "cir 512 512 .1" -c "x y pha pi" myview "*.ev"
+
+adds to data base:
+
+	view	file		columns		filter		format
+	----	-----------	-----------	-----------	--------------
+	myview	*.ev		x y pha pi	cir 512 512 .1	I=%.3d
+
+You then explicitly use a template view for a file that matches the template
+by specifying the actual filename instead of the view:
+
+	fundisp v:foo.ev
+
+Since foo.ev matches the *.ev template, this will use the myview
+columns, filter, and format on the foo.ev data file.
+
+Note possible conflict if a file name matches a view but ... we can
+avoid this to some extent by not allowing view names to contain ".",
+":", "/" (same sort of restriction in xpa).
+
+A view GUI can have a selectable option to tell funtools programs
+always to assume a file is a view, even if v: is not prepended. This means
+that, if you start the GUI and select the "auto view" switch, then:
+
+      fundisp v:myview
+and
+      fundisp myview
+
+will be handled identically. Without the GUI running, the v: prefix will
+be required.
+
+There should be a way to specify which view to use for a given file:
+	
+	fundisp v(fi3):foo.fits
+
+means use view fi3 on file foo.fits
+
+	fundisp v(reg1,col2):foo.fits
+
+means use views reg1 and col2 (in that order) on file foo.fits.
+
+Wild idea: we can implement a join of files as a view (just like the
+big boy databases).  I think we get this for free if FunOpen just
+opens a pipe: pseudo file (which is implemented and might actually
+work) instead of a disk file.
+
+Example: we define a view called "myjoin" to be:
+
+	view	file		...
+	----	-----------	...
+	myjoin	"funjoin -j1 key -j2 key2 cat1.fits cat2.fits" ...
+
+where the implied output arg is stdout and then we can execute:
+
+	fundisp v:myjoin
+
+and get the joined result. View filters and columns would be applied to the
+output and it all just might work ...
+
+Good points:
+
+Explicit specification of a pseudo-file "v:[viewid]" is unambiguous
+and alleviates the need to rely on a separate process to prevent
+accident use of a view.  If you don't want to use the view, just don't
+specify it.
+
+Easy flexibility for overriding view parameters.
+
+Support for file templates allows default setups of filter, column,
+and formats (can even use "*" as the file for a global setup).
+
+Uses standard funtools file type paradigm (socket:, pipe:, shm: etc).
+
+If we can implement arbitrary commands as a view file type via pipe:
+(particularly funjoin), then our views will be similar (superset?) to
+SQL views (see below).
+
+Bad points:
+
+The original idea of changing getenv would have made extensions to the
+list of view parameters automatic. We now have to implement each
+access of a view parameter explicitly, as opposed to having getenv do
+it automatically.  Actually, this is a two-edged sword: it also could
+be a bad thing that each and every getenv would look for a global value.
+
+Design:
+
+I. FunOpen("v:id", ....)	
+
+To process a view file spec inside FunOpen():
+
+1. Look for view file, either in default (~/.funtools.views) or
+pointed to by FUN_VIEWFILE. If it does not exist, return ERROR.
+
+2. If filespec contains a directory character ("/") go to 4.
+
+3. ID or simple file: Strip off and save filter spec. Look for ID
+name. If not found, go to 4. If found, replace input name with FILE
+name. If no filter spec is specified, use FILTER, otherwise use
+(override) filter spec. Save column spec for later use by
+FunColumnActivate.
+
+4. Pathname or Simple file: Try to match file spec again template
+FILEs. If found, replace input name with FILE name. If no filter spec
+is specified, use FILTER, otherwise use (override) filter spec. Save
+column spec for later use by FunColumnActivate.
+
+II. FunColumnActivate(fun, columns, plist)
+
+1. If columns != NULL, use columns (might be override).
+
+2. If columns == NULL and fun->view_columns, use it.
+
+3. If columns == NULL and !fun->view_columns, just return.
+
+-----------------------------------------------------------------------------
+
+			Views in SQL
+
+See: http://www.cs.unibo.it/~ciaccia/COURSES/RESOURCES/SQLTutorial/sqlch5.htm
+
+5.2 VIEWS. (Virtual Tables and Data Security)
+
+A View, as we will see, is a definition for a "virtual table" (virtual
+because there is no permanent allocation of storage space) which is
+assembled at reference time from selected rows and/or columns of one
+or more real tables.
+
+A view may be queried in exactly the same way as a real table.
+
+Views are useful for two main reasons:
+
+1) they enable users to see data, from a generalised database design, in the form most convenient for their needs.
+
+2) they may be employed as a security mechanism for restricting user access to specific tables columns and/or rows.
+
+The statement used to create a view has the following general form:
+
+CREATE VIEW view-name [( column-list )]
+
+AS SELECT column-list
+
+FROM table-list
+
+WHERE conditional-expression ;
+
+You can display the specification of the views you have created using
+the help statement whose general syntax is as follows:
+
+help view view_name {,view_name}
+
+When a view is no longer required it may be dropped from the database
+with the DROP statement:
+
+DROP VIEW table-name ;
+
+5.2.1 Views Designed to Simplify Queries.
+
+As we noted with Example 4.5.4, there are no direct flights from
+Heathrow (HROW) to Brussels (BRUS). To simplify the query required to
+list the departure times of interconnecting flights we will specify a
+view called Brussels-Link.
+
+CREATE VIEW Brussels_Link
+
+AS SELECT DISTINCT B.FlightNo,B.FromAirport,
+
+B.DepTime, B.ArrTime
+
+FROM Flight A, Flight B
+
+WHERE A.FromAirport = 'HROW'
+
+AND A.ToAirport = B.FromAirport
+
+AND B.ToAirport = 'BRUS' ;
+
+Example 5.2.1 - List the FlightNo's, Airport's, Departure and Arrival
+times for flights from 1500 which link Heathrow with Brussels.
+
+SELECT FromAirport, FlightNo, DepTime, ArrTime
+
+FROM Brussels_Link
+
+WHERE DepTime >= '15:00';
+
+Result:
+
+fromai   flightn  deptime   arrtime    
+
+BIRM     BD655    15:00     17:05      
+BIRM     BD657    17:30     19:35      
+BIRM     BD659    18:25     20:30      
+
+
+