summaryrefslogtreecommitdiffstats
path: root/doc/FileSystem.3
diff options
context:
space:
mode:
Diffstat (limited to 'doc/FileSystem.3')
-rw-r--r--doc/FileSystem.325
1 files changed, 15 insertions, 10 deletions
diff --git a/doc/FileSystem.3 b/doc/FileSystem.3
index 413c119..01a3585 100644
--- a/doc/FileSystem.3
+++ b/doc/FileSystem.3
@@ -4,7 +4,7 @@
'\" See the file "license.terms" for information on usage and redistribution
'\" of this file, and for a DISCLAIMER OF ALL WARRANTIES.
'\"
-'\" RCS: @(#) $Id: FileSystem.3,v 1.22 2002/05/02 20:15:20 vincentdarley Exp $
+'\" RCS: @(#) $Id: FileSystem.3,v 1.23 2002/05/07 18:03:04 vincentdarley Exp $
'\"
.so man.macros
.TH Filesystem 3 8.4 Tcl "Tcl Library Procedures"
@@ -241,15 +241,20 @@ filesystem which has been registered (through
media or access methods. This means that all of these functions (and
therefore the corresponding \fBfile\fR, \fBglob\fR, \fBpwd\fR, \fBcd\fR,
\fBopen\fR, etc. Tcl commands) may be operate on 'files' which are not
-native files in the native filesystem. If appropriate vfs's have been
-registered, the 'files' may, to give two examples, be remote (e.g.
-situated on a remote ftp server) or archived (e.g. lying inside a .zip
-archive). Such registered filesystems provide a lookup table of
-functions to implement all or some of the functionality listed here.
-Finally, the \fBTcl_FSStat\fR and \fBTcl_FSLstat\fR calls abstract
-away from what the 'struct stat' buffer buffer is actually declared to
-be, allowing the same code to be used both on systems with and systems
-without support for files larger than 2GB in size.
+native files in the native filesystem. This also means that any Tcl
+extension which accesses the filesystem through this API is
+automatically 'virtual filesystem aware'. Of course, if an extension
+accesses the native filesystem directly (through platform-specific
+APIs, for example), then Tcl cannot intercept such calls.
+.PP
+If appropriate vfs's have been registered, the 'files' may, to give two
+examples, be remote (e.g. situated on a remote ftp server) or archived
+(e.g. lying inside a .zip archive). Such registered filesystems provide
+a lookup table of functions to implement all or some of the functionality
+listed here. Finally, the \fBTcl_FSStat\fR and \fBTcl_FSLstat\fR calls
+abstract away from what the 'struct stat' buffer buffer is actually
+declared to be, allowing the same code to be used both on systems with
+and systems without support for files larger than 2GB in size.
.PP
The \fBTcl_FS...\fR are objectified and may cache internal
representations and other path-related strings (e.g. the current
generated by cgit v0.12 at 2024-08-20 11:49:16 (GMT)
span class="hl ppc"> fflush(stdout); \ fflush(stderr); \ err_flag = 0; \ } static void vrfy_elements(int* a, int* b, int size, int rank); static int find_writesize(int rank, int numprocs, int write_size); /* All writes are to non-overlapping locations in the file * Then, each task reads another tasks' data * */ static int allwrite_allread_blocks(int numprocs, int rank, int write_size) { MPI_File fh = MPI_FILE_NULL; int mpio_result; int amode, i; MPI_Offset offset = rank*write_size*sizeof(int); MPI_Status Status; int* writebuf = (int*)malloc(write_size*sizeof(int)); int* readbuf = (int*)malloc (write_size*sizeof(int)); for(i=0; i<write_size; i++) writebuf[i] = i; amode = MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_DELETE_ON_CLOSE; mpio_result = MPI_File_open(MPI_COMM_WORLD, testfile, amode, MPI_INFO_NULL, &fh); CHECK_SUCCESS(mpio_result); mpio_result = MPI_File_write_at(fh, offset, writebuf, write_size, MPI_INT, &Status); CHECK_SUCCESS(mpio_result); MPI_Barrier(MPI_COMM_WORLD); offset = ( (rank+(numprocs-1)) % numprocs)*write_size*sizeof(int); mpio_result = MPI_File_read_at(fh, offset, readbuf, write_size, MPI_INT, &Status); CHECK_SUCCESS(mpio_result); vrfy_elements(writebuf, readbuf, write_size, rank); mpio_result = MPI_File_close(&fh); CHECK_SUCCESS(mpio_result); HDfree(writebuf); HDfree(readbuf); return err_flag; } static int posix_allwrite_allread_blocks(int numprocs, int rank, int write_size) { int ret; int i; int offset = rank*write_size*sizeof(int); int* writebuf = (int*)malloc(write_size*sizeof(int)); int* readbuf = (int*)malloc (write_size*sizeof(int)); FILE* file = NULL; for(i=0; i<write_size; i++) writebuf[i] = i; if(rank==0) file = fopen(testfile, "w+"); MPI_Barrier(MPI_COMM_WORLD); if(rank != 0) file = fopen(testfile, "r+"); if(file == NULL) { fprintf(stderr, "Could not create testfile\n"); MPI_Abort(MPI_COMM_WORLD, 1); } ret = fseek(file, offset, SEEK_SET); if(ret == -1) { perror("fseek"); MPI_Abort(MPI_COMM_WORLD, 1); } ret = fwrite(writebuf, sizeof(int), write_size, file); if(ret != write_size) { perror("fwrite"); MPI_Abort(MPI_COMM_WORLD, 1); } MPI_Barrier(MPI_COMM_WORLD); offset = ( (rank+(numprocs-1)) % numprocs)*write_size*sizeof(int); ret = fseek(file, offset, SEEK_SET); if(ret == -1) { perror("fseek"); MPI_Abort(MPI_COMM_WORLD, 1); } ret = fread(readbuf, sizeof(int), write_size, file); if( (ret == 0) && feof(file)) printf("Process %d: Error. Prematurely reached end of file\n", rank); else if( (ret != write_size) && ferror(file)) { perror("Error encountered in fread"); MPI_Abort(MPI_COMM_WORLD, 1); } vrfy_elements(writebuf, readbuf, write_size, rank); fclose(file); MPI_Barrier(MPI_COMM_WORLD); if(rank == 0) unlink(testfile); HDfree(writebuf); HDfree(readbuf); return err_flag; } static int posix_onewrite_allread_blocks(int numprocs, int rank, int write_size) { int ret; int i; int offset = rank*write_size*sizeof(int); int* writebuf = (int*)malloc(write_size*sizeof(int)); int* readbuf = (int*)malloc (write_size*sizeof(int)); FILE* file = NULL; for(i=0; i<write_size; i++) writebuf[i] = i; if(rank==0) file = fopen(testfile, "w+"); MPI_Barrier(MPI_COMM_WORLD); if(rank != 0) file = fopen(testfile, "r+"); if(file == NULL) { fprintf(stderr, "Could not create testfile\n"); MPI_Abort(MPI_COMM_WORLD, 1); } if(rank == 0) { for(offset = 0; offset<numprocs*write_size*sizeof(int); offset+=(write_size*sizeof(int))) { ret = fseek(file, offset, SEEK_SET); if(ret == -1) { perror("fseek"); MPI_Abort(MPI_COMM_WORLD, 1); } ret = fwrite(writebuf, sizeof(int), write_size, file); if(ret != write_size) { perror("fwrite"); MPI_Abort(MPI_COMM_WORLD, 1); } } } MPI_Barrier(MPI_COMM_WORLD); offset = rank*write_size*sizeof(int); ret = fseek(file, offset, SEEK_SET); if(ret == -1) { perror("fseek"); MPI_Abort(MPI_COMM_WORLD, 1); } ret = fread(readbuf, sizeof(int), write_size, file); if( (ret == 0) && feof(file)) printf("Process %d: Error. Prematurely reached end of file\n", rank); else if( (ret != write_size) && ferror(file)) { perror("Error encountered in fread"); MPI_Abort(MPI_COMM_WORLD, 1); } vrfy_elements(writebuf, readbuf, write_size, rank); fclose(file); MPI_Barrier(MPI_COMM_WORLD); if(rank == 0) unlink(testfile); HDfree(writebuf); HDfree(readbuf); return err_flag; } static int posix_onewrite_allread_interlaced(int numprocs, int rank, int write_size) { int ret; int i, fill, index; int offset = rank*write_size*sizeof(int); int* writebuf = (int*)malloc(write_size*sizeof(int)); int* readbuf = (int*)malloc (write_size*sizeof(int)); FILE* file = NULL; if(rank==0) file = fopen(testfile, "w+"); MPI_Barrier(MPI_COMM_WORLD); if(rank != 0) file = fopen(testfile, "r+"); if(file == NULL) { fprintf(stderr, "Could not create testfile\n"); MPI_Abort(MPI_COMM_WORLD, 1); } if(rank == 0) { for(offset = 0; offset<numprocs*write_size*sizeof(int); offset+=(numprocs*sizeof(int))) { ret = fseek(file, offset, SEEK_SET); if(ret == -1) { perror("fseek"); MPI_Abort(MPI_COMM_WORLD, 1); } fill = offset / (numprocs*sizeof(int)); for(i=0; i<numprocs; i++) writebuf[i] = fill; ret = fwrite(writebuf, sizeof(int), numprocs, file); if(ret != numprocs) { perror("fwrite"); MPI_Abort(MPI_COMM_WORLD, 1); } } } MPI_Barrier(MPI_COMM_WORLD); index = 0; for(offset = rank*sizeof(int); offset<numprocs*write_size*sizeof(int); offset+=(numprocs*sizeof(int))) { ret = fseek(file, offset, SEEK_SET); if(ret == -1) { perror("fseek"); MPI_Abort(MPI_COMM_WORLD, 1); } ret = fread(readbuf+index, sizeof(int), 1, file); if( (ret == 0) && feof(file)) printf("Process %d: Error. Prematurely reached end of file\n", rank); else if( (ret != 1) && ferror(file)) { perror("Error encountered in fread"); MPI_Abort(MPI_COMM_WORLD, 1); } index++; } for(i=0; i<write_size; i++) writebuf[i] = i; vrfy_elements(writebuf, readbuf, write_size, rank); fclose(file); MPI_Barrier(MPI_COMM_WORLD); if(rank == 0) unlink(testfile); HDfree(writebuf); HDfree(readbuf); return err_flag; } /* Each proc wites out 0 1 2 3 with displacement i, so file contents are: * 0000 1111 2222 3333 etc. (with 4 procs) * * Each proc then reads in the whole file and verifies that the data is what it is supposed to be*/ static int allwrite_allread_interlaced(int numprocs, int rank, int write_size) { MPI_File fh = MPI_FILE_NULL; int mpio_result; int amode, i, counter = 0; MPI_Datatype filetype; MPI_Status Status; int* writebuf = (int*)malloc(write_size*sizeof(int)); int* readbuf = (int*) malloc(numprocs*sizeof(int)); int offset=0; for(i=0; i<write_size; i++) writebuf[i] = i; amode = MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_DELETE_ON_CLOSE; mpio_result = MPI_File_open(MPI_COMM_WORLD, testfile, amode, MPI_INFO_NULL, &fh); CHECK_SUCCESS(mpio_result); mpio_result = MPI_Type_vector(write_size, 1, numprocs, MPI_INT, &filetype); CHECK_SUCCESS(mpio_result); mpio_result = MPI_Type_commit(&filetype); CHECK_SUCCESS(mpio_result); mpio_result = MPI_File_set_view(fh, rank*sizeof(int), MPI_INT, filetype, "native", MPI_INFO_NULL); CHECK_SUCCESS(mpio_result); mpio_result = MPI_File_write(fh, writebuf, write_size, MPI_INT, &Status); CHECK_SUCCESS(mpio_result); MPI_Barrier(MPI_COMM_WORLD); mpio_result = MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE, "native", MPI_INFO_NULL); CHECK_SUCCESS(mpio_result); for(offset = 0; offset<(write_size*numprocs*sizeof(int)); offset+=(numprocs*sizeof(int))) { mpio_result = MPI_File_read_at(fh, offset, readbuf, numprocs, MPI_INT, &Status); CHECK_SUCCESS(mpio_result); for(i=0; i<numprocs; i++) { if(writebuf[offset/(numprocs*sizeof(int))] != readbuf[i]) { if( (rank == 0) && (counter == 0)) printf("\n"); if(counter++ < max_err_print) fprintf(stderr, "Arrays do not match! Prcoess %d, element %d: [%d, %d]\n", rank, i, writebuf[offset/(numprocs*sizeof(int))], readbuf[i]); else if(counter++ == max_err_print+1) fprintf(stderr, "Printed %d errors. Omitting the rest\n", max_err_print); err_flag = -1; } } } nmismatches += counter; mpio_result = MPI_File_close(&fh); CHECK_SUCCESS(mpio_result); HDfree(writebuf); HDfree(readbuf); return err_flag; } /* Overlapping pattern works as follows (this test requires at least 2 procs: * Writes: * Task 0: 0 2 4 6 etc... * Task 1: 1 3 5 7 etc... * Task 2: 0 3 6 etc.. * Task 3: 0 4 8 etc... * * The above describes only the pattern of the elements being written. The actual * number of elements written is going to be: * * Task i where i=(numprocs-1) writes write_size elements. All other tasks do: * x = ((write_size-1)*numprocs) * Task i's write_size is the smallest multiple of i<=x divided by i, with the exception * of tasks 0 and 1, for whom i is 2, since they are writing the even and odd multiples. * * So, if there are 5 tasks with write_size=4, the resulting pattern of writes is: * * Task 0: 0 2 4 6 8 10 12 14 * Task 1: 1 3 5 7 9 11 13 15 * Task 2: 0 3 6 9 12 15 * Task 3: 0 4 8 12 * Task 4: 0 5 10 15 * * * * * * All the entires that overlap will therefore be writing the same value * * At the end, all tasks read in the file and verify that it is correct should be * (1,2...((numprocs-1)*WRTE_SIZE). * */ static int allwrite_allread_overlap(int numprocs, int rank, int write_size) { MPI_File fh = MPI_FILE_NULL; int mpio_result; int amode, i, counter = 0; MPI_Datatype filetype; MPI_Status Status; int* writebuf = (int*) malloc(write_size*(numprocs-1)*sizeof(int)); /* An upper bound...not all the elements will be written */ int* readbuf = (int*) malloc(write_size*(numprocs-1)*sizeof(int)); if(numprocs < 2) { fprintf(stderr, "The allwrite_allread_overlap test requires at least 2 procs\n"); return -1; } if(rank == 0) { for(i=0; i<write_size*(numprocs-1); i++) writebuf[i] = 2*i; } else if(rank == 1) { for(i=0; i<write_size*(numprocs-1); i++) writebuf[i] = (2*i)+1; } else { for(i=0; i<write_size*(numprocs-1); i++) writebuf[i] = (rank+1)*i; } amode = MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_DELETE_ON_CLOSE; mpio_result = MPI_File_open(MPI_COMM_WORLD, testfile, amode, MPI_INFO_NULL, &fh); CHECK_SUCCESS(mpio_result); if( (rank == 0) || (rank == 1) ) mpio_result = MPI_Type_vector(write_size*(numprocs-1), 1, 2, MPI_INT, &filetype); else mpio_result = MPI_Type_vector(write_size*(numprocs-1), 1, rank+1, MPI_INT, &filetype); CHECK_SUCCESS(mpio_result); mpio_result = MPI_Type_commit(&filetype); CHECK_SUCCESS(mpio_result); if( rank == 1) mpio_result = MPI_File_set_view(fh, sizeof(int), MPI_INT, filetype, "native", MPI_INFO_NULL); else mpio_result = MPI_File_set_view(fh, 0, MPI_INT, filetype, "native", MPI_INFO_NULL); CHECK_SUCCESS(mpio_result); if( rank == (numprocs - 1)) mpio_result = MPI_File_write(fh, writebuf, write_size, MPI_INT, &Status); else mpio_result = MPI_File_write(fh, writebuf, find_writesize(rank, numprocs, write_size), MPI_INT, &Status); CHECK_SUCCESS(mpio_result); MPI_Barrier(MPI_COMM_WORLD); mpio_result = MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE, "native", MPI_INFO_NULL); CHECK_SUCCESS(mpio_result); mpio_result = MPI_File_read_at(fh, 0, readbuf, write_size*(numprocs-1), MPI_INT, &Status); CHECK_SUCCESS(mpio_result); for(i=0; i<write_size*(numprocs-1); i++) { if(i != readbuf[i]) { if( (rank == 0) && (counter == 0)) printf("\n"); if(counter++ < max_err_print) fprintf(stderr, "Arrays do not match! Prcoess %d, element %d: [%d, %d]\n", rank, i, i, readbuf[i]); else if(counter++ == max_err_print+1) fprintf(stderr, "Printed %d errors. Omitting the rest\n", max_err_print); err_flag = -1; } } nmismatches += counter; mpio_result = MPI_File_close(&fh); CHECK_SUCCESS(mpio_result); HDfree(writebuf); HDfree(readbuf); return err_flag; } /* A random process writes out the following to the file: * 0 1 2 3 4 0 1 2 3 4 0 1 2 3 4 (assuming write_size=5, and numprocs=3) * * Process i read's in write_size bytes at offset=i*write_size */ static int onewrite_allread_blocks(int numprocs, int rank, int write_size) { MPI_File fh = MPI_FILE_NULL; int mpio_result; int amode, i; MPI_Status Status; int* writebuf = (int*)malloc(write_size*sizeof(int)); int* readbuf = (int*)malloc (write_size*sizeof(int)); for(i=0; i<write_size; i++) writebuf[i] = i; amode = MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_DELETE_ON_CLOSE; mpio_result = MPI_File_open(MPI_COMM_WORLD, testfile, amode, MPI_INFO_NULL, &fh); CHECK_SUCCESS(mpio_result); /* A random process writes out all the data */ if(rank == (rand() % numprocs)) { for(i=0; i<numprocs; i++) { mpio_result = MPI_File_write_at(fh, write_size*i*sizeof(int), writebuf, write_size, MPI_INT, &Status); CHECK_SUCCESS(mpio_result); } } MPI_Barrier(MPI_COMM_WORLD); mpio_result = MPI_File_read_at(fh, write_size*rank*sizeof(int), readbuf, write_size, MPI_INT, &Status); CHECK_SUCCESS(mpio_result);