summaryrefslogtreecommitdiffstats
path: root/pablo/PabloHDF_RT.c
blob: f40fd5b43692b58a82b571fe206b7ad05034bcb8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
/*  This file is part of the Pablo Performance Analysis Environment
// 
//           (R)
//  The Pablo    Performance Analysis Environment software is NOT in
//  the public domain.  However, it is freely available without fee for
//  education, research, and non-profit purposes.  By obtaining copies
//  of this and other files that comprise the Pablo Performance Analysis
//  Environment, you, the Licensee, agree to abide by the following
//  conditions and understandings with respect to the copyrighted software:
//  
//  1.  The software is copyrighted in the name of the Board of Trustees
//      of the University of Illinois (UI), and ownership of the software
//      remains with the UI. 
// 
//  2.  Permission to use, copy, and modify this software and its documentation
//      for education, research, and non-profit purposes is hereby granted
//      to Licensee, provided that the copyright notice, the original author's
//      names and unit identification, and this permission notice appear on
//      all such copies, and that no charge be made for such copies.  Any
//      entity desiring permission to incorporate this software into commercial
//      products should contact:
// 
//           Professor Daniel A. Reed                 reed@cs.uiuc.edu
//           University of Illinois
//           Department of Computer Science
//           2413 Digital Computer Laboratory
//           1304 West Springfield Avenue
//           Urbana, Illinois  61801
//           USA
// 
//  3.  Licensee may not use the name, logo, or any other symbol of the UI
//      nor the names of any of its employees nor any adaptation thereof in
//      advertizing or publicity pertaining to the software without specific
//      prior written approval of the UI.
// 
//  4.  THE UI MAKES NO REPRESENTATIONS ABOUT THE SUITABILITY OF THE
//      SOFTWARE FOR ANY PURPOSE.  IT IS PROVIDED "AS IS" WITHOUT EXPRESS
//      OR IMPLIED WARRANTY.
// 
//  5.  The UI shall not be liable for any damages suffered by Licensee from
//      the use of this software.
// 
//  6.  The software was developed under agreements between the UI and the
//      Federal Government which entitle the Government to certain rights.
// 
// *************************************************************************
// 
//  Developed by: The Pablo Research Group
//                University of Illinois at Urbana-Champaign
//                Department of Computer Science
//                1304 W. Springfield Avenue
//                Urbana, IL     61801
// 
//                http://www-pablo.cs.uiuc.edu
// 
//  Send comments to: pablo-feedback@guitar.cs.uiuc.edu
// 
//  Copyright (c) 1987-1998
//  The University of Illinois Board of Trustees.
//       All Rights Reserved.
// 
//  PABLO is a registered trademark of
//  The Board of Trustees of the University of Illinois
//  registered in the U.S. Patent and Trademark Office.
// 
//  Project Manager and Principal Investigator:
//       Daniel A. Reed (reed@cs.uiuc.edu)
//
// Funded in part by the Defense Advanced Research Projects Agency 
// under DARPA contracts DABT63-94-C0049 (SIO Initiative), 
// F30602-96-C-0161, and DABT63-96-C-0027 by the National Science 
// Foundation under the PACI program and grants NSF CDA 94-01124 and
// ASC 97-20202, and by the Department of Energy under contracts
// DOE B-341494, W-7405-ENG-48, and 1-B-333164.
*/ 
/*======================================================================*
// File:  PabloHDF_RT							*
// Purpose: support use of Pablo trace library to analyze HDF 		*
//	    performance							*
// Contents:								*
//  HDFinitTrace_RT        : initialize real-time tracing		*
//  HDFendTrace_RT         : complete trace 				*
//  initHDFProcTrace_RT    : sets up data structures at init time.	*
//  initproctracert_()	   : fortran interface				*
//  HDFtraceEvent_RT	   : called to record event information		*
//  HDFrecordSum 	   : adds fields of one record to those of 	*
//			     another					*
//  HDFnodeInit 	   : initializes linked list node		*
//  HDFrecordFileName	   : records named HDF identifiers 		*
//  BeginIOEventRecord     : initialize before I/O call			*
//  EndIOEventRecord 	   : finalize after I/O call			*
//  BeginMPIOEventRecord   : initialize before MPI-I/O call		*
//  EndMPIOEventRecord 	   : finalize after MPI-I/O call		*
//  BeginHDFEventRecord    : initialize before HDF call			*
//  EndHDFEventRecord 	   : finalizie after HDF call			*
//  HDFrecordFileName	   : record named identifier information	*
//  HDFassignPabloIDs	   : assigns a number to named identifiers	*
//  writeHDFNamePacketsRT  : write SDDF packets for identifier names	*
//  HDFupdateProcLists     : adds records in queue to entries in 	*
//			     tables					*
//  HDFupdateProcs	   : called by HDFupdateProcLists to do 	*
//			     addition					*
//  HDFSummarySDDF	   : write SDDF event summary packets		*
//  HDFnodeInit 	   : initialize event node			*
//  HDFrecordSum 	   : add one event record to another		*
//  getHDFFieldIndex	   : get Field Index for counts and times	*
//  getHDFByteFieldIndex   : get field index for bytes 			*
//  writeHDFRecDescrptrsRT : write HDF Record Descriptor packets	*
//  printFileMappingsRT	   : print map of named identifiers		*
//  _hdfNameDescriptor()   : writes SDDF descriptor packet for names	*
//======================================================================*/
#include "H5config.h"
#include "SystemDepend.h"
#include "Trace.h"
#include "TraceParam.h"
#include "ProcIDs.h"
#include "HDF5Trace.h"
#include "SDDFparam.h"
#include <string.h>
#include <stdio.h>
#include <fcntl.h>
/*======================================================================* 
// on ipsc/860 don't include this or you'll get multiply defined SEEK_  *
//======================================================================*/
#ifndef __NX
#include <unistd.h>
#endif
 
#ifndef SUCCESS
#define SUCCESS 0
#define FAILURE	1
#endif

#ifndef TRUE
#define TRUE 1
#define FALSE	0
#endif
#define NEG_THREAD_ID -999

#include "HDF5record_RT.h"

#ifdef HAVE_PARALLEL
#include "mpio.h"
#include "MPIO_Init.h"
#include "MPIO_EventArgs.h"
#include "MPIO_TraceParams.h"
#endif

#ifndef TRgetThreadID
#define TRgetThreadID  TRgetNode
#endif

#ifndef TRnumNodes
#define TRnumNodes 1
#endif

#define AllThreads -1

#define set_c_mappedID( fd ) (fd)
#define c_mappedID( fd ) (fd)
/*======================================================================*
//  User output file pointer.						*
//======================================================================*/
FILE *outP;
/*======================================================================*
// Data Structures:							*
//									*
// HDFQueues:   an array of linked list.  Each list corresponds to an	*
//              HDF event and contains an entry for each different 	*
//		thread and data set referenced by a call to that event  *
//									*
// CallStack: 	a stack of HDFnode_t objects.  At any given time, the   *
//		stack represents the calling stack of the HDF routines 	*
//									*
// HDFfileList: a linked list of named identifiers and identifier 	*
//		numbers.  This is processed later to assign a single 	*
//		numbers to identifiers with the same name.		*
//======================================================================*/
HDFnode_t **HDFQueues;
HDFnode_t *CallStack;
HDFnode_t *TagQueue;
fileRec_t *HDFfileList;
/*======================================================================*
// Internal Function prototypes						*
//======================================================================*/
void HDFinitTrace_RT( char *, unsigned );
void HDFendTrace_RT( void );
int initproctracert_( void );
int initHDFProcTrace_RT( void );
void HDFtraceEvent_RT( int , char *, unsigned ) ;
void BeginIOEventRecord ( int, double , void * );
void EndIOEventRecord ( int , double , void * );
void BeginMPIOEventRecord ( int, double, void *, int ); 
void EndMPIOEventRecord ( int , double , void *, int);
void BeginHDFEventRecord( int , double );
void EndHDFEventRecord ( int , double ,void *);
void HDFrecordFileName( HDFsetInfo * );
void HDFassignPabloIDs( int *, char *** );
void writeHDFNamePacketsRT( char **, int );
void HDFupdateProcLists( void );
void HDFupdateProcs( HDFnode_t * );
void HDFSummarySDDF( HDFnode_t *, int );
void HDFnodeInit ( HDFnode_t * ) ;
void HDFrecordSum ( HDFrec_t *, HDFrec_t * );
int getHDFFieldIndex( int );
int getHDFByteFieldIndex( int );
void writeHDFRecDescrptrsRT( void );
void printFileMappingsRT( char *, char **, int );
void _hdfNameDescriptor( void );
void _hdfDescriptorRT( char *, char *, int );
void HDFfinalTimeStamp( void );
/*======================================================================*
// Global variables           						*
//======================================================================*/
HDFnode_t InitNode;		/* Node used in initialization		*/
HDFrec_t Tally;			/* Node used to get total		*/
char *FileName;			/* Name of Trace file			*/
HDFsetInfo openInfo;		/* Info about file opened		*/
char openName[256];		/* Name of opened file			*/
extern char *hdfRecordPointer;
/*======================================================================*
// NAME									*
//     HDFinitTrace_RT-- initialize HDF real-time tracing		*
// USAGE								*
//     VOID HDFinitTrace_RT( fileName, procTraceMask)			*
//									*
//     char *fileName;		IN: name of output file			*
//     unsigned procTraceMask;	IN: families of procedures to trace	*
// RETURNS								*
//     None.								*
//======================================================================*/
void HDFinitTrace_RT( char *fileName, unsigned procTraceMask )
{
#ifdef  HAVE_PARALLEL
	int myNode;
#endif
	char *nameBuff;
	int error;
	TR_LOCK	criticalSection;
	TRgetClock( &epoch );
	criticalSection = TRlock();
	error = initHDFProcTrace_RT() ;
    	procTrace = procTraceMask;
	TRunlock( criticalSection );
	if ( error != SUCCESS ) {
	   fprintf (stderr,"Unable to Initialize properly.  Exiting program\n");
	   exit(-1);
	}
	FileName = ( char * ) malloc ( strlen( fileName ) + 10 );
#ifdef  HAVE_PARALLEL
        MPI_Comm_rank( MPI_COMM_WORLD, &myNode );
        setTraceProcessorNumber( myNode );
	sprintf(FileName,"%s.nd%d\0",fileName,myNode);
	/*==============================================================*
	// In the parallel case, initialize MPI-IO tracing.  This will	*
	// set the trace file name.					*
	//==============================================================*/
        /*initMPIOTrace( FileName, 1); */
#else
	/*==============================================================*
	// In the non-parallel case, set the trace file name and 	*
	// initialize the trace library.				*
	//==============================================================*/
	strcpy( FileName, fileName ) ;
#endif	/* HAVE_PARALLEL */
        setTraceFileName(FileName);
        basicLibraryInit( );         
}
/*======================================================================*
// NAME									*
//     HDFendTrace-- end HDF tracing					*
// USAGE								*
//     VOID HDFendTrace_RT(void)					*
// RETURNS								*
//     None.								*
//======================================================================*/
void HDFendTrace_RT( void )
{
	int j, numSetIDs;
	HDFnode_t *P;
	char **Names;
	char* mapFile;

	HDFfinalTimeStamp();
	/*==============================================================*
	//  Assing pablo ids to named identifiers and tag records	*
	//==============================================================*/
	HDFassignPabloIDs( &numSetIDs, &Names );
	mapFile = (char *)malloc( strlen(FileName) + 15 );
	sprintf( mapFile,"%s.map.nd%d\0", FileName, TRgetNode() );
        printFileMappingsRT( mapFile, Names, numSetIDs ); 
	/*==============================================================*
	// Print SDDF summary records					*
	//==============================================================*/
	writeHDFRecDescrptrsRT();
	writeHDFNamePacketsRT( Names, numSetIDs );
     	for ( j = 0; j < NumHDFProcs; ++j ) {
	   HDFSummarySDDF( HDFQueues[j], j );
	}  
	endTracing();
	/*==============================================================*
	// Clean up storage						*
	//==============================================================*/
	/* free( (void *)mapFile );
	for ( j = 0; j < numSetIDs; ++j ) {
	    if ( Names[j] != NULL ) {
               free((void *)Names[j]);
	    }
	}
	free( (void *)Names );
        P = CallStack;
        if ( P->ptr != NULL ) {
	   fprintf(stderr,"CallStack not empty at termination\n");
        } else {
           free((void *)P);
	}
	free((void *)HDFQueues) ; */
}
/*======================================================================*
// initHFDProcTrace_RT							*
//	This function initializes data structures specific to		* 
//	the HDF real-time procedure entry/exit tracing extensions of 	*
//      the Pablo instrumentation library.  				*
//======================================================================*/
int initproctracert_( void )

{
	return initHDFProcTrace_RT();
}

int initHDFProcTrace_RT( void )

{
	int i, j, size;
	int numProcs = NumHDFProcs;

        if ( traceProcessorNumber == -1 ) {
            traceProcessorNumber = TRgetDefaultProcessor();
        }
	/*==============================================================*
	// Initialize InitNode used for node initialization.		*
	//==============================================================*/
	InitNode.ptr = NULL;
        InitNode.eventID = 0;             
        InitNode.lastIOtime = 0;           
        InitNode.record.nCalls = 0;             
        InitNode.record.lastCall = 0;
        InitNode.record.incDur = 0;              
        InitNode.record.excDur = 0;              
        InitNode.record.hdfID = 0;             
        InitNode.record.xRef = 0;             
	for ( j = 0; j < nTallyFields; ++j ) {
           InitNode.record.times[j] = 0; 
	}
	for ( j = 0; j < nTallyFields; ++j ) {
           InitNode.record.counts[j] = 0; 
	}
	for ( j = 0; j < nByteFields; ++j ) {
           InitNode.record.bytes[j] = 0; 
	}
	for ( i = 0; i < nByteFields; ++i ) {
	   for ( j = 0; j < nBkts; ++j ) {
	      InitNode.record.Hists[i][j] = 0;
	   }
	}
	/*==============================================================*
	// initialize linked list used to keep track of named hdf 	*
	// identifiers.							*
	//==============================================================*/
	HDFfileList = NULL;
	/*==============================================================*
	// Allocate a one dimensional array of pointers to queues of 	*
	// HDFnodes.  There is one queue for each thread and one for 	*
	// each HDF procedure.  Each queue will be a list of summary 	*
	// records distinquished by file type and 			*
	//==============================================================*/
	size = (int)(numProcs*sizeof( HDFnode_t * ));
	HDFQueues = (HDFnode_t **)malloc( size );
	if ( HDFQueues == NULL ) {
	   fprintf(stderr,"Failed to allocate HDFQueues in initHDFProcTrace\n");
	   return FAILURE;
	}
	for ( j = 0; j < numProcs; ++j ) {
	   HDFQueues[j] = NULL;
	}
	/*==============================================================*
	// Initialize call stack to a dummy node and TagQueue to NULL   *
	//==============================================================*/
	CallStack = (HDFnode_t *)malloc( sizeof(HDFnode_t) );
	*CallStack = InitNode;
	TagQueue = NULL ;
	return SUCCESS;
}
/*======================================================================*
// This is called from the HDF and I/O routines when real-time summary	*
// tracing is used.  It sets up a call stack for the specific thread in *
// use if no stack is yet set up.  It then calls calls a routine to 	*
// handle the event based on whether it is an I/O or HDF call.		*
//======================================================================*/
void HDFtraceEvent_RT( int eventType, char *dataPtr, unsigned dataLen ) 
{
	TR_LOCK	criticalSection;
	CLOCK	currentTime;
	double  seconds;

	criticalSection = TRlock();
	currentTime = getClock();
	seconds = clockToSeconds( currentTime );

	if ( isBeginIOEvent ( eventType ) || eventType == ID_malloc ) {
	   BeginIOEventRecord ( eventType, seconds, dataPtr ) ;
	} else if ( isEndIOEvent( eventType )  || eventType == -ID_malloc) {
	   EndIOEventRecord ( eventType, seconds, dataPtr );
	} else if ( isBeginHDFEvent( eventType ) ) { 
	   BeginHDFEventRecord ( eventType , seconds ) ;
	} else if ( isEndHDFEvent( eventType ) ) {
	   EndHDFEventRecord ( eventType, seconds, dataPtr );
#ifdef  HAVE_PARALLEL
	} else if ( isBeginMPIOEvent( eventType ) ) { 
	   BeginMPIOEventRecord ( eventType, seconds, dataPtr, dataLen ) ;
	} else if ( isEndMPIOEvent( eventType ) ) {
	   EndMPIOEventRecord ( eventType, seconds, dataPtr, dataLen );
#endif  /* HAVE_PARALLEL */
	} else {
	   fprintf(stderr,"eventType %d, dataLen = %u\n",eventType,dataLen);
	} 
	TRunlock( criticalSection );
}
/*======================================================================* 
// BeginIOEventRecord:                                               	*
//  This routine simply records the time in the record on the top of 	*
//  the stack.								*
//======================================================================*/ 
void BeginIOEventRecord ( int eventType, double seconds, void *dataPtr  )
{
	char *name;
	/*==============================================================*
	// save the time value temporarily in top of stack		*
	// When the end record is received, the duration can be 	*
	// computed.							*
	//==============================================================*/
	CallStack->lastIOtime = seconds;
	/*==============================================================*
	// get the ID or name of the file accessed from the structure	*
	// passed as dataPtr.  						*
	//==============================================================*/
	switch ( eventType )
	{
		case fopenBeginID:
		case openBeginID:
		   name = (char *)(dataPtr) + 2*sizeof(int);
  	   	   strcpy( openName, name );
		   break;
		case fcloseBeginID:
		case closeBeginID:
	   	   CallStack->record.hdfID = *( long *)dataPtr;
		   break;
		case readBeginID:
		case freadBeginID:
		case writeBeginID:
		case fwriteBeginID:
		case lseekBeginID:
		case fseekBeginID:
		case fsetposBeginID:
		case rewindBeginID:
	   	   CallStack->record.hdfID = *(int *)dataPtr;
		   break;
		default:
		   break;
	}
}
/*======================================================================* 
// EndIOEventRecord:							*
//  This routine retrieves the entry time saved on the top of the stack *
//  and computes the duration of the I/O event.  This is added to the   *
//  record field corresponding to this type of I/O.  The Bytes field in *
//  the record is updated if this is a read or write operation.		*
//======================================================================*/ 
void EndIOEventRecord ( int eventType, double secs, void *dataPtr )
{
	double incDur;
	int i, Field, ByteField, bytes;

	incDur = secs - CallStack->lastIOtime;
	Field = getHDFFieldIndex( eventType ) ;
        CallStack->record.times[Field] += incDur;
        ++CallStack->record.counts[Field];
	ByteField = getHDFByteFieldIndex( Field ) ;
	switch ( eventType ) 
	{
		case readEndID:
		case freadEndID:
		case writeEndID:
		case fwriteEndID:
		case -ID_malloc:
	   	   bytes = *((int *)dataPtr);
	      	   CallStack->record.bytes[ByteField] += bytes;
	           /*====================================================
		   // update histogram					*
	           //===================================================*/
                   i = -1;
                   while ( bytes >= BktLim[i+1] ) ++i  ;
                   if ( i >= 0 ) ++CallStack->record.Hists[ByteField][i];
		   break;
		case fopenEndID:
		case openEndID:
		   openInfo.setName = openName;
		   openInfo.setID = (int)(*((long *)dataPtr));
		   CallStack->record.hdfID = openInfo.setID;
	           HDFrecordFileName ( &openInfo );
		   break;
		default:
		   break;
	}
			
}
#ifdef HAVE_PARALLEL
/*======================================================================*
// BeginMPIOEventRecord:                                               	*
//  This routine simply records the time in the record on the top of 	*
//  the stack.								*
//======================================================================*/ 
void BeginMPIOEventRecord( int eventType, 
	                   double seconds, 
	                   void *dataPtr,
	                   int dataLen )
{
	/*==============================================================*
	// save the time value temporarily in top of stack		*
	// When the end record is received, the duration can be 	*
	// computed.							*
	//==============================================================*/
	CallStack->lastIOtime = seconds;
	/*==============================================================*
	// get useful info from the structure pointed to by dataPtr.	*
	// in most cases, this is the file ID.  For mpiOpen, it is the	*
	// name of the file.  For mpiDelete, no information is of any	*
	// use.								*
	//==============================================================*/
        if ( dataLen == 0 ) return;
	switch ( eventType ) 
	{
	   case mpiGetSizeBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiGetSizeBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiGetGroupBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiGetGroupBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiGetAmodeBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiGetAmodeBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiGetViewBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiGetViewBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiGetPositionBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiGetPositionBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiGetByteOffsetBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiGetByteOffsetBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiGetTypeExtentBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiGetTypeExtentBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiGetAtomicityBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiGetAtomicityBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiOpenBeginID:
	      strcpy( openName,
		     ((struct mpiOpenBeginArgs *)dataPtr)->fileName);
	      break;
	   case mpiCloseBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiCloseBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiDeleteBeginID:
	      break;
	   case mpiSetSizeBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiSetSizeBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiPreallocateBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiPreallocateBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiSetViewBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiSetViewBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiReadAtBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiReadAtBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiReadAtAllBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiReadAtAllBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiWriteAtBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiWriteAtBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiWriteAtAllBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiWriteAtAllBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiIreadAtBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiIreadAtBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiIwriteAtBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiIwriteAtBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiReadBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiReadBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiReadAllBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiReadAllBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiWriteBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiWriteBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiWriteAllBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiWriteAllBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiIreadBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiIreadBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiIwriteBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiIwriteBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiSeekBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiSeekBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiSetAtomicityBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiSetAtomicityBeginArgs *)dataPtr)->fileID;
	      break;
	   case mpiSyncBeginID:
	      CallStack->record.hdfID 
		 = ((struct mpiSyncBeginArgs *)dataPtr)->fileID;
	      break;
	   default:
	      break;
	}
}
/*======================================================================*
// EndMPIOEventRecord:							*
//  This routine retrieves the entry time saved on the top of the stack *
//  and computes the duration of the MPI-I/O event.  This is added to   *
//  the record field corresponding MPI-I/O.  				*
//======================================================================*/ 
void EndMPIOEventRecord ( int eventType, 
	                  double seconds, 
	                  void *dataPtr,
	                  int dataLen )
{
	double incDur;

	incDur = seconds - CallStack->lastIOtime;
        CallStack->record.times[MPI] += incDur;
        ++CallStack->record.counts[MPI];
	if ( eventType == mpiOpenEndID && dataLen != 0 ) {
	   /*===========================================================*
	   // complete the file information for the case of a file 	*
	   // open and record the information.				*
	   //===========================================================*/
	   openInfo.setName = openName;
	   openInfo.setID = ((struct mpiOpenEndArgs *)dataPtr)->fileID;
	   CallStack->record.hdfID = openInfo.setID;
	   HDFrecordFileName ( &openInfo );
	}
}
#endif /* HAVE_PARALLEL */
/*======================================================================*
//   BeginHDFEventRecord:						* 
// 	This function puts a trace record on the stack corresponding to	*
//   	this thread.  If no stack exists, one is created.  If no record	* 
//   	exist, a record is created.                                   	* 
//======================================================================*/ 
void BeginHDFEventRecord( int eventID, double secs )
{
	HDFnode_t *HDFrec;
	/*==============================================================*
	// Create a record. Push it onto the call stack.                *
	//==============================================================*/
        HDFrec = (HDFnode_t *)malloc( sizeof(HDFnode_t) );
        HDFnodeInit( HDFrec ) ;
	HDFrec->eventID = eventID;
	HDFrec->ptr = CallStack;
	CallStack = HDFrec ;
	/*==============================================================*
	// save time stamp in record.					*
	//==============================================================*/
	HDFrec->record.lastCall = secs;
}
/*======================================================================* 
// EndHDFEventRecord:							*
//  This routine pops the HDF record from the top of the stack 		*
//  corresponding to this thread and computes the inclusive duration    *
//  and adds it to the inclusive duration field of this record and to   *
//  the HDF time field of the calling routines record.			*
//======================================================================*/ 
void EndHDFEventRecord ( int eventID, double seconds, void *dataPtr )
{
        HDFsetInfo 	*info;
	HDFnode_t	*HDFrec;
	double 		incSecs;
	static int	dummyIDs = -4;
	/*==============================================================*
	// pop record from top of the stack, compute inclusive duration	*
	// and set the corresponding record field and increment nCalls.	*
	//==============================================================*/
	HDFrec = CallStack;
	CallStack = CallStack->ptr;
	if ( CallStack == NULL ) {
	   fprintf(stderr,">>> EndHDFEventRecord: Call Stack is empty. <<<\n");
	   return;
	}
	incSecs = seconds - HDFrec->record.lastCall;
	HDFrec->record.incDur = +incSecs;
	++HDFrec->record.nCalls;
   	/*==============================================================*
	// add old record to chain to have its xRef field tagged.	*
	//==============================================================*/
	HDFrec->ptr = TagQueue;
	TagQueue = HDFrec;
	/*==============================================================*
	// Add set ID information.					*
	//==============================================================*/
	if ( dataPtr != NULL ) {
	   info = (HDFsetInfo *)dataPtr;
	   if ( info->setName != NULL ) {
	      if ( info->setID == 0 ) {
	         info->setID = dummyIDs--;
	      }
	      HDFrecordFileName ( info );
	   }
	   HDFrec->record.hdfID = info->setID;
	}
	/*==============================================================*
	// Update the HDF totals for the calling program.		*
	//==============================================================*/
        CallStack->record.times[ HDF_ ] += incSecs ;
        ++CallStack->record.counts[ HDF_ ] ;
	/*==============================================================*
	// If the stack has only one record it represents the main 	* 
	// program.  Tag all of the records on the TagQueue and tally   * 
	// them up.							* 
	//==============================================================*/
        if ( CallStack->ptr == NULL ) {
           HDFupdateProcLists( );
	}
}
/*======================================================================* 
// This routine keeps track of the identifier names and tags.  Some	*
// names may be associated with more than one tag.  This will be 	*
// rectified when final tallies are done.				*
//======================================================================*/
void HDFrecordFileName( HDFsetInfo *info )
{
	fileRec_t *P;
	char *t;
	int match; 
	long id;
	P = HDFfileList;
	match = FALSE;
	id = info->setID;
	while ( P != NULL && match == FALSE ) {
	   if ( strcmp( P->fileName, info->setName ) != 0 && P->hdfID == id ) {
	      match = TRUE;
	   } else {
	      P = P->ptr;
	   }
	}
	if ( match == FALSE ) {
	   P = ( fileRec_t *) malloc( sizeof( fileRec_t ) );
	   P->ptr = HDFfileList;
	   HDFfileList = P;
	   t = (char *)malloc( strlen( info->setName ) + 1 );
	   strcpy ( t, info->setName ) ;
	   P->fileName = t;
	   P->hdfID = info->setID;
	   P->PabloID = 0;
	} 
}  
/*======================================================================* 
// This routine assigns a unique Pablo ID to each unique name 		*
// regardless of the HDF tag.						*
// It then goes through the HDFRecordQueue and marks each record with   *
// the PabloID corresponding to the hdfID and xRef fields or 0.		*
//======================================================================*/
void HDFassignPabloIDs( int *nSetIDs, char ***Names )
{
	fileRec_t *F, *G;
	HDFnode_t *P;
	int j; 
	long PabloID = 1;
	long hdfID, xRef;
	char *fName, **T;

	F = HDFfileList;
        /*==============================================================*
        // Assign the same ID to identical names.			*
        //==============================================================*/
	while ( F != NULL ) {
	   if ( F->PabloID == 0 ) {
	      F->PabloID = PabloID++;
	      fName = F->fileName;
	      G = F->ptr;
	      while ( G != NULL ) {
	         if ( strcmp( G->fileName , fName ) == 0 ) {
	            G->PabloID = F->PabloID;
	         }
	         G = G->ptr;
	      }
	   }
	   F = F->ptr;
	}
	*nSetIDs = (int)(PabloID - 1);
        if ( *nSetIDs <= 0 ) return;
        /*==============================================================*
	// Repace hdfID and xRef fields with corresponding Pablo ID	*
        //==============================================================*/
   	for ( j = 0; j < NumHDFProcs; ++j ) {
   	   P = HDFQueues[j] ;
   	   while ( P != NULL ) {
   	      hdfID = P->record.hdfID;
   	      if ( hdfID != 0 ) {
   	         PabloID = 0;
                 F = HDFfileList;
   	         while ( F != NULL && PabloID == 0 ) {
                    if ( hdfID == F->hdfID ) {
   	               PabloID = F->PabloID;
   	            }
   	            F = F->ptr;
   	         }
   	         P->record.hdfID = PabloID;
   	      }
   	      xRef = P->record.xRef;
   	      if ( xRef != 0 ) {
   	         PabloID = 0;
                 F = HDFfileList;
   	         while ( F != NULL && PabloID == 0 ) {
                    if ( xRef == F->hdfID ) {
   	               PabloID = F->PabloID;
   	            }
   	            F = F->ptr;
   	         }
   	         P->record.xRef = PabloID;
   	      }
              P = P->ptr;
   	   } /* end while ( P != NULL ) */
        } /* end for */
	/*==============================================================*
	// get a list of all the unique names and order them according  *
	// to their Pablo IDs.						*
	//==============================================================*/
	T = ( char ** )malloc( (*nSetIDs+1) * sizeof( char * ) );
	for ( j = 0; j <= *nSetIDs; ++j ) {
	   T[j] = NULL;
	}
	F = HDFfileList;
	while ( F != NULL ) {
	   PabloID = F->PabloID  ;
	   if ( T[PabloID] == NULL ) {
	      T[PabloID] = ( char * )malloc( strlen( F->fileName ) + 1 );
	      strcpy( T[PabloID], F->fileName ) ;
	   }
	   free((void *)(F->fileName));
	   G = F;
	   F = F->ptr;
	   free ( (void *)G );
	}
	*Names = T;
}
/*======================================================================* 
// This routine writes SDDF packets to SDDF file containing information	*
// about the named identifiers found in the program.			*
//======================================================================*/
void writeHDFNamePacketsRT( char **Names, int numSetIDs )
{
	int j;
	HDFNamePacket_t NamePkt;
	char *BUFF, *fName;
	int buffSize;
	/*==============================================================*
	// Allocate a buffer to hold the packet.  Allow 80 chars for 	*
	// identifier name.						*
	//==============================================================*/
	buffSize = sizeof(HDFNamePacket_t) + 80;
	BUFF = (char *)malloc(buffSize);
	/*==============================================================*
	// Fill in constant information					*
	//==============================================================*/
	NamePkt.packetType = PKT_DATA;
	NamePkt.packetTag = FAMILY_NAME;
	/*==============================================================*
	// Fill in named identifier information and write to SDDF file	*
	//==============================================================*/
	for ( j = 1; j <= numSetIDs; ++j ) {
	   fName = Names[j];
	   NamePkt.packetLength = (int)(sizeof(NamePkt) + strlen(fName));
	   NamePkt.fileType = 0;		/* not currently used	*/
	   NamePkt.fileID = j;
	   NamePkt.nameLen = (int)strlen(fName) ;
	   if ( buffSize < NamePkt.packetLength ) {
	      free((void *)BUFF) ;
	      buffSize = NamePkt.packetLength + 80;
	      BUFF = (char *)malloc( buffSize ) ;
	   }
	   /*===========================================================*
	   // Copy packet data and tack on identifier name		*
	   //===========================================================*/
	   memcpy( BUFF, &NamePkt, sizeof(NamePkt) );
	   memcpy( BUFF + sizeof(NamePkt) , fName, strlen(fName) );
	   putBytes( BUFF , NamePkt.packetLength ) ;
	}
	free((void *)BUFF);
}
/*======================================================================*
// Tag xRef field of all records in this queue with the hdfID of the 	*
// highest level caller. Also						*
// This routine takes the records after they have been tagged and adds  *
// their fields to the apporopriate position in the HDFQueues structure *
//======================================================================*/
void HDFupdateProcLists( void )
{
	HDFnode_t *P, *Q;
	long hdfID;

	hdfID = TagQueue->record.hdfID;
	P = TagQueue;
	while ( P != NULL ) {
	   P->record.xRef = hdfID;
	   Q = P->ptr;
	   HDFupdateProcs( P );
	   P = Q;
	}
        TagQueue = NULL;
}
/*======================================================================*
// This routine takes as input a node pointer P  and looks for a Total  *
// record with this same eventID, hdfID and xRef.  If such a record     *
// exists, P is added to the record, otherwise a record is created and  *
// its values are set to P's.						*
//======================================================================*/
void HDFupdateProcs( HDFnode_t *P )
{
	int procIndex, eventID;
	long hdfID, xRef;
	HDFnode_t *Q;
	eventID = P->eventID;
	procIndex = ProcIndexForHDFEntry( eventID );
        hdfID = P->record.hdfID;
        xRef = P->record.xRef;
	Q = HDFQueues[ procIndex ];
	/*==============================================================*
	// First determine if a tally node exists that matches the     	*
	// eventID, hdfID and xRef of P.				*
	//==============================================================*/
	while ( Q != NULL && 
            (( Q->record.hdfID != hdfID ) || ( Q->record.xRef != xRef )) ) {
           Q = Q->ptr;
	}
	if ( Q == NULL ) {
	   /*===========================================================*
	   // No tally record matches the hdfID and xRef so put P in    *
	   // the queue.                 				*
	   //===========================================================*/
	   P->ptr = HDFQueues[ procIndex ];
	   HDFQueues[ procIndex ] = P;
	} else {
	   /*===========================================================*
	   // add P to the exiting record and free it.			*
	   //===========================================================*/
           HDFrecordSum ( &Q->record , &P->record );
	   free((void *)P);
	}
}
/*======================================================================*
// Print SDDF records for all records in this linked list.		*
//======================================================================*/
void HDFSummarySDDF( HDFnode_t *P, int procIndex )
{
	int i, arrayLen;
	char buff[1024];
	char *Packet;
	HDFnode_t *Q;
	struct {
		int packetLen,
		    packetType,
		    packetTag,
	            eventID,
		    threadID,
	            nCalls;
                 double Seconds, 
                        IncDur,
                        ExcDur;
	         long HDFid,
	              XREFid;
	} Header;

	Header.packetLen = sizeof(Header) 
	                 + sizeof(int)			 /* array len   */
			 + nTallyFields*sizeof(double)	 /* times array */
	                 + sizeof(int)			 /* array len   */
	                 + nTallyFields*sizeof(int) 	 /* count array */
	                 + sizeof(int)			 /* array len   */
	                 + nByteFields*sizeof(int) 	 /* bytes array */
	                 + nByteFields*sizeof(int)	 /* array lens  */
	                 + nByteFields*nBkts*sizeof(int) /* byte hist   */
	                 + sizeof(int) ;                 /* Name len    */
	Header.packetTag = HDF_SUMMARY_FAMILY +
			   ( procIndex + 1 )*8 + RECORD_TRACE ;
	Header.packetType = PKT_DATA;
	Header.threadID = TRgetNode();
        while ( P != NULL ) {
	   Q = P->ptr;
	   Header.eventID = P->eventID;
	   Header.nCalls  = P->record.nCalls;
           Header.Seconds = P->record.lastCall;
	   Header.IncDur  = P->record.incDur;
	   Header.ExcDur  = P->record.excDur;
	   Header.HDFid   = P->record.hdfID;
	   Header.XREFid  = P->record.xRef;
	   Packet = buff;
	   memcpy( Packet, &Header, sizeof(Header) );
	   Packet += sizeof(Header);
	   /*===========================================================*
	   // copy length of times array and times array to Packet.	*
	   //===========================================================*/
	   arrayLen = nTallyFields;
	   memcpy( Packet, &arrayLen, sizeof(int) );
	   Packet += sizeof(int);
	   memcpy( Packet, P->record.times, nTallyFields*sizeof(double) );
	   Packet += nTallyFields*sizeof(double);
	   /*===========================================================*
	   // copy length of counts array and counts array to Packet.	*
	   //===========================================================*/
	   arrayLen = nTallyFields;
	   memcpy( Packet, &arrayLen, sizeof(int) );
	   Packet += sizeof(int);
	   memcpy( Packet, P->record.counts, nTallyFields*sizeof(int) );
	   Packet += nTallyFields*sizeof(int);
	   /*===========================================================*
	   // copy length of bytes array and bytes array to Packet.	*
	   //===========================================================*/
	   arrayLen = nByteFields;
	   memcpy( Packet, &arrayLen, sizeof(int) );
	   Packet += sizeof(int);
	   memcpy( Packet, P->record.bytes, nByteFields*sizeof(int) );
	   Packet += nByteFields*sizeof(int);
	   /*===========================================================*
	   // copy length of historgram arrays and arrays to Packet.	*
	   //===========================================================*/
	   arrayLen = nBkts;
	   for ( i = 0; i < nByteFields; ++i ) {
	      memcpy( Packet, &arrayLen, sizeof(int) );
	      Packet += sizeof(int);
	      memcpy( Packet, P->record.Hists[i], nBkts*sizeof(int) );
	      Packet += nBkts*sizeof(int);
	   }
	   arrayLen = 0;	/* name length */
	   memcpy( Packet, &arrayLen, sizeof(int) );
	   putBytes( buff, Header.packetLen ); 
           free((void *)P);
           P = Q;
	} 
}
/*======================================================================*
// Initialize a node.							*
//======================================================================*/
void HDFnodeInit ( HDFnode_t *S ) 
{
	*S = InitNode;
}
/*======================================================================*
//      Compute IO totals, exclusive durations of the input record T    *
//      then add the fields of T to that of S.                          *
//======================================================================*/
void HDFrecordSum ( HDFrec_t *S, HDFrec_t *T )
{
        int i, j, allIOCount;
        double allIOTime;
        allIOTime = 0;
        for ( j = FirstIO; j <= LastIO; ++j ) {
           allIOTime += T->times[j];
        }
        T->times[AllIO] = allIOTime;
 
        allIOCount = 0;
        for ( j = FirstIO; j <= LastIO; ++j ) {
           allIOCount += T->counts[j];
        }
        T->counts[AllIO] = allIOCount;
 
        T->excDur = T->incDur - ( T->times[HDF_] + T->times[MPI]
                                                 + T->times[AllIO] );
 
        S->nCalls    += T->nCalls;
        S->lastCall  =  max( S->lastCall, T->lastCall );
        S->incDur    += T->incDur;
        S->excDur    += T->excDur;
        for ( j = 0; j < nTallyFields; ++j ) {
           S->times[j] += T->times[j] ;
        }
        for ( j = 0; j < nTallyFields; ++j ) {
           S->counts[j] += T->counts[j] ;
        }
        for ( j = 0; j < nByteFields; ++j ) {
           S->bytes[j] += T->bytes[j] ;
        }
        for ( j = 0; j < nByteFields; ++j ) {
           for ( i = 0; i < nBkts; ++i ) {
              S->Hists[j][i] += T->Hists[j][i] ;
           }
        }
}
/*======================================================================*
// Return the field index corresponding to an IO event ID.  The fields  *
// are specified in an enum statement in an include file.		*
//======================================================================*/
int getHDFFieldIndex( int eventID )
{
	int result = -1;
	switch ( eventID )
	{
	        case ID_malloc:
	        case -ID_malloc:
			result = Malloc;
			break;
		case openBeginID:
		case openEndID:
		case fopenBeginID:
		case fopenEndID:
			result = Open;
			break;
		case closeBeginID:
		case closeEndID:
		case fcloseBeginID:
		case fcloseEndID:
			result = Close;
			break;
		case readBeginID:
		case readEndID:
		case freadBeginID:
		case freadEndID:
			result = Read;
			break;
		case lseekBeginID:
		case lseekEndID:
		case fseekBeginID:
		case fseekEndID:
			result = Seek;
			break;
		case writeBeginID:
		case writeEndID:
		case fwriteBeginID:
		case fwriteEndID:
			result = Write;
			break;
		case fflushBeginID:
		case fflushEndID:
		case flushBeginID:
		case flushEndID:
			result = Misc;
			break;
		case rewindBeginID:
		case rewindEndID:
		case fsetposBeginID:
		case fsetposEndID:
			result = Misc;
			break;
#ifdef	creadBeginID
		case creadBeginID:
		case creadEndID:
		case creadvBeginID:
		case creadvEndID:
			result = Read;
 			break;
		case cwriteBeginID:
		case cwriteEndID:
		case cwritevBeginID:
		case cwritevEndID:
			result = Write;
 			break;
		case ireadBeginID:
		case ireadEndID:
		case ireadvBeginID:
		case ireadvEndID:
			result = ARead;
 			break;
		case iwriteBeginID:
		case iwriteEndID:
		case iwritevBeginID:
		case iwritevEndID:
			result = AWrite;
 			break;
		case iowaitBeginID:
		case iowaitEndID:
			result = Wait;
 			break;
		case iodoneBeginID:
		case iodoneEndID:
			result = Misc;
 			break;
		case gopenBeginID:
		case gopenEndID:
			result = Open;
 			break;
		case iomodeBeginID:
		case iomodeEndID:
		case setiomodeBeginID:
		case setiomodeEndID:
		case lsizeBeginID:
		case lsizeEndID:
		case forflushBeginID:
		case forflushEndID:
			result = Misc;
 			break;
#endif	
	}
	return result;
}
/*======================================================================*
// This routine determines the field index in the bytes array of the 	*
// HDF records which correspond to a given IO operation.  If the  	*
// operation does not transfer bytes, (e.g., open operation), -1 is	*
// returned.								*
//======================================================================*/
int getHDFByteFieldIndex( int Operation ) 
{
	int result;
	switch ( Operation )
	{
		case Malloc:
	 		result = MallocBytes;
			break;
		case Read:
	 		result = ReadBytes;
			break;
		case Write:
	 		result = WriteBytes;
			break;
		case ARead:
	 		result = AReadBytes;
			break;
		case AWrite:
	 		result = AWriteBytes;
			break;
		default:
			result = -1;
			break;
	}
	return result;
}
/*======================================================================*
// This routine writes the SDDF packet descriptors for the HDF summary	*
// records to the output file.						*
//======================================================================*/
void _hdfDescriptorRT( char *recordName, char *recordDescription, 
                                                       int recordFamily )
{
    static char	recordBuffer[ 4096 ];
    int		recordLength;

    hdfRecordPointer = recordBuffer;
    /*==================================================================*
    // Allow space at the beginning of the record for the packet 	*
    //length which will be computed after the packet is complete.	*
    //==================================================================*/
    sddfWriteInteger( &hdfRecordPointer, 0 );
    /*==================================================================* 
    // The record type, tag, and name 					* 
    //==================================================================*/
    sddfWriteInteger( &hdfRecordPointer, PKT_DESCRIPTOR );
    sddfWriteInteger( &hdfRecordPointer, ( recordFamily | RECORD_TRACE ) );
    sddfWriteString( &hdfRecordPointer, recordName );
    /*==================================================================* 
    // The record attribute count and string pair 			* 
    //==================================================================*/
    sddfWriteInteger( &hdfRecordPointer, 1 );
    sddfWriteString( &hdfRecordPointer, "description" );
    sddfWriteString( &hdfRecordPointer, recordDescription );
    /*==================================================================*
    // The record field count 						*
    //==================================================================*/
    sddfWriteInteger( &hdfRecordPointer, 17 );
    WRITE_HDF_FIELD( "Event Identifier", 
	             "Event ID",
                     "Corresponding Event",
	             INTEGER, 0 );
    WRITE_HDF_FIELD( "Processor Number", 
		     "Node", 
		     "Processor number", 
		     INTEGER, 0 );
    WRITE_HDF_FIELD( "N Calls", 
		     "N Calls", 
		     "Number of Calls to this Proc", 
		     INTEGER, 0 );
    WRITE_HDF_FIELD( "Seconds", 
	   	     "Seconds", 
                     "Floating Point Timestamp", 
		      DOUBLE, 0 );
    WRITE_HDF_FIELD( "Inclusive Duration", 
  	             "Inclusive Duration", 
	             "Inclusive Duration of this Procedure",
		      DOUBLE, 0 );
    WRITE_HDF_FIELD( "Exclusive Duration", 
  	             "Exclusive Duration", 
	             "Excludes IO, MPI-IO and other HDF calls",
		      DOUBLE, 0 );
    WRITE_HDF_FIELD2("HDF ID",
                     "HDF ID", "Identifier number",
                     "0", "No HDF ID specified",
                     LONG, 0 );
    WRITE_HDF_FIELD( "Xref ID",
                     "Cross Reference", 
                     "Index of related HDF ID or 0 if none",
		     LONG, 0 );
    WRITE_HDF_FIELD( "Times Array",
                     "Times Array",
	             "Array of Total Operation Times",
		     DOUBLE, 1 );
    WRITE_HDF_FIELD( "Counts Array",
                     "Counts Array",
	             "Array of Total Operation Counts",
		     INTEGER, 1 );
    WRITE_HDF_FIELD( "Bytes Array",
                     "Bytes Array",
	             "Array of Total Bytes Transferred",
		     INTEGER, 1 );
    WRITE_HDF_FIELD( "Malloc Histogram",
                     "Malloc Histogram",
	             "Historgram of size Malloc Requests",
		     INTEGER, 1 );
    WRITE_HDF_FIELD( "Read Histogram",
                     "Read Histogram",
	             "Historgram of size Read Requests",
		     INTEGER, 1 );
    WRITE_HDF_FIELD( "Write Histogram",
                     "Write Histogram",
	             "Historgram of size Write Requests",
		     INTEGER, 1 );
    WRITE_HDF_FIELD( "ARead Histogram",
                     "ARead Histogram",
	             "Historgram of size Asynch Read Requests",
		     INTEGER, 1 );
    WRITE_HDF_FIELD( "AWrite Histogram",
                     "AWrite Histogram",
	             "Historgram of size Asynch Write Requests",
		     INTEGER, 1 );
    WRITE_HDF_FIELD( "HDF Name",
                     "HDF Name", 
	             "Name of File,Data Set or Dim accessed",
                     CHARACTER, 1 ); 
    /*=================================================================== 
    // The entire record descriptor packet has been written.		* 
    // Compute and update the record length.				* 
    // Write the completed record.					* 
    //==================================================================*/
    recordLength = (int)(hdfRecordPointer - recordBuffer);

    hdfRecordPointer = recordBuffer;
    sddfWriteInteger( &hdfRecordPointer, recordLength );

    putBytes( recordBuffer, (unsigned) recordLength );
}

/*======================================================================* 
//   Internal Routine:  writeHDFRecDescrptrsRT	                        * 
//                      Writes record descriptors for the HDF events.   * 
//======================================================================*/
void writeHDFRecDescrptrsRT( void ) 
{
	char HDFProcNames[][40] = {
#	include "HDFentryNames.h"
	"HDF_Last_Entry"
	};
	int j, FAMILY;
        char BUF1[256], BUF2[256] ;
	_hdfNameDescriptor();	/* Descriptor for named identifiers	*/
        for ( j = 0; j < NumHDFProcs; ++j ) {
           if ( HDFQueues[j] != NULL ) {
              strcpy( BUF2, "HDF ");
              strcat( BUF2, HDFProcNames[j] );
              strcat( BUF2, " Procedure Summary");
              strcpy( BUF1, BUF2 );
              strcat( BUF1, " Trace");
	      FAMILY = HDF_SUMMARY_FAMILY + (j + 1)*8;
              _hdfDescriptorRT( BUF1, BUF2, FAMILY );
           }
        }
        return;
}
/*======================================================================*
// This routine prints the Pablo IDs assigned to named HDF identifiers  *
//======================================================================*/
void printFileMappingsRT( char *mapFile, char **Names, int nPabloIDs )
{
	int i;
	FILE *ptr;
	ptr = fopen( mapFile, "w" );
 
        if ( ptr == NULL ) {
	    fprintf(stderr,
                    "Couldn't open map file %s - none created.\n",mapFile);
            return;
        }
 
	fprintf(ptr,"\n\nPablo ID to HDF Name mappings:\n");
	fprintf(ptr,"------------------------------\n");
        for ( i = 1; i <= nPabloIDs; i++ ) {
	     fprintf(ptr,"%4d %s\n",i,Names[i] );
	}
	fprintf(ptr,"\n\n");
	fclose( ptr );
}
/************************************************************************/
/*	_hdfNameDescriptor	     					*/
/*	   Generate a SDDF binary format record descriptor for the	*/
/*	   named identifiers used during execution.              	*/
/************************************************************************/
void _hdfNameDescriptor( void )
{
    static char recordBuffer[ 4096 ];
    int         recordLength;

#ifdef DEBUG
	fprintf( debugFile, "_hdfExitTraceDescriptor entered\n" );
	fflush( debugFile );
#endif /* DEBUG */
    hdfRecordPointer = recordBuffer;
    /********************************************************************/
    /* Allow space at the beginning of the record for the packet        */
    /*length which will be computed after the packet is complete.       */
    /********************************************************************/
    sddfWriteInteger( &hdfRecordPointer, 0 );
    /********************************************************************/
    /* The record type, tag, and name                                   */
    /********************************************************************/
    sddfWriteInteger( &hdfRecordPointer, PKT_DESCRIPTOR );
    sddfWriteInteger( &hdfRecordPointer, ( FAMILY_NAME ) );
    sddfWriteString( &hdfRecordPointer, "HDF Name Identifier Record" );
    /********************************************************************/
    /* The record attribute count and string pair                       */
    /********************************************************************/
    sddfWriteInteger( &hdfRecordPointer, 1 );
    sddfWriteString( &hdfRecordPointer, "description" );
    sddfWriteString( &hdfRecordPointer, "HDF Name Identifier Record" );
    /********************************************************************/
    /* The record field count                                           */
    /********************************************************************/
    sddfWriteInteger( &hdfRecordPointer, 3);
    /********************************************************************/
    /* Create fields                                               	*/
    /********************************************************************/
    WRITE_HDF_FIELD(  "Identifier Type", 
		      "Data Set Type", 
		      "Data Set Identifier Type", 
		      INTEGER, 0 );
    WRITE_HDF_FIELD2( "HDF ID",
                      "HDF ID", "File, Data Set or Dim Identifier number",
                      "0", "No HDF ID specified",
                      INTEGER, 0 ); 
    WRITE_HDF_FIELD( "HDF Name",
                     "HDF Name", "Name of File, Data Set or Dim",
                      CHARACTER, 1 );

    recordLength = (int)(hdfRecordPointer - recordBuffer);

    hdfRecordPointer = recordBuffer;
    sddfWriteInteger( &hdfRecordPointer, recordLength );

    putBytes( recordBuffer, (unsigned) recordLength );
}