[svn-r17177] In H5Odtype.c, the decoding function used to mark the datatype as on disk (see bug

#1585).  I changed it to undefined and let the caller functions decide the location 
of the datatype.  For H5Tdecode, it should mark the datatype as in memory.  For other 
callers like H5Dopen or H5Aopen, they should makr it as on disk.

Tested it on jam, smirom, linew.

11 files changed, 62 insertions, 13 deletions

diff --git a/src/H5Dint.c b/src/H5Dint.c
index 196ff04..d36dc95 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -1345,6 +1345,10 @@ H5D_open_oid(H5D_t *dataset, hid_t dapl_id, hid_t dxpl_id)
     /* Get the type and space */
     if(NULL == (dataset->shared->type = (H5T_t *)H5O_msg_read(&(dataset->oloc), H5O_DTYPE_ID, NULL, dxpl_id)))
         HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to load type info from dataset header")
+
+    if(H5T_set_loc(dataset->shared->type, dataset->oloc.file, H5T_LOC_DISK) < 0)
+        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "invalid datatype location")
+
     if(NULL == (dataset->shared->space = H5S_read(&(dataset->oloc), dxpl_id)))
         HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to load dataspace info from dataset header")
 
diff --git a/src/H5HG.c b/src/H5HG.c
index 9042146..ba3e968 100644
--- a/src/H5HG.c
+++ b/src/H5HG.c
@@ -320,6 +320,8 @@ H5HG_alloc (H5F_t *f, H5HG_heap_t *heap, size_t size, unsigned * heap_flags_ptr)
                 break;
     } /* end else */
 
+    HDassert(idx < heap->nused);
+
     /* Check if we need more room to store heap objects */
     if(idx>=heap->nalloc) {
         size_t new_alloc;       /* New allocation number */
diff --git a/src/H5HGcache.c b/src/H5HGcache.c
index 747c41e..22374ce 100644
--- a/src/H5HGcache.c
+++ b/src/H5HGcache.c
@@ -244,6 +244,8 @@ H5HG_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * udata1,
     else
         heap->nused = 1;
 
+    HDassert(max_idx < heap->nused);
+
     /*
      * Add the new heap to the CWFS list, removing some other entry if
      * necessary to make room. We remove the right-most entry that has less
diff --git a/src/H5Oattr.c b/src/H5Oattr.c
index 1e85c69..c474681 100644
--- a/src/H5Oattr.c
+++ b/src/H5Oattr.c
@@ -633,7 +633,7 @@ H5O_attr_pre_copy_file(H5F_t UNUSED *file_src, const void UNUSED *native_src,
  *-------------------------------------------------------------------------
  */
 static void *
-H5O_attr_copy_file(H5F_t UNUSED *file_src, const H5O_msg_class_t UNUSED *mesg_type,
+H5O_attr_copy_file(H5F_t *file_src, const H5O_msg_class_t UNUSED *mesg_type,
     void *native_src, H5F_t *file_dst, hbool_t *recompute_size,
     H5O_copy_t *cpy_info, void UNUSED *udata, hid_t dxpl_id)
 {
@@ -647,6 +647,12 @@ H5O_attr_copy_file(H5F_t UNUSED *file_src, const H5O_msg_class_t UNUSED *mesg_ty
     HDassert(cpy_info);
     HDassert(!cpy_info->copy_without_attr);
 
+    /* Mark datatype as being on disk now.  This step used to be done in a lower level 
+     * by H5O_dtype_decode.  But it has been moved up.  Not an ideal place, but no better 
+     * place than here. */
+    if(H5T_set_loc(((H5A_t *)native_src)->shared->dt, file_src, H5T_LOC_DISK) < 0)
+        HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, NULL, "invalid datatype location")
+
     if ( NULL == (ret_value=H5A_attr_copy_file((H5A_t *)native_src, file_dst, recompute_size, cpy_info,  dxpl_id)))
         HGOTO_ERROR(H5E_ATTR, H5E_CANTCOPY, NULL, "can't copy attribute")
 
diff --git a/src/H5Oattribute.c b/src/H5Oattribute.c
index 81c2d3b..f752a6f 100644
--- a/src/H5Oattribute.c
+++ b/src/H5Oattribute.c
@@ -536,6 +536,11 @@ H5O_attr_open_by_name(const H5O_loc_t *loc, const char *name, hid_t dxpl_id)
             HDassert(udata.attr);
             ret_value = udata.attr;
         } /* end else */
+
+        /* Mark datatype as being on disk now */
+        if(H5T_set_loc(ret_value->shared->dt, loc->file, H5T_LOC_DISK) < 0)
+            HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, "invalid datatype location")
+
     } /* end else */
 
 done:
@@ -641,6 +646,10 @@ H5O_attr_open_by_idx(const H5O_loc_t *loc, H5_index_t idx_type,
                 HGOTO_ERROR(H5E_ATTR, H5E_CANTCLOSEOBJ, NULL, "can't close attribute")
             if(NULL == (ret_value = H5A_copy(NULL, exist_attr)))
                 HGOTO_ERROR(H5E_ATTR, H5E_CANTCOPY, NULL, "can't copy existing attribute")
+        } else {
+            /* Mark datatype as being on disk now */
+            if(H5T_set_loc(ret_value->shared->dt, loc->file, H5T_LOC_DISK) < 0)
+                HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, NULL, "invalid datatype location")
         } /* end if */
     } /* end if */
 
diff --git a/src/H5Odtype.c b/src/H5Odtype.c
index fe8c627..47fdd7e 100644
--- a/src/H5Odtype.c
+++ b/src/H5Odtype.c
@@ -452,8 +452,9 @@ H5O_dtype_decode_helper(H5F_t *f, unsigned *ioflags/*in,out*/, const uint8_t **p
 
             /* Set extra information for object references, so the hobj_ref_t gets swizzled correctly */
             if(dt->shared->u.atomic.u.r.rtype == H5R_OBJECT) {
-                /* This type is on disk */
-                dt->shared->u.atomic.u.r.loc = H5T_LOC_DISK;
+                /* Mark location this type as undefined for now.  The caller function should
+                 * decide the location. */
+                dt->shared->u.atomic.u.r.loc = H5T_LOC_BADLOC;
 
                 /* This type needs conversion */
                 dt->shared->force_conv = TRUE;
@@ -518,9 +519,16 @@ H5O_dtype_decode_helper(H5F_t *f, unsigned *ioflags/*in,out*/, const uint8_t **p
                 ioflags, "vlen", FAIL)
 
             dt->shared->force_conv=TRUE;
-            /* Mark this type as on disk */
-            if(H5T_set_loc(dt, f, H5T_LOC_DISK) < 0)
+
+#ifdef TMP
+            /* Mark location this type as undefined for now.  The caller function should
+             * decide the location. */
+            if(H5T_set_loc(dt, f, H5T_LOC_BADLOC) < 0)
+                HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "invalid datatype location")
+else
+            if(H5T_set_loc(dt, NULL, H5T_LOC_MEMORY) < 0)
                 HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "invalid datatype location")
+#endif
             break;
 
         case H5T_ARRAY:  /* Array datatypes */
diff --git a/src/H5T.c b/src/H5T.c
index 200d5c1..a8b4fcd 100644
--- a/src/H5T.c
+++ b/src/H5T.c
@@ -2923,6 +2923,10 @@ H5T_decode(const unsigned char *buf)
     if((ret_value = H5O_msg_decode(f, H5AC_dxpl_id, NULL, H5O_DTYPE_ID, buf)) == NULL)
 	HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, NULL, "can't decode object")
 
+    /* Mark datatype as being in memory now */
+    if(H5T_set_loc(ret_value, NULL, H5T_LOC_MEMORY) < 0)
+        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "invalid datatype location")
+
 done:
     /* Release fake file structure */
     if(f && H5F_fake_free(f) < 0)
@@ -4029,8 +4033,8 @@ H5T_cmp(const H5T_t *dt1, const H5T_t *dt2, hbool_t superset)
         case H5T_VLEN:
             assert(dt1->shared->u.vlen.type>H5T_VLEN_BADTYPE && dt1->shared->u.vlen.type<H5T_VLEN_MAXTYPE);
             assert(dt2->shared->u.vlen.type>H5T_VLEN_BADTYPE && dt2->shared->u.vlen.type<H5T_VLEN_MAXTYPE);
-            assert(dt1->shared->u.vlen.loc>H5T_LOC_BADLOC && dt1->shared->u.vlen.loc<H5T_LOC_MAXLOC);
-            assert(dt2->shared->u.vlen.loc>H5T_LOC_BADLOC && dt2->shared->u.vlen.loc<H5T_LOC_MAXLOC);
+            assert(dt1->shared->u.vlen.loc>=H5T_LOC_BADLOC && dt1->shared->u.vlen.loc<H5T_LOC_MAXLOC);
+            assert(dt2->shared->u.vlen.loc>=H5T_LOC_BADLOC && dt2->shared->u.vlen.loc<H5T_LOC_MAXLOC);
 
             /* Arbitrarily sort sequence VL datatypes before string VL datatypes */
             if (dt1->shared->u.vlen.type==H5T_VLEN_SEQUENCE &&
@@ -4047,7 +4051,11 @@ H5T_cmp(const H5T_t *dt1, const H5T_t *dt2, hbool_t superset)
             } else if (dt1->shared->u.vlen.loc==H5T_LOC_DISK &&
                     dt2->shared->u.vlen.loc==H5T_LOC_MEMORY) {
                 HGOTO_DONE(1);
+            } else if (dt1->shared->u.vlen.loc==H5T_LOC_BADLOC &&
+                    dt2->shared->u.vlen.loc!=H5T_LOC_BADLOC) {
+                HGOTO_DONE(1);
             }
+
             /* Don't allow VL types in different files to compare as equal */
             if (dt1->shared->u.vlen.f < dt2->shared->u.vlen.f)
                 HGOTO_DONE(-1);
@@ -4969,7 +4977,7 @@ H5T_set_loc(H5T_t *dt, H5F_t *f, H5T_loc_t loc)
     FUNC_ENTER_NOAPI(H5T_set_loc, FAIL);
 
     assert(dt);
-    assert(loc>H5T_LOC_BADLOC && loc<H5T_LOC_MAXLOC);
+    assert(loc>=H5T_LOC_BADLOC && loc<H5T_LOC_MAXLOC);
 
     /* Datatypes can't change in size if the force_conv flag is not set */
     if(dt->shared->force_conv) {
diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c
index df35789..f580c15 100644
--- a/src/H5Tcommit.c
+++ b/src/H5Tcommit.c
@@ -741,6 +741,10 @@ H5T_open(const H5G_loc_t *loc, hid_t dxpl_id)
         /* Point to shared datatype info */
         dt->shared = shared_fo;
 
+        /* Mark any datatypes as being in memory now */
+        if(H5T_set_loc(dt, NULL, H5T_LOC_MEMORY) < 0)
+            HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "invalid datatype location")
+
         /* Increment ref. count on shared info */
         shared_fo->fo_count++;
 
diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c
index 27eb0f6..412012a 100644
--- a/src/H5Tvlen.c
+++ b/src/H5Tvlen.c
@@ -214,7 +214,7 @@ H5T_vlen_set_loc(const H5T_t *dt, H5F_t *f, H5T_loc_t loc)
 
     /* check parameters */
     HDassert(dt);
-    HDassert(loc > H5T_LOC_BADLOC && loc < H5T_LOC_MAXLOC);
+    HDassert(loc >= H5T_LOC_BADLOC && loc < H5T_LOC_MAXLOC);
 
     /* Only change the location if it's different */
     if(loc != dt->shared->u.vlen.loc || f != dt->shared->u.vlen.f) {
@@ -280,6 +280,12 @@ H5T_vlen_set_loc(const H5T_t *dt, H5F_t *f, H5T_loc_t loc)
                 /* Set file ID (since this VL is on disk) */
                 dt->shared->u.vlen.f = f;
                 break;
+            
+            case H5T_LOC_BADLOC:
+                /* Allow undefined location. In H5Odtype.c, H5O_dtype_decode sets undefined
+                 * location for VL type and leaves it for the caller to decide.
+                 */  
+                break;
 
             default:
                 HGOTO_ERROR(H5E_DATATYPE, H5E_BADRANGE, FAIL, "invalid VL datatype location")
diff --git a/tools/testfiles/tvlstr.h5 b/tools/testfiles/tvlstr.h5
index 769d811..acd8a1f 100644
--- a/tools/testfiles/tvlstr.h5
+++ b/tools/testfiles/tvlstr.h5
diff --git a/tools/testfiles/tvlstr.h5.xml b/tools/testfiles/tvlstr.h5.xml
index 5999abd..507f0eb 100644
--- a/tools/testfiles/tvlstr.h5.xml
+++ b/tools/testfiles/tvlstr.h5.xml
@@ -3,19 +3,19 @@ Expected output for 'h5dump --xml tvlstr.h5'
 #############################
 <?xml version="1.0" encoding="UTF-8"?>
 <hdf5:HDF5-File xmlns:hdf5="http://hdfgroup.org/DTDs/HDF5-File" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://hdfgroup.org/DTDs/HDF5-File http://www.hdfgroup.org/DTDs/HDF5-File.xsd">
-<hdf5:RootGroup OBJ-XID="xid_928" H5Path="/">
+<hdf5:RootGroup OBJ-XID="xid_96" H5Path="/">
    <hdf5:Attribute Name="test_scalar">
       <hdf5:Dataspace>
          <hdf5:ScalarDataspace />
       </hdf5:Dataspace>
-      <hdf5:NamedDataTypePtr OBJ-XID="xid_1576" H5Path="/vl_string_type" />
+      <hdf5:NamedDataTypePtr OBJ-XID="xid_1400" H5Path="/vl_string_type" />
       <hdf5:Data>
          <hdf5:DataFromFile>
          "This is the string for the attribute"
          </hdf5:DataFromFile>
       </hdf5:Data>
    </hdf5:Attribute>
-   <hdf5:Dataset Name="Dataset1" OBJ-XID="xid_976" H5Path= "/Dataset1" Parents="xid_928" H5ParentPaths="/">
+   <hdf5:Dataset Name="Dataset1" OBJ-XID="xid_800" H5Path= "/Dataset1" Parents="xid_96" H5ParentPaths="/">
       <hdf5:StorageLayout>
          <hdf5:ContiguousLayout/>
       </hdf5:StorageLayout>
@@ -46,7 +46,7 @@ Expected output for 'h5dump --xml tvlstr.h5'
          </hdf5:DataFromFile>
       </hdf5:Data>
    </hdf5:Dataset>
-   <hdf5:NamedDataType Name="vl_string_type" OBJ-XID="xid_1576" H5Path="/vl_string_type" Parents="xid_928" H5ParentPaths="/">
+   <hdf5:NamedDataType Name="vl_string_type" OBJ-XID="xid_1400" H5Path="/vl_string_type" Parents="xid_96" H5ParentPaths="/">
       <hdf5:DataType>
          <hdf5:AtomicType>
             <hdf5:StringType Cset="H5T_CSET_ASCII" StrSize="H5T_VARIABLE" StrPad="H5T_STR_NULLPAD"/>