Merge topic 'ExternalData-multiple-hashes'

33a9aaa8 ExternalData: Add support for multiple hash algorithms

1 files changed, 91 insertions, 19 deletions

diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake
index b00de14..07cd114 100644
--- a/Modules/ExternalData.cmake
+++ b/Modules/ExternalData.cmake
@@ -201,6 +201,11 @@ For example, the argument ``DATA{img.png}`` may be satisfied by either a
 real ``img.png`` file in the current source directory or a ``img.png.md5``
 file containing its MD5 sum.
 
+Multiple content links of the same name with different hash algorithms
+are supported (e.g. ``img.png.sha256`` and ``img.png.sha1``) so long as
+they all correspond to the same real file.  This allows objects to be
+fetched from sources indexed by different hash algorithms.
+
 Referencing File Series
 """""""""""""""""""""""
 
@@ -429,8 +434,10 @@ function(ExternalData_add_target target)
     string(REPLACE "|" ";" tuple "${entry}")
     list(GET tuple 0 file)
     list(GET tuple 1 name)
-    list(GET tuple 2 ext)
-    set(stamp "${ext}-stamp")
+    list(GET tuple 2 exts)
+    string(REPLACE "+" ";" exts_list "${exts}")
+    list(GET exts_list 0 first_ext)
+    set(stamp "-hash-stamp")
     if(NOT DEFINED "_ExternalData_FILE_${file}")
       set("_ExternalData_FILE_${file}" 1)
       get_property(added DIRECTORY PROPERTY "_ExternalData_FILE_${file}")
@@ -446,12 +453,12 @@ function(ExternalData_add_target target)
           OUTPUT "${file}${stamp}" "${file}"
           # Run the data fetch/update script.
           COMMAND ${CMAKE_COMMAND} -Drelative_top=${CMAKE_BINARY_DIR}
-                                   -Dfile=${file} -Dname=${name} -Dext=${ext}
+                                   -Dfile=${file} -Dname=${name} -Dexts=${exts}
                                    -DExternalData_ACTION=fetch
                                    -DExternalData_CONFIG=${config}
                                    -P ${_ExternalData_SELF}
           # Update whenever the object hash changes.
-          MAIN_DEPENDENCY "${name}${ext}"
+          MAIN_DEPENDENCY "${name}${first_ext}"
           )
       endif()
       list(APPEND files "${file}${stamp}")
@@ -798,6 +805,7 @@ function(_ExternalData_arg_find_files glob pattern regex)
   cmake_policy(SET CMP0009 NEW)
   file(${glob} globbed RELATIVE "${top_src}" "${top_src}/${pattern}*")
   cmake_policy(POP)
+  set(externals_count -1)
   foreach(entry IN LISTS globbed)
     if("x${entry}" MATCHES "^x(.*)(\\.(${_ExternalData_REGEX_EXT}))$")
       set(relname "${CMAKE_MATCH_1}")
@@ -817,7 +825,11 @@ function(_ExternalData_arg_find_files glob pattern regex)
         set(name "${top_src}/${relname}")
         set(file "${top_bin}/${relname}")
         if(alg)
-          list(APPEND external "${file}|${name}|${alg}")
+          if(NOT "${external_${externals_count}_file_name}" STREQUAL "${file}|${name}")
+            math(EXPR externals_count "${externals_count} + 1")
+            set(external_${externals_count}_file_name "${file}|${name}")
+          endif()
+          list(APPEND external_${externals_count}_algs "${alg}")
         elseif(ExternalData_LINK_CONTENT)
           _ExternalData_link_content("${name}" alg)
           list(APPEND external "${file}|${name}|${alg}")
@@ -830,6 +842,14 @@ function(_ExternalData_arg_find_files glob pattern regex)
       endif()
     endif()
   endforeach()
+  if(${externals_count} GREATER -1)
+    foreach(ii RANGE ${externals_count})
+      string(REPLACE ";" "+" algs_delim "${external_${ii}_algs}")
+      list(APPEND external "${external_${ii}_file_name}|${algs_delim}")
+      unset(external_${ii}_algs)
+      unset(external_${ii}_file_name)
+    endforeach()
+  endif()
   set(external "${external}" PARENT_SCOPE)
   set(internal "${internal}" PARENT_SCOPE)
   set(have_original "${have_original}" PARENT_SCOPE)
@@ -947,13 +967,28 @@ function(_ExternalData_custom_fetch key loc file err_var msg_var)
   set("${msg_var}" "${msg}" PARENT_SCOPE)
 endfunction()
 
-function(_ExternalData_download_object name hash algo var_obj)
+function(_ExternalData_get_from_object_store hash algo var_obj var_success)
+  # Search all object stores for an existing object.
+  foreach(dir ${ExternalData_OBJECT_STORES})
+    set(obj "${dir}/${algo}/${hash}")
+    if(EXISTS "${obj}")
+      message(STATUS "Found object: \"${obj}\"")
+      set("${var_obj}" "${obj}" PARENT_SCOPE)
+      set("${var_success}" 1 PARENT_SCOPE)
+      return()
+    endif()
+  endforeach()
+endfunction()
+
+function(_ExternalData_download_object name hash algo var_obj var_success var_errorMsg)
   # Search all object stores for an existing object.
+  set(success 1)
   foreach(dir ${ExternalData_OBJECT_STORES})
     set(obj "${dir}/${algo}/${hash}")
     if(EXISTS "${obj}")
       message(STATUS "Found object: \"${obj}\"")
       set("${var_obj}" "${obj}" PARENT_SCOPE)
+      set("${var_success}" "${success}" PARENT_SCOPE)
       return()
     endif()
   endforeach()
@@ -1008,6 +1043,7 @@ function(_ExternalData_download_object name hash algo var_obj)
   get_filename_component(dir "${name}" PATH)
   set(staged "${dir}/.ExternalData_${algo}_${hash}")
 
+  set(success 1)
   if(found)
     file(RENAME "${tmp}" "${obj}")
     message(STATUS "Downloaded object: \"${obj}\"")
@@ -1018,38 +1054,74 @@ function(_ExternalData_download_object name hash algo var_obj)
     if(NOT tried)
       set(tried "\n  (No ExternalData_URL_TEMPLATES given)")
     endif()
-    message(FATAL_ERROR "Object ${algo}=${hash} not found at:${tried}")
+    set(success 0)
+    set("${var_errorMsg}" "Object ${algo}=${hash} not found at:${tried}" PARENT_SCOPE)
   endif()
 
   set("${var_obj}" "${obj}" PARENT_SCOPE)
+  set("${var_success}" "${success}" PARENT_SCOPE)
 endfunction()
 
 if("${ExternalData_ACTION}" STREQUAL "fetch")
-  foreach(v ExternalData_OBJECT_STORES file name ext)
+  foreach(v ExternalData_OBJECT_STORES file name exts)
     if(NOT DEFINED "${v}")
       message(FATAL_ERROR "No \"-D${v}=\" value provided!")
     endif()
   endforeach()
 
-  file(READ "${name}${ext}" hash)
-  string(STRIP "${hash}" hash)
-
-  if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$")
-    string(TOUPPER "${CMAKE_MATCH_1}" algo)
-  else()
-    message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"")
-  endif()
+  string(REPLACE "+" ";" exts_list "${exts}")
+  set(succeeded 0)
+  set(errorMsg "")
+  set(hash_list )
+  set(algo_list )
+  set(hash )
+  set(algo )
+  foreach(ext ${exts_list})
+    file(READ "${name}${ext}" hash)
+    string(STRIP "${hash}" hash)
+
+    if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$")
+      string(TOUPPER "${CMAKE_MATCH_1}" algo)
+    else()
+      message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"")
+    endif()
 
-  _ExternalData_download_object("${name}" "${hash}" "${algo}" obj)
+    list(APPEND hash_list ${hash})
+    list(APPEND algo_list ${algo})
+  endforeach()
 
+  list(LENGTH exts_list num_extensions)
+  math(EXPR exts_range "${num_extensions} - 1")
+  foreach(ii RANGE 0 ${exts_range})
+    list(GET hash_list ${ii} hash)
+    list(GET algo_list ${ii} algo)
+    _ExternalData_get_from_object_store("${hash}" "${algo}" obj succeeded)
+    if(succeeded)
+      break()
+    endif()
+  endforeach()
+  if(NOT succeeded)
+    foreach(ii RANGE 0 ${exts_range})
+      list(GET hash_list ${ii} hash)
+      list(GET algo_list ${ii} algo)
+      _ExternalData_download_object("${name}" "${hash}" "${algo}"
+        obj succeeded algoErrorMsg)
+      set(errorMsg "${errorMsg}\n${algoErrorMsg}")
+      if(succeeded)
+        break()
+      endif()
+    endforeach()
+  endif()
+  if(NOT succeeded)
+    message(FATAL_ERROR "${errorMsg}")
+  endif()
   # Check if file already corresponds to the object.
-  set(stamp "${ext}-stamp")
+  set(stamp "-hash-stamp")
   set(file_up_to_date 0)
   if(EXISTS "${file}" AND EXISTS "${file}${stamp}")
     file(READ "${file}${stamp}" f_hash)
     string(STRIP "${f_hash}" f_hash)
     if("${f_hash}" STREQUAL "${hash}")
-      #message(STATUS "File already corresponds to object")
       set(file_up_to_date 1)
     endif()
   endif()