From 0fe4d8bb3b9e8074f7efd95ae92fa32fe29a2c02 Mon Sep 17 00:00:00 2001 From: Brad King Date: Tue, 13 Jan 2015 10:03:56 -0500 Subject: ExternalData: Add support for custom download scripts Add support for a special URL template to map the fetch operation to a project-specified .cmake script insead of using file(DOWNLOAD). Extend the Module.ExternalData test to cover the behavior. Extend the RunCMake.ExternalData test to cover error cases. --- Modules/ExternalData.cmake | 105 +++++++++++++++++++++- Modules/ExternalData_config.cmake.in | 1 + Tests/Module/ExternalData/CMakeLists.txt | 3 + Tests/Module/ExternalData/Data1Check.cmake | 4 + Tests/Module/ExternalData/DataScript.dat.md5 | 1 + Tests/Module/ExternalData/MyScript1.cmake | 5 ++ Tests/RunCMake/ExternalData/BadCustom1-result.txt | 1 + Tests/RunCMake/ExternalData/BadCustom1-stderr.txt | 9 ++ Tests/RunCMake/ExternalData/BadCustom1.cmake | 5 ++ Tests/RunCMake/ExternalData/BadCustom2-result.txt | 1 + Tests/RunCMake/ExternalData/BadCustom2-stderr.txt | 9 ++ Tests/RunCMake/ExternalData/BadCustom2.cmake | 5 ++ Tests/RunCMake/ExternalData/BadCustom3-result.txt | 1 + Tests/RunCMake/ExternalData/BadCustom3-stderr.txt | 7 ++ Tests/RunCMake/ExternalData/BadCustom3.cmake | 5 ++ Tests/RunCMake/ExternalData/BadCustom4-result.txt | 1 + Tests/RunCMake/ExternalData/BadCustom4-stderr.txt | 7 ++ Tests/RunCMake/ExternalData/BadCustom4.cmake | 6 ++ Tests/RunCMake/ExternalData/RunCMakeTest.cmake | 4 + 19 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 Tests/Module/ExternalData/DataScript.dat.md5 create mode 100644 Tests/Module/ExternalData/MyScript1.cmake create mode 100644 Tests/RunCMake/ExternalData/BadCustom1-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom1-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom1.cmake create mode 100644 Tests/RunCMake/ExternalData/BadCustom2-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom2-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom2.cmake create mode 100644 Tests/RunCMake/ExternalData/BadCustom3-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom3-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom3.cmake create mode 100644 Tests/RunCMake/ExternalData/BadCustom4-result.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom4-stderr.txt create mode 100644 Tests/RunCMake/ExternalData/BadCustom4.cmake diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index 4f84d53..2d5088a 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -99,6 +99,12 @@ calling any of the functions provided by this module. default is ``CMAKE_BINARY_DIR``. The directory layout will mirror that of content links under ``ExternalData_SOURCE_ROOT``. +.. variable:: ExternalData_CUSTOM_SCRIPT_ + + Specify a full path to a ``.cmake`` custom fetch script identified by + ```` in entries of the ``ExternalData_URL_TEMPLATES`` list. + See `Custom Fetch Scripts`_. + .. variable:: ExternalData_LINK_CONTENT The ``ExternalData_LINK_CONTENT`` variable may be set to the name of a @@ -246,6 +252,44 @@ The following hash algorithms are supported:: Note that the hashes are used only for unique data identification and download verification. + +Custom Fetch Scripts +^^^^^^^^^^^^^^^^^^^^ + +When a data file must be fetched from one of the URL templates +specified in the ``ExternalData_URL_TEMPLATES`` variable, it is +normally downloaded using the :command:`file(DOWNLOAD)` command. +One may specify usage of a custom fetch script by using a URL +template of the form ``ExternalDataCustomScript:///``. +The ```` must be a C identifier, and the ```` must +contain the ``%(algo)`` and ``%(hash)`` placeholders. +A variable corresponding to the key, ``ExternalData_CUSTOM_SCRIPT_``, +must be set to the full path to a ``.cmake`` script file. The script +will be included to perform the actual fetch, and provided with +the following variables: + +.. variable:: ExternalData_CUSTOM_LOCATION + + When a custom fetch script is loaded, this variable is set to the + location part of the URL, which will contain the substituted hash + algorithm name and content hash value. + +.. variable:: ExternalData_CUSTOM_FILE + + When a custom fetch script is loaded, this variable is set to the + full path to a file in which the script must store the fetched + content. The name of the file is unspecified and should not be + interpreted in any way. + +The custom fetch script is expected to store fetched content in the +file or set a variable: + +.. variable:: ExternalData_CUSTOM_ERROR + + When a custom fetch script fails to fetch the requested content, + it must set this variable to a short one-line message describing + the reason for failure. + #]=======================================================================] #============================================================================= @@ -275,6 +319,37 @@ function(ExternalData_add_target target) if(NOT ExternalData_OBJECT_STORES) set(ExternalData_OBJECT_STORES ${CMAKE_BINARY_DIR}/ExternalData/Objects) endif() + set(_ExternalData_CONFIG_CODE "") + + # Store custom script configuration. + foreach(url_template IN LISTS ExternalData_URL_TEMPLATES) + if("${url_template}" MATCHES "^ExternalDataCustomScript://([^/]*)/(.*)$") + set(key "${CMAKE_MATCH_1}") + if(key MATCHES "^[A-Za-z_][A-Za-z0-9_]*$") + if(ExternalData_CUSTOM_SCRIPT_${key}) + if(IS_ABSOLUTE "${ExternalData_CUSTOM_SCRIPT_${key}}") + string(CONCAT _ExternalData_CONFIG_CODE "${_ExternalData_CONFIG_CODE}\n" + "set(ExternalData_CUSTOM_SCRIPT_${key} \"${ExternalData_CUSTOM_SCRIPT_${key}}\")") + else() + message(FATAL_ERROR + "No ExternalData_CUSTOM_SCRIPT_${key} is not set to a full path:\n" + " ${ExternalData_CUSTOM_SCRIPT_${key}}") + endif() + else() + message(FATAL_ERROR + "No ExternalData_CUSTOM_SCRIPT_${key} is set for URL template:\n" + " ${url_template}") + endif() + else() + message(FATAL_ERROR + "Bad ExternalDataCustomScript key '${key}' in URL template:\n" + " ${url_template}\n" + "The key must be a valid C identifier.") + endif() + endif() + endforeach() + + # Store configuration for use by build-time script. set(config ${CMAKE_CURRENT_BINARY_DIR}/${target}_config.cmake) configure_file(${_ExternalData_SELF_DIR}/ExternalData_config.cmake.in ${config} @ONLY) @@ -781,6 +856,30 @@ function(_ExternalData_download_file url file err_var msg_var) set("${msg_var}" "${msg}" PARENT_SCOPE) endfunction() +function(_ExternalData_custom_fetch key loc file err_var msg_var) + if(NOT ExternalData_CUSTOM_SCRIPT_${key}) + set(err 1) + set(msg "No ExternalData_CUSTOM_SCRIPT_${key} set!") + elseif(NOT EXISTS "${ExternalData_CUSTOM_SCRIPT_${key}}") + set(err 1) + set(msg "No '${ExternalData_CUSTOM_SCRIPT_${key}}' exists!") + else() + set(ExternalData_CUSTOM_LOCATION "${loc}") + set(ExternalData_CUSTOM_FILE "${file}") + unset(ExternalData_CUSTOM_ERROR) + include("${ExternalData_CUSTOM_SCRIPT_${key}}") + if(DEFINED ExternalData_CUSTOM_ERROR) + set(err 1) + set(msg "${ExternalData_CUSTOM_ERROR}") + else() + set(err 0) + set(msg "no error") + endif() + endif() + set("${err_var}" "${err}" PARENT_SCOPE) + set("${msg_var}" "${msg}" PARENT_SCOPE) +endfunction() + function(_ExternalData_download_object name hash algo var_obj) # Search all object stores for an existing object. foreach(dir ${ExternalData_OBJECT_STORES}) @@ -804,7 +903,11 @@ function(_ExternalData_download_object name hash algo var_obj) string(REPLACE "%(hash)" "${hash}" url_tmp "${url_template}") string(REPLACE "%(algo)" "${algo}" url "${url_tmp}") message(STATUS "Fetching \"${url}\"") - _ExternalData_download_file("${url}" "${tmp}" err errMsg) + if(url MATCHES "^ExternalDataCustomScript://([A-Za-z_][A-Za-z0-9_]*)/(.*)$") + _ExternalData_custom_fetch("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}" "${tmp}" err errMsg) + else() + _ExternalData_download_file("${url}" "${tmp}" err errMsg) + endif() set(tried "${tried}\n ${url}") if(err) set(tried "${tried} (${errMsg})") diff --git a/Modules/ExternalData_config.cmake.in b/Modules/ExternalData_config.cmake.in index 0858f53..4434e4b 100644 --- a/Modules/ExternalData_config.cmake.in +++ b/Modules/ExternalData_config.cmake.in @@ -2,3 +2,4 @@ set(ExternalData_OBJECT_STORES "@ExternalData_OBJECT_STORES@") set(ExternalData_URL_TEMPLATES "@ExternalData_URL_TEMPLATES@") set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@") set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@") +@_ExternalData_CONFIG_CODE@ diff --git a/Tests/Module/ExternalData/CMakeLists.txt b/Tests/Module/ExternalData/CMakeLists.txt index ebca48e..f99f6af 100644 --- a/Tests/Module/ExternalData/CMakeLists.txt +++ b/Tests/Module/ExternalData/CMakeLists.txt @@ -10,7 +10,9 @@ if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" MATCHES "^/") endif() set(ExternalData_URL_TEMPLATES "file://${slash}${CMAKE_CURRENT_SOURCE_DIR}/%(algo)/%(hash)" + "ExternalDataCustomScript://MyScript1/%(algo)/%(hash)" ) +set(ExternalData_CUSTOM_SCRIPT_MyScript1 "${CMAKE_CURRENT_SOURCE_DIR}/MyScript1.cmake") set(ExternalData_BINARY_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ExternalData") file(REMOVE_RECURSE ${ExternalData_BINARY_ROOT}) # clean test @@ -23,6 +25,7 @@ ExternalData_Add_Test(Data1 COMMAND ${CMAKE_COMMAND} -D Data=DATA{Data.dat} ${Data1CheckSpaces} + -D DataScript=DATA{DataScript.dat} -D DataMissing=DATA{DataMissing.dat} -D DataMissingWithAssociated=DATA{DataMissing.dat,Data.dat} -D SeriesA=DATA{SeriesA.dat,:} diff --git a/Tests/Module/ExternalData/Data1Check.cmake b/Tests/Module/ExternalData/Data1Check.cmake index 485b5c6..a7aa4ae 100644 --- a/Tests/Module/ExternalData/Data1Check.cmake +++ b/Tests/Module/ExternalData/Data1Check.cmake @@ -8,6 +8,10 @@ if(DEFINED DataSpace) message(SEND_ERROR "Input file:\n ${DataSpace}\ndoes not have expected content, but [[${lines}]]") endif() endif() +file(STRINGS "${DataScript}" lines LIMIT_INPUT 1024) +if(NOT "x${lines}" STREQUAL "xDataScript") + message(SEND_ERROR "Input file:\n ${DataScript}\ndoes not have expected content, but [[${lines}]]") +endif() if(DataMissing) if(EXISTS "${DataMissing}") message(SEND_ERROR diff --git a/Tests/Module/ExternalData/DataScript.dat.md5 b/Tests/Module/ExternalData/DataScript.dat.md5 new file mode 100644 index 0000000..74b4616 --- /dev/null +++ b/Tests/Module/ExternalData/DataScript.dat.md5 @@ -0,0 +1 @@ +fd95c03719e8626c0d10a818f9996dc5 diff --git a/Tests/Module/ExternalData/MyScript1.cmake b/Tests/Module/ExternalData/MyScript1.cmake new file mode 100644 index 0000000..242c64d --- /dev/null +++ b/Tests/Module/ExternalData/MyScript1.cmake @@ -0,0 +1,5 @@ +if(ExternalData_CUSTOM_LOCATION STREQUAL "MD5/fd95c03719e8626c0d10a818f9996dc5") + file(WRITE "${ExternalData_CUSTOM_FILE}" "DataScript") +else() + set(ExternalData_CUSTOM_ERROR "no ${ExternalData_CUSTOM_LOCATION} known") +endif() diff --git a/Tests/RunCMake/ExternalData/BadCustom1-result.txt b/Tests/RunCMake/ExternalData/BadCustom1-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom1-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadCustom1-stderr.txt b/Tests/RunCMake/ExternalData/BadCustom1-stderr.txt new file mode 100644 index 0000000..5d2986d --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom1-stderr.txt @@ -0,0 +1,9 @@ +^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\): + Bad ExternalDataCustomScript key '0BadKey' in URL template: + + ExternalDataCustomScript://0BadKey/%\(algo\)/%\(hash\) + + The key must be a valid C identifier. +Call Stack \(most recent call first\): + BadCustom1.cmake:[0-9]+ \(ExternalData_Add_Target\) + CMakeLists.txt:[0-9]+ \(include\)$ diff --git a/Tests/RunCMake/ExternalData/BadCustom1.cmake b/Tests/RunCMake/ExternalData/BadCustom1.cmake new file mode 100644 index 0000000..ec94fc1 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom1.cmake @@ -0,0 +1,5 @@ +include(ExternalData) +set(ExternalData_URL_TEMPLATES + "ExternalDataCustomScript://0BadKey/%(algo)/%(hash)" + ) +ExternalData_Add_Target(Data) diff --git a/Tests/RunCMake/ExternalData/BadCustom2-result.txt b/Tests/RunCMake/ExternalData/BadCustom2-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom2-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadCustom2-stderr.txt b/Tests/RunCMake/ExternalData/BadCustom2-stderr.txt new file mode 100644 index 0000000..4d59ca9 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom2-stderr.txt @@ -0,0 +1,9 @@ +^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\): + Bad ExternalDataCustomScript key '' in URL template: + + ExternalDataCustomScript:///%\(algo\)/%\(hash\) + + The key must be a valid C identifier. +Call Stack \(most recent call first\): + BadCustom2.cmake:[0-9]+ \(ExternalData_Add_Target\) + CMakeLists.txt:[0-9]+ \(include\)$ diff --git a/Tests/RunCMake/ExternalData/BadCustom2.cmake b/Tests/RunCMake/ExternalData/BadCustom2.cmake new file mode 100644 index 0000000..1ed7646 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom2.cmake @@ -0,0 +1,5 @@ +include(ExternalData) +set(ExternalData_URL_TEMPLATES + "ExternalDataCustomScript:///%(algo)/%(hash)" + ) +ExternalData_Add_Target(Data) diff --git a/Tests/RunCMake/ExternalData/BadCustom3-result.txt b/Tests/RunCMake/ExternalData/BadCustom3-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom3-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadCustom3-stderr.txt b/Tests/RunCMake/ExternalData/BadCustom3-stderr.txt new file mode 100644 index 0000000..460814b --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom3-stderr.txt @@ -0,0 +1,7 @@ +^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\): + No ExternalData_CUSTOM_SCRIPT_MissingKey is set for URL template: + + ExternalDataCustomScript://MissingKey/%\(algo\)/%\(hash\) +Call Stack \(most recent call first\): + BadCustom3.cmake:[0-9]+ \(ExternalData_Add_Target\) + CMakeLists.txt:[0-9]+ \(include\)$ diff --git a/Tests/RunCMake/ExternalData/BadCustom3.cmake b/Tests/RunCMake/ExternalData/BadCustom3.cmake new file mode 100644 index 0000000..b4f2fb8 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom3.cmake @@ -0,0 +1,5 @@ +include(ExternalData) +set(ExternalData_URL_TEMPLATES + "ExternalDataCustomScript://MissingKey/%(algo)/%(hash)" + ) +ExternalData_Add_Target(Data) diff --git a/Tests/RunCMake/ExternalData/BadCustom4-result.txt b/Tests/RunCMake/ExternalData/BadCustom4-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom4-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/ExternalData/BadCustom4-stderr.txt b/Tests/RunCMake/ExternalData/BadCustom4-stderr.txt new file mode 100644 index 0000000..b83bcee --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom4-stderr.txt @@ -0,0 +1,7 @@ +^CMake Error at .*/Modules/ExternalData.cmake:[0-9]+ \(message\): + No ExternalData_CUSTOM_SCRIPT_RelPathKey is not set to a full path: + + RelPathScript.cmake +Call Stack \(most recent call first\): + BadCustom4.cmake:[0-9]+ \(ExternalData_Add_Target\) + CMakeLists.txt:[0-9]+ \(include\)$ diff --git a/Tests/RunCMake/ExternalData/BadCustom4.cmake b/Tests/RunCMake/ExternalData/BadCustom4.cmake new file mode 100644 index 0000000..0cc5521 --- /dev/null +++ b/Tests/RunCMake/ExternalData/BadCustom4.cmake @@ -0,0 +1,6 @@ +include(ExternalData) +set(ExternalData_URL_TEMPLATES + "ExternalDataCustomScript://RelPathKey/%(algo)/%(hash)" + ) +set(ExternalData_CUSTOM_SCRIPT_RelPathKey "RelPathScript.cmake") +ExternalData_Add_Target(Data) diff --git a/Tests/RunCMake/ExternalData/RunCMakeTest.cmake b/Tests/RunCMake/ExternalData/RunCMakeTest.cmake index 04e3d59..7afd289 100644 --- a/Tests/RunCMake/ExternalData/RunCMakeTest.cmake +++ b/Tests/RunCMake/ExternalData/RunCMakeTest.cmake @@ -1,5 +1,9 @@ include(RunCMake) +run_cmake(BadCustom1) +run_cmake(BadCustom2) +run_cmake(BadCustom3) +run_cmake(BadCustom4) run_cmake(BadHashAlgo1) run_cmake(BadOption1) run_cmake(BadOption2) -- cgit v0.12