From 0aea435aa1196fd7c32d328059c1f6a2ef3ac382 Mon Sep 17 00:00:00 2001 From: Craig Scott Date: Wed, 12 Feb 2020 17:48:01 +1100 Subject: ExternalProject: Provide choice of git update strategies Fixes: #16528 Co-Authored-By: Michael Wake --- Modules/ExternalProject-gitupdate.cmake.in | 58 ++++++++++++++++++---- Modules/ExternalProject.cmake | 53 +++++++++++++++++++- Tests/ExternalProjectUpdate/CMakeLists.txt | 2 + .../ExternalProjectUpdateTest.cmake | 55 ++++++++++++++++---- 4 files changed, 148 insertions(+), 20 deletions(-) diff --git a/Modules/ExternalProject-gitupdate.cmake.in b/Modules/ExternalProject-gitupdate.cmake.in index b7d484b..e993c3c 100644 --- a/Modules/ExternalProject-gitupdate.cmake.in +++ b/Modules/ExternalProject-gitupdate.cmake.in @@ -59,7 +59,7 @@ if(error_code OR is_remote_ref OR NOT ("${tag_sha}" STREQUAL "${head_sha}")) message(FATAL_ERROR "Failed to fetch repository '@git_repository@'") endif() - if(is_remote_ref) + if(is_remote_ref AND NOT "@git_update_strategy@" STREQUAL "CHECKOUT") # Check if stash is needed execute_process( COMMAND "@git_EXECUTABLE@" status --porcelain @@ -90,21 +90,61 @@ if(error_code OR is_remote_ref OR NOT ("${tag_sha}" STREQUAL "${head_sha}")) COMMAND "@git_EXECUTABLE@" rebase "${git_remote}/${git_tag}" WORKING_DIRECTORY "@work_dir@" RESULT_VARIABLE error_code + OUTPUT_VARIABLE rebase_output + ERROR_VARIABLE rebase_output ) if(error_code) - # Rebase failed: Restore previous state. + # Rebase failed, undo the rebase attempt before continuing execute_process( COMMAND "@git_EXECUTABLE@" rebase --abort WORKING_DIRECTORY "@work_dir@" ) - if(need_stash) - execute_process( - COMMAND "@git_EXECUTABLE@" stash pop --index --quiet - WORKING_DIRECTORY "@work_dir@" - ) + + if(NOT "@git_update_strategy@" STREQUAL "REBASE_CHECKOUT") + # Not allowed to do a checkout as a fallback, so cannot proceed + if(need_stash) + execute_process( + COMMAND "@git_EXECUTABLE@" stash pop --index --quiet + WORKING_DIRECTORY "@work_dir@" + ) + endif() + message(FATAL_ERROR "\nFailed to rebase in: '@work_dir@'." + "\nOutput from the attempted rebase follows:" + "\n${rebase_output}" + "\n\nYou will have to resolve the conflicts manually") endif() - message(FATAL_ERROR "\nFailed to rebase in: '@work_dir@'." - "\nYou will have to resolve the conflicts manually") + + # Fall back to checkout. We create an annotated tag so that the user + # can manually inspect the situation and revert if required. + # We can't log the failed rebase output because MSVC sees it and + # intervenes, causing the build to fail even though it completes. + # Write it to a file instead. + string(TIMESTAMP tag_timestamp "%Y%m%dT%H%M%S" UTC) + set(tag_name _cmake_ExternalProject_moved_from_here_${tag_timestamp}Z) + set(error_log_file ${CMAKE_CURRENT_LIST_DIR}/rebase_error_${tag_timestamp}Z.log) + file(WRITE ${error_log_file} "${rebase_output}") + message(WARNING "Rebase failed, output has been saved to ${error_log_file}" + "\nFalling back to checkout, previous commit tagged as ${tag_name}") + execute_process( + COMMAND "@git_EXECUTABLE@" tag -a + -m "ExternalProject attempting to move from here to ${git_remote}/${git_tag}" + ${tag_name} + WORKING_DIRECTORY "@work_dir@" + RESULT_VARIABLE error_code + ) + if(error_code) + message(FATAL_ERROR "Failed to add marker tag") + endif() + + execute_process( + COMMAND "@git_EXECUTABLE@" checkout ${git_remote}/${git_tag} + WORKING_DIRECTORY "@work_dir@" + RESULT_VARIABLE error_code + ) + if(error_code) + message(FATAL_ERROR "Failed to checkout : '${git_remote}/${git_tag}'") + endif() + endif() if(need_stash) diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake index 9074b18..9b1963f 100644 --- a/Modules/ExternalProject.cmake +++ b/Modules/ExternalProject.cmake @@ -294,6 +294,42 @@ External Project Definition ``git clone`` command line, with each option required to be in the form ``key=value``. + ``GIT_REMOTE_UPDATE_STRATEGY `` + When ``GIT_TAG`` refers to a remote branch, this option can be used to + specify how the update step behaves. The ```` must be one of + the following: + + ``CHECKOUT`` + Ignore the local branch and always checkout the branch specified by + ``GIT_TAG``. + + ``REBASE`` + Try to rebase the current branch to the one specified by ``GIT_TAG``. + If there are local uncommitted changes, they will be stashed first + and popped again after rebasing. If rebasing or popping stashed + changes fail, abort the rebase and halt with an error. + When ``GIT_REMOTE_UPDATE_STRATEGY`` is not present, this is the + default strategy unless the default has been overridden with + ``CMAKE_EP_GIT_REMOTE_UPDATE_STRATEGY`` (see below). + + ``REBASE_CHECKOUT`` + Same as ``REBASE`` except if the rebase fails, an annotated tag will + be created at the original ``HEAD`` position from before the rebase + and then checkout ``GIT_TAG`` just like the ``CHECKOUT`` strategy. + The message stored on the annotated tag will give information about + what was attempted and the tag name will include a timestamp so that + each failed run will add a new tag. This strategy ensures no changes + will be lost, but updates should always succeed if ``GIT_TAG`` refers + to a valid ref unless there are uncommitted changes that cannot be + popped successfully. + + The variable ``CMAKE_EP_GIT_REMOTE_UPDATE_STRATEGY`` can be set to + override the default strategy. This variable should not be set by a + project, it is intended for the user to set. It is primarily intended + for use in continuous integration scripts to ensure that when history + is rewritten on a remote branch, the build doesn't end up with unintended + changes or failed builds resulting from conflicts during rebase operations. + *Subversion* ``SVN_REPOSITORY `` URL of the Subversion repository. @@ -938,6 +974,7 @@ The custom step could then be triggered from the main build like so:: cmake_policy(PUSH) cmake_policy(SET CMP0054 NEW) # if() quoted variables not dereferenced +cmake_policy(SET CMP0057 NEW) # if() supports IN_LIST # Pre-compute a regex to match documented keywords for each command. math(EXPR _ep_documentation_line_count "${CMAKE_CURRENT_LIST_LINE} - 4") @@ -1242,7 +1279,7 @@ endif() endfunction() -function(_ep_write_gitupdate_script script_filename git_EXECUTABLE git_tag git_remote_name init_submodules git_submodules_recurse git_submodules git_repository work_dir) +function(_ep_write_gitupdate_script script_filename git_EXECUTABLE git_tag git_remote_name init_submodules git_submodules_recurse git_submodules git_repository work_dir git_update_strategy) if("${git_tag}" STREQUAL "") message(FATAL_ERROR "Tag for git checkout should not be empty.") endif() @@ -2631,10 +2668,22 @@ function(_ep_add_update_command name) endif() endif() + get_property(git_update_strategy TARGET ${name} PROPERTY _EP_GIT_REMOTE_UPDATE_STRATEGY) + if(NOT git_update_strategy) + set(git_update_strategy "${CMAKE_EP_GIT_REMOTE_UPDATE_STRATEGY}") + endif() + if(NOT git_update_strategy) + set(git_update_strategy REBASE) + endif() + set(strategies CHECKOUT REBASE REBASE_CHECKOUT) + if(NOT git_update_strategy IN_LIST strategies) + message(FATAL_ERROR "'${git_update_strategy}' is not one of the supported strategies: ${strategies}") + endif() + _ep_get_git_submodules_recurse(git_submodules_recurse) _ep_write_gitupdate_script(${tmp_dir}/${name}-gitupdate.cmake - ${GIT_EXECUTABLE} ${git_tag} ${git_remote_name} ${git_init_submodules} "${git_submodules_recurse}" "${git_submodules}" ${git_repository} ${work_dir} + ${GIT_EXECUTABLE} ${git_tag} ${git_remote_name} ${git_init_submodules} "${git_submodules_recurse}" "${git_submodules}" ${git_repository} ${work_dir} ${git_update_strategy} ) set(cmd ${CMAKE_COMMAND} -P ${tmp_dir}/${name}-gitupdate.cmake) set(always 1) diff --git a/Tests/ExternalProjectUpdate/CMakeLists.txt b/Tests/ExternalProjectUpdate/CMakeLists.txt index dbf26c8..9dddae2 100644 --- a/Tests/ExternalProjectUpdate/CMakeLists.txt +++ b/Tests/ExternalProjectUpdate/CMakeLists.txt @@ -78,6 +78,8 @@ if(do_git_tests) ExternalProject_Add(${proj} GIT_REPOSITORY "${local_git_repo}" GIT_TAG ${TEST_GIT_TAG} + GIT_CONFIG "user.email=testauthor@cmake.org" + "user.name=testauthor" CMAKE_GENERATOR "${CMAKE_GENERATOR}" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= INSTALL_COMMAND "" diff --git a/Tests/ExternalProjectUpdate/ExternalProjectUpdateTest.cmake b/Tests/ExternalProjectUpdate/ExternalProjectUpdateTest.cmake index 8ea4452..ba0c598 100644 --- a/Tests/ExternalProjectUpdate/ExternalProjectUpdateTest.cmake +++ b/Tests/ExternalProjectUpdate/ExternalProjectUpdateTest.cmake @@ -2,7 +2,7 @@ # resulting checked out version is resulting_sha and rebuild. # This check's the correct behavior of the ExternalProject UPDATE_COMMAND. # Also verify that a fetch only occurs when fetch_expected is 1. -macro(check_a_tag desired_tag resulting_sha fetch_expected) +macro(check_a_tag desired_tag resulting_sha fetch_expected update_strategy) message( STATUS "Checking ExternalProjectUpdate to tag: ${desired_tag}" ) # Remove the FETCH_HEAD file, so we can check if it gets replaced with a 'git @@ -10,11 +10,16 @@ macro(check_a_tag desired_tag resulting_sha fetch_expected) set( FETCH_HEAD_file ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals/Source/TutorialStep1-GIT/.git/FETCH_HEAD ) file( REMOVE ${FETCH_HEAD_file} ) + # Give ourselves a marker in the output. It is difficult to tell where we + # are up to without this + message(STATUS "===> check_a_tag ${desired_tag} ${resulting_sha} ${fetch_expected} ${update_strategy}") + # Configure execute_process(COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} -T "${CMAKE_GENERATOR_TOOLSET}" -A "${CMAKE_GENERATOR_PLATFORM}" -DTEST_GIT_TAG:STRING=${desired_tag} + -DCMAKE_EP_GIT_REMOTE_UPDATE_STRATEGY:STRING=${update_strategy} ${ExternalProjectUpdate_SOURCE_DIR} WORKING_DIRECTORY ${ExternalProjectUpdate_BINARY_DIR} RESULT_VARIABLE error_code @@ -176,16 +181,48 @@ if(GIT_EXECUTABLE) endif() endif() +# When re-running tests locally, this ensures we always start afresh +file(REMOVE_RECURSE ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals) + if(do_git_tests) - check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1) - check_a_tag(tag1 d1970730310fe8bc07e73f15dc570071f9f9654a 1) + check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE) + check_a_tag(tag1 d1970730310fe8bc07e73f15dc570071f9f9654a 1 REBASE) # With the Git UPDATE_COMMAND performance patch, this will not required a # 'git fetch' - check_a_tag(tag1 d1970730310fe8bc07e73f15dc570071f9f9654a 0) - check_a_tag(tag2 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1) - check_a_tag(d19707303 d1970730310fe8bc07e73f15dc570071f9f9654a 1) - check_a_tag(d19707303 d1970730310fe8bc07e73f15dc570071f9f9654a 0) - check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1) + check_a_tag(tag1 d1970730310fe8bc07e73f15dc570071f9f9654a 0 REBASE) + check_a_tag(tag2 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE) + check_a_tag(d19707303 d1970730310fe8bc07e73f15dc570071f9f9654a 1 REBASE) + check_a_tag(d19707303 d1970730310fe8bc07e73f15dc570071f9f9654a 0 REBASE) + check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE) # This is a remote symbolic ref, so it will always trigger a 'git fetch' - check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1) + check_a_tag(origin/master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE) + + foreach(strategy IN ITEMS CHECKOUT REBASE_CHECKOUT) + # Move local master back, then apply a change that will cause a conflict + # during rebase. We want to test the fallback to checkout. + check_a_tag(master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 REBASE) + execute_process(COMMAND ${GIT_EXECUTABLE} reset --hard tag1 + WORKING_DIRECTORY ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals/Source/TutorialStep1-GIT + RESULT_VARIABLE error_code + ) + if(error_code) + message(FATAL_ERROR "Could not reset local master back to tag1.") + endif() + set(cmlFile ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals/Source/TutorialStep1-GIT/CMakeLists.txt) + file(READ ${cmlFile} contents) + string(REPLACE "find TutorialConfig.h" "find TutorialConfig.h (conflict here)" + conflictingContent "${contents}" + ) + file(WRITE ${cmlFile} "${conflictingContent}") + execute_process(COMMAND ${GIT_EXECUTABLE} commit -a -m "This should cause a conflict" + WORKING_DIRECTORY ${ExternalProjectUpdate_BINARY_DIR}/CMakeExternals/Source/TutorialStep1-GIT + RESULT_VARIABLE error_code + ) + if(error_code) + message(FATAL_ERROR "Could not commit conflicting change.") + endif() + # This should discard our commit but leave behind an annotated tag + check_a_tag(master 5842b503ba4113976d9bb28d57b5aee1ad2736b7 1 ${strategy}) + endforeach() + endif() -- cgit v0.12