cmListFileLexer: Allow a leading UTF-8 Byte-Order-Mark (#11137)

Teach the lexer to read a UTF-8, UTF-16 BE/LE, or UTF-32 BE/LE
Byte-Order-Mark from the start of a file if any is present.  Report an
error on files using UTF-16 or UTF-32 and accept a UTF-8 or missing BOM.

15 files changed, 35 insertions, 0 deletions

diff --git a/Tests/RunCMake/Syntax/BOM-UTF-16-BE-result.txt b/Tests/RunCMake/Syntax/BOM-UTF-16-BE-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-16-BE-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-16-BE-stderr.txt b/Tests/RunCMake/Syntax/BOM-UTF-16-BE-stderr.txt
new file mode 100644
index 0000000..b3f1e47
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-16-BE-stderr.txt
@@ -0,0 +1,6 @@
+CMake Error at CMakeLists.txt:3 \(include\):
+  File
+
+    .*/Tests/RunCMake/Syntax/BOM-UTF-16-BE.cmake
+
+  starts with a Byte-Order-Mark that is not UTF-8.
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-16-BE.cmake b/Tests/RunCMake/Syntax/BOM-UTF-16-BE.cmake
new file mode 100644
index 0000000..c51f6e6
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-16-BE.cmake
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-16-LE-result.txt b/Tests/RunCMake/Syntax/BOM-UTF-16-LE-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-16-LE-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-16-LE-stderr.txt b/Tests/RunCMake/Syntax/BOM-UTF-16-LE-stderr.txt
new file mode 100644
index 0000000..c08c902
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-16-LE-stderr.txt
@@ -0,0 +1,6 @@
+CMake Error at CMakeLists.txt:3 \(include\):
+  File
+
+    .*/Tests/RunCMake/Syntax/BOM-UTF-16-LE.cmake
+
+  starts with a Byte-Order-Mark that is not UTF-8.
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-16-LE.cmake b/Tests/RunCMake/Syntax/BOM-UTF-16-LE.cmake
new file mode 100644
index 0000000..b57446f
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-16-LE.cmake
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-32-BE-result.txt b/Tests/RunCMake/Syntax/BOM-UTF-32-BE-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-32-BE-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-32-BE-stderr.txt b/Tests/RunCMake/Syntax/BOM-UTF-32-BE-stderr.txt
new file mode 100644
index 0000000..5dde4e3
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-32-BE-stderr.txt
@@ -0,0 +1,6 @@
+CMake Error at CMakeLists.txt:3 \(include\):
+  File
+
+    .*/Tests/RunCMake/Syntax/BOM-UTF-32-BE.cmake
+
+  starts with a Byte-Order-Mark that is not UTF-8.
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-32-BE.cmake b/Tests/RunCMake/Syntax/BOM-UTF-32-BE.cmake
new file mode 100644
index 0000000..23c57f3
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-32-BE.cmake
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-32-LE-result.txt b/Tests/RunCMake/Syntax/BOM-UTF-32-LE-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-32-LE-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-32-LE-stderr.txt b/Tests/RunCMake/Syntax/BOM-UTF-32-LE-stderr.txt
new file mode 100644
index 0000000..eb054ec
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-32-LE-stderr.txt
@@ -0,0 +1,6 @@
+CMake Error at CMakeLists.txt:3 \(include\):
+  File
+
+    .*/Tests/RunCMake/Syntax/BOM-UTF-32-LE.cmake
+
+  starts with a Byte-Order-Mark that is not UTF-8.
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-32-LE.cmake b/Tests/RunCMake/Syntax/BOM-UTF-32-LE.cmake
new file mode 100644
index 0000000..c330f5b
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-32-LE.cmake
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-8-stdout.txt b/Tests/RunCMake/Syntax/BOM-UTF-8-stdout.txt
new file mode 100644
index 0000000..5776d6e
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-8-stdout.txt
@@ -0,0 +1 @@
+-- message
diff --git a/Tests/RunCMake/Syntax/BOM-UTF-8.cmake b/Tests/RunCMake/Syntax/BOM-UTF-8.cmake
new file mode 100644
index 0000000..bdff83b
--- /dev/null
+++ b/Tests/RunCMake/Syntax/BOM-UTF-8.cmake
@@ -0,0 +1 @@
+message(STATUS "message")
diff --git a/Tests/RunCMake/Syntax/RunCMakeTest.cmake b/Tests/RunCMake/Syntax/RunCMakeTest.cmake
index 2d87328..d1a15c8 100644
--- a/Tests/RunCMake/Syntax/RunCMakeTest.cmake
+++ b/Tests/RunCMake/Syntax/RunCMakeTest.cmake
@@ -1,5 +1,10 @@
 include(RunCMake)
 
+run_cmake(BOM-UTF-8)
+run_cmake(BOM-UTF-16-LE)
+run_cmake(BOM-UTF-16-BE)
+run_cmake(BOM-UTF-32-LE)
+run_cmake(BOM-UTF-32-BE)
 run_cmake(CommandSpaces)
 run_cmake(CommandTabs)
 run_cmake(CommandNewlines)