From bba2b1c89b32030289623cd9f5cc68d7c40ab0bb Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 14 Mar 2019 11:02:40 -0400 Subject: testUTF8: comment why sequences are invalid --- Tests/CMakeLib/testUTF8.cxx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Tests/CMakeLib/testUTF8.cxx b/Tests/CMakeLib/testUTF8.cxx index c99c46d..eb37c7e 100644 --- a/Tests/CMakeLib/testUTF8.cxx +++ b/Tests/CMakeLib/testUTF8.cxx @@ -30,8 +30,12 @@ static test_utf8_entry const good_entry[] = { }; static test_utf8_char const bad_chars[] = { - "\x80\x00\x00\x00", "\xC0\x00\x00\x00", "\xE0\x00\x00\x00", - "\xE0\x80\x80\x00", "\xF0\x80\x80\x80", { 0, 0, 0, 0, 0 } + "\x80\x00\x00\x00", /* Leading continuation byte. */ + "\xC0\x00\x00\x00", /* Overlong encoding and missing continuation byte. */ + "\xE0\x00\x00\x00", /* Missing continuation bytes. */ + "\xE0\x80\x80\x00", /* Overlong encoding. */ + "\xF0\x80\x80\x80", /* Overlong encoding. */ + { 0, 0, 0, 0, 0 } }; static void report_good(bool passed, test_utf8_char const c) -- cgit v0.12 From 7111873efd91c4ed7c38af117e50ff2ea4d8f0e5 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 14 Mar 2019 11:02:54 -0400 Subject: testUTF8: add more test cases --- Tests/CMakeLib/testUTF8.cxx | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Tests/CMakeLib/testUTF8.cxx b/Tests/CMakeLib/testUTF8.cxx index eb37c7e..e225152 100644 --- a/Tests/CMakeLib/testUTF8.cxx +++ b/Tests/CMakeLib/testUTF8.cxx @@ -21,17 +21,27 @@ struct test_utf8_entry }; static test_utf8_entry const good_entry[] = { - { 1, "\x20\x00\x00\x00", 0x0020 }, /* Space. */ - { 2, "\xC2\xA9\x00\x00", 0x00A9 }, /* Copyright. */ - { 3, "\xE2\x80\x98\x00", 0x2018 }, /* Open-single-quote. */ - { 3, "\xE2\x80\x99\x00", 0x2019 }, /* Close-single-quote. */ - { 4, "\xF0\xA3\x8E\xB4", 0x233B4 }, /* Example from RFC 3629. */ + { 1, "\x20\x00\x00\x00", 0x0020 }, /* Space. */ + { 2, "\xC2\xA9\x00\x00", 0x00A9 }, /* Copyright. */ + { 3, "\xE2\x80\x98\x00", 0x2018 }, /* Open-single-quote. */ + { 3, "\xE2\x80\x99\x00", 0x2019 }, /* Close-single-quote. */ + { 4, "\xF0\xA3\x8E\xB4", 0x233B4 }, /* Example from RFC 3629. */ + { 3, "\xED\x80\x80\x00", 0xD000 }, /* Valid 0xED prefixed codepoint. */ + { 4, "\xF4\x8F\xBF\xBF", 0x10FFFF }, /* Highest valid RFC codepoint. */ + /* These are invalid according to the RFC, but accepted here. */ + { 3, "\xED\xA0\x80\x00", 0xD800 }, /* UTF-16 surrogate half. */ + { 3, "\xED\xBF\xBF\x00", 0xDFFF }, /* UTF-16 surrogate half. */ + { 4, "\xF4\x90\x80\x80", 0x110000 }, /* Lowest out-of-range codepoint. */ + { 4, "\xF5\x80\x80\x80", + 0x140000 }, /* Prefix forces out-of-range codepoints. */ { 0, { 0, 0, 0, 0, 0 }, 0 } }; static test_utf8_char const bad_chars[] = { "\x80\x00\x00\x00", /* Leading continuation byte. */ - "\xC0\x00\x00\x00", /* Overlong encoding and missing continuation byte. */ + "\xC0\x80\x00\x00", /* Overlong encoding. */ + "\xC1\x80\x00\x00", /* Overlong encoding. */ + "\xC2\x00\x00\x00", /* Missing continuation byte. */ "\xE0\x00\x00\x00", /* Missing continuation bytes. */ "\xE0\x80\x80\x00", /* Overlong encoding. */ "\xF0\x80\x80\x80", /* Overlong encoding. */ -- cgit v0.12 From a11e5e021b5c30a0d6490cfff0399a14a3fb798a Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 14 Mar 2019 11:01:27 -0400 Subject: cm_utf8: reject UTF-16 surrogate half codepoints --- Source/cm_utf8.c | 5 +++++ Tests/CMakeLib/testUTF8.cxx | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Source/cm_utf8.c b/Source/cm_utf8.c index 52af4a6..2459c02 100644 --- a/Source/cm_utf8.c +++ b/Source/cm_utf8.c @@ -71,6 +71,11 @@ const char* cm_utf8_decode_character(const char* first, const char* last, return 0; } + /* UTF-16 surrogate halves. */ + if (0xD800 <= uc && uc <= 0xDFFF) { + return 0; + } + *pc = uc; return first; } diff --git a/Tests/CMakeLib/testUTF8.cxx b/Tests/CMakeLib/testUTF8.cxx index e225152..f1da6df 100644 --- a/Tests/CMakeLib/testUTF8.cxx +++ b/Tests/CMakeLib/testUTF8.cxx @@ -29,8 +29,6 @@ static test_utf8_entry const good_entry[] = { { 3, "\xED\x80\x80\x00", 0xD000 }, /* Valid 0xED prefixed codepoint. */ { 4, "\xF4\x8F\xBF\xBF", 0x10FFFF }, /* Highest valid RFC codepoint. */ /* These are invalid according to the RFC, but accepted here. */ - { 3, "\xED\xA0\x80\x00", 0xD800 }, /* UTF-16 surrogate half. */ - { 3, "\xED\xBF\xBF\x00", 0xDFFF }, /* UTF-16 surrogate half. */ { 4, "\xF4\x90\x80\x80", 0x110000 }, /* Lowest out-of-range codepoint. */ { 4, "\xF5\x80\x80\x80", 0x140000 }, /* Prefix forces out-of-range codepoints. */ @@ -45,6 +43,8 @@ static test_utf8_char const bad_chars[] = { "\xE0\x00\x00\x00", /* Missing continuation bytes. */ "\xE0\x80\x80\x00", /* Overlong encoding. */ "\xF0\x80\x80\x80", /* Overlong encoding. */ + "\xED\xA0\x80\x00", /* UTF-16 surrogate half. */ + "\xED\xBF\xBF\x00", /* UTF-16 surrogate half. */ { 0, 0, 0, 0, 0 } }; -- cgit v0.12 From fb5de060bc99635a5b18b3389cc15e9937b19a0e Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 14 Mar 2019 13:26:21 -0400 Subject: cm_utf8: reject codepoints above 0x10FFFF These are invalid because the Unicode standard says so (because UTF-16 as specified today cannot encode them). --- Source/cm_utf8.c | 5 +++++ Tests/CMakeLib/testUTF8.cxx | 6 ++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Source/cm_utf8.c b/Source/cm_utf8.c index 2459c02..d41d097 100644 --- a/Source/cm_utf8.c +++ b/Source/cm_utf8.c @@ -76,6 +76,11 @@ const char* cm_utf8_decode_character(const char* first, const char* last, return 0; } + /* Invalid codepoints. */ + if (0x10FFFF < uc) { + return 0; + } + *pc = uc; return first; } diff --git a/Tests/CMakeLib/testUTF8.cxx b/Tests/CMakeLib/testUTF8.cxx index f1da6df..7f52c82 100644 --- a/Tests/CMakeLib/testUTF8.cxx +++ b/Tests/CMakeLib/testUTF8.cxx @@ -28,10 +28,6 @@ static test_utf8_entry const good_entry[] = { { 4, "\xF0\xA3\x8E\xB4", 0x233B4 }, /* Example from RFC 3629. */ { 3, "\xED\x80\x80\x00", 0xD000 }, /* Valid 0xED prefixed codepoint. */ { 4, "\xF4\x8F\xBF\xBF", 0x10FFFF }, /* Highest valid RFC codepoint. */ - /* These are invalid according to the RFC, but accepted here. */ - { 4, "\xF4\x90\x80\x80", 0x110000 }, /* Lowest out-of-range codepoint. */ - { 4, "\xF5\x80\x80\x80", - 0x140000 }, /* Prefix forces out-of-range codepoints. */ { 0, { 0, 0, 0, 0, 0 }, 0 } }; @@ -45,6 +41,8 @@ static test_utf8_char const bad_chars[] = { "\xF0\x80\x80\x80", /* Overlong encoding. */ "\xED\xA0\x80\x00", /* UTF-16 surrogate half. */ "\xED\xBF\xBF\x00", /* UTF-16 surrogate half. */ + "\xF4\x90\x80\x80", /* Lowest out-of-range codepoint. */ + "\xF5\x80\x80\x80", /* Prefix forces out-of-range codepoints. */ { 0, 0, 0, 0, 0 } }; -- cgit v0.12