diff options
-rw-r--r-- | src/shared/utf8.c | 36 | ||||
-rw-r--r-- | src/shared/utf8.h | 12 | ||||
-rw-r--r-- | test/test-utf8.c | 13 |
3 files changed, 58 insertions, 3 deletions
diff --git a/src/shared/utf8.c b/src/shared/utf8.c index a3e0738cef..fec2f4df34 100644 --- a/src/shared/utf8.c +++ b/src/shared/utf8.c @@ -224,12 +224,42 @@ char *utf16_to_utf8(const void *s, size_t length) { const uint8_t *f; char *r, *t; - r = new(char, (length*3+1)/2 + 1); + r = new(char, (length * 4 + 1) / 2 + 1); if (!r) return NULL; - for (f = s, t = r; f < (const uint8_t*) s + length; f += 2) - t += utf8_encode_unichar(t, (f[1] << 8) | f[0]); + f = s; + t = r; + + while (f < (const uint8_t*) s + length) { + uint16_t w1, w2; + + /* see RFC 2781 section 2.2 */ + + w1 = f[1] << 8 | f[0]; + f += 2; + + if (!utf16_is_surrogate(w1)) { + t += utf8_encode_unichar(t, w1); + + continue; + } + + if (utf16_is_trailing_surrogate(w1)) + continue; + else if (f >= (const uint8_t*) s + length) + break; + + w2 = f[1] << 8 | f[0]; + f += 2; + + if (!utf16_is_trailing_surrogate(w2)) { + f -= 2; + continue; + } + + t += utf8_encode_unichar(t, utf16_surrogate_pair_to_unichar(w1, w2)); + } *t = 0; return r; diff --git a/src/shared/utf8.h b/src/shared/utf8.h index 2d9b5f7011..143b0ba08c 100644 --- a/src/shared/utf8.h +++ b/src/shared/utf8.h @@ -35,3 +35,15 @@ char *utf16_to_utf8(const void *s, size_t length); int utf8_encoded_valid_unichar(const char *str); int utf8_encoded_to_unichar(const char *str); + +static inline bool utf16_is_surrogate(uint16_t c) { + return (0xd800 <= c && c <= 0xdfff); +} + +static inline bool utf16_is_trailing_surrogate(uint16_t c) { + return (0xdc00 <= c && c <= 0xdfff); +} + +static inline uint32_t utf16_surrogate_pair_to_unichar(uint16_t lead, uint16_t trail) { + return ((lead - 0xd800) << 10) + (trail - 0xdc00) + 0x10000; +} diff --git a/test/test-utf8.c b/test/test-utf8.c index bd8ca86834..2f0a95f1c0 100644 --- a/test/test-utf8.c +++ b/test/test-utf8.c @@ -44,10 +44,23 @@ static void test_utf8_encoded_valid_unichar(void) { } +static void test_utf16_to_utf8(void) { + char *a = NULL; + const uint16_t utf16[] = { 'a', 0xd800, 'b', 0xdc00, 'c', 0xd801, 0xdc37 }; + const char utf8[] = { 'a', 'b', 'c', 0xf0, 0x90, 0x90, 0xb7, 0 }; + + a = utf16_to_utf8(utf16, 14); + assert_se(a); + assert_se(streq(a, utf8)); + + free(a); +} + int main(int argc, char *argv[]) { test_utf8_is_printable(); test_ascii_is_valid(); test_utf8_encoded_valid_unichar(); + test_utf16_to_utf8(); return 0; } |