shared: utf8 - support decoding the full utf16

We originally only supported the BMP (i.e., we treated UTF-16 as UCS-2).

2 files changed, 45 insertions, 3 deletions

diff --git a/src/shared/utf8.c b/src/shared/utf8.c
index ab57f4276b..013c110f07 100644
--- a/src/shared/utf8.c
+++ b/src/shared/utf8.c
@@ -310,12 +310,42 @@ char *utf16_to_utf8(const void *s, size_t length) {
         const uint8_t *f;
         char *r, *t;
 
-        r = new(char, (length*3+1)/2 + 1);
+        r = new(char, (length * 4 + 1) / 2 + 1);
         if (!r)
                 return NULL;
 
-        for (f = s, t = r; f < (const uint8_t*) s + length; f += 2)
-                t += utf8_encode_unichar(t, (f[1] << 8) | f[0]);
+        f = s;
+        t = r;
+
+        while (f < (const uint8_t*) s + length) {
+                uint16_t w1, w2;
+
+                /* see RFC 2781 section 2.2 */
+
+                w1 = f[1] << 8 | f[0];
+                f += 2;
+
+                if (!utf16_is_surrogate(w1)) {
+                        t += utf8_encode_unichar(t, w1);
+
+                        continue;
+                }
+
+                if (utf16_is_trailing_surrogate(w1))
+                        continue;
+                else if (f >= (const uint8_t*) s + length)
+                        break;
+
+                w2 = f[1] << 8 | f[0];
+                f += 2;
+
+                if (!utf16_is_trailing_surrogate(w2)) {
+                        f -= 2;
+                        continue;
+                }
+
+                t += utf8_encode_unichar(t, utf16_surrogate_pair_to_unichar(w1, w2));
+        }
 
         *t = 0;
         return r;
diff --git a/src/shared/utf8.h b/src/shared/utf8.h
index 3d5a4c3a9c..77f663438e 100644
--- a/src/shared/utf8.h
+++ b/src/shared/utf8.h
@@ -41,3 +41,15 @@ char *utf16_to_utf8(const void *s, size_t length);
 
 int utf8_encoded_valid_unichar(const char *str);
 int utf8_encoded_to_unichar(const char *str);
+
+static inline bool utf16_is_surrogate(uint16_t c) {
+        return (0xd800 <= c && c <= 0xdfff);
+}
+
+static inline bool utf16_is_trailing_surrogate(uint16_t c) {
+        return (0xdc00 <= c && c <= 0xdfff);
+}
+
+static inline uint32_t utf16_surrogate_pair_to_unichar(uint16_t lead, uint16_t trail) {
+                return ((lead - 0xd800) << 10) + (trail - 0xdc00) + 0x10000;
+}