summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorTom Gundersen <teg@jklm.no>2014-12-22 12:57:05 +0100
committerTom Gundersen <teg@jklm.no>2014-12-22 20:26:53 +0100
commit04166cb7dd90918385835f246c43d8ec22af0d68 (patch)
treef5877cd3e9c5edd3b37c6cdc9d229c79a40ade31 /src
parent2bb4c7e384c31de4727f1330da3f4de2f0bb7784 (diff)
shared: utf8 - support decoding the full utf16
We originally only supported the BMP (i.e., we treated UTF-16 as UCS-2).
Diffstat (limited to 'src')
-rw-r--r--src/shared/utf8.c36
-rw-r--r--src/shared/utf8.h12
-rw-r--r--src/test/test-utf8.c13
3 files changed, 58 insertions, 3 deletions
diff --git a/src/shared/utf8.c b/src/shared/utf8.c
index ab57f4276b..013c110f07 100644
--- a/src/shared/utf8.c
+++ b/src/shared/utf8.c
@@ -310,12 +310,42 @@ char *utf16_to_utf8(const void *s, size_t length) {
const uint8_t *f;
char *r, *t;
- r = new(char, (length*3+1)/2 + 1);
+ r = new(char, (length * 4 + 1) / 2 + 1);
if (!r)
return NULL;
- for (f = s, t = r; f < (const uint8_t*) s + length; f += 2)
- t += utf8_encode_unichar(t, (f[1] << 8) | f[0]);
+ f = s;
+ t = r;
+
+ while (f < (const uint8_t*) s + length) {
+ uint16_t w1, w2;
+
+ /* see RFC 2781 section 2.2 */
+
+ w1 = f[1] << 8 | f[0];
+ f += 2;
+
+ if (!utf16_is_surrogate(w1)) {
+ t += utf8_encode_unichar(t, w1);
+
+ continue;
+ }
+
+ if (utf16_is_trailing_surrogate(w1))
+ continue;
+ else if (f >= (const uint8_t*) s + length)
+ break;
+
+ w2 = f[1] << 8 | f[0];
+ f += 2;
+
+ if (!utf16_is_trailing_surrogate(w2)) {
+ f -= 2;
+ continue;
+ }
+
+ t += utf8_encode_unichar(t, utf16_surrogate_pair_to_unichar(w1, w2));
+ }
*t = 0;
return r;
diff --git a/src/shared/utf8.h b/src/shared/utf8.h
index 3d5a4c3a9c..77f663438e 100644
--- a/src/shared/utf8.h
+++ b/src/shared/utf8.h
@@ -41,3 +41,15 @@ char *utf16_to_utf8(const void *s, size_t length);
int utf8_encoded_valid_unichar(const char *str);
int utf8_encoded_to_unichar(const char *str);
+
+static inline bool utf16_is_surrogate(uint16_t c) {
+ return (0xd800 <= c && c <= 0xdfff);
+}
+
+static inline bool utf16_is_trailing_surrogate(uint16_t c) {
+ return (0xdc00 <= c && c <= 0xdfff);
+}
+
+static inline uint32_t utf16_surrogate_pair_to_unichar(uint16_t lead, uint16_t trail) {
+ return ((lead - 0xd800) << 10) + (trail - 0xdc00) + 0x10000;
+}
diff --git a/src/test/test-utf8.c b/src/test/test-utf8.c
index 3399f2ba9c..befa385754 100644
--- a/src/test/test-utf8.c
+++ b/src/test/test-utf8.c
@@ -93,6 +93,18 @@ static void test_utf8_escaping_printable(void) {
assert_se(utf8_is_valid(p6));
}
+static void test_utf16_to_utf8(void) {
+ char *a = NULL;
+ const uint16_t utf16[] = { 'a', 0xd800, 'b', 0xdc00, 'c', 0xd801, 0xdc37 };
+ const char utf8[] = { 'a', 'b', 'c', 0xf0, 0x90, 0x90, 0xb7, 0 };
+
+ a = utf16_to_utf8(utf16, 14);
+ assert_se(a);
+ assert_se(streq(a, utf8));
+
+ free(a);
+}
+
int main(int argc, char *argv[]) {
test_utf8_is_valid();
test_utf8_is_printable();
@@ -100,6 +112,7 @@ int main(int argc, char *argv[]) {
test_utf8_encoded_valid_unichar();
test_utf8_escaping();
test_utf8_escaping_printable();
+ test_utf16_to_utf8();
return 0;
}