sailfork / t3code-github / apps/ mobile/ modules/ t3-highlights/ tests/ utf16-to-utf8.cpp

#include <cstdint>
#include <cstdio>
#include <random>
#include <string>

#include "Utf16ToUtf8.hpp"

using margelo::nitro::t3highlights::appendUtf16AsUtf8;

static int failures = 0;

static std::string encode(const std::u16string& source, std::string out = {}) {
  appendUtf16AsUtf8(out, source.data(), source.size());
  return out;
}

static void expectBytes(const char* name, const std::string& actual, const std::string& expected) {
  if (actual == expected) return;
  failures++;
  std::fprintf(stderr, "%s: expected", name);
  for (unsigned char byte : expected) std::fprintf(stderr, " %02x", byte);
  std::fprintf(stderr, ", got");
  for (unsigned char byte : actual) std::fprintf(stderr, " %02x", byte);
  std::fprintf(stderr, "\n");
}

// One code unit or valid pair at a time, as a slow reference.
static std::string reference(const std::u16string& source) {
  std::string out;
  for (size_t at = 0; at < source.size(); at++) {
    uint32_t point = source[at];
    if (point >= 0xd800 && point <= 0xdbff && at + 1 < source.size() && source[at + 1] >= 0xdc00 &&
        source[at + 1] <= 0xdfff) {
      point = 0x10000 + ((point - 0xd800) << 10) + (source[++at] - 0xdc00);
    } else if (point >= 0xd800 && point <= 0xdfff) {
      point = 0xfffd;
    }
    if (point < 0x80) {
      out += static_cast<char>(point);
    } else if (point < 0x800) {
      out += static_cast<char>(0xc0 | (point >> 6));
      out += static_cast<char>(0x80 | (point & 0x3f));
    } else if (point < 0x10000) {
      out += static_cast<char>(0xe0 | (point >> 12));
      out += static_cast<char>(0x80 | ((point >> 6) & 0x3f));
      out += static_cast<char>(0x80 | (point & 0x3f));
    } else {
      out += static_cast<char>(0xf0 | (point >> 18));
      out += static_cast<char>(0x80 | ((point >> 12) & 0x3f));
      out += static_cast<char>(0x80 | ((point >> 6) & 0x3f));
      out += static_cast<char>(0x80 | (point & 0x3f));
    }
  }
  return out;
}

int main() {
  expectBytes("empty", encode(u""), "");
  expectBytes("nul", encode(std::u16string(u"a\0b", 3)), std::string("a\0b", 3));
  expectBytes("two-byte", encode(u"\u00e9\u07ff"), "\xc3\xa9\xdf\xbf");
  expectBytes("three-byte", encode(u"\u0800\u65e5\uffff"), "\xe0\xa0\x80\xe6\x97\xa5\xef\xbf\xbf");
  expectBytes("pair", encode(u"\U0001F389"), "\xf0\x9f\x8e\x89");
  expectBytes("lone high at end", encode(u"a\xd83c"), "a\xef\xbf\xbd");
  expectBytes("lone high before text", encode(u"\xd83cz"), "\xef\xbf\xbdz");
  expectBytes("high high low", encode(u"\xd83c\xd83c\xdf89"), "\xef\xbf\xbd\xf0\x9f\x8e\x89");
  expectBytes("lone low", encode(u"\xdf89"), "\xef\xbf\xbd");
  expectBytes("appends", encode(u"\u00e9", "ok "), "ok \xc3\xa9");
  // A high surrogate at the end of the input must not pair with the unit after it.
  const std::u16string pair = u"\U0001F389";
  std::string split;
  appendUtf16AsUtf8(split, pair.data(), 1);
  expectBytes("high at input end", split, "\xef\xbf\xbd");

  // Runs either side of the four-unit ASCII fast path, with a non-ASCII unit at every offset.
  for (size_t length = 0; length <= 12; length++) {
    for (size_t at = 0; at <= length; at++) {
      std::u16string source(length, u'x');
      if (at < length) source[at] = 0x00e9;
      expectBytes("fast path boundary", encode(source), reference(source));
    }
  }

  const char16_t pool[] = {u'a', u'\n', 0x7f, 0x80, 0x7ff, 0x800, 0xd7ff, 0xd800, 0xdbff, 0xdc00, 0xdfff, 0xe000, 0xffff};
  std::mt19937 random(7);
  for (int run = 0; run < 20000; run++) {
    std::u16string source(random() % 24, u' ');
    for (auto& unit : source) unit = pool[random() % (sizeof(pool) / sizeof(pool[0]))];
    expectBytes("random", encode(source), reference(source));
  }

  if (failures > 0) return 1;
  std::printf("utf16-to-utf8: ok\n");
  return 0;
}