string_private_utf8.h

#pragma once
 
_meta_inline _Pure uint32 _strUTF8SeqLen(uint8 u)
{
    // single byte encoding aka ASCII
    if (u < 0x80)
        return 1;
 
    if (u >= 0x80 && u <= 0xbf)
        return 0;   // continuation byte, not valid here!
    else if (u == 0xc0 || u == 0xc1)
        return 0;   // overlong encoding of code point < 0x80
    else if (u >= 0xc2 && u <= 0xdf)
        return 2;
    else if (u >= 0xe0 && u <= 0xef)
        return 3;
    else if (u >= 0xf0 && u <= 0xf4)
        return 4;
 
    return 0;
}
 
_meta_inline bool _strUTF8DecodeSeq(striter* _Nonnull it, uint32 len, uint8 ch,
                                    int32* _Nullable codepoint)
{
    int32 ret = 0;
    // the loop below runs len down to 1, so the overlong checks need the original
    uint32 seqlen = len;
 
    if (len == 2)
        ret = ch & 0x1f;
    else if (len == 3)
        ret = ch & 0x0f;
    else if (len == 4)
        ret = ch & 0x07;
    else
        return false;
 
    for (; len > 1; --len) {
        if (!striChar(it, (uint8*)&ch))
            return false;
 
        if (ch < 0x80 || ch > 0xbf)
            return false;   // continuation byte must follow
 
        ret = (ret << 6) | (ch & 0x3f);
    }
 
    if (ret > 0x10ffff ||                     // outside unicode range
        (ret >= 0xd800 && ret <= 0xdfff) ||   // UTF-16 surrogate pairs
        (seqlen == 2 && ret < 0x80) ||        // overlong encodings
        (seqlen == 3 && ret < 0x800) || (seqlen == 4 && ret < 0x10000))
        return false;
 
    if (codepoint)
        *codepoint = ret;
 
    return true;
}
 
_meta_inline uint32 _strUTF8Decode(striter* _Nonnull it, int32* _Nullable codepoint)
{
    uint8 first;
    if (!striChar(it, (uint8*)&first))
        return 0;
 
    uint32 len = _strUTF8SeqLen(first);
 
    if (len == 1) {
        if (codepoint)
            *codepoint = first;
        return 1;
    }
 
    if (_strUTF8DecodeSeq(it, len, first, codepoint))
        return len;
    return 0;
}
 
// Returns 0 for anything that isn't a legal encoding
_meta_inline uint32 _strUTF8Encode(uint8* _Nonnull buffer, int32 codepoint)
{
    if (codepoint < 0 || codepoint > 0x10ffff || (codepoint >= 0xd800 && codepoint <= 0xdfff))
        return 0;
 
    if (codepoint < 0x80) {
        buffer[0] = (uint8)codepoint;
        return 1;
    } else if (codepoint < 0x800) {
        buffer[0] = 0xc0 | ((codepoint & 0x7c0) >> 6);
        buffer[1] = 0x80 | ((codepoint & 0x03f));
        return 2;
    } else if (codepoint < 0x10000) {
        buffer[0] = 0xe0 | ((codepoint & 0xf000) >> 12);
        buffer[1] = 0x80 | ((codepoint & 0x0fc0) >> 6);
        buffer[2] = 0x80 | ((codepoint & 0x003f));
        return 3;
    }
 
    buffer[0] = 0xf0 | ((codepoint & 0x1c0000) >> 18);
    buffer[1] = 0x80 | ((codepoint & 0x03f000) >> 12);
    buffer[2] = 0x80 | ((codepoint & 0x000fc0) >> 6);
    buffer[3] = 0x80 | ((codepoint & 0x00003f));
    return 4;
}