/* $Id: utf-8.cpp $ */
/** @file
 * IPRT - UTF-8 Decoding.
 */

/*
 * Copyright (C) 2006-2024 Oracle and/or its affiliates.
 *
 * This file is part of VirtualBox base platform packages, as
 * available from https://www.virtualbox.org.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation, in version 3 of the
 * License.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see <https://www.gnu.org/licenses>.
 *
 * The contents of this file may alternatively be used under the terms
 * of the Common Development and Distribution License Version 1.0
 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
 * in the VirtualBox distribution, in which case the provisions of the
 * CDDL are applicable instead of those of the GPL.
 *
 * You may elect to license modified versions of this file under the
 * terms and conditions of either the GPL or the CDDL or both.
 *
 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
 */


/*********************************************************************************************************************************
*   Header Files                                                                                                                 *
*********************************************************************************************************************************/
#include <iprt/string.h>
#include <iprt/latin1.h>
#include "internal/iprt.h"

#include <iprt/uni.h>
#include <iprt/asm.h>
#include <iprt/alloc.h>
#include <iprt/assert.h>
#include <iprt/err.h>
#include "internal/string.h"



/**
 * Get get length in code points of a UTF-8 encoded string.
 * The string is validated while doing this.
 *
 * @returns IPRT status code.
 * @param   psz             Pointer to the UTF-8 string.
 * @param   cch             The max length of the string. (btw cch = cb)
 *                          Use RTSTR_MAX if all of the string is to be examined.
 * @param   pcuc            Where to store the length in unicode code points.
 * @param   pcchActual      Where to store the actual size of the UTF-8 string
 *                          on success (cch = cb again). Optional.
 */
DECLHIDDEN(int) rtUtf8Length(const char *psz, size_t cch, size_t *pcuc, size_t *pcchActual)
{
    const unsigned char *puch = (const unsigned char *)psz;
    size_t cCodePoints = 0;
    while (cch > 0)
    {
        const unsigned char uch = *puch;
        if (!uch)
            break;
        if (uch & RT_BIT(7))
        {
            /* figure sequence length and validate the first byte */
/** @todo RT_USE_RTC_3629 */
            unsigned cb;
            if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
                cb = 2;
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
                cb = 3;
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
                cb = 4;
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
                cb = 5;
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
                cb = 6;
            else
            {
                RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
                return VERR_INVALID_UTF8_ENCODING;
            }

            /* check length */
            if (cb > cch)
            {
                RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
                return VERR_INVALID_UTF8_ENCODING;
            }

            /* validate the rest */
            switch (cb)
            {
                case 6:
                    RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RT_FALL_THRU();
                case 5:
                    RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RT_FALL_THRU();
                case 4:
                    RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RT_FALL_THRU();
                case 3:
                    RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RT_FALL_THRU();
                case 2:
                    RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                   break;
            }

            /* validate the code point. */
            RTUNICP uc;
            switch (cb)
            {
                case 6:
                    uc =            (puch[5] & 0x3f)
                        | ((RTUNICP)(puch[4] & 0x3f) << 6)
                        | ((RTUNICP)(puch[3] & 0x3f) << 12)
                        | ((RTUNICP)(puch[2] & 0x3f) << 18)
                        | ((RTUNICP)(puch[1] & 0x3f) << 24)
                        | ((RTUNICP)(uch     & 0x01) << 30);
                    RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    break;
                case 5:
                    uc =            (puch[4] & 0x3f)
                        | ((RTUNICP)(puch[3] & 0x3f) << 6)
                        | ((RTUNICP)(puch[2] & 0x3f) << 12)
                        | ((RTUNICP)(puch[1] & 0x3f) << 18)
                        | ((RTUNICP)(uch     & 0x03) << 24);
                    RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    break;
                case 4:
                    uc =            (puch[3] & 0x3f)
                        | ((RTUNICP)(puch[2] & 0x3f) << 6)
                        | ((RTUNICP)(puch[1] & 0x3f) << 12)
                        | ((RTUNICP)(uch     & 0x07) << 18);
                    RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    break;
                case 3:
                    uc =            (puch[2] & 0x3f)
                        | ((RTUNICP)(puch[1] & 0x3f) << 6)
                        | ((RTUNICP)(uch     & 0x0f) << 12);
                    RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
                                         uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
                    RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
                    break;
                case 2:
                    uc =            (puch[1] & 0x3f)
                        | ((RTUNICP)(uch     & 0x1f) << 6);
                    RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    break;
            }

            /* advance */
            cch -= cb;
            puch += cb;
        }
        else
        {
             /* one ASCII byte */
            puch++;
            cch--;
        }
        cCodePoints++;
    }

    /* done */
    *pcuc = cCodePoints;
    if (pcchActual)
        *pcchActual = puch - (unsigned char const *)psz;
    return VINF_SUCCESS;
}


/**
 * Decodes and UTF-8 string into an array of unicode code point.
 *
 * Since we know the input is valid, we do *not* perform encoding or length checks.
 *
 * @returns iprt status code.
 * @param   psz     The UTF-8 string to recode. This is a valid encoding.
 * @param   cch     The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
 *                  The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
 * @param   paCps   Where to store the code points array.
 * @param   cCps    The number of RTUNICP items the paCps buffer can hold, excluding the terminator ('\\0').
 */
static int rtUtf8Decode(const char *psz, size_t cch, PRTUNICP paCps, size_t cCps)
{
    int                     rc   = VINF_SUCCESS;
    const unsigned char    *puch = (const unsigned char *)psz;
    PRTUNICP                pCp  = paCps;
    while (cch > 0)
    {
        /* read the next char and check for terminator. */
        const unsigned char uch = *puch;
        if (uch)
        { /* we only break once, so consider this the likely branch. */ }
        else
            break;

        /* check for output overflow */
        if (RT_LIKELY(cCps >= 1))
        { /* likely */ }
        else
        {
            rc = VERR_BUFFER_OVERFLOW;
            break;
        }
        cCps--;

        /* decode and recode the code point */
        if (!(uch & RT_BIT(7)))
        {
            *pCp++ = uch;
            puch++;
            cch--;
        }
#ifdef RT_STRICT
        else if (!(uch & RT_BIT(6)))
            AssertMsgFailed(("Internal error!\n"));
#endif
        else if (!(uch & RT_BIT(5)))
        {
            *pCp++ = (puch[1] & 0x3f)
                   | ((uint16_t)(uch     & 0x1f) << 6);
            puch += 2;
            cch -= 2;
        }
        else if (!(uch & RT_BIT(4)))
        {
            *pCp++ = (puch[2] & 0x3f)
                   | ((uint16_t)(puch[1] & 0x3f) << 6)
                   | ((uint16_t)(uch     & 0x0f) << 12);
            puch += 3;
            cch -= 3;
        }
        else if (!(uch & RT_BIT(3)))
        {
            *pCp++ = (puch[3] & 0x3f)
                   | ((RTUNICP)(puch[2] & 0x3f) << 6)
                   | ((RTUNICP)(puch[1] & 0x3f) << 12)
                   | ((RTUNICP)(uch     & 0x07) << 18);
            puch += 4;
            cch -= 4;
        }
        else if (!(uch & RT_BIT(2)))
        {
            *pCp++ = (puch[4] & 0x3f)
                   | ((RTUNICP)(puch[3] & 0x3f) << 6)
                   | ((RTUNICP)(puch[2] & 0x3f) << 12)
                   | ((RTUNICP)(puch[1] & 0x3f) << 18)
                   | ((RTUNICP)(uch     & 0x03) << 24);
            puch += 5;
            cch -= 6;
        }
        else
        {
            Assert(!(uch & RT_BIT(1)));
            *pCp++ = (puch[5] & 0x3f)
                   | ((RTUNICP)(puch[4] & 0x3f) << 6)
                   | ((RTUNICP)(puch[3] & 0x3f) << 12)
                   | ((RTUNICP)(puch[2] & 0x3f) << 18)
                   | ((RTUNICP)(puch[1] & 0x3f) << 24)
                   | ((RTUNICP)(uch     & 0x01) << 30);
            puch += 6;
            cch -= 6;
        }
    }

    /* done */
    *pCp = 0;
    return rc;
}


RTDECL(size_t) RTStrUniLen(const char *psz)
{
    size_t cCodePoints;
    int rc = rtUtf8Length(psz, RTSTR_MAX, &cCodePoints, NULL);
    return RT_SUCCESS(rc) ? cCodePoints : 0;
}
RT_EXPORT_SYMBOL(RTStrUniLen);


RTDECL(int) RTStrUniLenEx(const char *psz, size_t cch, size_t *pcCps)
{
    size_t cCodePoints;
    int rc = rtUtf8Length(psz, cch, &cCodePoints, NULL);
    if (pcCps)
        *pcCps = RT_SUCCESS(rc) ? cCodePoints : 0;
    return rc;
}
RT_EXPORT_SYMBOL(RTStrUniLenEx);


RTDECL(int) RTStrValidateEncoding(const char *psz)
{
    return RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
}
RT_EXPORT_SYMBOL(RTStrValidateEncoding);


RTDECL(int) RTStrValidateEncodingEx(const char *psz, size_t cch, uint32_t fFlags)
{
    AssertReturn(!(fFlags & ~(RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED | RTSTR_VALIDATE_ENCODING_EXACT_LENGTH)),
                 VERR_INVALID_PARAMETER);
    AssertPtr(psz);

    /*
     * Use rtUtf8Length for the job.
     */
    size_t cchActual;
    size_t cCpsIgnored;
    int rc = rtUtf8Length(psz, cch, &cCpsIgnored, &cchActual);
    if (RT_SUCCESS(rc))
    {
        if (fFlags & RTSTR_VALIDATE_ENCODING_EXACT_LENGTH)
        {
            if (fFlags & RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)
                cchActual++;
            if (cchActual == cch)
                rc = VINF_SUCCESS;
            else if (cchActual < cch)
                rc = VERR_BUFFER_UNDERFLOW;
            else
                rc = VERR_BUFFER_OVERFLOW;
        }
        else if (    (fFlags & RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)
                 &&  cchActual >= cch)
            rc = VERR_BUFFER_OVERFLOW;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTStrValidateEncodingEx);


RTDECL(bool) RTStrIsValidEncoding(const char *psz)
{
    int rc = RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
    return RT_SUCCESS(rc);
}
RT_EXPORT_SYMBOL(RTStrIsValidEncoding);


RTDECL(size_t) RTStrPurgeEncoding(char *psz)
{
    size_t cErrors = 0;
    for (;;)
    {
        RTUNICP Cp;
        int rc = RTStrGetCpEx((const char **)&psz, &Cp);
        if (RT_SUCCESS(rc))
        {
            if (!Cp)
                break;
        }
        else
        {
            psz[-1] = '?';
            cErrors++;
        }
    }
    return cErrors;
}
RT_EXPORT_SYMBOL(RTStrPurgeEncoding);


/**
 * Helper for RTStrPurgeComplementSet.
 *
 * @returns true if @a Cp is valid, false if not.
 * @param   Cp              The code point to validate.
 * @param   puszValidPairs  Pair of valid code point sets.
 * @param   cValidPairs     Number of pairs.
 */
DECLINLINE(bool) rtStrPurgeIsInSet(RTUNICP Cp, PCRTUNICP puszValidPairs, uint32_t cValidPairs)
{
    while (cValidPairs-- > 0)
    {
        if (   Cp >= puszValidPairs[0]
            && Cp <= puszValidPairs[1])
            return true;
        puszValidPairs += 2;
    }
    return false;
}


RTDECL(ssize_t) RTStrPurgeComplementSet(char *psz, PCRTUNICP puszValidPairs, char chReplacement)
{
    AssertReturn(chReplacement && (unsigned)chReplacement < 128, -1);

    /*
     * Calc valid pairs and check that we've got an even number.
     */
    uint32_t cValidPairs = 0;
    while (puszValidPairs[cValidPairs * 2])
    {
        AssertReturn(puszValidPairs[cValidPairs * 2 + 1], -1);
        AssertMsg(puszValidPairs[cValidPairs * 2] <= puszValidPairs[cValidPairs * 2 + 1],
                  ("%#x vs %#x\n", puszValidPairs[cValidPairs * 2], puszValidPairs[cValidPairs * 2 + 1]));
        cValidPairs++;
    }

    /*
     * Do the replacing.
     */
    ssize_t cReplacements = 0;
    for (;;)
    {
        char    *pszCur = psz;
        RTUNICP  Cp;
        int rc = RTStrGetCpEx((const char **)&psz, &Cp);
        if (RT_SUCCESS(rc))
        {
            if (Cp)
            {
                if (!rtStrPurgeIsInSet(Cp, puszValidPairs, cValidPairs))
                {
                    for (; pszCur != psz; ++pszCur)
                        *pszCur = chReplacement;
                    ++cReplacements;
                }
            }
            else
                break;
        }
        else
            return -1;
    }
    return cReplacements;
}
RT_EXPORT_SYMBOL(RTStrPurgeComplementSet);


RTDECL(int) RTStrToUni(const char *pszString, PRTUNICP *ppaCps)
{
    /*
     * Validate input.
     */
    AssertPtr(pszString);
    AssertPtr(ppaCps);
    *ppaCps = NULL;

    /*
     * Validate the UTF-8 input and count its code points.
     */
    size_t cCps;
    int rc = rtUtf8Length(pszString, RTSTR_MAX, &cCps, NULL);
    if (RT_SUCCESS(rc))
    {
        /*
         * Allocate buffer.
         */
        PRTUNICP paCps = (PRTUNICP)RTMemAlloc((cCps + 1) * sizeof(RTUNICP));
        if (paCps)
        {
            /*
             * Decode the string.
             */
            rc = rtUtf8Decode(pszString, RTSTR_MAX, paCps, cCps);
            if (RT_SUCCESS(rc))
            {
                *ppaCps = paCps;
                return rc;
            }
            RTMemFree(paCps);
        }
        else
            rc = VERR_NO_CODE_POINT_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTStrToUni);


RTDECL(int)  RTStrToUniEx(const char *pszString, size_t cchString, PRTUNICP *ppaCps, size_t cCps, size_t *pcCps)
{
    /*
     * Validate input.
     */
    AssertPtr(pszString);
    AssertPtr(ppaCps);
    AssertPtrNull(pcCps);

    /*
     * Validate the UTF-8 input and count the code points.
     */
    size_t cCpsResult;
    int rc = rtUtf8Length(pszString, cchString, &cCpsResult, NULL);
    if (RT_SUCCESS(rc))
    {
        if (pcCps)
            *pcCps = cCpsResult;

        /*
         * Check buffer size / Allocate buffer.
         */
        bool fShouldFree;
        PRTUNICP paCpsResult;
        if (cCps > 0 && *ppaCps)
        {
            fShouldFree = false;
            if (cCps <= cCpsResult)
                return VERR_BUFFER_OVERFLOW;
            paCpsResult = *ppaCps;
        }
        else
        {
            *ppaCps = NULL;
            fShouldFree = true;
            cCps = RT_MAX(cCpsResult + 1, cCps);
            paCpsResult = (PRTUNICP)RTMemAlloc(cCps * sizeof(RTUNICP));
        }
        if (paCpsResult)
        {
            /*
             * Encode the UTF-16 string.
             */
            rc = rtUtf8Decode(pszString, cchString, paCpsResult, cCps - 1);
            if (RT_SUCCESS(rc))
            {
                *ppaCps = paCpsResult;
                return rc;
            }
            if (fShouldFree)
                RTMemFree(paCpsResult);
        }
        else
            rc = VERR_NO_CODE_POINT_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTStrToUniEx);


/**
 * Calculates the UTF-16 length of a string, validating the encoding while doing so.
 *
 * @returns IPRT status code.
 * @param   psz     Pointer to the UTF-8 string.
 * @param   cch     The max length of the string. (btw cch = cb)
 * @param   pcwc    Where to store the length of the UTF-16 string as a number
 *                  of RTUTF16 characters.
 * @sa      rtUtf8CalcUtf16Length
 */
static int rtUtf8CalcUtf16LengthN(const char *psz, size_t cch, size_t *pcwc)
{
    const unsigned char *puch = (const unsigned char *)psz;
    size_t cwc = 0;
    while (cch > 0)
    {
        const unsigned char uch = *puch;
        if (!(uch & RT_BIT(7)))
        {
            /* one ASCII byte */
            if (uch)
            {
                cwc++;
                puch++;
                cch--;
            }
            else
                break;
        }
        else
        {
            /*
             * Multibyte sequence is more complicated when we have length
             * restrictions on the input.
             */
            /* figure sequence length and validate the first byte */
            unsigned cb;
            if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
                cb = 2;
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
                cb = 3;
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
                cb = 4;
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
                cb = 5;
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
                cb = 6;
            else
            {
                RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
                return VERR_INVALID_UTF8_ENCODING;
            }

            /* check length */
            if (cb > cch)
            {
                RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
                return VERR_INVALID_UTF8_ENCODING;
            }

            /* validate the rest */
            switch (cb)
            {
                case 6:
                    RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RT_FALL_THRU();
                case 5:
                    RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RT_FALL_THRU();
                case 4:
                    RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RT_FALL_THRU();
                case 3:
                    RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RT_FALL_THRU();
                case 2:
                    RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                   break;
            }

            /* validate the code point. */
            RTUNICP uc;
            switch (cb)
            {
                case 6:
                    uc =            (puch[5] & 0x3f)
                        | ((RTUNICP)(puch[4] & 0x3f) << 6)
                        | ((RTUNICP)(puch[3] & 0x3f) << 12)
                        | ((RTUNICP)(puch[2] & 0x3f) << 18)
                        | ((RTUNICP)(puch[1] & 0x3f) << 24)
                        | ((RTUNICP)(uch     & 0x01) << 30);
                    RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
                    return VERR_CANT_RECODE_AS_UTF16;
                case 5:
                    uc =            (puch[4] & 0x3f)
                        | ((RTUNICP)(puch[3] & 0x3f) << 6)
                        | ((RTUNICP)(puch[2] & 0x3f) << 12)
                        | ((RTUNICP)(puch[1] & 0x3f) << 18)
                        | ((RTUNICP)(uch     & 0x03) << 24);
                    RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
                    return VERR_CANT_RECODE_AS_UTF16;
                case 4:
                    uc =            (puch[3] & 0x3f)
                        | ((RTUNICP)(puch[2] & 0x3f) << 6)
                        | ((RTUNICP)(puch[1] & 0x3f) << 12)
                        | ((RTUNICP)(uch     & 0x07) << 18);
                    RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    RTStrAssertMsgReturn(uc <= 0x0010ffff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CANT_RECODE_AS_UTF16);
                    cwc++;
                    break;
                case 3:
                    uc =            (puch[2] & 0x3f)
                        | ((RTUNICP)(puch[1] & 0x3f) << 6)
                        | ((RTUNICP)(uch     & 0x0f) << 12);
                    RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
                                         uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
                    RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
                    break;
                case 2:
                    uc =            (puch[1] & 0x3f)
                        | ((RTUNICP)(uch     & 0x1f) << 6);
                    RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
                                         ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                    break;
            }

            /* advance */
            cch -= cb;
            puch += cb;
            cwc++;
        }
    }

    /* done */
    *pcwc = cwc;
    return VINF_SUCCESS;
}


/**
 * Calculates the UTF-16 length of a string, validating the encoding while doing so.
 *
 * @returns IPRT status code.
 * @param   psz     Pointer to the UTF-8 string.
 * @param   pcwc    Where to store the length of the UTF-16 string as a number
 *                  of RTUTF16 characters.
 * @sa      rtUtf8CalcUtf16LengthN
 */
static int rtUtf8CalcUtf16Length(const char *psz, size_t *pcwc)
{
    const unsigned char *puch = (const unsigned char *)psz;
    size_t cwc = 0;
    for (;;)
    {
        const unsigned char uch = *puch;
        if (!(uch & RT_BIT(7)))
        {
            /* one ASCII byte */
            if (uch)
            {
                cwc++;
                puch++;
            }
            else
                break;
        }
        else
        {
            /*
             * Figure sequence length, implicitly validate the first byte.
             * Then validate the additional bytes.
             * Finally validate the code point.
             */
            unsigned cb;
            RTUNICP uc;
            if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
            {
                RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                uc =            (puch[1] & 0x3f)
                    | ((RTUNICP)(uch     & 0x1f) << 6);
                RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                cb = 2;
            }
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
            {
                RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                uc =            (puch[2] & 0x3f)
                    | ((RTUNICP)(puch[1] & 0x3f) << 6)
                    | ((RTUNICP)(uch     & 0x0f) << 12);
                RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
                                     uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
                cb = 3;
            }
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
            {
                RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                uc =            (puch[3] & 0x3f)
                    | ((RTUNICP)(puch[2] & 0x3f) << 6)
                    | ((RTUNICP)(puch[1] & 0x3f) << 12)
                    | ((RTUNICP)(uch     & 0x07) << 18);
                RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn(uc <= 0x0010ffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CANT_RECODE_AS_UTF16);
                cwc++;
                cb = 4;
            }
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
            {
                RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                uc =            (puch[4] & 0x3f)
                    | ((RTUNICP)(puch[3] & 0x3f) << 6)
                    | ((RTUNICP)(puch[2] & 0x3f) << 12)
                    | ((RTUNICP)(puch[1] & 0x3f) << 18)
                    | ((RTUNICP)(uch     & 0x03) << 24);
                RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
                return VERR_CANT_RECODE_AS_UTF16;
                //cb = 5;
            }
            else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
            {
                RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                uc =            (puch[5] & 0x3f)
                    | ((RTUNICP)(puch[4] & 0x3f) << 6)
                    | ((RTUNICP)(puch[3] & 0x3f) << 12)
                    | ((RTUNICP)(puch[2] & 0x3f) << 18)
                    | ((RTUNICP)(puch[1] & 0x3f) << 24)
                    | ((RTUNICP)(uch     & 0x01) << 30);
                RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
                RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
                return VERR_CANT_RECODE_AS_UTF16;
                //cb = 6;
            }
            else
            {
                RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
                return VERR_INVALID_UTF8_ENCODING;
            }

            /* advance */
            puch += cb;
            cwc++;
        }
    }

    /* done */
    *pcwc = cwc;
    return VINF_SUCCESS;
}



/**
 * Recodes a valid UTF-8 string as UTF-16.
 *
 * Since we know the input is valid, we do *not* perform encoding or length checks.
 *
 * @returns iprt status code.
 * @param   psz     The UTF-8 string to recode. This is a valid encoding.
 * @param   cch     The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
 *                  The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
 * @param   pwsz    Where to store the UTF-16 string.
 * @param   cwc     The number of RTUTF16 items the pwsz buffer can hold, excluding the terminator ('\\0').
 *
 * @note    rtUtf8RecodeAsUtf16Big is a duplicate with RT_H2BE_U16 applied.
 */
static int rtUtf8RecodeAsUtf16(const char *psz, size_t cch, PRTUTF16 pwsz, size_t cwc)
{
    int                     rc   = VINF_SUCCESS;
    const unsigned char    *puch = (const unsigned char *)psz;
    PRTUTF16                pwc  = pwsz;
    while (cch > 0)
    {
        /* read the next char and check for terminator. */
        const unsigned char uch = *puch;
        if (uch)
        { /* we only break once, so consider this the likely branch. */ }
        else
            break;

        /* check for output overflow */
        if (RT_LIKELY(cwc >= 1))
        { /* likely */ }
        else
        {
            rc = VERR_BUFFER_OVERFLOW;
            break;
        }
        cwc--;

        /* decode and recode the code point */
        if (!(uch & RT_BIT(7)))
        {
            *pwc++ = uch;
            puch++;
            cch--;
        }
        else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
        {
            uint16_t uc = (puch[1] & 0x3f)
                    | ((uint16_t)(uch     & 0x1f) << 6);
            *pwc++ = uc;
            puch += 2;
            cch -= 2;
        }
        else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
        {
            uint16_t uc = (puch[2] & 0x3f)
                    | ((uint16_t)(puch[1] & 0x3f) << 6)
                    | ((uint16_t)(uch     & 0x0f) << 12);
            *pwc++ = uc;
            puch += 3;
            cch -= 3;
        }
        else
        {
            /* generate surrogate pair */
            Assert((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)));
            RTUNICP uc =           (puch[3] & 0x3f)
                       | ((RTUNICP)(puch[2] & 0x3f) << 6)
                       | ((RTUNICP)(puch[1] & 0x3f) << 12)
                       | ((RTUNICP)(uch     & 0x07) << 18);
            if (RT_UNLIKELY(cwc < 1))
            {
                rc = VERR_BUFFER_OVERFLOW;
                break;
            }
            cwc--;

            uc -= 0x10000;
            *pwc++ = 0xd800 | (uc >> 10);
            *pwc++ = 0xdc00 | (uc & 0x3ff);
            puch += 4;
            cch -= 4;
        }
    }

    /* done */
    *pwc = '\0';
    return rc;
}


/**
 * Recodes a valid UTF-8 string as UTF-16BE.
 *
 * Since we know the input is valid, we do *not* perform encoding or length checks.
 *
 * @returns iprt status code.
 * @param   psz     The UTF-8 string to recode. This is a valid encoding.
 * @param   cch     The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
 *                  The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
 * @param   pwsz    Where to store the UTF-16BE string.
 * @param   cwc     The number of RTUTF16 items the pwsz buffer can hold, excluding the terminator ('\\0').
 *
 * @note    This is a copy of rtUtf8RecodeAsUtf16 with RT_H2BE_U16 applied.
 */
static int rtUtf8RecodeAsUtf16Big(const char *psz, size_t cch, PRTUTF16 pwsz, size_t cwc)
{
    int                     rc   = VINF_SUCCESS;
    const unsigned char    *puch = (const unsigned char *)psz;
    PRTUTF16                pwc  = pwsz;
    while (cch > 0)
    {
        /* read the next char and check for terminator. */
        const unsigned char uch = *puch;
        if (uch)
        { /* we only break once, so consider this the likely branch. */ }
        else
            break;

        /* check for output overflow */
        if (RT_LIKELY(cwc >= 1))
        { /* likely */ }
        else
        {
            rc = VERR_BUFFER_OVERFLOW;
            break;
        }
        cwc--;

        /* decode and recode the code point */
        if (!(uch & RT_BIT(7)))
        {
            *pwc++ = RT_H2BE_U16((RTUTF16)uch);
            puch++;
            cch--;
        }
        else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
        {
            uint16_t uc = (puch[1] & 0x3f)
                    | ((uint16_t)(uch     & 0x1f) << 6);
            *pwc++ = RT_H2BE_U16(uc);
            puch += 2;
            cch -= 2;
        }
        else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
        {
            uint16_t uc = (puch[2] & 0x3f)
                    | ((uint16_t)(puch[1] & 0x3f) << 6)
                    | ((uint16_t)(uch     & 0x0f) << 12);
            *pwc++ = RT_H2BE_U16(uc);
            puch += 3;
            cch -= 3;
        }
        else
        {
            /* generate surrogate pair */
            Assert((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)));
            RTUNICP uc =           (puch[3] & 0x3f)
                       | ((RTUNICP)(puch[2] & 0x3f) << 6)
                       | ((RTUNICP)(puch[1] & 0x3f) << 12)
                       | ((RTUNICP)(uch     & 0x07) << 18);
            if (RT_UNLIKELY(cwc < 1))
            {
                rc = VERR_BUFFER_OVERFLOW;
                break;
            }
            cwc--;

            uc -= 0x10000;
            *pwc++ = RT_H2BE_U16(0xd800 | (uc >> 10));
            *pwc++ = RT_H2BE_U16(0xdc00 | (uc & 0x3ff));
            puch += 4;
            cch -= 4;
        }
    }

    /* done */
    *pwc = '\0';
    return rc;
}


RTDECL(int) RTStrToUtf16Tag(const char *pszString, PRTUTF16 *ppwszString, const char *pszTag)
{
    /*
     * Validate input.
     */
    AssertPtr(ppwszString);
    AssertPtr(pszString);
    *ppwszString = NULL;

    /*
     * Validate the UTF-8 input and calculate the length of the UTF-16 string.
     */
    size_t cwc;
    int rc = rtUtf8CalcUtf16Length(pszString, &cwc);
    if (RT_SUCCESS(rc))
    {
        /*
         * Allocate buffer.
         */
        PRTUTF16 pwsz = (PRTUTF16)RTMemAllocTag((cwc + 1) * sizeof(RTUTF16), pszTag);
        if (pwsz)
        {
            /*
             * Encode the UTF-16 string.
             */
            rc = rtUtf8RecodeAsUtf16(pszString, RTSTR_MAX, pwsz, cwc);
            if (RT_SUCCESS(rc))
            {
                *ppwszString = pwsz;
                return rc;
            }
            RTMemFree(pwsz);
        }
        else
            rc = VERR_NO_UTF16_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTStrToUtf16Tag);


RTDECL(int) RTStrToUtf16BigTag(const char *pszString, PRTUTF16 *ppwszString, const char *pszTag)
{
    /*
     * Validate input.
     */
    AssertPtr(ppwszString);
    AssertPtr(pszString);
    *ppwszString = NULL;

    /*
     * Validate the UTF-8 input and calculate the length of the UTF-16 string.
     */
    size_t cwc;
    int rc = rtUtf8CalcUtf16Length(pszString, &cwc);
    if (RT_SUCCESS(rc))
    {
        /*
         * Allocate buffer.
         */
        PRTUTF16 pwsz = (PRTUTF16)RTMemAllocTag((cwc + 1) * sizeof(RTUTF16), pszTag);
        if (pwsz)
        {
            /*
             * Encode the UTF-16 string.
             */
            rc = rtUtf8RecodeAsUtf16Big(pszString, RTSTR_MAX, pwsz, cwc);
            if (RT_SUCCESS(rc))
            {
                *ppwszString = pwsz;
                return rc;
            }
            RTMemFree(pwsz);
        }
        else
            rc = VERR_NO_UTF16_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTStrToUtf16BigTag);


RTDECL(int)  RTStrToUtf16ExTag(const char *pszString, size_t cchString,
                               PRTUTF16 *ppwsz, size_t cwc, size_t *pcwc, const char *pszTag)
{
    /*
     * Validate input.
     */
    AssertPtr(pszString);
    AssertPtr(ppwsz);
    AssertPtrNull(pcwc);

    /*
     * Validate the UTF-8 input and calculate the length of the UTF-16 string.
     */
    size_t cwcResult;
    int rc;
    if (cchString != RTSTR_MAX)
        rc = rtUtf8CalcUtf16LengthN(pszString, cchString, &cwcResult);
    else
        rc = rtUtf8CalcUtf16Length(pszString, &cwcResult);
    if (RT_SUCCESS(rc))
    {
        if (pcwc)
            *pcwc = cwcResult;

        /*
         * Check buffer size / Allocate buffer.
         */
        bool fShouldFree;
        PRTUTF16 pwszResult;
        if (cwc > 0 && *ppwsz)
        {
            fShouldFree = false;
            if (cwc <= cwcResult)
                return VERR_BUFFER_OVERFLOW;
            pwszResult = *ppwsz;
        }
        else
        {
            *ppwsz = NULL;
            fShouldFree = true;
            cwc = RT_MAX(cwcResult + 1, cwc);
            pwszResult = (PRTUTF16)RTMemAllocTag(cwc * sizeof(RTUTF16), pszTag);
        }
        if (pwszResult)
        {
            /*
             * Encode the UTF-16 string.
             */
            rc = rtUtf8RecodeAsUtf16(pszString, cchString, pwszResult, cwc - 1);
            if (RT_SUCCESS(rc))
            {
                *ppwsz = pwszResult;
                return rc;
            }
            if (fShouldFree)
                RTMemFree(pwszResult);
        }
        else
            rc = VERR_NO_UTF16_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTStrToUtf16ExTag);


RTDECL(int)  RTStrToUtf16BigExTag(const char *pszString, size_t cchString,
                                  PRTUTF16 *ppwsz, size_t cwc, size_t *pcwc, const char *pszTag)
{
    /*
     * Validate input.
     */
    AssertPtr(pszString);
    AssertPtr(ppwsz);
    AssertPtrNull(pcwc);

    /*
     * Validate the UTF-8 input and calculate the length of the UTF-16 string.
     */
    size_t cwcResult;
    int rc;
    if (cchString != RTSTR_MAX)
        rc = rtUtf8CalcUtf16LengthN(pszString, cchString, &cwcResult);
    else
        rc = rtUtf8CalcUtf16Length(pszString, &cwcResult);
    if (RT_SUCCESS(rc))
    {
        if (pcwc)
            *pcwc = cwcResult;

        /*
         * Check buffer size / Allocate buffer.
         */
        bool fShouldFree;
        PRTUTF16 pwszResult;
        if (cwc > 0 && *ppwsz)
        {
            fShouldFree = false;
            if (cwc <= cwcResult)
                return VERR_BUFFER_OVERFLOW;
            pwszResult = *ppwsz;
        }
        else
        {
            *ppwsz = NULL;
            fShouldFree = true;
            cwc = RT_MAX(cwcResult + 1, cwc);
            pwszResult = (PRTUTF16)RTMemAllocTag(cwc * sizeof(RTUTF16), pszTag);
        }
        if (pwszResult)
        {
            /*
             * Encode the UTF-16BE string.
             */
            rc = rtUtf8RecodeAsUtf16Big(pszString, cchString, pwszResult, cwc - 1);
            if (RT_SUCCESS(rc))
            {
                *ppwsz = pwszResult;
                return rc;
            }
            if (fShouldFree)
                RTMemFree(pwszResult);
        }
        else
            rc = VERR_NO_UTF16_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTStrToUtf16BigExTag);


RTDECL(size_t) RTStrCalcUtf16Len(const char *psz)
{
    size_t cwc;
    int rc = rtUtf8CalcUtf16Length(psz, &cwc);
    return RT_SUCCESS(rc) ? cwc : 0;
}
RT_EXPORT_SYMBOL(RTStrCalcUtf16Len);


RTDECL(int) RTStrCalcUtf16LenEx(const char *psz, size_t cch, size_t *pcwc)
{
    size_t cwc;
    int rc;
    if (cch != RTSTR_MAX)
        rc = rtUtf8CalcUtf16LengthN(psz, cch, &cwc);
    else
        rc = rtUtf8CalcUtf16Length(psz, &cwc);
    if (pcwc)
        *pcwc = RT_SUCCESS(rc) ? cwc : ~(size_t)0;
    return rc;
}
RT_EXPORT_SYMBOL(RTStrCalcUtf16LenEx);


/**
 * Calculates the length of the UTF-8 encoding of a Latin-1 string.
 *
 * @returns iprt status code.
 * @param   psz         The Latin-1 string.
 * @param   cchIn       The max length of the Latin-1 string to consider.
 * @param   pcch        Where to store the length (excluding '\\0') of the UTF-8 string. (cch == cb, btw)
 */
static int rtLatin1CalcUtf8Length(const char *psz, size_t cchIn, size_t *pcch)
{
    size_t  cch = 0;
    for (;;)
    {
        RTUNICP Cp;
        int rc = RTLatin1GetCpNEx(&psz, &cchIn, &Cp);
        if (Cp == 0 || rc == VERR_END_OF_STRING)
            break;
        if (RT_FAILURE(rc))
            return rc;
        cch += RTStrCpSize(Cp); /* cannot fail */
    }

    /* done */
    *pcch = cch;
    return VINF_SUCCESS;
}


/**
 * Recodes a Latin-1 string as UTF-8.
 *
 * @returns iprt status code.
 * @param   pszIn       The Latin-1 string.
 * @param   cchIn       The number of characters to process from psz. The recoding
 *                      will stop when cch or '\\0' is reached.
 * @param   psz         Where to store the UTF-8 string.
 * @param   cch         The size of the UTF-8 buffer, excluding the terminator.
 */
static int rtLatin1RecodeAsUtf8(const char *pszIn, size_t cchIn, char *psz, size_t cch)
{
    int rc;
    for (;;)
    {
        RTUNICP Cp;
        size_t cchCp;
        rc = RTLatin1GetCpNEx(&pszIn, &cchIn, &Cp);
        if (Cp == 0 || RT_FAILURE(rc))
            break;
        cchCp = RTStrCpSize(Cp);
        if (RT_UNLIKELY(cch < cchCp))
        {
            RTStrAssertMsgFailed(("Buffer overflow! 1\n"));
            rc = VERR_BUFFER_OVERFLOW;
            break;
        }
        cch -= cchCp;
        psz = RTStrPutCp(psz, Cp);
    }

    /* done */
    if (rc == VERR_END_OF_STRING)
        rc = VINF_SUCCESS;
    *psz = '\0';
    return rc;
}



RTDECL(int)  RTLatin1ToUtf8Tag(const char *pszString, char **ppszString, const char *pszTag)
{
    /*
     * Validate input.
     */
    AssertPtr(ppszString);
    AssertPtr(pszString);
    *ppszString = NULL;

    /*
     * Calculate the length of the UTF-8 encoding of the Latin-1 string.
     */
    size_t cch;
    int rc = rtLatin1CalcUtf8Length(pszString, RTSTR_MAX, &cch);
    if (RT_SUCCESS(rc))
    {
        /*
         * Allocate buffer and recode it.
         */
        char *pszResult = (char *)RTMemAllocTag(cch + 1, pszTag);
        if (pszResult)
        {
            rc = rtLatin1RecodeAsUtf8(pszString, RTSTR_MAX, pszResult, cch);
            if (RT_SUCCESS(rc))
            {
                *ppszString = pszResult;
                return rc;
            }

            RTMemFree(pszResult);
        }
        else
            rc = VERR_NO_STR_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTLatin1ToUtf8Tag);


RTDECL(int)  RTLatin1ToUtf8ExTag(const char *pszString, size_t cchString, char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
{
    /*
     * Validate input.
     */
    AssertPtr(pszString);
    AssertPtr(ppsz);
    AssertPtrNull(pcch);

    /*
     * Calculate the length of the UTF-8 encoding of the Latin-1 string.
     */
    size_t cchResult;
    int rc = rtLatin1CalcUtf8Length(pszString, cchString, &cchResult);
    if (RT_SUCCESS(rc))
    {
        if (pcch)
            *pcch = cchResult;

        /*
         * Check buffer size / Allocate buffer and recode it.
         */
        bool fShouldFree;
        char *pszResult;
        if (cch > 0 && *ppsz)
        {
            fShouldFree = false;
            if (RT_UNLIKELY(cch <= cchResult))
                return VERR_BUFFER_OVERFLOW;
            pszResult = *ppsz;
        }
        else
        {
            *ppsz = NULL;
            fShouldFree = true;
            cch = RT_MAX(cch, cchResult + 1);
            pszResult = (char *)RTStrAllocTag(cch, pszTag);
        }
        if (pszResult)
        {
            rc = rtLatin1RecodeAsUtf8(pszString, cchString, pszResult, cch - 1);
            if (RT_SUCCESS(rc))
            {
                *ppsz = pszResult;
                return rc;
            }

            if (fShouldFree)
                RTStrFree(pszResult);
        }
        else
            rc = VERR_NO_STR_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTLatin1ToUtf8ExTag);


RTDECL(size_t) RTLatin1CalcUtf8Len(const char *psz)
{
    size_t cch;
    int rc = rtLatin1CalcUtf8Length(psz, RTSTR_MAX, &cch);
    return RT_SUCCESS(rc) ? cch : 0;
}
RT_EXPORT_SYMBOL(RTLatin1CalcUtf8Len);


RTDECL(int) RTLatin1CalcUtf8LenEx(const char *psz, size_t cchIn, size_t *pcch)
{
    size_t cch;
    int rc = rtLatin1CalcUtf8Length(psz, cchIn, &cch);
    if (pcch)
        *pcch = RT_SUCCESS(rc) ? cch : ~(size_t)0;
    return rc;
}
RT_EXPORT_SYMBOL(RTLatin1CalcUtf8LenEx);


/**
 * Calculates the Latin-1 length of a string, validating the encoding while
 * doing so.
 *
 * @returns IPRT status code.
 * @param   psz     Pointer to the UTF-8 string.
 * @param   cchIn   The max length of the string. (btw cch = cb)
 *                  Use RTSTR_MAX if all of the string is to be examined.
 * @param   pcch    Where to store the length of the Latin-1 string in bytes.
 */
static int rtUtf8CalcLatin1Length(const char *psz, size_t cchIn, size_t *pcch)
{
    size_t  cch = 0;
    for (;;)
    {
        RTUNICP Cp;
        size_t cchCp;
        int rc = RTStrGetCpNEx(&psz, &cchIn, &Cp);
        if (Cp == 0 || rc == VERR_END_OF_STRING)
            break;
        if (RT_FAILURE(rc))
            return rc;
        cchCp = RTLatin1CpSize(Cp);
        if (cchCp == 0)
            return VERR_NO_TRANSLATION;
        cch += cchCp;
    }

    /* done */
    *pcch = cch;
    return VINF_SUCCESS;
}


/**
 * Recodes a valid UTF-8 string as Latin-1.
 *
 * Since we know the input is valid, we do *not* perform encoding or length checks.
 *
 * @returns iprt status code.
 * @param   pszIn   The UTF-8 string to recode. This is a valid encoding.
 * @param   cchIn   The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
 *                  The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
 * @param   psz     Where to store the Latin-1 string.
 * @param   cch     The number of characters the pszOut buffer can hold, excluding the terminator ('\\0').
 */
static int rtUtf8RecodeAsLatin1(const char *pszIn, size_t cchIn, char *psz, size_t cch)
{
    int rc;
    for (;;)
    {
        RTUNICP Cp;
        size_t cchCp;
        rc = RTStrGetCpNEx(&pszIn, &cchIn, &Cp);
        if (Cp == 0 || RT_FAILURE(rc))
            break;
        cchCp = RTLatin1CpSize(Cp);
        if (RT_UNLIKELY(cch < cchCp))
        {
            RTStrAssertMsgFailed(("Buffer overflow! 1\n"));
            rc = VERR_BUFFER_OVERFLOW;
            break;
        }
        cch -= cchCp;
        psz = RTLatin1PutCp(psz, Cp);
    }

    /* done */
    if (rc == VERR_END_OF_STRING)
        rc = VINF_SUCCESS;
    *psz = '\0';
    return rc;
}



RTDECL(int) RTStrToLatin1Tag(const char *pszString, char **ppszString, const char *pszTag)
{
    /*
     * Validate input.
     */
    AssertPtr(ppszString);
    AssertPtr(pszString);
    *ppszString = NULL;

    /*
     * Validate the UTF-8 input and calculate the length of the Latin-1 string.
     */
    size_t cch;
    int rc = rtUtf8CalcLatin1Length(pszString, RTSTR_MAX, &cch);
    if (RT_SUCCESS(rc))
    {
        /*
         * Allocate buffer.
         */
        char *psz = (char *)RTMemAllocTag(cch + 1, pszTag);
        if (psz)
        {
            /*
             * Encode the UTF-16 string.
             */
            rc = rtUtf8RecodeAsLatin1(pszString, RTSTR_MAX, psz, cch);
            if (RT_SUCCESS(rc))
            {
                *ppszString = psz;
                return rc;
            }
            RTMemFree(psz);
        }
        else
            rc = VERR_NO_STR_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTStrToLatin1Tag);


RTDECL(int)  RTStrToLatin1ExTag(const char *pszString, size_t cchString,
                                char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
{
    /*
     * Validate input.
     */
    AssertPtr(pszString);
    AssertPtr(ppsz);
    AssertPtrNull(pcch);

    /*
     * Validate the UTF-8 input and calculate the length of the UTF-16 string.
     */
    size_t cchResult;
    int rc = rtUtf8CalcLatin1Length(pszString, cchString, &cchResult);
    if (RT_SUCCESS(rc))
    {
        if (pcch)
            *pcch = cchResult;

        /*
         * Check buffer size / Allocate buffer.
         */
        bool fShouldFree;
        char *pszResult;
        if (cch > 0 && *ppsz)
        {
            fShouldFree = false;
            if (cch <= cchResult)
                return VERR_BUFFER_OVERFLOW;
            pszResult = *ppsz;
        }
        else
        {
            *ppsz = NULL;
            fShouldFree = true;
            cch = RT_MAX(cchResult + 1, cch);
            pszResult = (char *)RTMemAllocTag(cch, pszTag);
        }
        if (pszResult)
        {
            /*
             * Encode the Latin-1 string.
             */
            rc = rtUtf8RecodeAsLatin1(pszString, cchString, pszResult, cch - 1);
            if (RT_SUCCESS(rc))
            {
                *ppsz = pszResult;
                return rc;
            }
            if (fShouldFree)
                RTMemFree(pszResult);
        }
        else
            rc = VERR_NO_STR_MEMORY;
    }
    return rc;
}
RT_EXPORT_SYMBOL(RTStrToLatin1ExTag);


RTDECL(size_t) RTStrCalcLatin1Len(const char *psz)
{
    size_t cch;
    int rc = rtUtf8CalcLatin1Length(psz, RTSTR_MAX, &cch);
    return RT_SUCCESS(rc) ? cch : 0;
}
RT_EXPORT_SYMBOL(RTStrCalcLatin1Len);


RTDECL(int) RTStrCalcLatin1LenEx(const char *psz, size_t cchIn, size_t *pcch)
{
    size_t cch;
    int rc = rtUtf8CalcLatin1Length(psz, cchIn, &cch);
    if (pcch)
        *pcch = RT_SUCCESS(rc) ? cch : ~(size_t)0;
    return rc;
}
RT_EXPORT_SYMBOL(RTStrCalcLatin1LenEx);


/**
 * Handle invalid encodings passed to RTStrGetCp() and RTStrGetCpEx().
 * @returns rc
 * @param   ppsz        The pointer to the string position point.
 * @param   pCp         Where to store RTUNICP_INVALID.
 * @param   rc          The iprt error code.
 */
static int rtStrGetCpExFailure(const char **ppsz, PRTUNICP pCp, int rc)
{
    /*
     * Try find a valid encoding.
     */
    (*ppsz)++; /** @todo code this! */
    *pCp = RTUNICP_INVALID;
    return rc;
}


RTDECL(RTUNICP) RTStrGetCpInternal(const char *psz)
{
    RTUNICP Cp;
    RTStrGetCpExInternal(&psz, &Cp);
    return Cp;
}
RT_EXPORT_SYMBOL(RTStrGetCpInternal);


RTDECL(int) RTStrGetCpExInternal(const char **ppsz, PRTUNICP pCp)
{
    const unsigned char *puch = (const unsigned char *)*ppsz;
    const unsigned char uch = *puch;
    RTUNICP             uc;

    /* ASCII ? */
    if (!(uch & RT_BIT(7)))
    {
        uc = uch;
        puch++;
    }
    else if (uch & RT_BIT(6))
    {
        /* figure the length and validate the first octet. */
/** @todo RT_USE_RTC_3629 */
        unsigned cb;
        if (!(uch & RT_BIT(5)))
            cb = 2;
        else if (!(uch & RT_BIT(4)))
            cb = 3;
        else if (!(uch & RT_BIT(3)))
            cb = 4;
        else if (!(uch & RT_BIT(2)))
            cb = 5;
        else if (!(uch & RT_BIT(1)))
            cb = 6;
        else
        {
            RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
            return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
        }

        /* validate the rest */
        switch (cb)
        {
            case 6:
                RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
                RT_FALL_THRU();
            case 5:
                RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
                RT_FALL_THRU();
            case 4:
                RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
                RT_FALL_THRU();
            case 3:
                RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
                RT_FALL_THRU();
            case 2:
                RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
               break;
        }

        /* get and validate the code point. */
        switch (cb)
        {
            case 6:
                uc =            (puch[5] & 0x3f)
                    | ((RTUNICP)(puch[4] & 0x3f) << 6)
                    | ((RTUNICP)(puch[3] & 0x3f) << 12)
                    | ((RTUNICP)(puch[2] & 0x3f) << 18)
                    | ((RTUNICP)(puch[1] & 0x3f) << 24)
                    | ((RTUNICP)(uch     & 0x01) << 30);
                RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
                break;
            case 5:
                uc =            (puch[4] & 0x3f)
                    | ((RTUNICP)(puch[3] & 0x3f) << 6)
                    | ((RTUNICP)(puch[2] & 0x3f) << 12)
                    | ((RTUNICP)(puch[1] & 0x3f) << 18)
                    | ((RTUNICP)(uch     & 0x03) << 24);
                RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
                break;
            case 4:
                uc =            (puch[3] & 0x3f)
                    | ((RTUNICP)(puch[2] & 0x3f) << 6)
                    | ((RTUNICP)(puch[1] & 0x3f) << 12)
                    | ((RTUNICP)(uch     & 0x07) << 18);
                RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
                break;
            case 3:
                uc =            (puch[2] & 0x3f)
                    | ((RTUNICP)(puch[1] & 0x3f) << 6)
                    | ((RTUNICP)(uch     & 0x0f) << 12);
                RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
                RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_CODE_POINT_SURROGATE));
                break;
            case 2:
                uc =            (puch[1] & 0x3f)
                    | ((RTUNICP)(uch     & 0x1f) << 6);
                RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
                break;
            default: /* impossible, but GCC is bitching. */
                uc = RTUNICP_INVALID;
                break;
        }
        puch += cb;
    }
    else
    {
        /* 6th bit is always set. */
        RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
        return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
    }
    *pCp = uc;
    *ppsz = (const char *)puch;
    return VINF_SUCCESS;
}
RT_EXPORT_SYMBOL(RTStrGetCpExInternal);


/**
 * Handle invalid encodings passed to RTStrGetCpNEx().
 * @returns rc
 * @param   ppsz        The pointer to the string position point.
 * @param   pcch        Pointer to the string length.
 * @param   pCp         Where to store RTUNICP_INVALID.
 * @param   rc          The iprt error code.
 */
static int rtStrGetCpNExFailure(const char **ppsz, size_t *pcch, PRTUNICP pCp, int rc)
{
    /*
     * Try find a valid encoding.
     */
    (*ppsz)++; /** @todo code this! */
    (*pcch)--;
    *pCp = RTUNICP_INVALID;
    return rc;
}


RTDECL(int) RTStrGetCpNExInternal(const char **ppsz, size_t *pcch, PRTUNICP pCp)
{
    const unsigned char *puch = (const unsigned char *)*ppsz;
    const unsigned char uch = *puch;
    size_t              cch = *pcch;
    RTUNICP             uc;

    if (cch == 0)
    {
        *pCp = RTUNICP_INVALID;
        return VERR_END_OF_STRING;
    }

    /* ASCII ? */
    if (!(uch & RT_BIT(7)))
    {
        uc = uch;
        puch++;
        cch--;
    }
    else if (uch & RT_BIT(6))
    {
        /* figure the length and validate the first octet. */
/** @todo RT_USE_RTC_3629 */
        unsigned cb;
        if (!(uch & RT_BIT(5)))
            cb = 2;
        else if (!(uch & RT_BIT(4)))
            cb = 3;
        else if (!(uch & RT_BIT(3)))
            cb = 4;
        else if (!(uch & RT_BIT(2)))
            cb = 5;
        else if (!(uch & RT_BIT(1)))
            cb = 6;
        else
        {
            RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
            return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
        }

        if (cb > cch)
            return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);

        /* validate the rest */
        switch (cb)
        {
            case 6:
                RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
                RT_FALL_THRU();
            case 5:
                RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
                RT_FALL_THRU();
            case 4:
                RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
                RT_FALL_THRU();
            case 3:
                RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
                RT_FALL_THRU();
            case 2:
                RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
               break;
        }

        /* get and validate the code point. */
        switch (cb)
        {
            case 6:
                uc =            (puch[5] & 0x3f)
                    | ((RTUNICP)(puch[4] & 0x3f) << 6)
                    | ((RTUNICP)(puch[3] & 0x3f) << 12)
                    | ((RTUNICP)(puch[2] & 0x3f) << 18)
                    | ((RTUNICP)(puch[1] & 0x3f) << 24)
                    | ((RTUNICP)(uch     & 0x01) << 30);
                RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
                break;
            case 5:
                uc =            (puch[4] & 0x3f)
                    | ((RTUNICP)(puch[3] & 0x3f) << 6)
                    | ((RTUNICP)(puch[2] & 0x3f) << 12)
                    | ((RTUNICP)(puch[1] & 0x3f) << 18)
                    | ((RTUNICP)(uch     & 0x03) << 24);
                RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
                break;
            case 4:
                uc =            (puch[3] & 0x3f)
                    | ((RTUNICP)(puch[2] & 0x3f) << 6)
                    | ((RTUNICP)(puch[1] & 0x3f) << 12)
                    | ((RTUNICP)(uch     & 0x07) << 18);
                RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
                break;
            case 3:
                uc =            (puch[2] & 0x3f)
                    | ((RTUNICP)(puch[1] & 0x3f) << 6)
                    | ((RTUNICP)(uch     & 0x0f) << 12);
                RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
                RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_CODE_POINT_SURROGATE));
                break;
            case 2:
                uc =            (puch[1] & 0x3f)
                    | ((RTUNICP)(uch     & 0x1f) << 6);
                RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
                                     ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
                                     rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
                break;
            default: /* impossible, but GCC is bitching. */
                uc = RTUNICP_INVALID;
                break;
        }
        puch += cb;
        cch  -= cb;
    }
    else
    {
        /* 6th bit is always set. */
        RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
        return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
    }
    *pCp = uc;
    *ppsz = (const char *)puch;
    (*pcch) = cch;
    return VINF_SUCCESS;
}
RT_EXPORT_SYMBOL(RTStrGetCpNExInternal);


RTDECL(char *) RTStrPutCpInternal(char *psz, RTUNICP uc)
{
    unsigned char *puch = (unsigned char *)psz;
    if (uc < 0x80)
        *puch++ = (unsigned char )uc;
    else if (uc < 0x00000800)
    {
        *puch++ = 0xc0 | (uc >> 6);
        *puch++ = 0x80 | (uc & 0x3f);
    }
    else if (uc < 0x00010000)
    {
/** @todo RT_USE_RTC_3629 */
        if (   uc < 0x0000d8000
             || (   uc > 0x0000dfff
                 && uc < 0x0000fffe))
        {
            *puch++ = 0xe0 | (uc >> 12);
            *puch++ = 0x80 | ((uc >> 6) & 0x3f);
            *puch++ = 0x80 | (uc & 0x3f);
        }
        else
        {
            AssertMsgFailed(("Invalid code point U+%05x!\n", uc));
            *puch++ = 0x7f;
        }
    }
/** @todo RT_USE_RTC_3629 */
    else if (uc < 0x00200000)
    {
        *puch++ = 0xf0 | (uc >> 18);
        *puch++ = 0x80 | ((uc >> 12) & 0x3f);
        *puch++ = 0x80 | ((uc >> 6) & 0x3f);
        *puch++ = 0x80 | (uc & 0x3f);
    }
    else if (uc < 0x04000000)
    {
        *puch++ = 0xf8 | (uc >> 24);
        *puch++ = 0x80 | ((uc >> 18) & 0x3f);
        *puch++ = 0x80 | ((uc >> 12) & 0x3f);
        *puch++ = 0x80 | ((uc >> 6) & 0x3f);
        *puch++ = 0x80 | (uc & 0x3f);
    }
    else if (uc <= 0x7fffffff)
    {
        *puch++ = 0xfc | (uc >> 30);
        *puch++ = 0x80 | ((uc >> 24) & 0x3f);
        *puch++ = 0x80 | ((uc >> 18) & 0x3f);
        *puch++ = 0x80 | ((uc >> 12) & 0x3f);
        *puch++ = 0x80 | ((uc >> 6) & 0x3f);
        *puch++ = 0x80 | (uc & 0x3f);
    }
    else
    {
        AssertMsgFailed(("Invalid code point U+%08x!\n", uc));
        *puch++ = 0x7f;
    }

    return (char *)puch;
}
RT_EXPORT_SYMBOL(RTStrPutCpInternal);


RTDECL(char *) RTStrPrevCp(const char *pszStart, const char *psz)
{
    if (pszStart < psz)
    {
        /* simple char? */
        const unsigned char *puch = (const unsigned char *)psz;
        unsigned uch = *--puch;
        if (!(uch & RT_BIT(7)))
            return (char *)puch;
        RTStrAssertMsgReturn(!(uch & RT_BIT(6)), ("uch=%#x\n", uch), (char *)pszStart);

        /* two or more. */
        uint32_t uMask = 0xffffffc0;
        while (     (const unsigned char *)pszStart < puch
               &&   !(uMask & 1))
        {
            uch = *--puch;
            if ((uch & 0xc0) != 0x80)
            {
                RTStrAssertMsgReturn((uch & (uMask >> 1)) == (uMask & 0xff),
                                     ("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz),
                                     (char *)pszStart);
                return (char *)puch;
            }
            uMask >>= 1;
        }
        RTStrAssertMsgFailed(("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz));
    }
    return (char *)pszStart;
}
RT_EXPORT_SYMBOL(RTStrPrevCp);

