ember::nfc Namespace Reference

Namespaces
namespace	tables

Functions
uint8_t	ccc (uint32_t c)
void	decompose (uint32_t c, std::vector< uint32_t > &out)
uint32_t	compose (uint32_t a, uint32_t b)
std::vector< uint32_t >	nfc (const std::vector< uint32_t > &in)
std::string	nfc_utf8 (const std::string &text)

Variables
constexpr uint32_t	kSBase = 0xAC00
constexpr uint32_t	kLBase = 0x1100
constexpr uint32_t	kVBase = 0x1161
constexpr uint32_t	kTBase = 0x11A7
constexpr uint32_t	kLCount = 19
constexpr uint32_t	kVCount = 21
constexpr uint32_t	kTCount = 28
constexpr uint32_t	kNCount = kVCount * kTCount
constexpr uint32_t	kSCount = kLCount * kNCount

Function Documentation

ccc()

uint8_t ember::nfc::ccc ( uint32_t  c )

inline

Definition at line 26 of file unicode_nfc.h.

                               {
    if (c < tables::kCccCp[0]) return 0;
    const uint32_t* end = tables::kCccCp + tables::kCccCount;
    const uint32_t* it = std::lower_bound(tables::kCccCp, end, c);
    return it != end && *it == c ? tables::kCccVal[it - tables::kCccCp] : 0;
}

References c, j, ember::nfc::tables::kCccCount, ember::nfc::tables::kCccCp, and ember::nfc::tables::kCccVal.

Referenced by nfc().

compose()

uint32_t ember::nfc::compose ( uint32_t  a, uint32_t  b  )

inline

Definition at line 56 of file unicode_nfc.h.

                                                {
    if (a >= kLBase && a < kLBase + kLCount && b >= kVBase && b < kVBase + kVCount)
        return kSBase + ((a - kLBase) * kVCount + (b - kVBase)) * kTCount;
    if (a >= kSBase && a < kSBase + kSCount && (a - kSBase) % kTCount == 0 && b > kTBase && b < kTBase + kTCount)
        return a + (b - kTBase);
    const uint64_t key = (static_cast<uint64_t>(a) << 32) | b;
    const uint64_t* end = tables::kCompKey + tables::kCompCount;
    const uint64_t* it = std::lower_bound(tables::kCompKey, end, key);
    return it != end && *it == key ? tables::kCompVal[it - tables::kCompKey] : 0;
}

References a, j, ember::nfc::tables::kCompCount, ember::nfc::tables::kCompKey, ember::nfc::tables::kCompVal, kLBase, kSBase, kSCount, kTBase, kTCount, kVBase, and kVCount.

Referenced by nfc().

decompose()

void ember::nfc::decompose ( uint32_t  c, std::vector< uint32_t > &  out  )

inline

Definition at line 34 of file unicode_nfc.h.

                                                            {
    if (c >= kSBase && c < kSBase + kSCount) {
        const uint32_t s = c - kSBase;
        out.push_back(kLBase + s / kNCount);
        out.push_back(kVBase + (s % kNCount) / kTCount);
        if (s % kTCount) out.push_back(kTBase + s % kTCount);
        return;
    }
    if (c >= tables::kDecompCp[0]) {
        const uint32_t* end = tables::kDecompCp + tables::kDecompCount;
        const uint32_t* it = std::lower_bound(tables::kDecompCp, end, c);
        if (it != end && *it == c) {
            const size_t i = static_cast<size_t>(it - tables::kDecompCp);
            out.insert(out.end(), tables::kDecompData + tables::kDecompOff[i],
                       tables::kDecompData + tables::kDecompOff[i] + tables::kDecompLen[i]);
            return;
        }
    }
    out.push_back(c);
}

References c, j, ember::nfc::tables::kDecompCount, ember::nfc::tables::kDecompCp, ember::nfc::tables::kDecompData, ember::nfc::tables::kDecompLen, ember::nfc::tables::kDecompOff, kLBase, kNCount, kSBase, kSCount, kTBase, kTCount, kVBase, and out.

Referenced by nfc().

nfc()

std::vector< uint32_t > ember::nfc::nfc ( const std::vector< uint32_t > &  in )

inline

Definition at line 68 of file unicode_nfc.h.

                                                              {
    std::vector<uint32_t> d;
    d.reserve(in.size() + 8);
    for (uint32_t c : in) decompose(c, d);
    // Canonical ordering: each run of non-starters stably sorted by class.
    for (size_t i = 0; i < d.size();) {
        if (ccc(d[i]) == 0) {
            ++i;
            continue;
        }
        size_t j = i;
        while (j < d.size() && ccc(d[j]) != 0) ++j;
        std::stable_sort(d.begin() + static_cast<std::ptrdiff_t>(i), d.begin() + static_cast<std::ptrdiff_t>(j),
                         [](uint32_t a, uint32_t b) { return ccc(a) < ccc(b); });
        i = j;
    }
    // Canonical composition: c joins the last starter unless blocked (a character between them of
    // class 0 or of class >= c's).
    std::vector<uint32_t> out;
    out.reserve(d.size());
    size_t starter = 0;
    bool have_starter = false;
    int last = 0;  // class of out.back()
    for (uint32_t c : d) {
        const int cc = ccc(c);
        if (have_starter) {
            const bool adjacent = out.size() == starter + 1;
            if (adjacent || (last != 0 && last < cc)) {
                if (const uint32_t p = compose(out[starter], c)) {
                    out[starter] = p;
                    continue;
                }
            }
        }
        if (cc == 0) {
            starter = out.size();
            have_starter = true;
        }
        last = cc;
        out.push_back(c);
    }
    return out;
}

References a, c, ccc(), compose(), decompose(), in, j, nfc(), out, and p.

Referenced by nfc(), and nfc_utf8().

nfc_utf8()

std::string ember::nfc::nfc_utf8 ( const std::string &  text )

inline

Definition at line 113 of file unicode_nfc.h.

                                                 {
    bool ascii = true;
    for (unsigned char b : text)
        if (b >= 0x80) {
            ascii = false;
            break;
        }
    if (ascii) return text;
    const std::vector<uint32_t> cp = microtorch::llama3::cpts_from_utf8(text);
    const std::vector<uint32_t> n = nfc(cp);
    if (n == cp) return text;
    std::string out;
    out.reserve(text.size() + 4);
    for (uint32_t c : n) microtorch::llama3::append_utf8(c, out);
    return out;
}

References microtorch::llama3::append_utf8(), c, microtorch::llama3::cpts_from_utf8(), j, n, nfc(), out, and text.

Referenced by ember::QwenNfcTokenizer::encode().

Variable Documentation

kLBase

constexpr uint32_t ember::nfc::kLBase = 0x1100

Definition at line 22 of file unicode_nfc.h.

Referenced by compose(), and decompose().

kLCount

constexpr uint32_t ember::nfc::kLCount = 19

inlineconstexpr

Definition at line 23 of file unicode_nfc.h.

kNCount

constexpr uint32_t ember::nfc::kNCount = kVCount * kTCount

Definition at line 23 of file unicode_nfc.h.

Referenced by decompose().

kSBase

constexpr uint32_t ember::nfc::kSBase = 0xAC00

inlineconstexpr

Definition at line 22 of file unicode_nfc.h.

Referenced by compose(), and decompose().

kSCount

constexpr uint32_t ember::nfc::kSCount = kLCount * kNCount

Definition at line 24 of file unicode_nfc.h.

Referenced by compose(), and decompose().

kTBase

constexpr uint32_t ember::nfc::kTBase = 0x11A7

Definition at line 22 of file unicode_nfc.h.

Referenced by compose(), and decompose().

kTCount

constexpr uint32_t ember::nfc::kTCount = 28

Definition at line 23 of file unicode_nfc.h.

Referenced by compose(), and decompose().

kVBase

constexpr uint32_t ember::nfc::kVBase = 0x1161

Definition at line 22 of file unicode_nfc.h.

Referenced by compose(), and decompose().

kVCount

constexpr uint32_t ember::nfc::kVCount = 21

Definition at line 23 of file unicode_nfc.h.

Referenced by compose().