unicode_nfc.h File Reference

#include <algorithm>
#include <cstdint>
#include <string>
#include <vector>
#include "llama3_tokenizer.h"
#include "unicode_nfc_tables.h"

Include dependency graph for unicode_nfc.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Namespaces
namespace	ember
namespace	ember::nfc

Functions
uint8_t	ember::nfc::ccc (uint32_t c)
void	ember::nfc::decompose (uint32_t c, std::vector< uint32_t > &out)
uint32_t	ember::nfc::compose (uint32_t a, uint32_t b)
std::vector< uint32_t >	ember::nfc::nfc (const std::vector< uint32_t > &in)
std::string	ember::nfc::nfc_utf8 (const std::string &text)

Variables
constexpr uint32_t	ember::nfc::kSBase = 0xAC00
constexpr uint32_t	ember::nfc::kLBase = 0x1100
constexpr uint32_t	ember::nfc::kVBase = 0x1161
constexpr uint32_t	ember::nfc::kTBase = 0x11A7
constexpr uint32_t	ember::nfc::kLCount = 19
constexpr uint32_t	ember::nfc::kVCount = 21
constexpr uint32_t	ember::nfc::kTCount = 28
constexpr uint32_t	ember::nfc::kNCount = kVCount * kTCount
constexpr uint32_t	ember::nfc::kSCount = kLCount * kNCount