microtorch::bpe Namespace Reference

Classes
struct	GgufTokenizer
class	Tokenizer

Functions
const std::vector< std::string > &	specials ()
GgufTokenizer	read_gguf_tokenizer (const std::string &path)
Tokenizer	load_gguf (const std::string &path)

Function Documentation

load_gguf()

Tokenizer microtorch::bpe::load_gguf ( const std::string &  path )

inline

Definition at line 229 of file bpe_tokenizer.h.

                                                  {
    GgufTokenizer g = read_gguf_tokenizer(path);
    if (g.model != "bpe") throw std::runtime_error(path + ": tokenizer.ggml.model is '" + g.model + "', not 'bpe'");
    return Tokenizer(std::move(g.tokens), g.merges);
}

References j, microtorch::bpe::GgufTokenizer::merges, microtorch::bpe::GgufTokenizer::model, read_gguf_tokenizer(), and microtorch::bpe::GgufTokenizer::tokens.

read_gguf_tokenizer()

GgufTokenizer microtorch::bpe::read_gguf_tokenizer ( const std::string &  path )

inline

Definition at line 164 of file bpe_tokenizer.h.

                                                                {
    std::ifstream f(path, std::ios::binary);
    if (!f) throw std::runtime_error("cannot open " + path);
    std::string buf((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
    size_t p = 0;
    auto need = [&](size_t n) {
        if (p + n > buf.size()) throw std::runtime_error(path + ": truncated GGUF");
    };
    auto u32 = [&]() { need(4); uint32_t v; std::memcpy(&v, buf.data() + p, 4); p += 4; return v; };
    auto u64 = [&]() { need(8); uint64_t v; std::memcpy(&v, buf.data() + p, 8); p += 8; return v; };
    auto str = [&]() { const uint64_t n = u64(); need(n); std::string s = buf.substr(p, n); p += n; return s; };
    if (u32() != 0x46554747u) throw std::runtime_error(path + ": not GGUF");
    u32();
    u64();
    const uint64_t nkv = u64();
    std::vector<std::string> tokens, merges;
    std::string model, pre;
    std::vector<int32_t> token_type;
    std::function<void(uint32_t)> skip = [&](uint32_t t) {
        static const size_t sz[] = {1, 1, 2, 2, 4, 4, 4, 1, 0, 0, 8, 8, 8};
        if (t == 8) {
            str();
        } else if (t == 9) {
            const uint32_t et = u32();
            const uint64_t n = u64();
            for (uint64_t i = 0; i < n; ++i) skip(et);
        } else if (t < 13) {
            need(sz[t]);
            p += sz[t];
        } else {
            throw std::runtime_error(path + ": unknown GGUF type");
        }
    };
    for (uint64_t i = 0; i < nkv; ++i) {
        const std::string k = str();
        const uint32_t t = u32();
        if ((k == "tokenizer.ggml.tokens" || k == "tokenizer.ggml.merges") && t == 9) {
            const uint32_t et = u32();
            const uint64_t n = u64();
            if (et != 8) throw std::runtime_error(path + ": " + k + " is not a string array");
            auto& dst = k == "tokenizer.ggml.tokens" ? tokens : merges;
            for (uint64_t j = 0; j < n; ++j) dst.push_back(str());
        } else if (k == "tokenizer.ggml.token_type" && t == 9) {
            const uint32_t et = u32();
            const uint64_t n = u64();
            for (uint64_t j = 0; j < n; ++j) {  // another element type is skipped, not an error
                if (et == 5)
                    token_type.push_back(static_cast<int32_t>(u32()));
                else
                    skip(et);
            }
        } else if (k == "tokenizer.ggml.model" && t == 8) {
            model = str();
        } else if (k == "tokenizer.ggml.pre" && t == 8) {
            pre = str();
        } else {
            skip(t);
        }
    }
    return {std::move(model), std::move(tokens), std::move(merges), std::move(pre), std::move(token_type)};
}

References j, k, n, p, str, and v.

Referenced by ember::chat_vocab_from_gguf(), load_gguf(), microtorch::llama3::load_gguf(), and tinyllama::TinyLlamaSession::TinyLlamaSession().

specials()

const std::vector< std::string > & microtorch::bpe::specials ( )

inline

Definition at line 37 of file bpe_tokenizer.h.

                                              {
    static const std::vector<std::string> s = {"<unk>", "<|endoftext|>", "<|user|>", "<|assistant|>"};
    return s;
}

References j.

Referenced by microtorch::bpe::Tokenizer::encode_chunk(), microtorch::bpe::Tokenizer::pretokenize(), and microtorch::bpe::Tokenizer::Tokenizer().