#include <llama3_tokenizer.h>

Collaboration diagram for microtorch::llama3::Tokenizer:

Classes
struct	Bigram

struct	Later

struct	Symbol

Public Member Functions
	Tokenizer (std::vector< std::string > tokens, const std::vector< std::string > &merges, const std::vector< int32_t > &token_type, size_t max_digits=3)

std::vector< int >	encode (const std::string &text) const

std::string	decode (const std::vector< int > &ids) const

size_t	size () const

const std::vector< std::string > &	tokens () const

Private Member Functions
void	encode_word (const std::string &word, std::vector< int > &out) const

Static Private Member Functions
static size_t	utf8_len (unsigned char c)

Private Attributes
std::vector< std::string >	tokens_

size_t	max_digits_ = 3

std::unordered_map< std::string, int >	id_

std::unordered_map< std::string, int >	rank_

std::vector< bool >	special_

std::vector< int >	specials_

std::unordered_map< std::string, unsigned char >	unbyte_

Detailed Description

Definition at line 262 of file llama3_tokenizer.h.

Constructor & Destructor Documentation

◆ Tokenizer()

microtorch::llama3::Tokenizer::Tokenizer	(	std::vector< std::string >	tokens,
		const std::vector< std::string > &	merges,
		const std::vector< int32_t > &	token_type,
		size_t	max_digits = `3`
	)

inline

Definition at line 268 of file llama3_tokenizer.h.

        : tokens_(std::move(tokens)), max_digits_(max_digits) {
        // A repeated string maps to its last id, as llama.cpp's token_to_id.
        for (size_t i = 0; i < tokens_.size(); ++i) id_[tokens_[i]] = static_cast<int>(i);
        for (size_t r = 0; r < merges.size(); ++r) rank_.emplace(merges[r], static_cast<int>(r));
        special_.assign(tokens_.size(), false);
        for (size_t i = 0; i < token_type.size() && i < tokens_.size(); ++i) {
            const int t = token_type[i];
            if (t == 2 || t == 3 || t == 4) {
                special_[i] = true;
                specials_.push_back(static_cast<int>(i));
            }
        }
        // Longest first, as llama.cpp's special-token cache (ties by id).
        std::stable_sort(specials_.begin(), specials_.end(),
                         [&](int a, int b) { return tokens_[a].size() > tokens_[b].size(); });
        for (int c = 0; c < 256; ++c) unbyte_.emplace(byte_to_unicode()[c], static_cast<unsigned char>(c));
    }

References a, microtorch::llama3::byte_to_unicode(), c, id_, j, rank_, special_, specials_, tokens_, and unbyte_.

Member Function Documentation

◆ decode()

std::string microtorch::llama3::Tokenizer::decode ( const std::vector< int > & ids ) const

inline

Definition at line 320 of file llama3_tokenizer.h.

                                                      {
        std::string s;
        for (int id : ids) {
            if (id < 0 || static_cast<size_t>(id) >= tokens_.size()) continue;
            if (special_[id]) {
                s += tokens_[id];
                continue;
            }
            const std::string& t = tokens_[id];
            for (size_t i = 0; i < t.size();) {
                const size_t len = utf8_len(static_cast<unsigned char>(t[i]));
                const auto it = unbyte_.find(t.substr(i, len));
                if (it != unbyte_.end()) s.push_back(static_cast<char>(it->second));
                i += len;
            }
        }
        return s;
    }

References id, j, special_, tokens_, unbyte_, and utf8_len().

◆ encode()

std::vector< int > microtorch::llama3::Tokenizer::encode ( const std::string & text ) const

inline

Definition at line 288 of file llama3_tokenizer.h.

                                                       {
        std::vector<std::pair<std::string, int>> frags = {{text, -1}};
        for (int sid : specials_) {
            const std::string& s = tokens_[sid];
            if (s.empty()) continue;
            std::vector<std::pair<std::string, int>> next;
            for (auto& [t, id] : frags) {
                if (id >= 0) {
                    next.emplace_back(std::move(t), id);
                    continue;
                }
                size_t at = 0;
                for (size_t m; (m = t.find(s, at)) != std::string::npos; at = m + s.size()) {
                    if (m > at) next.emplace_back(t.substr(at, m - at), -1);
                    next.emplace_back(std::string(), sid);
                }
                if (at < t.size()) next.emplace_back(t.substr(at), -1);
            }
            frags = std::move(next);
        }
        std::vector<int> out;
        for (const auto& [t, id] : frags) {
            if (id >= 0) {
                out.push_back(id);
                continue;
            }
            for (const std::string& w : split_words(t, max_digits_)) encode_word(byte_encode(w), out);
        }
        return out;
    }

References microtorch::llama3::byte_encode(), encode_word(), id, j, max_digits_, out, specials_, microtorch::llama3::split_words(), text, and tokens_.

◆ encode_word()

void microtorch::llama3::Tokenizer::encode_word	(	const std::string &	word,
		std::vector< int > &	out
	)		const

inlineprivate

Definition at line 363 of file llama3_tokenizer.h.

                                                                       {
        std::vector<Symbol> sym;
        if (id_.count(word)) {  // ignore_merges: a whole-word token wins
            sym.push_back({-1, -1, 0, word.size()});
        } else {
            for (size_t off = 0; off < word.size();) {
                const size_t len = std::min(word.size() - off, utf8_len(static_cast<unsigned char>(word[off])));
                const int idx = static_cast<int>(sym.size());
                sym.push_back({idx - 1, off + len == word.size() ? -1 : idx + 1, off, len});
                off += len;
            }
        }
        std::priority_queue<Bigram, std::vector<Bigram>, Later> q;
        const auto text = [&](int i) { return word.substr(sym[i].start, sym[i].n); };
        const auto push = [&](int l, int r) {
            if (l < 0 || r < 0) return;
            const std::string a = text(l), b = text(r);
            const auto it = rank_.find(a + " " + b);
            if (it != rank_.end()) q.push({l, r, it->second, a + b});
        };
        for (int i = 1; i < static_cast<int>(sym.size()); ++i) push(i - 1, i);
        while (!q.empty()) {
            const Bigram bg = q.top();
            q.pop();
            Symbol& L = sym[bg.left];
            Symbol& R = sym[bg.right];
            if (L.n == 0 || R.n == 0 || text(bg.left) + text(bg.right) != bg.text) continue;  // outdated
            L.n += R.n;
            R.n = 0;
            L.next = R.next;
            if (R.next >= 0) sym[R.next].prev = bg.left;
            push(L.prev, bg.left);
            push(bg.left, L.next);
        }
        for (const Symbol& s : sym) {
            if (s.n == 0) continue;
            const std::string t = word.substr(s.start, s.n);
            const auto it = id_.find(t);
            if (it != id_.end()) {
                out.push_back(it->second);
                continue;
            }
            for (char c : t) {  // llama.cpp's fallback: single bytes that are tokens
                const auto b = id_.find(std::string(1, c));
                if (b != id_.end()) out.push_back(b->second);
            }
        }
    }

References a, c, id_, j, n, microtorch::llama3::Tokenizer::Symbol::n, off, out, rank_, text, and utf8_len().

Referenced by encode().

◆ size()

size_t microtorch::llama3::Tokenizer::size ( ) const

inline

Definition at line 339 of file llama3_tokenizer.h.

339{ return tokens_.size(); }

References tokens_.

◆ tokens()

const std::vector< std::string > & microtorch::llama3::Tokenizer::tokens ( ) const

inline

Definition at line 340 of file llama3_tokenizer.h.

340{ return tokens_; }

References tokens_.

◆ utf8_len()

static size_t microtorch::llama3::Tokenizer::utf8_len ( unsigned char c )

inlinestaticprivate

Definition at line 343 of file llama3_tokenizer.h.

                                            {
        static const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};
        return lookup[c >> 4];
    }

References c.

Referenced by decode(), and encode_word().

Member Data Documentation

◆ id_

std::unordered_map<std::string, int> microtorch::llama3::Tokenizer::id_

private

Definition at line 414 of file llama3_tokenizer.h.

Referenced by encode_word(), and Tokenizer().

◆ max_digits_

size_t microtorch::llama3::Tokenizer::max_digits_ = 3

private

Definition at line 413 of file llama3_tokenizer.h.

Referenced by encode().

◆ rank_

std::unordered_map<std::string, int> microtorch::llama3::Tokenizer::rank_

private

Definition at line 414 of file llama3_tokenizer.h.

Referenced by encode_word(), and Tokenizer().

◆ special_

std::vector<bool> microtorch::llama3::Tokenizer::special_

private

Definition at line 415 of file llama3_tokenizer.h.

Referenced by decode(), and Tokenizer().

◆ specials_

std::vector<int> microtorch::llama3::Tokenizer::specials_

private

Definition at line 416 of file llama3_tokenizer.h.

Referenced by encode(), and Tokenizer().

◆ tokens_

std::vector<std::string> microtorch::llama3::Tokenizer::tokens_

private

Definition at line 412 of file llama3_tokenizer.h.

Referenced by decode(), encode(), size(), Tokenizer(), and tokens().

◆ unbyte_

std::unordered_map<std::string, unsigned char> microtorch::llama3::Tokenizer::unbyte_

private

Definition at line 417 of file llama3_tokenizer.h.

Referenced by decode(), and Tokenizer().

The documentation for this class was generated from the following file:

llama3_tokenizer.h

Classes

Public Member Functions

Private Member Functions

Static Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ Tokenizer()

Member Function Documentation

◆ decode()

◆ encode()

◆ encode_word()

◆ size()

◆ tokens()

◆ utf8_len()

Member Data Documentation

◆ id_

◆ max_digits_

◆ rank_

◆ special_

◆ specials_

◆ tokens_

◆ unbyte_