ember::SpmTokenizer Class Reference

#include <spm_tokenizer.h>

Collaboration diagram for ember::SpmTokenizer:

Classes
struct	Bigram
struct	Lower
struct	Symbol

Public Member Functions
	SpmTokenizer (std::vector< std::string > tokens, std::vector< float > scores, std::vector< int32_t > types, bool add_space_prefix)
std::vector< int >	encode (const std::string &text) const
std::string	decode (const std::vector< int > &ids) const
size_t	size () const

Private Member Functions
int	lookup (const std::string &s) const
void	encode_fragment (const std::string &text, std::vector< int > &out) const

Static Private Member Functions
static std::string	escape (const std::string &s)

Private Attributes
std::vector< std::string >	tokens_
std::vector< float >	scores_
std::vector< int32_t >	types_
bool	add_space_prefix_
std::unordered_map< std::string, int >	id_
std::vector< int >	specials_
int	byte_id_ [256]

Static Private Attributes
static constexpr int32_t	kUnknown = 2
static constexpr int32_t	kControl = 3
static constexpr int32_t	kUserDefined = 4
static constexpr int32_t	kByte = 6
static constexpr const char *	kSpace = "\xe2\x96\x81"

Detailed Description

Definition at line 30 of file spm_tokenizer.h.

Constructor & Destructor Documentation

SpmTokenizer()

ember::SpmTokenizer::SpmTokenizer ( std::vector< std::string >  tokens, std::vector< float >  scores, std::vector< int32_t >  types, bool  add_space_prefix  )

inline

Definition at line 32 of file spm_tokenizer.h.

        : tokens_(std::move(tokens)), scores_(std::move(scores)), types_(std::move(types)),
          add_space_prefix_(add_space_prefix) {
        if (scores_.size() != tokens_.size()) throw std::runtime_error("SentencePiece GGUF: scores and tokens differ in length");
        if (types_.size() != tokens_.size()) throw std::runtime_error("SentencePiece GGUF: token types and tokens differ in length");
        id_.reserve(tokens_.size());
        for (size_t i = 0; i < tokens_.size(); ++i) id_[tokens_[i]] = static_cast<int>(i);  // the last one wins
        for (size_t i = 0; i < tokens_.size(); ++i)
            if ((types_[i] == kControl || types_[i] == kUserDefined || types_[i] == kUnknown) && !tokens_[i].empty())
                specials_.push_back(static_cast<int>(i));
        // Longest text first (llama.cpp sorts by length with std::sort; among equal lengths the
        // order only matters for overlapping specials, which the gate against HF would show).
        std::stable_sort(specials_.begin(), specials_.end(),
                         [&](int a, int b) { return tokens_[a].size() > tokens_[b].size(); });
        for (int b = 0; b < 256; ++b) {
            char buf[8];
            std::snprintf(buf, sizeof buf, "<0x%02X>", b);
            auto it = id_.find(buf);
            if (it == id_.end()) it = id_.find(std::string(1, static_cast<char>(b)));
            byte_id_[b] = it == id_.end() ? -1 : it->second;
        }
    }

References a, byte_id_, id_, j, kControl, kUnknown, kUserDefined, scores_, specials_, tokens_, and types_.

Member Function Documentation

decode()

std::string ember::SpmTokenizer::decode ( const std::vector< int > &  ids ) const

inline

Definition at line 102 of file spm_tokenizer.h.

                                                      {
        std::string s;
        for (int id : ids) {
            if (id < 0 || static_cast<size_t>(id) >= tokens_.size()) continue;
            const std::string& t = tokens_[id];
            if (types_[id] == kByte && t.size() == 6 && t.compare(0, 3, "<0x") == 0) {
                s += static_cast<char>(std::stoi(t.substr(3, 2), nullptr, 16));
                continue;
            }
            for (size_t i = 0; i < t.size();) {
                if (t.compare(i, 3, kSpace) == 0) {
                    s += ' ';
                    i += 3;
                } else {
                    s += t[i++];
                }
            }
        }
        return s;
    }

References id, j, kByte, kSpace, tokens_, and types_.

encode()

std::vector< int > ember::SpmTokenizer::encode ( const std::string &  text ) const

inline

Definition at line 56 of file spm_tokenizer.h.

                                                       {
        // Fragments: (special token id, or -1 with [offset, offset + length) of text).
        struct Frag {
            int id;
            size_t off, len;
        };
        std::vector<Frag> frags = {{-1, 0, text.size()}};
        for (int sid : specials_) {
            const std::string& s = tokens_[sid];
            std::vector<Frag> next;
            next.reserve(frags.size());
            for (const Frag& f : frags) {
                if (f.id >= 0) {
                    next.push_back(f);
                    continue;
                }
                size_t at = f.off;
                const size_t end = f.off + f.len;
                while (at < end) {
                    const size_t hit = text.find(s, at);
                    if (hit == std::string::npos || hit + s.size() > end) break;
                    if (hit > at) next.push_back({-1, at, hit - at});
                    next.push_back({sid, hit, s.size()});
                    at = hit + s.size();
                }
                if (at < end) next.push_back({-1, at, end - at});
            }
            frags.swap(next);
        }
        std::vector<int> out;
        bool prev_special = true;
        for (const Frag& f : frags) {
            if (f.id >= 0) {
                out.push_back(f.id);
                prev_special = true;
                continue;
            }
            if (f.len == 0) continue;
            std::string piece = add_space_prefix_ && prev_special ? std::string(" ") : std::string();
            piece += text.substr(f.off, f.len);
            encode_fragment(escape(piece), out);
            prev_special = false;
        }
        return out;
    }

References add_space_prefix_, encode_fragment(), escape(), id, j, off, out, specials_, text, and tokens_.

encode_fragment()

void ember::SpmTokenizer::encode_fragment ( const std::string &  text, std::vector< int > &  out  ) const

inlineprivate

Definition at line 159 of file spm_tokenizer.h.

                                                                           {
        static const uint8_t utf8_len[16] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};
        std::vector<Symbol> sym;
        for (size_t off = 0; off < text.size();) {
            const size_t len = std::min<size_t>(utf8_len[static_cast<uint8_t>(text[off]) >> 4], text.size() - off);
            const int index = static_cast<int>(sym.size());
            sym.push_back({index - 1, off + len == text.size() ? -1 : index + 1, off, len});
            off += len;
        }
        std::priority_queue<Bigram, std::vector<Bigram>, Lower> queue;
        std::unordered_map<std::string, std::pair<int, int>> rev_merge;
        auto try_add = [&](int left, int right) {
            if (left == -1 || right == -1) return;
            const std::string s = text.substr(sym[left].off, sym[left].n + sym[right].n);
            const int id = lookup(s);
            if (id < 0) return;
            queue.push({left, right, scores_[id], s.size()});
            rev_merge[s] = {left, right};
        };
        for (int i = 1; i < static_cast<int>(sym.size()); ++i) try_add(i - 1, i);
        while (!queue.empty()) {
            const Bigram b = queue.top();
            queue.pop();
            Symbol& l = sym[b.left];
            Symbol& r = sym[b.right];
            if (l.n == 0 || r.n == 0 || l.n + r.n != b.size) continue;
            l.n += r.n;
            r.n = 0;
            l.next = r.next;
            if (r.next >= 0) sym[r.next].prev = b.left;
            try_add(l.prev, b.left);
            try_add(b.left, l.next);
        }
        std::function<void(const Symbol&)> resegment = [&](const Symbol& s) {
            const std::string t = text.substr(s.off, s.n);
            const int id = lookup(t);
            if (id >= 0) {
                out.push_back(id);
                return;
            }
            const auto p = rev_merge.find(t);
            if (p == rev_merge.end()) {
                for (size_t j = 0; j < s.n; ++j) {
                    const int bid = byte_id_[static_cast<uint8_t>(text[s.off + j])];
                    if (bid < 0) throw std::runtime_error("SentencePiece GGUF has no byte token for a byte of the text");
                    out.push_back(bid);
                }
                return;
            }
            resegment(sym[p->second.first]);
            resegment(sym[p->second.second]);
        };
        for (int i = sym.empty() ? -1 : 0; i != -1; i = sym[i].next) resegment(sym[i]);
    }

References byte_id_, id, index, j, lookup(), n, ember::SpmTokenizer::Symbol::n, off, out, p, scores_, and text.

Referenced by encode().

escape()

static std::string ember::SpmTokenizer::escape ( const std::string &  s )

inlinestaticprivate

Definition at line 129 of file spm_tokenizer.h.

                                                {
        std::string o;
        o.reserve(s.size() + s.size() / 2);
        for (char c : s) {
            if (c == ' ') o += kSpace;
            else o += c;
        }
        return o;
    }

References c, j, and kSpace.

Referenced by encode().

lookup()

int ember::SpmTokenizer::lookup ( const std::string &  s ) const

inlineprivate

Definition at line 154 of file spm_tokenizer.h.

                                         {
        auto it = id_.find(s);
        return it == id_.end() ? -1 : it->second;
    }

References id_, and j.

Referenced by encode_fragment().

size()

size_t ember::SpmTokenizer::size ( ) const

inline

Definition at line 123 of file spm_tokenizer.h.

{ return tokens_.size(); }

References tokens_.

Member Data Documentation

add_space_prefix_

bool ember::SpmTokenizer::add_space_prefix_

private

Definition at line 217 of file spm_tokenizer.h.

Referenced by encode().

byte_id_

int ember::SpmTokenizer::byte_id_[256]

private

Definition at line 220 of file spm_tokenizer.h.

Referenced by encode_fragment(), and SpmTokenizer().

id_

std::unordered_map<std::string, int> ember::SpmTokenizer::id_

private

Definition at line 218 of file spm_tokenizer.h.

Referenced by lookup(), and SpmTokenizer().

kByte

constexpr int32_t ember::SpmTokenizer::kByte = 6

staticprivate

Definition at line 126 of file spm_tokenizer.h.

Referenced by decode().

kControl

constexpr int32_t ember::SpmTokenizer::kControl = 3

staticprivate

Definition at line 126 of file spm_tokenizer.h.

Referenced by SpmTokenizer().

kSpace

constexpr const char* ember::SpmTokenizer::kSpace = "\xe2\x96\x81"

staticconstexprprivate

Definition at line 127 of file spm_tokenizer.h.

Referenced by decode(), and escape().

kUnknown

constexpr int32_t ember::SpmTokenizer::kUnknown = 2

staticconstexprprivate

Definition at line 126 of file spm_tokenizer.h.

Referenced by SpmTokenizer().

kUserDefined

constexpr int32_t ember::SpmTokenizer::kUserDefined = 4

staticprivate

Definition at line 126 of file spm_tokenizer.h.

Referenced by SpmTokenizer().

scores_

std::vector<float> ember::SpmTokenizer::scores_

private

Definition at line 215 of file spm_tokenizer.h.

Referenced by encode_fragment(), and SpmTokenizer().

specials_

std::vector<int> ember::SpmTokenizer::specials_

private

Definition at line 219 of file spm_tokenizer.h.

Referenced by encode(), and SpmTokenizer().

tokens_

std::vector<std::string> ember::SpmTokenizer::tokens_

private

Definition at line 214 of file spm_tokenizer.h.

Referenced by decode(), encode(), size(), and SpmTokenizer().

types_

std::vector<int32_t> ember::SpmTokenizer::types_

private

Definition at line 216 of file spm_tokenizer.h.

Referenced by decode(), and SpmTokenizer().

---

The documentation for this class was generated from the following file:

• spm_tokenizer.h