microtorch::bpe::Tokenizer Class Reference

#include <bpe_tokenizer.h>

Collaboration diagram for microtorch::bpe::Tokenizer:

Classes
struct	Merge

Public Member Functions
	Tokenizer (std::vector< std::string > tokens, const std::vector< std::string > &merges)
std::vector< int >	encode (const std::string &text) const
std::string	decode (const std::vector< int > &ids) const
size_t	size () const
const std::vector< std::string > &	tokens () const

Static Public Member Functions
static bool	letter (unsigned char c)
static bool	digit (unsigned char c)
static bool	space (unsigned char c)
static std::vector< std::pair< std::string, int > >	pretokenize (const std::string &b)

Private Member Functions
void	encode_chunk (const std::string &chunk, std::vector< int > &out) const

Static Private Member Functions
static uint64_t	key (int a, int b)

Private Attributes
std::vector< std::string >	tokens_
std::unordered_map< uint64_t, Merge >	rank_

Detailed Description

Definition at line 42 of file bpe_tokenizer.h.

Constructor & Destructor Documentation

Tokenizer()

microtorch::bpe::Tokenizer::Tokenizer ( std::vector< std::string >  tokens, const std::vector< std::string > &  merges  )

inline

Definition at line 46 of file bpe_tokenizer.h.

                                                                               : tokens_(std::move(tokens)) {
        const int base = static_cast<int>(specials().size()) + 256;
        for (size_t r = 0; r < merges.size(); ++r) {
            const auto sp = merges[r].find(' ');
            const int a = std::stoi(merges[r].substr(0, sp)), b = std::stoi(merges[r].substr(sp + 1));
            rank_[key(a, b)] = {static_cast<int>(r), base + static_cast<int>(r)};
        }
    }

References a, j, key(), rank_, and microtorch::bpe::specials().

Member Function Documentation

decode()

std::string microtorch::bpe::Tokenizer::decode ( const std::vector< int > &  ids ) const

inline

Definition at line 67 of file bpe_tokenizer.h.

                                                      {
        std::string s;
        for (int id : ids)
            if (id >= 0 && static_cast<size_t>(id) < tokens_.size()) s += tokens_[id];
        return s;
    }

References j, and tokens_.

digit()

static bool microtorch::bpe::Tokenizer::digit ( unsigned char  c )

inlinestatic

Definition at line 80 of file bpe_tokenizer.h.

{ return c >= '0' && c <= '9'; }

References c.

Referenced by pretokenize().

encode()

std::vector< int > microtorch::bpe::Tokenizer::encode ( const std::string &  text ) const

inline

Definition at line 55 of file bpe_tokenizer.h.

                                                       {
        std::vector<int> out;
        for (const auto& [chunk, special] : pretokenize(text)) {
            if (special >= 0) {
                out.push_back(special);
                continue;
            }
            encode_chunk(chunk, out);
        }
        return out;
    }

References encode_chunk(), j, out, pretokenize(), and text.

encode_chunk()

void microtorch::bpe::Tokenizer::encode_chunk ( const std::string &  chunk, std::vector< int > &  out  ) const

inlineprivate

Definition at line 129 of file bpe_tokenizer.h.

                                                                         {
        const int off = static_cast<int>(specials().size());
        std::vector<int> ids;
        for (unsigned char c : chunk) ids.push_back(off + c);
        while (ids.size() > 1) {
            int best = -1, at = -1, id = -1;
            for (size_t k = 0; k + 1 < ids.size(); ++k) {
                const auto it = rank_.find(key(ids[k], ids[k + 1]));
                if (it != rank_.end() && (best < 0 || it->second.rank < best)) {
                    best = it->second.rank;
                    at = static_cast<int>(k);
                    id = it->second.id;
                }
            }
            if (best < 0) break;
            ids[at] = id;
            ids.erase(ids.begin() + at + 1);
        }
        out.insert(out.end(), ids.begin(), ids.end());
    }

References c, id, j, k, key(), off, out, rank_, and microtorch::bpe::specials().

Referenced by encode().

key()

static uint64_t microtorch::bpe::Tokenizer::key ( int  a, int  b  )

inlinestaticprivate

Definition at line 127 of file bpe_tokenizer.h.

{ return (static_cast<uint64_t>(a) << 32) | static_cast<uint32_t>(b); }

References a, and j.

Referenced by encode_chunk(), and Tokenizer().

letter()

static bool microtorch::bpe::Tokenizer::letter ( unsigned char  c )

inlinestatic

Definition at line 77 of file bpe_tokenizer.h.

                                        {
        return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '\'' || c >= 0x80;
    }

References c.

Referenced by pretokenize().

pretokenize()

static std::vector< std::pair< std::string, int > > microtorch::bpe::Tokenizer::pretokenize ( const std::string &  b )

inlinestatic

Definition at line 84 of file bpe_tokenizer.h.

                                                                              {
        std::vector<std::pair<std::string, int>> out;
        const size_t n = b.size();
        auto special_at = [&](size_t i) -> int {
            for (size_t k = 1; k < specials().size(); ++k)
                if (b.compare(i, specials()[k].size(), specials()[k]) == 0) return static_cast<int>(k);
            return -1;
        };
        size_t i = 0;
        while (i < n) {
            const int sp = special_at(i);
            if (sp >= 0) {
                out.emplace_back(specials()[sp], sp);
                i += specials()[sp].size();
                continue;
            }
            const unsigned char c = b[i];
            const size_t j = (c == ' ' && i + 1 < n && letter(b[i + 1])) ? i + 1 : i;
            size_t k;
            if (letter(b[j])) {
                k = j;
                while (k < n && letter(b[k])) ++k;
            } else if (digit(c)) {
                k = i + 1;
            } else if (space(c)) {
                k = i;
                while (k < n && space(b[k]) && !(b[k] == ' ' && k + 1 < n && letter(b[k + 1]))) ++k;
                if (k == i) k = i + 1;
            } else {
                k = i;
                while (k < n && !(space(b[k]) || letter(b[k]) || digit(b[k]) || special_at(k) >= 0)) ++k;
                if (k == i) k = i + 1;
            }
            out.emplace_back(b.substr(i, k - i), -1);
            i = k;
        }
        return out;
    }

References c, digit(), j, k, letter(), n, out, size(), space(), and microtorch::bpe::specials().

Referenced by encode().

size()

size_t microtorch::bpe::Tokenizer::size ( ) const

inline

Definition at line 74 of file bpe_tokenizer.h.

{ return tokens_.size(); }

References tokens_.

Referenced by pretokenize().

space()

static bool microtorch::bpe::Tokenizer::space ( unsigned char  c )

inlinestatic

Definition at line 81 of file bpe_tokenizer.h.

{ return c == ' ' || (c >= '\t' && c <= '\r'); }

References c.

Referenced by pretokenize().

tokens()

const std::vector< std::string > & microtorch::bpe::Tokenizer::tokens ( ) const

inline

Definition at line 75 of file bpe_tokenizer.h.

{ return tokens_; }

References tokens_.

Member Data Documentation

rank_

std::unordered_map<uint64_t, Merge> microtorch::bpe::Tokenizer::rank_

private

Definition at line 151 of file bpe_tokenizer.h.

Referenced by encode_chunk(), and Tokenizer().

tokens_

std::vector<std::string> microtorch::bpe::Tokenizer::tokens_

private

Definition at line 150 of file bpe_tokenizer.h.

Referenced by decode(), size(), and tokens().

---

The documentation for this class was generated from the following file:

• bpe_tokenizer.h