#include <chat_tokenizer.h>

Collaboration diagram for ember::QwenNfcTokenizer:

Public Member Functions
	QwenNfcTokenizer (std::vector< std::string > tokens, const std::vector< std::string > &merges, const std::vector< int32_t > &token_type)

std::vector< int >	encode (const std::string &text) const

std::string	decode (const std::vector< int > &ids) const

size_t	size () const

Private Attributes
std::unique_ptr< microtorch::llama3::Tokenizer >	t_

std::vector< int >	specials_

std::vector< std::string >	texts_

Detailed Description

Definition at line 165 of file chat_tokenizer.h.

Constructor & Destructor Documentation

◆ QwenNfcTokenizer()

ember::QwenNfcTokenizer::QwenNfcTokenizer	(	std::vector< std::string >	tokens,
		const std::vector< std::string > &	merges,
		const std::vector< int32_t > &	token_type
	)

inline

Definition at line 167 of file chat_tokenizer.h.

                                                           {
        for (size_t i = 0; i < token_type.size() && i < tokens.size(); ++i) {
            const int t = token_type[i];
            if (t != 2 && t != 3 && t != 4) continue;
            for (const char* c = nfc::tables::kNfcAsciiImages; *c; ++c)
                if (tokens[i].find(*c) != std::string::npos)
                    throw std::runtime_error("Qwen vocabulary: added token '" + tokens[i] + "' holds '" +
                                             std::string(1, *c) +
                                             "', which NFC writes for another character; ember's NFC cut "
                                             "would differ from HF's there");
            specials_.push_back(static_cast<int>(i));
            texts_.push_back(tokens[i]);
        }
        std::vector<size_t> order(specials_.size());
        for (size_t k = 0; k < order.size(); ++k) order[k] = k;
        std::stable_sort(order.begin(), order.end(),
                         [&](size_t a, size_t b) { return texts_[a].size() > texts_[b].size(); });
        std::vector<int> s;
        std::vector<std::string> t;
        for (size_t k : order) {
            s.push_back(specials_[k]);
            t.push_back(texts_[k]);
        }
        specials_ = std::move(s);
        texts_ = std::move(t);
        t_ = std::make_unique<microtorch::llama3::Tokenizer>(std::move(tokens), merges, token_type, 1);
    }

References a, c, j, k, ember::nfc::tables::kNfcAsciiImages, specials_, t_, and texts_.

Member Function Documentation

◆ decode()

std::string ember::QwenNfcTokenizer::decode ( const std::vector< int > & ids ) const

inline

Definition at line 229 of file chat_tokenizer.h.

229{ return t_->decode(ids); }

References t_.

◆ encode()

std::vector< int > ember::QwenNfcTokenizer::encode ( const std::string & text ) const

inline

Definition at line 196 of file chat_tokenizer.h.

                                                       {
        if (nfc::nfc_utf8(text) == text) return t_->encode(text);  // already NFC: the port's ids
        std::vector<std::pair<std::string, int>> frags = {{text, -1}};
        for (size_t k = 0; k < specials_.size(); ++k) {
            const std::string& s = texts_[k];
            if (s.empty()) continue;
            std::vector<std::pair<std::string, int>> next;
            for (auto& [t, id] : frags) {
                if (id >= 0) {
                    next.emplace_back(std::move(t), id);
                    continue;
                }
                size_t at = 0;
                for (size_t m; (m = t.find(s, at)) != std::string::npos; at = m + s.size()) {
                    if (m > at) next.emplace_back(t.substr(at, m - at), -1);
                    next.emplace_back(std::string(), specials_[k]);
                }
                if (at < t.size()) next.emplace_back(t.substr(at), -1);
            }
            frags = std::move(next);
        }
        std::vector<int> out;
        for (const auto& [t, id] : frags) {
            if (id >= 0) {
                out.push_back(id);
                continue;
            }
            const std::vector<int> ids = t_->encode(nfc::nfc_utf8(t));
            out.insert(out.end(), ids.begin(), ids.end());
        }
        return out;
    }