bpe_tokenizer.h File Reference

#include <cstdint>
#include <cstring>
#include <fstream>
#include <stdexcept>
#include <functional>
#include <iterator>
#include <map>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>

Include dependency graph for bpe_tokenizer.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
class	microtorch::bpe::Tokenizer
struct	microtorch::bpe::Tokenizer::Merge
struct	microtorch::bpe::GgufTokenizer

Namespaces
namespace	microtorch
namespace	microtorch::bpe

Functions
const std::vector< std::string > &	microtorch::bpe::specials ()
GgufTokenizer	microtorch::bpe::read_gguf_tokenizer (const std::string &path)
Tokenizer	microtorch::bpe::load_gguf (const std::string &path)