llama3_tokenizer.h File Reference

#include <algorithm>
#include <cstdint>
#include <queue>
#include <stdexcept>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "bpe_tokenizer.h"
#include "llama3_unicode_tables.h"

Include dependency graph for llama3_tokenizer.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	microtorch::llama3::CptFlags
class	microtorch::llama3::Tokenizer
struct	microtorch::llama3::Tokenizer::Symbol
struct	microtorch::llama3::Tokenizer::Bigram
struct	microtorch::llama3::Tokenizer::Later

Namespaces
namespace	microtorch
namespace	microtorch::llama3

Functions
uint16_t	microtorch::llama3::category_bits (uint32_t cpt)
CptFlags	microtorch::llama3::flags_of (uint32_t cpt)
std::vector< uint32_t >	microtorch::llama3::cpts_from_utf8 (const std::string &s)
void	microtorch::llama3::append_utf8 (uint32_t cpt, std::string &out)
std::vector< std::string >	microtorch::llama3::split_words (const std::string &text, size_t max_digits=3)
const std::vector< std::string > &	microtorch::llama3::byte_to_unicode ()
std::string	microtorch::llama3::byte_encode (const std::string &word)
Tokenizer	microtorch::llama3::load_gguf (const std::string &path)