microtorch::llama3::Tokenizer Member List

This is the complete list of members for microtorch::llama3::Tokenizer, including all inherited members.

decode(const std::vector< int > &ids) const	microtorch::llama3::Tokenizer	inline
encode(const std::string &text) const	microtorch::llama3::Tokenizer	inline
encode_word(const std::string &word, std::vector< int > &out) const	microtorch::llama3::Tokenizer	inlineprivate
id_	microtorch::llama3::Tokenizer	private
max_digits_	microtorch::llama3::Tokenizer	private
rank_	microtorch::llama3::Tokenizer	private
size() const	microtorch::llama3::Tokenizer	inline
special_	microtorch::llama3::Tokenizer	private
specials_	microtorch::llama3::Tokenizer	private
Tokenizer(std::vector< std::string > tokens, const std::vector< std::string > &merges, const std::vector< int32_t > &token_type, size_t max_digits=3)	microtorch::llama3::Tokenizer	inline
tokens() const	microtorch::llama3::Tokenizer	inline
tokens_	microtorch::llama3::Tokenizer	private
unbyte_	microtorch::llama3::Tokenizer	private
utf8_len(unsigned char c)	microtorch::llama3::Tokenizer	inlineprivatestatic