microtorch::bpe::Tokenizer Member List

This is the complete list of members for microtorch::bpe::Tokenizer, including all inherited members.

decode(const std::vector< int > &ids) const	microtorch::bpe::Tokenizer	inline
digit(unsigned char c)	microtorch::bpe::Tokenizer	inlinestatic
encode(const std::string &text) const	microtorch::bpe::Tokenizer	inline
encode_chunk(const std::string &chunk, std::vector< int > &out) const	microtorch::bpe::Tokenizer	inlineprivate
key(int a, int b)	microtorch::bpe::Tokenizer	inlineprivatestatic
letter(unsigned char c)	microtorch::bpe::Tokenizer	inlinestatic
pretokenize(const std::string &b)	microtorch::bpe::Tokenizer	inlinestatic
rank_	microtorch::bpe::Tokenizer	private
size() const	microtorch::bpe::Tokenizer	inline
space(unsigned char c)	microtorch::bpe::Tokenizer	inlinestatic
Tokenizer(std::vector< std::string > tokens, const std::vector< std::string > &merges)	microtorch::bpe::Tokenizer	inline
tokens() const	microtorch::bpe::Tokenizer	inline
tokens_	microtorch::bpe::Tokenizer	private