chat_tokenizer.h File Reference

#include <memory>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
#include "bpe_tokenizer.h"
#include "chat_template.h"
#include "llama3_tokenizer.h"
#include "spm_tokenizer.h"
#include "unicode_nfc.h"

Include dependency graph for chat_tokenizer.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	ember::ChatFormat
class	ember::ChatTokenizer
class	ember::ChatTokenizerOf< T >
class	ember::QwenNfcTokenizer
struct	ember::ChatVocab
struct	ember::ChatSetup

Namespaces
namespace	ember

Functions
int	ember::token_id (const std::vector< std::string > &tokens, const std::string &text)
bool	ember::has_chat_specials (const std::vector< std::string > &tokens)
void	ember::control_ids_check (const chat::Template &t, const ChatTokenizer &tok, const std::vector< std::pair< std::string, int > > &want)
ChatSetup	ember::make_chat_setup (ChatVocab v)