#include <chat_tokenizer.h>

Collaboration diagram for ember::ChatFormat:

Public Member Functions
bool	is_eos (int id) const

bool	is_stop (int id) const

std::vector< int >	stop_ids () const

bool	v1_markers () const

std::string	prompt (const std::vector< std::pair< std::string, std::string > > &turns, size_t from, const std::string &user, const std::string *system=nullptr, int thinking=-1) const

Static Public Member Functions
static ChatFormat	of (const chat::Template &t, const std::vector< std::string > &tokens, std::string why)

Public Attributes
chat::Template	tmpl

std::string	name

std::string	why

std::vector< int >	eos_ids

std::vector< int >	turn_ids

int	tool_call_id = -1

Detailed Description

Definition at line 53 of file chat_tokenizer.h.

Member Function Documentation

◆ is_eos()

bool ember::ChatFormat::is_eos ( int id ) const

inline

Definition at line 89 of file chat_tokenizer.h.

                              {
        for (int e : eos_ids)
            if (e == id) return true;
        return false;
    }

References eos_ids, and j.

Referenced by is_stop().

◆ is_stop()

bool ember::ChatFormat::is_stop ( int id ) const

inline

Definition at line 94 of file chat_tokenizer.h.

                               {
        if (is_eos(id)) return true;
        for (int t : turn_ids)
            if (t == id) return true;
        return false;
    }

References is_eos(), j, and turn_ids.

◆ of()

static ChatFormat ember::ChatFormat::of	(	const chat::Template &	t,
		const std::vector< std::string > &	tokens,
		std::string	why
	)

inlinestatic

Definition at line 61 of file chat_tokenizer.h.

                                                                                                   {
        ChatFormat f;
        f.tmpl = t;
        f.name = t.id();
        f.why = std::move(why);
        auto ids = [&](const std::vector<std::string>& stops, const char* field) {
            std::vector<int> v;
            for (const auto& s : stops) {
                const int id = token_id(tokens, s);
                if (id < 0)
                    throw chat::Error("chat template " + t.id() + ": stop." + field + " '" + s +
                                      "' is not a single token of this vocabulary (ember stops on single tokens only)");
                v.push_back(id);
            }
            return v;
        };
        f.eos_ids = ids(t.stop_eos, "eos");
        f.turn_ids = ids(t.stop_turn, "turn");
        if (f.eos_ids.empty()) throw chat::Error("chat template " + t.id() + " has no stop.eos token");
        if (t.has_tool) {
            f.tool_call_id = token_id(tokens, t.tool_call_open);
            if (f.tool_call_id < 0)
                throw chat::Error("chat template " + t.id() + ": tool_call.open '" + t.tool_call_open +
                                  "' is not a single token of this vocabulary");
        }
        return f;
    }

References j, microtorch::chat::Template::name, tmpl, ember::token_id(), v, and why.

Referenced by ember::make_chat_setup().

◆ prompt()

std::string ember::ChatFormat::prompt	(	const std::vector< std::pair< std::string, std::string > > &	turns,
		size_t	from,
		const std::string &	user,
		const std::string *	system = `nullptr`,
		int	thinking = `-1`
	)		const

inline

Definition at line 119 of file chat_tokenizer.h.

                                                                                                          {
        const std::vector<std::pair<std::string, std::string>> kept(turns.begin() + static_cast<std::ptrdiff_t>(from),
                                                                    turns.end());
        return chat::render(chat::from_pairs(kept, &user, system), tmpl, /*generation_prompt=*/true, thinking);
    }

References microtorch::chat::from_pairs(), j, microtorch::chat::render(), and tmpl.

◆ stop_ids()

std::vector< int > ember::ChatFormat::stop_ids ( ) const

inline

Definition at line 100 of file chat_tokenizer.h.

                                    {
        std::vector<int> v = eos_ids;
        v.insert(v.end(), turn_ids.begin(), turn_ids.end());
        return v;
    }

References eos_ids, turn_ids, and v.

◆ v1_markers()

bool ember::ChatFormat::v1_markers ( ) const

inline

Definition at line 107 of file chat_tokenizer.h.

                            {
        const chat::Template v = chat::v1();
        return tmpl.name == v.name && tmpl.version == v.version && tmpl.bos.empty() && tmpl.content == "raw" &&
               tmpl.user_open == v.user_open && tmpl.user_close == v.user_close &&
               tmpl.assistant_open == v.assistant_open && tmpl.assistant_close == v.assistant_close &&
               tmpl.generation_prompt == v.generation_prompt && tmpl.system_mode != "turn" &&
               tmpl.stop_eos == v.stop_eos && tmpl.stop_turn == v.stop_turn;
    }

Member Data Documentation

◆ eos_ids

std::vector<int> ember::ChatFormat::eos_ids

Definition at line 57 of file chat_tokenizer.h.

Referenced by is_eos(), main(), and stop_ids().

◆ name

std::string ember::ChatFormat::name

Definition at line 55 of file chat_tokenizer.h.

◆ tmpl

chat::Template ember::ChatFormat::tmpl

Definition at line 54 of file chat_tokenizer.h.

Referenced by of(), prompt(), and v1_markers().

◆ tool_call_id

int ember::ChatFormat::tool_call_id = -1

Definition at line 58 of file chat_tokenizer.h.

◆ turn_ids

std::vector<int> ember::ChatFormat::turn_ids

Definition at line 57 of file chat_tokenizer.h.

Referenced by is_stop(), and stop_ids().

◆ why

std::string ember::ChatFormat::why

Definition at line 56 of file chat_tokenizer.h.

Referenced by of().

The documentation for this struct was generated from the following file:

chat_tokenizer.h

Public Member Functions

Static Public Member Functions

Public Attributes

Detailed Description

Member Function Documentation

◆ is_eos()

◆ is_stop()

◆ of()

◆ prompt()

◆ stop_ids()

◆ v1_markers()

Member Data Documentation

◆ eos_ids

◆ name

◆ tmpl

◆ tool_call_id

◆ turn_ids

◆ why