ember Namespace Reference

Namespaces
namespace	detail
namespace	nfc

Classes
struct	ChatFormat
struct	ChatPrompt
struct	ChatReply
struct	ChatSetup
class	ChatTokenizer
class	ChatTokenizerOf
struct	ChatVocab
struct	Conversation
class	QwenNfcTokenizer
class	ReplyBuilder
class	ReplyStream
struct	RequestError
struct	ScoreItem
struct	ScoreModel
class	SpmTokenizer

Typedefs
using	ChatDecode = std::function< std::string(const std::vector< int > &)>

Enumerations
enum class	Utf8Part { whole , ill_formed , cut }

Functions
ChatVocab	chat_vocab_from_gguf (const GGUFData &gd, const std::string &path)
std::string	strip_reply_ws (const std::string &s)
ChatReply	reply_of (const std::vector< int > &ids, int call_id, const ChatDecode &decode)
template<class Json >
Json	tool_calls_json (const std::vector< chat::ToolCall > &calls)
chat::json	ordered_tools (const std::string &body)
Conversation	chat_conversation_tools (const nlohmann::json &r, const chat::json &tools)
Conversation	chat_conversation (const nlohmann::json &r, const chat::Template *t=nullptr, const chat::json &tools=chat::json())
std::string	openai_content (const nlohmann::json &m, size_t i)
Conversation	openai_conversation_tools (const nlohmann::json &body, const chat::json &tools)
Conversation	openai_conversation (const nlohmann::json &body, const chat::Template *t=nullptr, const chat::json &tools=chat::json())
template<class Count >
ChatPrompt	build_general_prompt (const ChatFormat &f, const Conversation &c, int max_new, int ctx, Count count)
template<class Count >
ChatPrompt	build_chat_prompt (const ChatFormat &f, const Conversation &c, int max_new, int ctx, Count count)
int	token_id (const std::vector< std::string > &tokens, const std::string &text)
bool	has_chat_specials (const std::vector< std::string > &tokens)
void	control_ids_check (const chat::Template &t, const ChatTokenizer &tok, const std::vector< std::pair< std::string, int > > &want)
ChatSetup	make_chat_setup (ChatVocab v)
bool	deterministic ()
void	deterministic_cublas_env ()
template<class F >
void	parallel_for (int64_t n, F &&fn)
std::string	score_state_ids_error (const std::vector< int > &ids, const ScoreModel &m)
std::string	parse_score_body (const nlohmann::json &body, const ScoreModel &m, std::vector< ScoreItem > &out)
std::vector< char >	score_banned (const std::vector< std::string > &tokens, size_t vocab)
void	score_fill (nlohmann::json &rec, const std::vector< int > &c, const float *rows, size_t V, const std::vector< char > &banned)
std::vector< int >	insert_state_blocks (const std::vector< int > &ids, int vocab, int k, int eot, int user, int assistant)
size_t	utf8_part (const std::string &s, size_t i, Utf8Part &part)
std::string	utf8_valid (const std::string &s)
size_t	utf8_settled (const std::string &s, size_t from)
template<class Json >
Json	utf8_valid_json (const Json &j)
template<class Json >
std::string	dump_valid (const Json &j)

Typedef Documentation

ChatDecode

using ember::ChatDecode = typedef std::function<std::string(const std::vector<int>&)>

Definition at line 30 of file chat_reply.h.

Enumeration Type Documentation

Utf8Part

enum class ember::Utf8Part

strong

Enumerator
whole
ill_formed
cut

Definition at line 26 of file utf8_text.h.

{ whole, ill_formed, cut };

Function Documentation

build_chat_prompt()

template<class Count >

ChatPrompt ember::build_chat_prompt	(	const ChatFormat &	f,
		const Conversation &	c,
		int	max_new,
		int	ctx,
		Count	count
	)

Definition at line 449 of file chat_request.h.

                                                                                                            {
    if (c.general) return build_general_prompt(f, c, max_new, ctx, count);
    const std::string* sys = c.has_system ? &c.system : nullptr;
    auto build = [&](size_t from) { return f.prompt(c.turns, from, c.user, sys, c.thinking); };
    size_t from = 0;
    ChatPrompt out;
    out.prompt = build(from);
    while (from < c.turns.size() && count(out.prompt) + max_new > ctx) out.prompt = build(++from);
    out.prompt_tokens = count(out.prompt);
    out.turns_used = static_cast<int>(c.turns.size() - from);
    if (out.prompt_tokens >= ctx)
        out.error = "message is " + std::to_string(out.prompt_tokens) + " tokens; the model's context is " +
                    std::to_string(ctx);
    return out;
}

References build_general_prompt(), c, j, out, and ember::ChatPrompt::prompt.

Referenced by bpe_build_prompt().

build_general_prompt()

template<class Count >

ChatPrompt ember::build_general_prompt	(	const ChatFormat &	f,
		const Conversation &	c,
		int	max_new,
		int	ctx,
		Count	count
	)

Definition at line 411 of file chat_request.h.

                                                                                                               {
    const std::vector<chat::Message>& m = c.messages;
    const size_t sys = !m.empty() && m[0].role == "system" ? 1 : 0;
    const auto ex = chat::exchanges(m);
    auto kept = [&](size_t k) {
        if (k == 0) return m;  // the whole conversation: anything before the first user message is refused by render
        std::vector<chat::Message> out(m.begin(), m.begin() + static_cast<std::ptrdiff_t>(sys));
        if (k < ex.size()) out.insert(out.end(), m.begin() + static_cast<std::ptrdiff_t>(ex[k].first), m.end());
        return out;
    };
    auto build = [&](size_t k) { return chat::render(kept(k), f.tmpl, /*generation_prompt=*/true, c.thinking, c.tools); };
    size_t k = 0;
    ChatPrompt out;
    out.prompt = build(k);  // the whole conversation first: the record's refusals come from here
    while (k + 1 < ex.size() && count(out.prompt) + max_new > ctx) out.prompt = build(++k);
    out.prompt_tokens = count(out.prompt);
    out.turns_used = ex.empty() ? 0 : static_cast<int>(ex.size() - 1 - k);
    if (out.prompt_tokens >= ctx) {
        std::string head;  // the pinned system turn (with the tool definitions), as rendered
        for (const auto& p : chat::render_pieces(kept(k), f.tmpl, true, c.thinking, c.tools))
            if (p.role == "system" || p.kind == "bos") head += p.text;
        const int pinned = head.empty() ? 0 : count(head);
        out.error = pinned > 0 ? "the system turn and the newest exchange are " + std::to_string(out.prompt_tokens) +
                                     " tokens (the system turn " + std::to_string(pinned) +
                                     "); the model's context is " + std::to_string(ctx) +
                                     " (an exchange is never cut)"
                               : "the newest exchange is " + std::to_string(out.prompt_tokens) +
                                     " tokens; the model's context is " + std::to_string(ctx) +
                                     " (an exchange is never cut)";
    }
    return out;
}

References c, microtorch::chat::exchanges(), j, k, out, p, ember::ChatPrompt::prompt, microtorch::chat::render(), and microtorch::chat::render_pieces().

Referenced by build_chat_prompt().

chat_conversation()

Conversation ember::chat_conversation ( const nlohmann::json &  r, const chat::Template *  t = nullptr, const chat::json &  tools = chat::json()  )

inline

Definition at line 210 of file chat_request.h.

                                                                          {
    if (detail::tool_turns(t)) return chat_conversation_tools(r, tools);
    Conversation c;
    if (!r.contains("user_input") || !r["user_input"].is_string())
        throw RequestError("Missing 'user_input' field in request JSON");
    c.user = r["user_input"].get<std::string>();
    if (r.contains("system")) {
        if (!r["system"].is_string()) throw RequestError("system must be a string");
        c.system = r["system"].get<std::string>();
        c.has_system = true;
    }
    if (r.contains("history") && r["history"].is_array()) {
        const auto& hist = r["history"];
        std::string pending_user;
        bool have_user = false;
        for (size_t i = 0; i < hist.size(); ++i) {
            const auto& h = hist[i];
            if (!h.is_object()) throw RequestError("history[" + std::to_string(i) + "] must be an object");
            if (h.contains("role")) {
                if (!h["role"].is_string()) throw RequestError("history[" + std::to_string(i) + "].role must be a string");
                const std::string role = h["role"].get<std::string>();
                detail::refuse_tool_members_v1(h, "history[" + std::to_string(i) + "]");
                const std::string content = detail::history_content(h, i);
                if (role == "system") {
                    if (i != 0) throw RequestError("a system message is allowed only as the first history entry");
                    if (c.has_system)
                        throw RequestError("give the system message once: the 'system' field or a leading history entry");
                    c.system = content;
                    c.has_system = true;
                } else if (role == "user") {
                    if (have_user)
                        throw RequestError("history has two user messages in a row: user and assistant messages "
                                           "must alternate (section 3 point 3a)");
                    pending_user = content;
                    have_user = true;
                } else if (role == "assistant") {
                    if (!have_user)
                        throw RequestError("history has an assistant message without a user message before it: user "
                                           "and assistant messages must alternate, starting with user (section 3 "
                                           "point 3a)");
                    c.turns.emplace_back(pending_user, content);
                    have_user = false;
                } else {
                    throw RequestError("history role '" + role +
                                       "' is not allowed here (user, assistant; system only first)");
                }
            } else {
                if (have_user)
                    throw RequestError("history has a user message without its assistant reply before a {user, "
                                       "assistant} pair");
                c.turns.push_back(detail::history_pair(h, i));
            }
        }
        if (have_user)
            throw RequestError("history ends with a user message: it must end with an assistant reply (the new "
                               "message goes in user_input)");
    }
    c.has_system = c.has_system && !c.system.empty();
    if (r.contains("thinking")) c.thinking = detail::thinking_field(r["thinking"], "thinking");
    detail::refuse_tools_v1(r);
    return c;
}

References c, chat_conversation_tools(), ember::detail::history_content(), ember::detail::history_pair(), j, ember::detail::refuse_tool_members_v1(), ember::detail::refuse_tools_v1(), ember::detail::thinking_field(), and ember::detail::tool_turns().

Referenced by main().

chat_conversation_tools()

Conversation ember::chat_conversation_tools ( const nlohmann::json &  r, const chat::json &  tools  )

inline

Definition at line 146 of file chat_request.h.

                                                                                          {
    Conversation c;
    c.general = true;
    std::string system;
    bool has_system = false;
    if (r.contains("system")) {
        if (!r["system"].is_string()) throw RequestError("system must be a string");
        system = r["system"].get<std::string>();
        has_system = true;
    }
    if (r.contains("history") && r["history"].is_array()) {
        const auto& hist = r["history"];
        for (size_t i = 0; i < hist.size(); ++i) {
            const auto& h = hist[i];
            if (!h.is_object()) throw RequestError("history[" + std::to_string(i) + "] must be an object");
            if (h.contains("role")) {
                if (!h["role"].is_string()) throw RequestError("history[" + std::to_string(i) + "].role must be a string");
                const std::string role = h["role"].get<std::string>();
                const std::string content = detail::history_content(h, i, /*calls_may_omit=*/true);
                if (role == "system") {
                    if (h.contains("tool_calls") || h.contains("tool_call_id"))
                        throw RequestError("history[" + std::to_string(i) +
                                           "]: tool_calls belong on assistant messages, tool_call_id on tool messages");
                    if (i != 0) throw RequestError("a system message is allowed only as the first history entry");
                    if (has_system)
                        throw RequestError("give the system message once: the 'system' field or a leading history entry");
                    system = content;
                    has_system = true;
                } else if (role == "user" || role == "assistant" || role == "tool") {
                    c.messages.push_back(detail::tool_turn_message(role, content, h));
                } else {
                    throw RequestError("history role '" + role +
                                       "' is not allowed here (user, assistant, tool; system only first)");
                }
            } else {
                const auto pr = detail::history_pair(h, i);
                c.messages.push_back({"user", pr.first});
                c.messages.push_back({"assistant", pr.second});
            }
        }
    }
    if (r.contains("user_input")) {
        if (!r["user_input"].is_string()) throw RequestError("user_input must be a string");
        c.messages.push_back({"user", r["user_input"].get<std::string>()});
    } else if (c.messages.empty() || c.messages.back().role != "tool") {
        throw RequestError("Missing 'user_input' field in request JSON (it may be left out only when the history "
                           "ends on a tool message)");
    }
    detail::push_system(c.messages, system);
    c.tools = detail::tool_definitions(r, tools);
    if (r.contains("thinking")) c.thinking = detail::thinking_field(r["thinking"], "thinking");
    return c;
}

References c, ember::Conversation::general, ember::detail::history_content(), ember::detail::history_pair(), j, ember::detail::push_system(), ember::detail::thinking_field(), ember::detail::tool_definitions(), and ember::detail::tool_turn_message().

Referenced by chat_conversation().

chat_vocab_from_gguf()

ChatVocab ember::chat_vocab_from_gguf ( const GGUFData &  gd, const std::string &  path  )

inline

Definition at line 13 of file chat_gguf.h.

                                                                                 {
    ChatVocab v;
    auto str = [&](const char* key) {
        auto it = gd.metadata.find(key);
        const std::string* s = it == gd.metadata.end() ? nullptr : std::get_if<std::string>(&it->second);
        return s ? *s : std::string();
    };
    v.arch = str("general.architecture");
    v.chat_template = str("tokenizer.chat_template");
    v.declared = str("paperkiln.chat_template");
    auto asp = gd.metadata.find("tokenizer.ggml.add_space_prefix");
    if (asp != gd.metadata.end())
        if (const bool* b = std::get_if<bool>(&asp->second)) v.add_space_prefix = *b;
    // Qwen and SentencePiece vocabularies come from the metadata already parsed:
    // read_gguf_tokenizer reads the whole file into memory (1.3 GB for Qwen2.5-0.5B f16, 1.1 GB for
    // Gemma-3-270m f32). paperkiln files keep that reader.
    const std::string model = str("tokenizer.ggml.model");
    if ((str("tokenizer.ggml.pre") == "qwen2" || model == "llama") && !gd.tokenizer_tokens.empty()) {
        v.g.model = model;
        v.g.pre = str("tokenizer.ggml.pre");
        v.g.tokens = gd.tokenizer_tokens;
        v.g.merges = gd.tokenizer_merges;
        v.g.token_type.assign(gd.tokenizer_token_types.begin(), gd.tokenizer_token_types.end());
        v.scores = gd.tokenizer_scores;
    } else {
        v.g = microtorch::bpe::read_gguf_tokenizer(path);
    }
    return v;
}

References j, microtorch::bpe::read_gguf_tokenizer(), str, and v.

Referenced by tinyllama::TinyLlamaSession::TinyLlamaSession().

control_ids_check()

void ember::control_ids_check ( const chat::Template &  t, const ChatTokenizer &  tok, const std::vector< std::pair< std::string, int > > &  want  )

inline

Definition at line 269 of file chat_tokenizer.h.

                                                                              {
    for (const auto& [text, id] : want) {
        const std::vector<int> got = tok.encode(text);
        if (got.size() != 1 || got[0] != id)
            throw std::runtime_error("chat template " + t.id() + ": the " + tok.kind() + " tokenizer does not encode '" +
                                     text + "' as its control token " + std::to_string(id) + ": the file is refused");
    }
}

References id, j, text, and tok.

Referenced by make_chat_setup().

deterministic()

bool ember::deterministic ( )

inline

Definition at line 22 of file determinism.h.

                            {
  static const bool on = [] {
    const char* v = std::getenv("EMBER_DETERMINISTIC");
    return v != nullptr && *v != '\0' && std::strcmp(v, "0") != 0;
  }();
  return on;
}

References j, and v.

Referenced by attention_cuda_optimized(), attention_devpos_cuda(), deterministic_cublas_env(), TinyLlamaModel::initialize_gpu_and_rope(), main(), and rmsnorm_vector_cuda_optimized().

deterministic_cublas_env()

void ember::deterministic_cublas_env ( )

inline

Definition at line 32 of file determinism.h.

                                       {
  if (!deterministic() || std::getenv("CUBLAS_WORKSPACE_CONFIG") != nullptr) return;
#ifdef _WIN32
  _putenv_s("CUBLAS_WORKSPACE_CONFIG", ":4096:8");
#else
  setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8", 0);
#endif
}

References deterministic(), and j.

Referenced by TinyLlamaModel::initialize_gpu_and_rope().

dump_valid()

template<class Json >

std::string ember::dump_valid

(

const Json &

j

)

Definition at line 153 of file utf8_text.h.

                                    {
  try {
    return j.dump();
  } catch (const typename Json::type_error&) {
    return utf8_valid_json(j).dump();
  }
}

References j, and utf8_valid_json().

Referenced by json_out().

has_chat_specials()

bool ember::has_chat_specials ( const std::vector< std::string > &  tokens )

inline

Definition at line 239 of file chat_tokenizer.h.

                                                                  {
    if (tokens.size() < chat::kV1Specials.size()) return false;
    for (size_t i = 0; i < chat::kV1Specials.size(); ++i)
        if (tokens[i] != chat::kV1Specials[i]) return false;
    return true;
}

References j, and microtorch::chat::kV1Specials.

insert_state_blocks()

std::vector< int > ember::insert_state_blocks ( const std::vector< int > &  ids, int  vocab, int  k, int  eot, int  user, int  assistant  )

inline

Definition at line 24 of file state_tokens.h.

                                                           {
  if (k <= 0) {
    for (size_t i = 0; i < ids.size(); ++i)
      if (ids[i] >= vocab)
        throw std::invalid_argument("token id " + std::to_string(ids[i]) + " at position " + std::to_string(i) +
                                    " is past the vocabulary (" + std::to_string(vocab) + ") and the model has no state tokens");
    return ids;
  }
  std::vector<int> out;
  out.reserve(ids.size() + 4 * static_cast<size_t>(k));
  auto block = [&] {
    for (int j = 0; j < k; ++j) out.push_back(vocab + j);
  };
  // paperkiln's serialiser (tools/state_tokens.py, nn::insert_state_ids): an exchange opens at
  // <|user|>, has a reply once <|assistant|> follows it, and ends at the next <|endoftext|>.
  enum { kNone, kUser, kReply } phase = kNone;
  const size_t n = ids.size();
  for (size_t i = 0; i < n; ++i) {
    const int id = ids[i];
    if (id < 0) throw std::invalid_argument("token id " + std::to_string(id) + " at position " + std::to_string(i) + " is negative");
    if (id >= vocab)
      throw std::invalid_argument("state id " + std::to_string(id) + " at position " + std::to_string(i) +
                                  " is not part of a complete state block directly after an <|endoftext|> that ends "
                                  "an assistant reply");
    if (id == user) {
      if (phase == kReply) {  // the reply stopped early: close it as training does
        out.push_back(eot);
        block();
      }
      phase = kUser;
      out.push_back(id);
    } else if (id == assistant) {
      if (phase == kUser) phase = kReply;
      out.push_back(id);
    } else if (id == eot) {
      const bool closes = phase == kReply;
      phase = kNone;
      out.push_back(id);
      if (!closes) continue;
      if (i + 1 < n && ids[i + 1] >= vocab) {  // a state block given with the ids: it must be complete
        for (int j = 0; j < k; ++j)
          if (i + 1 + j >= n || ids[i + 1 + j] != vocab + j)
            throw std::invalid_argument("the state block after the <|endoftext|> at position " + std::to_string(i) +
                                        " is not the " + std::to_string(k) + " ids " + std::to_string(vocab) + " .. " +
                                        std::to_string(vocab + k - 1) + " in order");
        i += static_cast<size_t>(k);
      }
      block();
    } else {
      out.push_back(id);
    }
  }
  return out;
}

References j, k, n, and out.

Referenced by tinyllama::TinyLlamaSession::bpe_prompt_ids().

make_chat_setup()

ChatSetup ember::make_chat_setup ( ChatVocab  v )

inline

Definition at line 279 of file chat_tokenizer.h.

                                              {
    ChatSetup s;
    chat::Resolution r;
    try {
        r = chat::resolve(v.declared, v.g.tokens, v.g.model, v.chat_template, v.arch);
    } catch (const std::exception& e) {
        s.why = std::string("paperkiln.chat_template refused (") + e.what() + "): raw completion only";
        return s;
    }
    s.why = r.why;
    if (!r.chat) return s;
    s.resolved = true;
    s.tmpl = r.tmpl;
    auto& g = v.g;
    std::vector<std::pair<std::string, int>> control;  // tool-turn records: each reserved string and its id
    if (r.tmpl.has_tool) {
        try {
            chat::check_control_tokens(r.tmpl, g.tokens, g.token_type);
        } catch (const chat::Error& e) {
            throw std::runtime_error(std::string(e.what()) + ": the file is refused");
        }
        for (const auto& x : r.tmpl.reserved) control.emplace_back(x, token_id(g.tokens, x));
    }
    const bool bpe = g.model == "bpe";
    const bool l3 = g.model == "gpt2" && (g.pre == "llama-bpe" || g.pre == "qwen2");
    const bool spm = g.model == "llama";
    if (!bpe && !l3 && !spm) {
        if (g.model != "word") s.why += "; ember has no chat tokenizer for tokenizer.ggml.model '" + g.model + "'";
        return s;
    }
    ChatFormat f;
    try {
        f = ChatFormat::of(r.tmpl, g.tokens, r.why);
    } catch (const chat::Error& e) {
        s.resolved = false;
        s.why = r.why + "; " + e.what() + ": raw completion only";
        return s;
    }
    if (bpe) {
        if (g.merges.empty()) throw std::runtime_error("BPE GGUF has no tokenizer.ggml.merges");
        s.tok = std::make_unique<ChatTokenizerOf<microtorch::bpe::Tokenizer>>(
            microtorch::bpe::Tokenizer(std::move(g.tokens), g.merges), "bpe", std::move(f));
    } else if (l3) {
        if (g.merges.empty()) throw std::runtime_error(g.pre + " GGUF has no tokenizer.ggml.merges");
        if (g.pre == "qwen2")
            s.tok = std::make_unique<ChatTokenizerOf<QwenNfcTokenizer>>(
                QwenNfcTokenizer(std::move(g.tokens), g.merges, g.token_type), "qwen2_bpe", std::move(f));
        else
            s.tok = std::make_unique<ChatTokenizerOf<microtorch::llama3::Tokenizer>>(
                microtorch::llama3::Tokenizer(std::move(g.tokens), g.merges, g.token_type, 3), "llama3_bpe",
                std::move(f));
    } else {
        s.tok = std::make_unique<ChatTokenizerOf<SpmTokenizer>>(
            SpmTokenizer(std::move(g.tokens), std::move(v.scores), g.token_type, v.add_space_prefix), "spm",
            std::move(f));
    }
    if (!control.empty()) control_ids_check(s.tmpl, *s.tok, control);
    return s;
}

References microtorch::chat::check_control_tokens(), control_ids_check(), j, ember::ChatFormat::of(), microtorch::chat::resolve(), token_id(), and v.

Referenced by tinyllama::TinyLlamaSession::TinyLlamaSession().

openai_content()

std::string ember::openai_content ( const nlohmann::json &  m, size_t  i  )

inline

Definition at line 275 of file chat_request.h.

                                                                 {
    const std::string at = "messages[" + std::to_string(i) + "]";
    if (!m.contains("content") || m["content"].is_null()) throw RequestError(at + " has no content");
    const auto& c = m["content"];
    if (c.is_string()) return c.get<std::string>();
    if (!c.is_array()) throw RequestError(at + ".content must be a string or a list of text parts");
    std::string s;
    for (const auto& part : c) {
        if (!part.is_object() || part.value("type", "") != "text" || !part.contains("text") || !part["text"].is_string())
            throw RequestError(at + ".content: only text parts are supported (this model takes text only)");
        s += part["text"].get<std::string>();
    }
    return s;
}

References c, and j.

Referenced by openai_conversation(), and openai_conversation_tools().

openai_conversation()

Conversation ember::openai_conversation ( const nlohmann::json &  body, const chat::Template *  t = nullptr, const chat::json &  tools = chat::json()  )

inline

Definition at line 353 of file chat_request.h.

                                                                            {
    if (!body.is_object() || !body.contains("messages") || !body["messages"].is_array() || body["messages"].empty())
        throw RequestError("body must be JSON with a non-empty 'messages' array");
    if (detail::tool_turns(t)) return openai_conversation_tools(body, tools);
    const auto& msgs = body["messages"];
    Conversation c;
    std::vector<chat::Message> m;
    for (size_t i = 0; i < msgs.size(); ++i) {
        const auto& x = msgs[i];
        if (!x.is_object() || !x.contains("role") || !x["role"].is_string())
            throw RequestError("messages[" + std::to_string(i) + "] needs a string role");
        const std::string role = x["role"].get<std::string>();
        if (role == "tool")
            throw RequestError("tool messages are not supported: this model's chat template has no tool role");
        detail::refuse_tool_members_v1(x, "messages[" + std::to_string(i) + "]");
        if (role == "system" || role == "developer") {
            if (!m.empty())
                throw RequestError("a " + role +
                                   " message is allowed only before the first user message (this model's chat "
                                   "template has one system message, first)");
            const std::string s = openai_content(x, i);
            if (s.empty()) continue;
            c.system += (c.has_system ? "\n\n" : "") + s;
            c.has_system = true;
            continue;
        }
        if (role != "user" && role != "assistant")
            throw RequestError("messages[" + std::to_string(i) + "] has unknown role '" + role + "'");
        m.push_back({role, openai_content(x, i)});
    }
    if (m.empty() || m.back().role != "user") throw RequestError("the last message must be from the user");
    for (size_t i = 0; i < m.size(); ++i)
        if (m[i].role != (i % 2 == 0 ? "user" : "assistant"))
            throw RequestError("messages must alternate user and assistant, starting from the user (message " +
                               std::to_string(i) + " is " + m[i].role + ")");
    for (size_t i = 0; i + 1 < m.size(); i += 2) c.turns.emplace_back(m[i].content, m[i + 1].content);
    c.user = m.back().content;
    detail::openai_thinking(body, c);
    detail::refuse_tools_v1(body);
    return c;
}

References body, c, j, openai_content(), openai_conversation_tools(), ember::detail::openai_thinking(), ember::detail::refuse_tool_members_v1(), ember::detail::refuse_tools_v1(), and ember::detail::tool_turns().

Referenced by main().

openai_conversation_tools()

Conversation ember::openai_conversation_tools ( const nlohmann::json &  body, const chat::json &  tools  )

inline

Definition at line 305 of file chat_request.h.

                                                                                               {
    const auto& msgs = body["messages"];
    Conversation c;
    c.general = true;
    std::string system;
    for (size_t i = 0; i < msgs.size(); ++i) {
        const auto& x = msgs[i];
        if (!x.is_object() || !x.contains("role") || !x["role"].is_string())
            throw RequestError("messages[" + std::to_string(i) + "] needs a string role");
        const std::string role = x["role"].get<std::string>();
        if (role == "system" || role == "developer") {
            if (!c.messages.empty())
                throw RequestError("a " + role +
                                   " message is allowed only before the first user message (this model's chat "
                                   "template has one system message, first)");
            if (x.contains("tool_calls") || x.contains("tool_call_id"))
                throw RequestError("messages[" + std::to_string(i) +
                                   "]: tool_calls belong on assistant messages, tool_call_id on tool messages");
            const std::string s = openai_content(x, i);
            if (s.empty()) continue;
            system += (system.empty() ? "" : "\n\n") + s;
            continue;
        }
        if (role != "user" && role != "assistant" && role != "tool")
            throw RequestError("messages[" + std::to_string(i) + "] has unknown role '" + role + "'");
        const bool calls = x.contains("tool_calls") && !x["tool_calls"].is_null();
        const bool no_content = !x.contains("content") || x["content"].is_null();
        const std::string content = role == "assistant" && calls && no_content ? std::string() : openai_content(x, i);
        c.messages.push_back(detail::tool_turn_message(role, content, x));
    }
    if (c.messages.empty() || (c.messages.back().role != "user" && c.messages.back().role != "tool"))
        throw RequestError("the last message must be from the user or a tool");
    detail::push_system(c.messages, system);
    c.tools = detail::tool_definitions(body, tools);
    if (body.contains("tool_choice") && !body["tool_choice"].is_null() &&
        !(body["tool_choice"].is_string() && body["tool_choice"].get<std::string>() == "auto"))
        throw RequestError("tool_choice is not supported except \"auto\" (the model decides whether to call a tool)");
    detail::openai_thinking(body, c);
    return c;
}

References body, c, ember::Conversation::general, j, openai_content(), ember::detail::openai_thinking(), ember::detail::push_system(), ember::detail::tool_definitions(), and ember::detail::tool_turn_message().

Referenced by openai_conversation().

ordered_tools()

chat::json ember::ordered_tools ( const std::string &  body )

inline

Definition at line 135 of file chat_request.h.

                                                     {
    chat::json o = chat::json::parse(body, nullptr, false);
    if (o.is_discarded() || !o.is_object() || !o.contains("tools")) return chat::json();
    return o["tools"];
}

References body, and j.

Referenced by main().

parallel_for()

template<class F >

void ember::parallel_for	(	int64_t	n,
		F &&	fn
	)

Definition at line 31 of file parallel_for.h.

                                     {
#pragma omp parallel for schedule(static)
  for (int64_t i = 0; i < n; ++i) fn(i);
}

References j, and n.

Referenced by attention_batch_cpu(), attention_bias_batch_cpu(), attention_relkv_batch_cpu(), attention_rows_batch_cpu(), attention_window_batch_cpu(), matmul_f32_f32_batch_cpu(), matmul_rows_f32_batch_cpu(), and matvec_f32_f32_vector_cpu().

parse_score_body()

std::string ember::parse_score_body ( const nlohmann::json &  body, const ScoreModel &  m, std::vector< ScoreItem > &  out  )

inline

Definition at line 92 of file score.h.

                                                                                                            {
  if (!body.is_object() || !body.contains("requests") || !body["requests"].is_array())
    return "body must be {\"requests\": [{\"key\", \"prompt_ids\", \"continuation_ids\"}, ...]}";
  const nlohmann::json& reqs = body["requests"];
  out.clear();
  out.reserve(reqs.size());
  for (size_t n = 0; n < reqs.size(); ++n) {
    const nlohmann::json& r = reqs[n];
    const std::string at = "request " + std::to_string(n);
    if (!r.is_object()) return at + " is not an object";
    if (!r.contains("prompt_ids") || !r.contains("continuation_ids"))
      return at + ": prompt_ids and continuation_ids are required (ember scores token ids; mtstudio's text fields "
                  "are not served)";
    ScoreItem it;
    try {
      it.prompt = r["prompt_ids"].get<std::vector<int>>();
      it.cont = r["continuation_ids"].get<std::vector<int>>();
    } catch (const nlohmann::json::exception& e) {
      return at + ": prompt_ids and continuation_ids must be lists of integers (" + e.what() + ")";
    }
    if (it.prompt.empty() || it.cont.empty()) return at + " has an empty prompt or continuation";
    for (int t : it.prompt)
      if (t < 0 || static_cast<size_t>(t) >= m.vocab + m.state_k) return at + ": prompt id out of range";
    for (int t : it.cont)
      if (t < 0 || static_cast<size_t>(t) >= m.vocab) return at + ": continuation id out of range";
    if (m.state_k > 0) {
      std::vector<int> ids = it.prompt;
      ids.insert(ids.end(), it.cont.begin(), it.cont.end());
      const std::string err = score_state_ids_error(ids, m);
      if (!err.empty()) return at + ": " + err;
    }
    it.rec = {{"i", n}, {"prompt_tokens", it.prompt.size()}, {"continuation_ids", it.cont}};
    if (r.contains("key")) it.rec["key"] = r["key"];
    it.over_ctx = it.prompt.size() + it.cont.size() > m.ctx + 1;
    out.push_back(std::move(it));
  }
  return "";
}

References body, j, n, out, ember::ScoreItem::prompt, and score_state_ids_error().

Referenced by main().

reply_of()

ChatReply ember::reply_of ( const std::vector< int > &  ids, int  call_id, const ChatDecode &  decode  )

inline

Definition at line 45 of file chat_reply.h.

                                                                                            {
    ChatReply r;
    if (call_id >= 0) {
        chat::ParsedReply p = chat::parse_reply(ids, call_id, decode);
        if (p.parsed && !p.tool_calls.empty()) {
            r.content = utf8_valid(strip_reply_ws(p.content));
            for (auto& c : p.tool_calls) {
                c.name = utf8_valid(c.name);
                c.arguments = utf8_valid(c.arguments);
                r.tool_calls.push_back(std::move(c));
            }
            return r;
        }
    }
    r.content = utf8_valid(strip_reply_ws(decode(ids)));
    return r;
}

References c, j, p, microtorch::chat::parse_reply(), strip_reply_ws(), and utf8_valid().

Referenced by bpe_run(), and ember::ReplyBuilder::finish().

score_banned()

std::vector< char > ember::score_banned ( const std::vector< std::string > &  tokens, size_t  vocab  )

inline

Definition at line 133 of file score.h.

                                                                                      {
  std::vector<char> banned(vocab, 0);
  const size_t n = tokens.size() < vocab ? tokens.size() : vocab;
  for (const char* sp : {"<unk>", "<s>", "</s>", "<pad>"})
    for (size_t k = 0; k < n; ++k)
      if (tokens[k] == sp) {
        banned[k] = 1;
        break;
      }
  return banned;
}

References j, k, and n.

Referenced by main().

score_fill()

void ember::score_fill ( nlohmann::json &  rec, const std::vector< int > &  c, const float *  rows, size_t  V, const std::vector< char > &  banned  )

inline

Definition at line 147 of file score.h.

                                                      {
  std::vector<double> lp;
  std::vector<int> am;
  lp.reserve(c.size());
  am.reserve(c.size());
  double sum = 0.0;
  for (size_t j = 0; j < c.size(); ++j) {
    const float* row = rows + j * V;
    double mx = -1e300;
    int best = -1;
    double best_v = -1e300;
    for (size_t k = 0; k < V; ++k) {
      const double x = row[k];
      mx = std::max(mx, x);
      if (!banned[k] && (best < 0 || x > best_v)) {
        best = static_cast<int>(k);
        best_v = x;
      }
    }
    double z = 0.0;
    for (size_t k = 0; k < V; ++k) z += std::exp(static_cast<double>(row[k]) - mx);
    const double l = static_cast<double>(row[static_cast<size_t>(c[j])]) - mx - std::log(z);
    lp.push_back(l);
    am.push_back(best);
    sum += l;
  }
  rec["over_ctx"] = false;
  rec["logprobs"] = lp;
  rec["argmax"] = am;
  rec["sum"] = sum;
}

References c, j, and k.

Referenced by main().

score_state_ids_error()

std::string ember::score_state_ids_error ( const std::vector< int > &  ids, const ScoreModel &  m  )

inline

Definition at line 58 of file score.h.

                                                                                       {
  enum { NONE, USER, REPLY } phase = NONE;
  bool closed = false;
  for (size_t p = 0; p < ids.size(); ++p) {
    const int id = ids[p];
    if (id < 0 || static_cast<size_t>(id) >= m.vocab + m.state_k)
      return "id " + std::to_string(id) + " at position " + std::to_string(p) + " is out of range";
    if (static_cast<size_t>(id) >= m.vocab) {
      if (!closed || static_cast<size_t>(id) != m.vocab)
        return "state id " + std::to_string(id) + " at position " + std::to_string(p) +
               " does not start a block after an <|endoftext|> that ends a reply";
      for (size_t j = 0; j < m.state_k; ++j)
        if (p + j >= ids.size() || static_cast<size_t>(ids[p + j]) != m.vocab + j)
          return "the state block at position " + std::to_string(p) + " is not the ids " + std::to_string(m.vocab) +
                 " .. " + std::to_string(m.vocab + m.state_k - 1) + " in order";
      p += m.state_k - 1;
      closed = false;
      continue;
    }
    closed = false;
    if (id == m.user) {
      phase = USER;
    } else if (id == m.assistant) {
      if (phase == USER) phase = REPLY;
    } else if (id == m.eot) {
      closed = phase == REPLY;
      phase = NONE;
    }
  }
  return "";
}

References j, and p.

Referenced by parse_score_body().

strip_reply_ws()

std::string ember::strip_reply_ws ( const std::string &  s )

inline

Definition at line 33 of file chat_reply.h.

                                                    {
    const char* ws = " \t\n\v\f\r";
    const auto a = s.find_first_not_of(ws), b = s.find_last_not_of(ws);
    return a == std::string::npos ? std::string() : s.substr(a, b - a + 1);
}

References a, and j.

Referenced by reply_of().

token_id()

int ember::token_id ( const std::vector< std::string > &  tokens, const std::string &  text  )

inline

Definition at line 41 of file chat_tokenizer.h.

                                                                               {
    for (size_t i = tokens.size(); i-- > 0;)
        if (tokens[i] == text) return static_cast<int>(i);
    return -1;
}

References j, and text.

Referenced by make_chat_setup(), and ember::ChatFormat::of().

tool_calls_json()

template<class Json >

Json ember::tool_calls_json

(

const std::vector< chat::ToolCall > &

calls

)

Definition at line 66 of file chat_reply.h.

                                                           {
    Json a = Json::array();
    for (const auto& c : calls)
        a.push_back(Json{{"id", c.id}, {"type", "function"}, {"function", Json{{"name", c.name}, {"arguments", c.arguments}}}});
    return a;
}

References a, c, and j.

utf8_part()

size_t ember::utf8_part ( const std::string &  s, size_t  i, Utf8Part &  part  )

inline

Definition at line 29 of file utf8_text.h.

                                                                      {
  const unsigned char c = static_cast<unsigned char>(s[i]);
  part = Utf8Part::whole;
  if (c < 0x80) return 1;
  // continuation bytes needed after the lead, and the range of the first one (Table 3-7)
  size_t need = 0;
  unsigned char lo = 0x80, hi = 0xBF;
  if (c >= 0xC2 && c <= 0xDF) {
    need = 1;
  } else if (c == 0xE0) {
    need = 2;
    lo = 0xA0;
  } else if ((c >= 0xE1 && c <= 0xEC) || c == 0xEE || c == 0xEF) {
    need = 2;
  } else if (c == 0xED) {
    need = 2;
    hi = 0x9F;
  } else if (c == 0xF0) {
    need = 3;
    lo = 0x90;
  } else if (c >= 0xF1 && c <= 0xF3) {
    need = 3;
  } else if (c == 0xF4) {
    need = 3;
    hi = 0x8F;
  } else {  // a continuation byte without a lead, C0, C1, F5..FF
    part = Utf8Part::ill_formed;
    return 1;
  }
  const size_t n = s.size();
  size_t j = 1;
  for (; j <= need; ++j) {
    if (i + j >= n) {  // the lead and the continuations that fitted, then the end of s
      part = Utf8Part::cut;
      return j;
    }
    const unsigned char d = static_cast<unsigned char>(s[i + j]);
    if (d < (j == 1 ? lo : 0x80) || d > (j == 1 ? hi : 0xBF)) {
      part = Utf8Part::ill_formed;  // the lead and the continuations that fitted: one maximal subpart
      return j;
    }
  }
  return j;
}

References c, cut, ill_formed, j, n, and whole.

Referenced by utf8_settled(), and utf8_valid().

utf8_settled()

size_t ember::utf8_settled ( const std::string &  s, size_t  from  )

inline

Definition at line 95 of file utf8_text.h.

                                                            {
  const size_t n = s.size();
  size_t i = from;
  while (i < n) {
    Utf8Part part;
    const size_t len = utf8_part(s, i, part);
    if (part == Utf8Part::cut) return i;
    i += len;
  }
  return n;
}

References cut, j, n, and utf8_part().

Referenced by ember::ReplyStream::next().

utf8_valid()

std::string ember::utf8_valid ( const std::string &  s )

inline

Definition at line 74 of file utf8_text.h.

                                                {
  const char rep[] = {static_cast<char>(0xEF), static_cast<char>(0xBF), static_cast<char>(0xBD)};
  std::string out;
  out.reserve(s.size());
  const size_t n = s.size();
  size_t i = 0;
  while (i < n) {
    Utf8Part part;
    const size_t len = utf8_part(s, i, part);
    if (part == Utf8Part::whole)
      out.append(s, i, len);
    else
      out.append(rep, 3);  // ill-formed, or cut by the end of s: one U+FFFD per maximal subpart
    i += len;
  }
  return out;
}

References j, n, out, utf8_part(), and whole.

Referenced by bpe_run(), tinyllama::TinyLlamaSession::generate(), main(), ember::ReplyStream::next(), PYBIND11_MODULE(), reply_of(), and utf8_valid_json().

utf8_valid_json()

template<class Json >

Json ember::utf8_valid_json

(

const Json &

j

)

Definition at line 135 of file utf8_text.h.

                                    {
  if (j.is_string()) return Json(utf8_valid(j.template get<std::string>()));
  if (j.is_array()) {
    Json a = Json::array();
    for (const auto& x : j) a.push_back(utf8_valid_json(x));
    return a;
  }
  if (j.is_object()) {
    Json o = Json::object();
    for (auto it = j.begin(); it != j.end(); ++it) o[utf8_valid(it.key())] = utf8_valid_json(it.value());
    return o;
  }
  return j;
}

References a, j, utf8_valid(), and utf8_valid_json().

Referenced by dump_valid(), and utf8_valid_json().