ember.cpp 0.1.0
C++ inference engine for Llama-family and paperkiln models (CPU, CUDA, WebAssembly)
Loading...
Searching...
No Matches
Namespaces | Classes | Functions
microtorch::llama3 Namespace Reference

Namespaces

namespace  tables
 

Classes

struct  CptFlags
 
class  Tokenizer
 

Functions

uint16_t category_bits (uint32_t cpt)
 
CptFlags flags_of (uint32_t cpt)
 
std::vector< uint32_t > cpts_from_utf8 (const std::string &s)
 
void append_utf8 (uint32_t cpt, std::string &out)
 
std::vector< std::string > split_words (const std::string &text, size_t max_digits=3)
 
const std::vector< std::string > & byte_to_unicode ()
 
std::string byte_encode (const std::string &word)
 
Tokenizer load_gguf (const std::string &path)
 

Function Documentation

◆ append_utf8()

void microtorch::llama3::append_utf8 ( uint32_t  cpt,
std::string &  out 
)
inline

Definition at line 122 of file llama3_tokenizer.h.

122 {
123 if (cpt <= 0x7F) {
124 out.push_back(static_cast<char>(cpt));
125 } else if (cpt <= 0x7FF) {
126 out.push_back(static_cast<char>(0xC0 | ((cpt >> 6) & 0x1F)));
127 out.push_back(static_cast<char>(0x80 | (cpt & 0x3F)));
128 } else if (cpt <= 0xFFFF) {
129 out.push_back(static_cast<char>(0xE0 | ((cpt >> 12) & 0x0F)));
130 out.push_back(static_cast<char>(0x80 | ((cpt >> 6) & 0x3F)));
131 out.push_back(static_cast<char>(0x80 | (cpt & 0x3F)));
132 } else if (cpt <= 0x10FFFF) {
133 out.push_back(static_cast<char>(0xF0 | ((cpt >> 18) & 0x07)));
134 out.push_back(static_cast<char>(0x80 | ((cpt >> 12) & 0x3F)));
135 out.push_back(static_cast<char>(0x80 | ((cpt >> 6) & 0x3F)));
136 out.push_back(static_cast<char>(0x80 | (cpt & 0x3F)));
137 } else {
138 throw std::invalid_argument("invalid codepoint");
139 }
140}
out
node out
Definition graph_ir_schema.h:9
j
size_t j
Definition graph_ir_validate.cpp:222

References j, and out.

Referenced by ember::nfc::nfc_utf8(), and split_words().

◆ byte_encode()

std::string microtorch::llama3::byte_encode ( const std::string &  word)
inline

Definition at line 256 of file llama3_tokenizer.h.

256 {
257 std::string out;
258 for (unsigned char c : word) out += byte_to_unicode()[c];
259 return out;
260}
c
float c
Definition graph_ir_expr.cpp:256
microtorch::llama3::byte_to_unicode
const std::vector< std::string > & byte_to_unicode()
Definition llama3_tokenizer.h:242

References byte_to_unicode(), c, j, and out.

Referenced by microtorch::llama3::Tokenizer::encode().

◆ byte_to_unicode()

const std::vector< std::string > & microtorch::llama3::byte_to_unicode ( )
inline

Definition at line 242 of file llama3_tokenizer.h.

242 {
243 static const std::vector<std::string> m = [] {
244 std::vector<std::string> v(256);
245 std::vector<bool> direct(256, false);
246 for (int c = 0x21; c <= 0x7E; ++c) direct[c] = true;
247 for (int c = 0xA1; c <= 0xAC; ++c) direct[c] = true;
248 for (int c = 0xAE; c <= 0xFF; ++c) direct[c] = true;
249 uint32_t n = 0;
250 for (int c = 0; c < 256; ++c) append_utf8(direct[c] ? static_cast<uint32_t>(c) : 256 + n++, v[c]);
251 return v;
252 }();
253 return m;
254}
n
int n
Definition graph_ir_cuda.cu:326
v
std::vector< float > v
Definition graph_ir_expr.cpp:257

References c, j, n, and v.

Referenced by byte_encode(), and microtorch::llama3::Tokenizer::Tokenizer().

◆ category_bits()

uint16_t microtorch::llama3::category_bits ( uint32_t  cpt)
inline

Definition at line 53 of file llama3_tokenizer.h.

53 {
54 if (cpt >= 0x110000) return 0x0001; // undefined
55 uint32_t lo = 0, hi = tables::kRangeCount - 1; // kRangeStart[lo] <= cpt < kRangeStart[hi]
56 while (hi - lo > 1) {
57 const uint32_t mid = (lo + hi) / 2;
58 (tables::kRangeStart[mid] <= cpt ? lo : hi) = mid;
59 }
60 return tables::kRangeFlags[lo];
61}

References j, microtorch::llama3::tables::kRangeCount, microtorch::llama3::tables::kRangeFlags, and microtorch::llama3::tables::kRangeStart.

Referenced by flags_of().

◆ cpts_from_utf8()

std::vector< uint32_t > microtorch::llama3::cpts_from_utf8 ( const std::string &  s)
inline

Definition at line 75 of file llama3_tokenizer.h.

75 {
76 std::vector<uint32_t> out;
77 out.reserve(s.size());
78 const auto b = [&](size_t i) { return static_cast<unsigned char>(s[i]); };
79 const auto cont = [&](size_t i) { return (b(i) & 0xC0) == 0x80; };
80 size_t i = 0;
81 while (i < s.size()) {
82 const unsigned char c = b(i);
83 if (!(c & 0x80)) {
84 out.push_back(c);
85 i += 1;
86 } else if (!(c & 0x40)) {
87 out.push_back(0xFFFD);
88 i += 1;
89 } else if (!(c & 0x20)) {
90 if (i + 1 >= s.size() || !cont(i + 1)) {
91 out.push_back(0xFFFD);
92 i += 1;
93 } else {
94 out.push_back(((c & 0x1Fu) << 6) | (b(i + 1) & 0x3Fu));
95 i += 2;
96 }
97 } else if (!(c & 0x10)) {
98 if (i + 2 >= s.size() || !cont(i + 1) || !cont(i + 2)) {
99 out.push_back(0xFFFD);
100 i += 1;
101 } else {
102 out.push_back(((c & 0x0Fu) << 12) | ((b(i + 1) & 0x3Fu) << 6) | (b(i + 2) & 0x3Fu));
103 i += 3;
104 }
105 } else if (!(c & 0x08)) {
106 if (i + 3 >= s.size() || !cont(i + 1) || !cont(i + 2) || !cont(i + 3)) {
107 out.push_back(0xFFFD);
108 i += 1;
109 } else {
110 out.push_back(((c & 0x07u) << 18) | ((b(i + 1) & 0x3Fu) << 12) | ((b(i + 2) & 0x3Fu) << 6) |
111 (b(i + 3) & 0x3Fu));
112 i += 4;
113 }
114 } else {
115 out.push_back(0xFFFD);
116 i += 1;
117 }
118 }
119 return out;
120}

References c, j, and out.

Referenced by ember::nfc::nfc_utf8(), and split_words().

◆ flags_of()

CptFlags microtorch::llama3::flags_of ( uint32_t  cpt)
inline

Definition at line 63 of file llama3_tokenizer.h.

63 {
64 const uint16_t c = category_bits(cpt);
65 CptFlags f;
66 f.number = c & 0x0002;
67 f.letter = c & 0x0004;
68 for (uint32_t i = 0; i < tables::kWhitespaceCount; ++i)
69 if (tables::kWhitespace[i] == cpt) f.whitespace = true;
70 f.any = true;
71 return f;
72}
microtorch::llama3::category_bits
uint16_t category_bits(uint32_t cpt)
Definition llama3_tokenizer.h:53

References c, category_bits(), j, microtorch::llama3::tables::kWhitespace, microtorch::llama3::tables::kWhitespaceCount, and microtorch::llama3::CptFlags::number.

Referenced by split_words().

◆ load_gguf()

Tokenizer microtorch::llama3::load_gguf ( const std::string &  path)
inline

Definition at line 422 of file llama3_tokenizer.h.

422 {
423 bpe::GgufTokenizer g = bpe::read_gguf_tokenizer(path);
424 if (g.model != "gpt2" || (g.pre != "llama-bpe" && g.pre != "qwen2"))
425 throw std::runtime_error(path + ": tokenizer is '" + g.model + "'/'" + g.pre +
426 "', not 'gpt2'/'llama-bpe' or 'gpt2'/'qwen2'");
427 return Tokenizer(std::move(g.tokens), g.merges, g.token_type, g.pre == "qwen2" ? 1 : 3);
428}
microtorch::bpe::GgufTokenizer::merges
std::vector< std::string > merges
Definition bpe_tokenizer.h:156
microtorch::bpe::GgufTokenizer::tokens
std::vector< std::string > tokens
Definition bpe_tokenizer.h:156
microtorch::bpe::GgufTokenizer::token_type
std::vector< int32_t > token_type
Definition bpe_tokenizer.h:158

References j, microtorch::bpe::GgufTokenizer::merges, microtorch::bpe::GgufTokenizer::model, microtorch::bpe::GgufTokenizer::pre, microtorch::bpe::read_gguf_tokenizer(), microtorch::bpe::GgufTokenizer::token_type, and microtorch::bpe::GgufTokenizer::tokens.

◆ split_words()

std::vector< std::string > microtorch::llama3::split_words ( const std::string &  text,
size_t  max_digits = 3 
)
inline

Definition at line 145 of file llama3_tokenizer.h.

145 {
146 const std::vector<uint32_t> cp = cpts_from_utf8(text);
147 const size_t n = cp.size();
148 constexpr uint32_t kOut = 0xFFFFFFFF;
149 const auto get = [&](size_t p) { return p < n ? cp[p] : kOut; };
150 const auto fl = [&](size_t p) { return p < n ? flags_of(cp[p]) : CptFlags{}; };
151 // Only ASCII capitals lowercase to s, t, r, e, v, m, l or d in llama.cpp's
152 // table (checked by tools/gen_llama3_tables.py's source), so ASCII suffices.
153 const auto lower = [](uint32_t c) { return c >= 'A' && c <= 'Z' ? c + 32 : c; };
154 std::vector<std::string> words;
155 size_t prev = 0;
156 const auto add = [&](size_t end) {
157 const size_t len = end - prev;
158 if (len > 0) {
159 words.emplace_back();
160 for (size_t i = prev; i < end; ++i) append_utf8(cp[i], words.back());
161 }
162 prev = end;
163 return len;
164 };
165 for (size_t pos = 0; pos < n;) {
166 const uint32_t c = get(pos);
167 const CptFlags f = fl(pos);
168 // (?i:'s|'t|'re|'ve|'m|'ll|'d)
169 if (c == '\'' && pos + 1 < n) {
170 const uint32_t c1 = lower(get(pos + 1));
171 if (c1 == 's' || c1 == 't' || c1 == 'm' || c1 == 'd') {
172 pos += add(pos + 2);
173 continue;
174 }
175 if (pos + 2 < n) {
176 const uint32_t c2 = lower(get(pos + 2));
177 if ((c1 == 'r' && c2 == 'e') || (c1 == 'v' && c2 == 'e') || (c1 == 'l' && c2 == 'l')) {
178 pos += add(pos + 3);
179 continue;
180 }
181 }
182 }
183 // [^\r\n\p{L}\p{N}]?\p{L}+
184 if (!(c == '\r' || c == '\n' || f.number)) {
185 if (f.letter || fl(pos + 1).letter) {
186 ++pos;
187 while (fl(pos).letter) ++pos;
188 add(pos);
189 continue;
190 }
191 }
192 // \p{N}{1,3} (\p{N} for qwen2)
193 if (f.number) {
194 size_t ini = pos;
195 while (fl(pos).number) {
196 if (++pos - ini >= max_digits) {
197 add(pos);
198 ini = pos;
199 }
200 }
201 add(pos);
202 continue;
203 }
204 // <space>?[^\s\p{L}\p{N}]+[\r\n]*
205 CptFlags f2 = c == ' ' ? fl(pos + 1) : f;
206 if (!(f2.whitespace || f2.letter || f2.number) && f.any) {
207 pos += c == ' ' ? 1 : 0;
208 while (!(f2.whitespace || f2.letter || f2.number) && f2.any) f2 = fl(++pos);
209 uint32_t c2 = get(pos);
210 while (c2 == '\r' || c2 == '\n') c2 = get(++pos);
211 add(pos);
212 continue;
213 }
214 size_t nws = 0, last_rn = 0;
215 while (fl(pos + nws).whitespace) {
216 const uint32_t c2 = get(pos + nws);
217 if (c2 == '\r' || c2 == '\n') last_rn = pos + nws + 1;
218 ++nws;
219 }
220 if (last_rn > 0) { // \s*[\r\n]+
221 pos = last_rn;
222 add(pos);
223 continue;
224 }
225 if (nws > 1 && get(pos + nws) != kOut) { // \s+(?!\S)
226 pos += nws - 1;
227 add(pos);
228 continue;
229 }
230 if (nws > 0) { // \s+
231 pos += nws;
232 add(pos);
233 continue;
234 }
235 add(++pos); // no match
236 }
237 return words;
238}
p
const float * p[kIrDepthSourcesMax]
Definition graph_ir_cuda.cu:118
text
std::string text
Definition graph_ir_expr.cpp:37
pos
int pos
Definition graph_ir_recurrence.cpp:139
add
node add
Definition graph_ir_schema.h:9
number
double number
Definition ir_digest.cpp:64
ember_ir::expr::lower
std::vector< Stage > lower(const Expr &e)
Definition graph_ir_expr.cpp:509
microtorch::chat::get
Template get(const std::string &id)
Definition chat_template.h:341
microtorch::llama3::cpts_from_utf8
std::vector< uint32_t > cpts_from_utf8(const std::string &s)
Definition llama3_tokenizer.h:75
microtorch::llama3::append_utf8
void append_utf8(uint32_t cpt, std::string &out)
Definition llama3_tokenizer.h:122

References add, append_utf8(), c, cpts_from_utf8(), flags_of(), j, n, number, p, pos, and text.

Referenced by microtorch::llama3::Tokenizer::encode().