graph_ir_cuda.cu File Reference

#include "graph_ir_cuda.h"
#include <stdexcept>
#include <string>

Go to the source code of this file.

Namespaces
namespace	ember_ir

Functions
void	ember_ir::ir_rows_gather_cuda (const float *src, const int *idx, float *dst, int n, int w, cudaStream_t stream)
void	ember_ir::ir_rows_scatter_cuda (const float *src, const int *idx, float *dst, int n, int w, cudaStream_t stream)
void	ember_ir::ir_mor_gate_cuda (const float *x_in, float *x, const float *g, int n, int w, cudaStream_t stream)
void	ember_ir::ir_rope_at_cuda (float *q, float *k, int T, int hq, int hk, int hd, const int *pos, const float *freqs, cudaStream_t stream)
void	ember_ir::ir_rope_rows_at_cuda (float *x, int T, int heads, int hd, int n_rot, bool neox, const int *pos, const float *freqs, cudaStream_t stream)
void	ember_ir::ir_turn_state_attention_cuda (const float *q, const float *K, const float *V, float *out, int T, int row0, const int *codes, int hq, int hkv, int hd, float scale, cudaStream_t stream, const int *drow)
void	ember_ir::ir_kv_compact_cuda (float *K, float *V, const int *idx, int w0, int n, int row, cudaStream_t stream)
void	ember_ir::ir_embedding_state_cuda (const float *table, const float *state, int vocab, const int *tokens, float *out, int T, int d, cudaStream_t stream)
void	ember_ir::ir_window_attention_cuda (const float *q, const float *K, const float *V, float *out, int T, int start_pos, int hq, int hkv, int hd, float scale, int window, int sinks, cudaStream_t stream, const int *dpos, float softcap)
void	ember_ir::ir_kv_write_cuda (float *K, float *V, const float *k, const float *v, int T, int row, int start_pos, cudaStream_t stream, const int *dpos)
void	ember_ir::ir_relu_cuda (const float *x, float *out, size_t n, cudaStream_t stream)
void	ember_ir::ir_sigmoid_cuda (const float *x, float *out, size_t n, cudaStream_t stream)
void	ember_ir::ir_affine_cuda (const float *x, float *out, size_t n, float scale, float shift, cudaStream_t stream)
void	ember_ir::ir_mul_col_cuda (const float *x, const float *c, float *out, int T, int w, cudaStream_t stream)
void	ember_ir::ir_scale_cuda (const float *x, const float *g, const float *b, float *out, int T, int w, cudaStream_t stream)
void	ember_ir::ir_rms_row_cuda (const float *x, float *out, int T, int w, float eps, cudaStream_t stream)
void	ember_ir::ir_depth_attend_cuda (const float *const *sources, int S, const float *w, float *out, int T, int d, float eps, cudaStream_t stream)
void	ember_ir::ir_linear_attention_cuda (const float *q, const float *k, const float *v, float *num, float *den, float *out, int T, int width, float den_floor, cudaStream_t stream)
void	ember_ir::ir_ssm_scan_cuda (const float *u, const float *A, const float *B, const float *C, const float *D, float *s, float *y, int T, int n, cudaStream_t stream)
void	ember_ir::ir_embedding_cuda (const float *table, const int *tokens, float *out, int T, int d, cudaStream_t stream)
void	ember_ir::ir_pos_embedding_cuda (const float *x, const float *P, float *out, int T, int d, int start_pos, cudaStream_t stream, const int *dpos)
void	ember_ir::ir_layernorm_cuda (const float *x, const float *w, const float *b, float *out, int T, int d, float eps, cudaStream_t stream)
void	ember_ir::ir_bias_add_cuda (float *y, const float *b, int T, int O, cudaStream_t stream)
void	ember_ir::ir_binary_cuda (const float *a, const float *b, float *out, size_t n, int op, cudaStream_t stream)
void	ember_ir::ir_silu_cuda (const float *x, float *out, size_t n, cudaStream_t stream)
void	ember_ir::ir_gelu_cuda (const float *x, float *out, size_t n, bool tanh_form, cudaStream_t stream)
void	ember_ir::ir_rope_cuda (float *q, float *k, int T, int hq, int hk, int hd, int start_pos, const float *freqs, cudaStream_t stream, const int *dpos)
void	ember_ir::ir_rope_rows_cuda (float *x, int T, int heads, int hd, int n_rot, bool neox, int start_pos, const float *freqs, cudaStream_t stream, const int *dpos)
void	ember_ir::ir_conv1d_causal_cuda (const float *x, const float *w, const float *bias, float *state, float *out, int T, int C, int K, cudaStream_t stream)
void	ember_ir::ir_selective_scan_cuda (const float *x, const float *dt, const float *B, const float *C, const float *A, const float *d, float *s, float *y, int T, int D, int N, bool softplus, cudaStream_t stream)
size_t	ember_ir::ir_expr_scratch_floats (const std::vector< expr::Stage > &stages, int T, int w)
void	ember_ir::ir_expr_cuda (const std::vector< expr::Stage > &stages, const float *const *ins, int n_ins, float *scratch, float *out, int T, int w, cudaStream_t stream)
void	ember_ir::ir_mla_kv_write_cuda (float *KV, const float *c, const float *kr, int T, int r, int dr, int start_pos, cudaStream_t stream, const int *dpos)
void	ember_ir::ir_mla_attention_cuda (const float *qn, const float *qr, const float *KV, const float *Kb, const float *Vb, float *out, int T, int start_pos, int H, int r, int dn, int dr, int dv, float scale, int window, int sinks, cudaStream_t stream, const int *dpos)

Variable Documentation

code

expr::Instr code[expr::kMaxCode]

Definition at line 327 of file graph_ir_cuda.cu.

Referenced by ember_ir::ir_expr_cuda().

col

const float* col[expr::kMaxColumns]

Definition at line 331 of file graph_ir_cuda.cu.

Referenced by ember_ir::Interpreter::run_nodes(), and ember_ir::cpu::split_groups().

in

node in

Definition at line 330 of file graph_ir_cuda.cu.

Referenced by ember_ir::build_ir_from_gguf(), ember::nfc::nfc(), ember_ir::Interpreter::run_nodes(), and ember_ir::rec::Recurrence::step().

n

int n

Definition at line 326 of file graph_ir_cuda.cu.

Referenced by add_residual_cuda(), add_residual_kernel(), add_vectors_cuda(), add_vectors_kernel(), ember_ir::cpu::affine(), ember_ir::Interpreter::AppIndex::AppIndex(), ember_ir::rec::Program::assign(), attention_bias_batch_cpu(), ember_ir::Interpreter::attention_window(), ember_ir::Interpreter::bind(), Tokenizer::bpe_tokenize_to_ids(), ember_ir::build_ir_from_gguf(), microtorch::llama3::byte_to_unicode(), ember_ir::cpu::depth_attend(), dequant_legacy_to_f32(), ember_kvq::dequant_rows_cuda(), dequant_span_to_f32(), TinyLlamaModel::dequantize_all_if_small(), dequantize_q8_k(), ember_ir::device_supported(), dot_f32_as_f64(), ember::SpmTokenizer::encode_fragment(), microtorch::llama3::Tokenizer::encode_word(), ember_ir::es_number_string(), ember_ir::rec::Program::eval(), tinyllama::TinyLlamaSession::generate(), ember_ir::has_state_tokens(), ember_ir::has_windowed_attention(), ember_ir::Interpreter::init(), ember::insert_state_blocks(), ember_ir::ir_affine_cuda(), ember_ir::ir_bias_add_cuda(), ember_ir::ir_binary_cuda(), ember_ir::ir_expr_cuda(), ember_ir::ir_gelu_cuda(), ember_ir::ir_kv_compact_cuda(), ember_ir::ir_kv_write_cuda(), ember_ir::ir_mla_kv_write_cuda(), ember_ir::ir_mor_gate_cuda(), ember_ir::ir_mul_col_cuda(), ember_ir::ir_relu_cuda(), ember_ir::ir_rows_gather_cuda(), ember_ir::ir_rows_scatter_cuda(), ember_ir::ir_scale_cuda(), ember_ir::ir_sigmoid_cuda(), ember_ir::ir_silu_cuda(), ember_ir::ir_ssm_scan_cuda(), ember_draft::DraftTable::load(), moe_ffn_cuda(), ember_ir::cpu::mor_gate(), ember::nfc::nfc_utf8(), ember::parallel_for(), ember::parse_score_body(), microtorch::bpe::Tokenizer::pretokenize(), tinyllama::prompt_lookup_draft(), microtorch::bpe::read_gguf_tokenizer(), ember_ir::Interpreter::rec_slot(), ember_ir::cpu::relu(), rmsnorm_apply_kernel(), rmsnorm_optimized_det_kernel(), rmsnorm_optimized_kernel(), rmsnorm_sum_squares_kernel(), rmsnorm_vector_cpu(), rmsnorm_vector_cuda(), rmsnorm_vector_cuda(), rmsnorm_vector_cuda_optimized(), ember_kvq::roundtrip_rows(), ember_ir::Interpreter::run_nodes(), ember_ir::Interpreter::save_state(), ember::score_banned(), tinyllama::TinyLlamaSession::score_rows(), ember_ir::cpu::selective_scan_row(), microtorch::chat::detail::sha256_hex(), ember_ir::cpu::sigmoid(), silu_cuda(), silu_kernel(), simd_dot_product(), simd_scaled_add(), softmax_exp_sum_kernel(), softmax_find_max_kernel(), softmax_normalize_kernel(), softmax_optimized_kernel(), softmax_vector_cpu(), softmax_vector_cuda(), softmax_vector_cuda_optimized(), microtorch::llama3::split_words(), ember_ir::ssm_scan_row_cpu(), microtorch::chat::strip_py_ws(), ember_ir::cpu::sub(), swiglu_cuda(), swiglu_kernel(), ember_tiled::tiled_grid_chunks(), ember_tiled::tiled_split_block(), ember_ir::Interpreter::truncate(), ember_ir::Interpreter::ts_begin(), ember::utf8_part(), ember::utf8_settled(), ember::utf8_valid(), vec_dot_q4_k_q8_k_cpu(), and vec_dot_q6_k_q8_k_cpu().

p

const float* p[kIrDepthSourcesMax]

Definition at line 118 of file graph_ir_cuda.cu.

Referenced by ember_ir::cpu::affine(), tinyllama::apply_contract_penalty(), attention_bias_batch_cpu(), attention_relkv_batch_cpu(), bpe_build_prompt(), bpe_run(), ember::build_general_prompt(), ember_ir::build_ir_from_gguf(), ember_ir::rec::Recurrence::compile(), ember_ir::cpu::depth_attend(), dequant_span_to_f32(), ember_draft::DraftTable::draft(), ember::SpmTokenizer::encode_fragment(), ember_ir::Interpreter::held_positions(), ember_ir::rec::Recurrence::init(), TinyLlamaModel::initialize_gpu_and_rope(), ember_ir::ir_expr_cuda(), ember_ir::ir_schema_errors(), ember_ir::Interpreter::kv_positions(), ember_ir::Interpreter::layer_name(), ember_draft::DraftTable::load(), SafeTensorsLoader::load_from_directory(), ember_ir::cpu::mla_attention(), ember_ir::cpu::mor_gate(), ember::nfc::nfc(), microtorch::bpe::read_gguf_tokenizer(), ember::reply_of(), ember::score_state_ids_error(), tinyllama::TinyLlamaSession::set_repeat_penalty(), tinyllama::TinyLlamaSession::set_repetition_penalty(), microtorch::llama3::split_words(), ember_ir::rec::Recurrence::step(), ember_tiled::tiled_merge_block(), ember_ir::Interpreter::ts_evict_below(), ember_ir::Interpreter::ts_seg_after(), ember_ir::Interpreter::ts_segment_start(), and TinyLlamaModel::~TinyLlamaModel().