kv_quant.cu File Reference

#include "kv_quant.h"
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <stdexcept>
#include <string>
#include "decode_attn_tiled.cuh"

Go to the source code of this file.

Namespaces
namespace	ember_kvq

Functions
void	ember_kvq::write_cuda (int f, void *Kd, void *Ks, void *Vd, void *Vs, const float *k, const float *v, int T, int row, int start_pos, cudaStream_t st, const int *dpos)
size_t	ember_kvq::partial_floats (int hq, int hd, int max_len)
void	ember_kvq::decode_cuda (int f, const float *q, const void *Kd, const void *Ks, const void *Vd, const void *Vs, float *out, int hq, int hkv, int hd, float scale, int max_len, int seq_len, float *part, cudaStream_t st, const int *dpos)
void	ember_kvq::dequant_rows_cuda (int f, const void *Kd, const void *Ks, const void *Vd, const void *Vs, float *K, float *V, int n, int row, cudaStream_t st)