weight_management.h File Reference

#include "gguf_structs.h"
#include "model.h"

Include dependency graph for weight_management.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Functions
bool	dequant_legacy_to_f32 (const GGUFTensorInfo &info, const uint8_t *src, std::vector< float > &out)
bool	dequant_span_to_f32 (GGMLType t, const uint8_t *src, size_t first, size_t n, float *out)

Function Documentation

dequant_legacy_to_f32()

bool dequant_legacy_to_f32	(	const GGUFTensorInfo &	info,
		const uint8_t *	src,
		std::vector< float > &	out
	)

Definition at line 767 of file weight_management.cpp.

                                                                                                  {
  const GGMLType t = info.type;
  const size_t n = info.num_elements;
  if (t == GGMLType::GGML_TYPE_F16 || t == GGMLType::GGML_TYPE_BF16) {
    out.resize(n);
    dequant_span_to_f32(t, src, 0, n, out.data());
    return true;
  }
  const bool q4_0 = t == GGMLType::GGML_TYPE_Q4_0, q4_1 = t == GGMLType::GGML_TYPE_Q4_1;
  const bool q5_0 = t == GGMLType::GGML_TYPE_Q5_0, q5_1 = t == GGMLType::GGML_TYPE_Q5_1;
  const bool k_quant = t == GGMLType::GGML_TYPE_Q4_K || t == GGMLType::GGML_TYPE_Q5_K || t == GGMLType::GGML_TYPE_Q6_K;
  if (!(q4_0 || q4_1 || q5_0 || q5_1 || k_quant)) return false;
  const size_t width = k_quant ? GGML_QK_K : 32;
  if (n % width != 0)
    throw std::runtime_error(std::string(ggml_type_name(t)) + " tensor with " + std::to_string(n) +
                             " elements (not a multiple of " + std::to_string(width) + ")");
  const size_t bsz = ggml_type_size(t), nb = n / width;
  if (info.size_in_bytes < nb * bsz) throw std::runtime_error("legacy-quantised tensor is shorter than its element count implies");
  out.resize(n);
  dequant_span_to_f32(t, src, 0, n, out.data());
  return true;
}

References dequant_span_to_f32(), GGML_QK_K, GGML_TYPE_BF16, GGML_TYPE_F16, ggml_type_name(), GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, ggml_type_size(), j, n, GGUFTensorInfo::num_elements, out, GGUFTensorInfo::size_in_bytes, and GGUFTensorInfo::type.

Referenced by ember_ir::Interpreter::bind_tensor().

dequant_span_to_f32()

bool dequant_span_to_f32	(	GGMLType	t,
		const uint8_t *	src,
		size_t	first,
		size_t	n,
		float *	out
	)

Definition at line 802 of file weight_management.cpp.

                                                                                             {
  if (t == GGMLType::GGML_TYPE_F16) {
    const float* table = fp16_table();
    const uint8_t* p = src + 2 * first;
    for (size_t i = 0; i < n; ++i) {
      uint16_t h;
      std::memcpy(&h, p + 2 * i, 2);
      out[i] = table[h];
    }
    return true;
  }
  if (t == GGMLType::GGML_TYPE_BF16) {  // the top 16 bits of an f32: exact
    for (size_t i = 0; i < n; ++i) {
      uint16_t h;
      std::memcpy(&h, src + 2 * (first + i), 2);
      const uint32_t bits = static_cast<uint32_t>(h) << 16;
      std::memcpy(out + i, &bits, 4);
    }
    return true;
  }
  if (t == GGMLType::GGML_TYPE_Q8_0) {
    const block_q8_0* blocks = reinterpret_cast<const block_q8_0*>(src) + first / GGML_QK8_0;
    for (size_t b = 0; b < n / GGML_QK8_0; ++b) dequantize_q8_0_block(blocks + b, out + b * GGML_QK8_0);
    return true;
  }
  // K-quants: 256-wide super-blocks (first and n multiples of GGML_QK_K), ember's block dequantisers.
  if (t == GGMLType::GGML_TYPE_Q4_K) {
    const block_q4_K* blocks = reinterpret_cast<const block_q4_K*>(src) + first / GGML_QK_K;
    for (size_t b = 0; b < n / GGML_QK_K; ++b) dequantize_q4_k_m(blocks + b, out + b * GGML_QK_K, GGML_QK_K);
    return true;
  }
  if (t == GGMLType::GGML_TYPE_Q5_K) {
    const block_q5_K* blocks = reinterpret_cast<const block_q5_K*>(src) + first / GGML_QK_K;
    for (size_t b = 0; b < n / GGML_QK_K; ++b) dequantize_q5_k(blocks + b, out + b * GGML_QK_K);
    return true;
  }
  if (t == GGMLType::GGML_TYPE_Q6_K) {
    const block_q6_K* blocks = reinterpret_cast<const block_q6_K*>(src) + first / GGML_QK_K;
    for (size_t b = 0; b < n / GGML_QK_K; ++b) dequantize_q6_k(blocks + b, out + b * GGML_QK_K, GGML_QK_K);
    return true;
  }
  const bool q4_0 = t == GGMLType::GGML_TYPE_Q4_0, q4_1 = t == GGMLType::GGML_TYPE_Q4_1;
  const bool q5_0 = t == GGMLType::GGML_TYPE_Q5_0, q5_1 = t == GGMLType::GGML_TYPE_Q5_1;
  if (!(q4_0 || q4_1 || q5_0 || q5_1)) return false;
  const bool has_min = q4_1 || q5_1, five_bit = q5_0 || q5_1;
  const size_t bsz = ggml_type_size(t), b0 = first / 32, nb = n / 32;
  for (size_t b = 0; b < nb; ++b) {
    const uint8_t* p = src + (b0 + b) * bsz;
    uint16_t dh, mh = 0;
    std::memcpy(&dh, p, 2);
    p += 2;
    if (has_min) { std::memcpy(&mh, p, 2); p += 2; }
    uint32_t qh = 0;
    if (five_bit) { std::memcpy(&qh, p, 4); p += 4; }
    const float d = fp16_to_fp32(dh, false), m = has_min ? fp16_to_fp32(mh, false) : 0.0f;
    const int offset = has_min ? 0 : (five_bit ? 16 : 8);
    float* y = out + b * 32;
    for (int j = 0; j < 16; ++j) {
      int lo = p[j] & 0x0F, hi = p[j] >> 4;
      if (five_bit) {
        lo |= ((qh >> j) & 1) << 4;
        hi |= ((qh >> (j + 16)) & 1) << 4;
      }
      // ggml's dequantize_row_q4_0 / q5_0 write the product alone, q4_1 / q5_1 the product plus m (paperkiln
      // spec/IR_SPEC.md 4.7): adding a +0 m to a -0 product would give +0, so the 0-offset forms add nothing.
      if (has_min) {
        y[j] = static_cast<float>(lo - offset) * d + m;
        y[j + 16] = static_cast<float>(hi - offset) * d + m;
      } else {
        y[j] = static_cast<float>(lo - offset) * d;
        y[j + 16] = static_cast<float>(hi - offset) * d;
      }
    }
  }
  return true;
}

References dequantize_q4_k_m(), dequantize_q5_k(), dequantize_q6_k(), dequantize_q8_0_block(), fp16_table(), fp16_to_fp32(), GGML_QK8_0, GGML_QK_K, GGML_TYPE_BF16, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_Q8_0, ggml_type_size(), j, n, out, and p.

Referenced by ember_ir::Interpreter::bind_tensor(), dequant_legacy_to_f32(), and ember_ir::Interpreter::run_nodes().