ember.cpp 0.1.0
C++ inference engine for Llama-family and paperkiln models (CPU, CUDA, WebAssembly)
Loading...
Searching...
No Matches
Macros | Functions
weight_management.cpp File Reference
#include "weight_management.h"
#include <algorithm>
#include <mutex>
#include "ggml_types.h"
#include "logger.h"
#include "quantization.h"
#include "utils.h"
Include dependency graph for weight_management.cpp:

Go to the source code of this file.

Macros

#define EMBER_RECOVERABLE(w)
 
#define LEGACY_OR_THROW(target_f32)
 
#define FAST_COPY_WEIGHT(target_vec, block_type)
 

Functions

bool dequant_legacy_to_f32 (const GGUFTensorInfo &info, const uint8_t *src, std::vector< float > &out)
 
static const float * fp16_table ()
 
bool dequant_span_to_f32 (GGMLType t, const uint8_t *src, size_t first, size_t n, float *out)
 
void map_gguf_weights (const GGUFData &gguf, TinyLlamaModel &model)
 

Macro Definition Documentation

◆ EMBER_RECOVERABLE

#define EMBER_RECOVERABLE (   w)
Value:
(!layer.w##_q4k.empty() || !layer.w##_q5k.empty() || !layer.w##_q6k.empty() || !layer.w##_q8k.empty() || \
!layer.w##_q8_0.empty() || !layer.w.empty())
j
size_t j
Definition graph_ir_validate.cpp:222

◆ FAST_COPY_WEIGHT

#define FAST_COPY_WEIGHT (   target_vec,
  block_type 
)
Value:
target_vec.resize(info.size_in_bytes / sizeof(block_type)); \
std::memcpy(target_vec.data(), tensor_data_ptr, info.size_in_bytes);

◆ LEGACY_OR_THROW

#define LEGACY_OR_THROW (   target_f32)
Value:
default: \
if (!dequant_legacy_to_f32(info, tensor_data_ptr, target_f32)) \
throw std::runtime_error(name + ": unsupported tensor type " + ggml_type_name(info.type)); \
break;
name
const char * name
Definition graph_ir_expr.cpp:20
ggml_type_name
const char * ggml_type_name(GGMLType type)
Gets the string name of a GGML type.
Definition quantization.cpp:634
dequant_legacy_to_f32
bool dequant_legacy_to_f32(const GGUFTensorInfo &info, const uint8_t *src, std::vector< float > &out)
Definition weight_management.cpp:767

Function Documentation

◆ dequant_legacy_to_f32()

bool dequant_legacy_to_f32 ( const GGUFTensorInfo &  info,
const uint8_t *  src,
std::vector< float > &  out 
)

Definition at line 767 of file weight_management.cpp.

767 {
768 const GGMLType t = info.type;
769 const size_t n = info.num_elements;
770 if (t == GGMLType::GGML_TYPE_F16 || t == GGMLType::GGML_TYPE_BF16) {
771 out.resize(n);
772 dequant_span_to_f32(t, src, 0, n, out.data());
773 return true;
774 }
775 const bool q4_0 = t == GGMLType::GGML_TYPE_Q4_0, q4_1 = t == GGMLType::GGML_TYPE_Q4_1;
776 const bool q5_0 = t == GGMLType::GGML_TYPE_Q5_0, q5_1 = t == GGMLType::GGML_TYPE_Q5_1;
777 const bool k_quant = t == GGMLType::GGML_TYPE_Q4_K || t == GGMLType::GGML_TYPE_Q5_K || t == GGMLType::GGML_TYPE_Q6_K;
778 if (!(q4_0 || q4_1 || q5_0 || q5_1 || k_quant)) return false;
779 const size_t width = k_quant ? GGML_QK_K : 32;
780 if (n % width != 0)
781 throw std::runtime_error(std::string(ggml_type_name(t)) + " tensor with " + std::to_string(n) +
782 " elements (not a multiple of " + std::to_string(width) + ")");
783 const size_t bsz = ggml_type_size(t), nb = n / width;
784 if (info.size_in_bytes < nb * bsz) throw std::runtime_error("legacy-quantised tensor is shorter than its element count implies");
785 out.resize(n);
786 dequant_span_to_f32(t, src, 0, n, out.data());
787 return true;
788}
GGMLType
GGMLType
Enumeration of GGML tensor data types.
Definition ggml_types.h:21
GGML_TYPE_BF16
@ GGML_TYPE_BF16
Definition ggml_types.h:40
GGML_TYPE_Q5_0
@ GGML_TYPE_Q5_0
Definition ggml_types.h:27
GGML_TYPE_F16
@ GGML_TYPE_F16
Definition ggml_types.h:23
GGML_TYPE_Q6_K
@ GGML_TYPE_Q6_K
Definition ggml_types.h:35
GGML_TYPE_Q4_1
@ GGML_TYPE_Q4_1
Definition ggml_types.h:25
GGML_TYPE_Q5_K
@ GGML_TYPE_Q5_K
Definition ggml_types.h:34
GGML_TYPE_Q5_1
@ GGML_TYPE_Q5_1
Definition ggml_types.h:28
GGML_TYPE_Q4_K
@ GGML_TYPE_Q4_K
Definition ggml_types.h:33
GGML_TYPE_Q4_0
@ GGML_TYPE_Q4_0
Definition ggml_types.h:24
GGML_QK_K
constexpr size_t GGML_QK_K
Block size constants for different quantization formats.
Definition gguf_parser.h:42
n
int n
Definition graph_ir_cuda.cu:326
out
node out
Definition graph_ir_schema.h:9
ggml_type_size
size_t ggml_type_size(GGMLType type)
Gets the size in bytes of a GGML type.
Definition quantization.cpp:679
GGUFTensorInfo::size_in_bytes
size_t size_in_bytes
Definition gguf_structs.h:69
GGUFTensorInfo::num_elements
size_t num_elements
Definition gguf_structs.h:68
dequant_span_to_f32
bool dequant_span_to_f32(GGMLType t, const uint8_t *src, size_t first, size_t n, float *out)
Definition weight_management.cpp:802

References dequant_span_to_f32(), GGML_QK_K, GGML_TYPE_BF16, GGML_TYPE_F16, ggml_type_name(), GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, ggml_type_size(), j, n, GGUFTensorInfo::num_elements, out, GGUFTensorInfo::size_in_bytes, and GGUFTensorInfo::type.

Referenced by ember_ir::Interpreter::bind_tensor().

◆ dequant_span_to_f32()

bool dequant_span_to_f32 ( GGMLType  t,
const uint8_t *  src,
size_t  first,
size_t  n,
float *  out 
)

Definition at line 802 of file weight_management.cpp.

802 {
803 if (t == GGMLType::GGML_TYPE_F16) {
804 const float* table = fp16_table();
805 const uint8_t* p = src + 2 * first;
806 for (size_t i = 0; i < n; ++i) {
807 uint16_t h;
808 std::memcpy(&h, p + 2 * i, 2);
809 out[i] = table[h];
810 }
811 return true;
812 }
813 if (t == GGMLType::GGML_TYPE_BF16) { // the top 16 bits of an f32: exact
814 for (size_t i = 0; i < n; ++i) {
815 uint16_t h;
816 std::memcpy(&h, src + 2 * (first + i), 2);
817 const uint32_t bits = static_cast<uint32_t>(h) << 16;
818 std::memcpy(out + i, &bits, 4);
819 }
820 return true;
821 }
822 if (t == GGMLType::GGML_TYPE_Q8_0) {
823 const block_q8_0* blocks = reinterpret_cast<const block_q8_0*>(src) + first / GGML_QK8_0;
824 for (size_t b = 0; b < n / GGML_QK8_0; ++b) dequantize_q8_0_block(blocks + b, out + b * GGML_QK8_0);
825 return true;
826 }
827 // K-quants: 256-wide super-blocks (first and n multiples of GGML_QK_K), ember's block dequantisers.
828 if (t == GGMLType::GGML_TYPE_Q4_K) {
829 const block_q4_K* blocks = reinterpret_cast<const block_q4_K*>(src) + first / GGML_QK_K;
830 for (size_t b = 0; b < n / GGML_QK_K; ++b) dequantize_q4_k_m(blocks + b, out + b * GGML_QK_K, GGML_QK_K);
831 return true;
832 }
833 if (t == GGMLType::GGML_TYPE_Q5_K) {
834 const block_q5_K* blocks = reinterpret_cast<const block_q5_K*>(src) + first / GGML_QK_K;
835 for (size_t b = 0; b < n / GGML_QK_K; ++b) dequantize_q5_k(blocks + b, out + b * GGML_QK_K);
836 return true;
837 }
838 if (t == GGMLType::GGML_TYPE_Q6_K) {
839 const block_q6_K* blocks = reinterpret_cast<const block_q6_K*>(src) + first / GGML_QK_K;
840 for (size_t b = 0; b < n / GGML_QK_K; ++b) dequantize_q6_k(blocks + b, out + b * GGML_QK_K, GGML_QK_K);
841 return true;
842 }
843 const bool q4_0 = t == GGMLType::GGML_TYPE_Q4_0, q4_1 = t == GGMLType::GGML_TYPE_Q4_1;
844 const bool q5_0 = t == GGMLType::GGML_TYPE_Q5_0, q5_1 = t == GGMLType::GGML_TYPE_Q5_1;
845 if (!(q4_0 || q4_1 || q5_0 || q5_1)) return false;
846 const bool has_min = q4_1 || q5_1, five_bit = q5_0 || q5_1;
847 const size_t bsz = ggml_type_size(t), b0 = first / 32, nb = n / 32;
848 for (size_t b = 0; b < nb; ++b) {
849 const uint8_t* p = src + (b0 + b) * bsz;
850 uint16_t dh, mh = 0;
851 std::memcpy(&dh, p, 2);
852 p += 2;
853 if (has_min) { std::memcpy(&mh, p, 2); p += 2; }
854 uint32_t qh = 0;
855 if (five_bit) { std::memcpy(&qh, p, 4); p += 4; }
856 const float d = fp16_to_fp32(dh, false), m = has_min ? fp16_to_fp32(mh, false) : 0.0f;
857 const int offset = has_min ? 0 : (five_bit ? 16 : 8);
858 float* y = out + b * 32;
859 for (int j = 0; j < 16; ++j) {
860 int lo = p[j] & 0x0F, hi = p[j] >> 4;
861 if (five_bit) {
862 lo |= ((qh >> j) & 1) << 4;
863 hi |= ((qh >> (j + 16)) & 1) << 4;
864 }
865 // ggml's dequantize_row_q4_0 / q5_0 write the product alone, q4_1 / q5_1 the product plus m (paperkiln
866 // spec/IR_SPEC.md 4.7): adding a +0 m to a -0 product would give +0, so the 0-offset forms add nothing.
867 if (has_min) {
868 y[j] = static_cast<float>(lo - offset) * d + m;
869 y[j + 16] = static_cast<float>(hi - offset) * d + m;
870 } else {
871 y[j] = static_cast<float>(lo - offset) * d;
872 y[j + 16] = static_cast<float>(hi - offset) * d;
873 }
874 }
875 }
876 return true;
877}
GGML_TYPE_Q8_0
@ GGML_TYPE_Q8_0
Definition ggml_types.h:29
GGML_QK8_0
constexpr size_t GGML_QK8_0
Definition gguf_parser.h:43
p
const float * p[kIrDepthSourcesMax]
Definition graph_ir_cuda.cu:118
dequantize_q4_k_m
void dequantize_q4_k_m(const block_q4_K *qblock, float *output, int num_weights_in_block, bool log_this_block)
Definition quantization.cpp:171
dequantize_q5_k
void dequantize_q5_k(const block_q5_K *qblock, float *output)
Definition quantization.cpp:226
fp16_to_fp32
float fp16_to_fp32(uint16_t h, bool is_gguf_scale_field)
Converts a 16-bit floating point number to 32-bit float.
Definition quantization.cpp:47
dequantize_q8_0_block
void dequantize_q8_0_block(const block_q8_0 *qblock, float *output)
Dequantizes a Q8_0 block to float32.
Definition quantization.cpp:1092
dequantize_q6_k
void dequantize_q6_k(const block_q6_K *qblock, float *output, int num_weights_in_block, bool log_this_block)
Definition quantization.cpp:256
block_q4_K
4-bit K-quantized block structure
Definition quantization.h:58
block_q5_K
5-bit K-quantized block structure (ggml's block_q5_K)
Definition quantization.h:73
block_q6_K
6-bit K-quantized block structure
Definition quantization.h:88
block_q8_0
Simple 8-bit quantized block structure.
Definition quantization.h:137
fp16_table
static const float * fp16_table()
Definition weight_management.cpp:793

References dequantize_q4_k_m(), dequantize_q5_k(), dequantize_q6_k(), dequantize_q8_0_block(), fp16_table(), fp16_to_fp32(), GGML_QK8_0, GGML_QK_K, GGML_TYPE_BF16, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_Q8_0, ggml_type_size(), j, n, out, and p.

Referenced by ember_ir::Interpreter::bind_tensor(), dequant_legacy_to_f32(), and ember_ir::Interpreter::run_nodes().

◆ fp16_table()

static const float * fp16_table ( )
static

Definition at line 793 of file weight_management.cpp.

793 {
794 static const std::vector<float> table = [] {
795 std::vector<float> t(65536);
796 for (uint32_t h = 0; h < 65536; ++h) t[h] = fp16_to_fp32(static_cast<uint16_t>(h), false);
797 return t;
798 }();
799 return table.data();
800}

References j.

Referenced by dequant_span_to_f32().

◆ map_gguf_weights()

void map_gguf_weights ( const GGUFData &  gguf,
TinyLlamaModel &  model 
)

Definition at line 879 of file weight_management.cpp.

879 {
880 Logger::info("Mapping GGUF weights to model fields (ULTRA-OPTIMIZED VERSION)...");
881
882 const uint8_t* actual_data_block_start = nullptr;
883
884 // Determine which data source to use
885 if (gguf.mapped_tensor_data != nullptr && gguf.mapped_tensor_data_size > 0) {
886 const uint8_t* mmap_buffer_start = static_cast<const uint8_t*>(gguf.mapped_tensor_data);
887 actual_data_block_start = mmap_buffer_start + gguf.offset_diff_for_mmap;
888 Logger::info("map_gguf_weights: Using mmap mode (ZERO-COPY). Size: " +
889 std::to_string(gguf.mapped_tensor_data_size) + " bytes.");
890 } else if (!gguf.tensor_data.empty()) {
891 actual_data_block_start = gguf.tensor_data.data();
892 Logger::info("map_gguf_weights: Using non-mmap mode. Size: " +
893 std::to_string(gguf.tensor_data.size()) + " bytes.");
894 } else {
895 Logger::error("GGUF tensor data is not available. Cannot map weights.");
896 return;
897 }
898
899 const size_t num_tensors = gguf.tensor_infos_map.size();
900 Logger::info("Processing " + std::to_string(num_tensors) + " tensors with ultra-optimized parallel mapping...");
901
902 // Pre-allocate containers to avoid reallocations during parallel processing
903 std::vector<std::pair<std::string, GGUFTensorInfo>> tensor_pairs;
904 tensor_pairs.reserve(num_tensors);
905 for (const auto& pair : gguf.tensor_infos_map) {
906 tensor_pairs.emplace_back(pair.first, pair.second);
907 }
908
909 // Reserve capacity for major model containers to reduce allocations
910 const size_t typical_blocks = 4096;
911 if (model.lm_head_q8_0.capacity() == 0) model.lm_head_q8_0.reserve(32768);
912 if (model.embed_tokens_q8_0.capacity() == 0) model.embed_tokens_q8_0.reserve(32768);
913
914 for (auto& layer : model.layers) {
915 if (layer.q_proj_q8_0.capacity() == 0) layer.q_proj_q8_0.reserve(typical_blocks);
916 if (layer.k_proj_q8_0.capacity() == 0) layer.k_proj_q8_0.reserve(typical_blocks);
917 if (layer.v_proj_q8_0.capacity() == 0) layer.v_proj_q8_0.reserve(typical_blocks);
918 if (layer.o_proj_q8_0.capacity() == 0) layer.o_proj_q8_0.reserve(typical_blocks);
919 if (layer.gate_proj_q8_0.capacity() == 0) layer.gate_proj_q8_0.reserve(typical_blocks);
920 if (layer.up_proj_q8_0.capacity() == 0) layer.up_proj_q8_0.reserve(typical_blocks);
921 if (layer.down_proj_q8_0.capacity() == 0) layer.down_proj_q8_0.reserve(typical_blocks);
922 }
923
924 // BLAZING FAST: Sort tensors by type and process in bulk
925 std::vector<size_t> global_tensor_indices;
926 std::vector<std::vector<size_t>> layer_tensor_indices(model.layers.size());
927
928 global_tensor_indices.reserve(10); // output.weight, token_embd.weight, output_norm.weight, etc.
929 for (auto& layer_indices : layer_tensor_indices) {
930 layer_indices.reserve(9); // 7 weights + 2 norms per layer
931 }
932
933 // ULTRA-FAST categorization without string operations
934 for (size_t i = 0; i < tensor_pairs.size(); ++i) {
935 const std::string& name = tensor_pairs[i].first;
936 if (name[0] == 'o' || name[0] == 't') { // output.weight, token_embd.weight, output_norm.weight
937 global_tensor_indices.push_back(i);
938 } else if (name.size() > 4 && name[0] == 'b' && name[1] == 'l' && name[2] == 'k' && name[3] == '.') {
939 // Extract layer index without substr - MUCH faster
940 size_t layer_start = 4;
941 size_t layer_end = name.find('.', layer_start);
942 if (layer_end != std::string::npos) {
943 int layer_idx = 0;
944 for (size_t pos = layer_start; pos < layer_end; ++pos) {
945 layer_idx = layer_idx * 10 + (name[pos] - '0');
946 }
947 if (layer_idx >= 0 && static_cast<size_t>(layer_idx) < model.layers.size()) {
948 layer_tensor_indices[layer_idx].push_back(i);
949 }
950 }
951 }
952 }
953
954 std::atomic<int> processed_count{0};
955 std::atomic<int> error_count{0};
956 std::mutex first_error_mu;
957 std::string first_error; // reported if any tensor fails to map
958
959 // Process global tensors sequentially (small count, avoid overhead)
960 for (size_t idx : global_tensor_indices) {
961 try {
962 const std::string& target_field_key = tensor_pairs[idx].first;
963 const GGUFTensorInfo& info = tensor_pairs[idx].second;
964 const uint8_t* tensor_data_ptr = actual_data_block_start + info.offset;
965
966 // Global tensors with optimized type dispatch
967 if (target_field_key == "output.weight") {
968 switch (info.type) {
969 case GGMLType::GGML_TYPE_Q6_K: {
970 size_t num_blocks = info.size_in_bytes / sizeof(block_q6_K);
971 model.lm_head_q6k.resize(num_blocks);
972 std::memcpy(model.lm_head_q6k.data(), tensor_data_ptr, info.size_in_bytes);
973 break;
974 }
975 case GGMLType::GGML_TYPE_Q4_K: {
976 size_t num_blocks = info.size_in_bytes / sizeof(block_q4_K);
977 model.lm_head_q4k.resize(num_blocks);
978 std::memcpy(model.lm_head_q4k.data(), tensor_data_ptr, info.size_in_bytes);
979 break;
980 }
981 case GGMLType::GGML_TYPE_Q5_K: {
982 model.lm_head_q5k.resize(info.size_in_bytes / sizeof(block_q5_K));
983 std::memcpy(model.lm_head_q5k.data(), tensor_data_ptr, info.size_in_bytes);
984 break;
985 }
986 case GGMLType::GGML_TYPE_Q8_0: {
987 size_t num_blocks = info.size_in_bytes / sizeof(block_q8_0);
988 model.lm_head_q8_0.resize(num_blocks);
989 std::memcpy(model.lm_head_q8_0.data(), tensor_data_ptr, info.size_in_bytes);
990 break;
991 }
992 case GGMLType::GGML_TYPE_Q8_K: {
993 size_t num_blocks = info.size_in_bytes / sizeof(block_q8_K);
994 model.lm_head_q8k.resize(num_blocks);
995 std::memcpy(model.lm_head_q8k.data(), tensor_data_ptr, info.size_in_bytes);
996 break;
997 }
998 case GGMLType::GGML_TYPE_F32: {
999 size_t num_elements = info.size_in_bytes / sizeof(float);
1000 model.lm_head_f32.resize(num_elements);
1001 std::memcpy(model.lm_head_f32.data(), tensor_data_ptr, info.size_in_bytes);
1002 break;
1003 }
1004 default:
1005 if (!dequant_legacy_to_f32(info, tensor_data_ptr, model.lm_head_f32)) throw std::runtime_error(target_field_key + ": unsupported tensor type " + ggml_type_name(info.type));
1006 break;
1007 }
1008 processed_count++;
1009 continue;
1010 }
1011
1012 if (target_field_key == "token_embd.weight") {
1013 switch (info.type) {
1014 case GGMLType::GGML_TYPE_Q4_K: {
1015 size_t num_blocks = info.size_in_bytes / sizeof(block_q4_K);
1016 model.embed_tokens_q4k.resize(num_blocks);
1017 std::memcpy(model.embed_tokens_q4k.data(), tensor_data_ptr, info.size_in_bytes);
1018 break;
1019 }
1020 case GGMLType::GGML_TYPE_Q5_K: {
1021 model.embed_tokens_q5k.resize(info.size_in_bytes / sizeof(block_q5_K));
1022 std::memcpy(model.embed_tokens_q5k.data(), tensor_data_ptr, info.size_in_bytes);
1023 break;
1024 }
1025 case GGMLType::GGML_TYPE_Q8_0: {
1026 size_t num_blocks = info.size_in_bytes / sizeof(block_q8_0);
1027 model.embed_tokens_q8_0.resize(num_blocks);
1028 std::memcpy(model.embed_tokens_q8_0.data(), tensor_data_ptr, info.size_in_bytes);
1029 break;
1030 }
1031 case GGMLType::GGML_TYPE_Q8_K: {
1032 size_t num_blocks = info.size_in_bytes / sizeof(block_q8_K);
1033 model.embed_tokens_q8k.resize(num_blocks);
1034 std::memcpy(model.embed_tokens_q8k.data(), tensor_data_ptr, info.size_in_bytes);
1035 break;
1036 }
1037 case GGMLType::GGML_TYPE_Q6_K: {
1038 size_t num_blocks = info.size_in_bytes / sizeof(block_q6_K);
1039 model.embed_tokens_q6k.resize(num_blocks);
1040 std::memcpy(model.embed_tokens_q6k.data(), tensor_data_ptr, info.size_in_bytes);
1041 break;
1042 }
1043 case GGMLType::GGML_TYPE_F32: {
1044 size_t num_elements = info.size_in_bytes / sizeof(float);
1045 model.embed_tokens_f32.resize(num_elements);
1046 std::memcpy(model.embed_tokens_f32.data(), tensor_data_ptr, info.size_in_bytes);
1047 break;
1048 }
1049 default:
1050 if (!dequant_legacy_to_f32(info, tensor_data_ptr, model.embed_tokens_f32)) throw std::runtime_error(target_field_key + ": unsupported tensor type " + ggml_type_name(info.type));
1051 break;
1052 }
1053 processed_count++;
1054 continue;
1055 }
1056
1057 if (target_field_key == "output_norm.weight") {
1058 if (info.type == GGMLType::GGML_TYPE_F32) {
1059 size_t num_elements = info.size_in_bytes / sizeof(float);
1060 model.final_norm_f32.resize(num_elements);
1061 std::memcpy(model.final_norm_f32.data(), tensor_data_ptr, info.size_in_bytes);
1062 } else if (!dequant_legacy_to_f32(info, tensor_data_ptr, model.final_norm_f32)) throw std::runtime_error(target_field_key + ": unsupported tensor type " + ggml_type_name(info.type));
1063 processed_count++;
1064 continue;
1065 }
1066
1067 } catch (const std::exception& e) {
1068 error_count++;
1069 std::lock_guard<std::mutex> lk(first_error_mu);
1070 if (first_error.empty()) first_error = e.what();
1071 }
1072 }
1073
1074 // ULTRA-FAST: Process layers in parallel with pre-sorted tensors
1075 #pragma omp parallel for schedule(static) if(model.layers.size() > 4)
1076 for (int layer_idx = 0; layer_idx < (int)layer_tensor_indices.size(); ++layer_idx) {
1077 const auto& layer_indices = layer_tensor_indices[layer_idx];
1078 if (layer_indices.empty()) continue;
1079
1080 LayerWeights& layer = model.layers[layer_idx];
1081
1082 try {
1083 for (size_t idx : layer_indices) {
1084 const std::string& name = tensor_pairs[idx].first;
1085 const GGUFTensorInfo& info = tensor_pairs[idx].second;
1086 const uint8_t* tensor_data_ptr = actual_data_block_start + info.offset;
1087
1088 // BLAZING FAST: Direct character matching (faster than hashing)
1089 const size_t last_dot = name.find_last_of('.');
1090 if (last_dot == std::string::npos) continue;
1091
1092 const char* field = name.c_str() + name.find('.', 4) + 1;
1093
1094 // BLAZING FAST: Direct character-based dispatch without string operations
1095 #define LEGACY_OR_THROW(target_f32) \
1096 default: \
1097 if (!dequant_legacy_to_f32(info, tensor_data_ptr, target_f32)) \
1098 throw std::runtime_error(name + ": unsupported tensor type " + ggml_type_name(info.type)); \
1099 break;
1100 #define FAST_COPY_WEIGHT(target_vec, block_type) \
1101 target_vec.resize(info.size_in_bytes / sizeof(block_type)); \
1102 std::memcpy(target_vec.data(), tensor_data_ptr, info.size_in_bytes);
1103
1104 // IMPROVED: Pattern matching based on tensor name structure
1105 const char* name_cstr = name.c_str();
1106 const size_t name_len = name.length();
1107
1108 if (name_len > 10 && name.find("attn_") != std::string::npos) {
1109 if (name.find("attn_q.weight") != std::string::npos) {
1110 switch (info.type) {
1111 case GGMLType::GGML_TYPE_Q8_0: FAST_COPY_WEIGHT(layer.q_proj_q8_0, block_q8_0); break;
1112 case GGMLType::GGML_TYPE_Q4_K: FAST_COPY_WEIGHT(layer.q_proj_q4k, block_q4_K); break;
1113 case GGMLType::GGML_TYPE_Q5_K: FAST_COPY_WEIGHT(layer.q_proj_q5k, block_q5_K); break;
1114 case GGMLType::GGML_TYPE_Q6_K: FAST_COPY_WEIGHT(layer.q_proj_q6k, block_q6_K); break;
1115 case GGMLType::GGML_TYPE_Q8_K: FAST_COPY_WEIGHT(layer.q_proj_q8k, block_q8_K); break;
1116 case GGMLType::GGML_TYPE_BF16: FAST_COPY_WEIGHT(layer.q_proj, uint16_t); break;
1117 case GGMLType::GGML_TYPE_F32: FAST_COPY_WEIGHT(layer.q_proj_f32, float);
1118 Logger::info("[MAP_DEBUG] " + name + " F32 copied, elems=" +
1119 std::to_string(layer.q_proj_f32.size()) +
1120 " into layer_idx=" + std::to_string(layer_idx));
1121 break; // transformer_cpp fp32 exports
1122 LEGACY_OR_THROW(layer.q_proj_f32)
1123 }
1124 } else if (name.find("attn_k.weight") != std::string::npos) {
1125 switch (info.type) {
1126 case GGMLType::GGML_TYPE_Q8_0: FAST_COPY_WEIGHT(layer.k_proj_q8_0, block_q8_0); break;
1127 case GGMLType::GGML_TYPE_Q4_K: FAST_COPY_WEIGHT(layer.k_proj_q4k, block_q4_K); break;
1128 case GGMLType::GGML_TYPE_Q5_K: FAST_COPY_WEIGHT(layer.k_proj_q5k, block_q5_K); break;
1129 case GGMLType::GGML_TYPE_Q6_K: FAST_COPY_WEIGHT(layer.k_proj_q6k, block_q6_K); break;
1130 case GGMLType::GGML_TYPE_Q8_K: FAST_COPY_WEIGHT(layer.k_proj_q8k, block_q8_K); break;
1131 case GGMLType::GGML_TYPE_BF16: FAST_COPY_WEIGHT(layer.k_proj, uint16_t); break;
1132 case GGMLType::GGML_TYPE_F32: FAST_COPY_WEIGHT(layer.k_proj_f32, float); break; // transformer_cpp fp32 exports
1133 LEGACY_OR_THROW(layer.k_proj_f32)
1134 }
1135 } else if (name.find("attn_v.weight") != std::string::npos) {
1136 switch (info.type) {
1137 case GGMLType::GGML_TYPE_Q8_0: FAST_COPY_WEIGHT(layer.v_proj_q8_0, block_q8_0); break;
1138 case GGMLType::GGML_TYPE_Q4_K: FAST_COPY_WEIGHT(layer.v_proj_q4k, block_q4_K); break;
1139 case GGMLType::GGML_TYPE_Q5_K: FAST_COPY_WEIGHT(layer.v_proj_q5k, block_q5_K); break;
1140 case GGMLType::GGML_TYPE_Q6_K: FAST_COPY_WEIGHT(layer.v_proj_q6k, block_q6_K); break;
1141 case GGMLType::GGML_TYPE_Q8_K: FAST_COPY_WEIGHT(layer.v_proj_q8k, block_q8_K); break;
1142 case GGMLType::GGML_TYPE_BF16: FAST_COPY_WEIGHT(layer.v_proj, uint16_t); break;
1143 case GGMLType::GGML_TYPE_F32: FAST_COPY_WEIGHT(layer.v_proj_f32, float); break; // transformer_cpp fp32 exports
1144 LEGACY_OR_THROW(layer.v_proj_f32)
1145 }
1146 } else if (name.find("attn_output.weight") != std::string::npos) {
1147 switch (info.type) {
1148 case GGMLType::GGML_TYPE_Q8_0: FAST_COPY_WEIGHT(layer.o_proj_q8_0, block_q8_0); break;
1149 case GGMLType::GGML_TYPE_Q4_K: FAST_COPY_WEIGHT(layer.o_proj_q4k, block_q4_K); break;
1150 case GGMLType::GGML_TYPE_Q5_K: FAST_COPY_WEIGHT(layer.o_proj_q5k, block_q5_K); break;
1151 case GGMLType::GGML_TYPE_Q6_K: FAST_COPY_WEIGHT(layer.o_proj_q6k, block_q6_K); break;
1152 case GGMLType::GGML_TYPE_Q8_K: FAST_COPY_WEIGHT(layer.o_proj_q8k, block_q8_K); break;
1153 case GGMLType::GGML_TYPE_BF16: FAST_COPY_WEIGHT(layer.o_proj, uint16_t); break;
1154 case GGMLType::GGML_TYPE_F32: FAST_COPY_WEIGHT(layer.o_proj_f32, float); break; // transformer_cpp fp32 exports
1155 LEGACY_OR_THROW(layer.o_proj_f32)
1156 }
1157 } else if (name.find("attn_norm.weight") != std::string::npos) {
1158 if (info.type == GGMLType::GGML_TYPE_F32) { FAST_COPY_WEIGHT(layer.input_layernorm_f32, float); }
1159 else if (!dequant_legacy_to_f32(info, tensor_data_ptr, layer.input_layernorm_f32))
1160 throw std::runtime_error(name + ": unsupported tensor type " + ggml_type_name(info.type));
1161 }
1162 } else if (name_len > 10 && name.find("ffn_") != std::string::npos) {
1163 if (name.find("ffn_gate_inp.weight") != std::string::npos) { // MoE router
1164 if (info.type == GGMLType::GGML_TYPE_F32) { FAST_COPY_WEIGHT(layer.moe_router_f32, float); }
1165 else if (info.type == GGMLType::GGML_TYPE_Q8_0) { FAST_COPY_WEIGHT(layer.moe_router_q8_0, block_q8_0); }
1166 else if (!dequant_legacy_to_f32(info, tensor_data_ptr, layer.moe_router_f32))
1167 throw std::runtime_error(name + ": unsupported tensor type " + ggml_type_name(info.type));
1168 } else if (name.find("_exps.weight") != std::string::npos) { // MoE expert stacks
1169 std::vector<float>& dst = name.find("ffn_gate_exps") != std::string::npos ? layer.moe_gate_exps_f32
1170 : name.find("ffn_up_exps") != std::string::npos ? layer.moe_up_exps_f32
1171 : layer.moe_down_exps_f32;
1172 if (info.type == GGMLType::GGML_TYPE_F32) { FAST_COPY_WEIGHT(dst, float); }
1173 else if (!dequant_legacy_to_f32(info, tensor_data_ptr, dst))
1174 throw std::runtime_error(name + ": MoE expert stacks must be F32, F16 or Q4_0/Q4_1/Q5_0/Q5_1 (got " +
1175 std::string(ggml_type_name(info.type)) + ")");
1176 } else if (name.find("ffn_gate.weight") != std::string::npos) {
1177 switch (info.type) {
1178 case GGMLType::GGML_TYPE_Q8_0: FAST_COPY_WEIGHT(layer.gate_proj_q8_0, block_q8_0); break;
1179 case GGMLType::GGML_TYPE_Q4_K: FAST_COPY_WEIGHT(layer.gate_proj_q4k, block_q4_K); break;
1180 case GGMLType::GGML_TYPE_Q5_K: FAST_COPY_WEIGHT(layer.gate_proj_q5k, block_q5_K); break;
1181 case GGMLType::GGML_TYPE_Q6_K: FAST_COPY_WEIGHT(layer.gate_proj_q6k, block_q6_K); break;
1182 case GGMLType::GGML_TYPE_Q8_K: FAST_COPY_WEIGHT(layer.gate_proj_q8k, block_q8_K); break;
1183 case GGMLType::GGML_TYPE_BF16: FAST_COPY_WEIGHT(layer.gate_proj, uint16_t); break;
1184 case GGMLType::GGML_TYPE_F32: FAST_COPY_WEIGHT(layer.gate_proj_f32, float); break; // transformer_cpp fp32 exports
1185 LEGACY_OR_THROW(layer.gate_proj_f32)
1186 }
1187 } else if (name.find("ffn_up.weight") != std::string::npos) {
1188 switch (info.type) {
1189 case GGMLType::GGML_TYPE_Q8_0: FAST_COPY_WEIGHT(layer.up_proj_q8_0, block_q8_0); break;
1190 case GGMLType::GGML_TYPE_Q4_K: FAST_COPY_WEIGHT(layer.up_proj_q4k, block_q4_K); break;
1191 case GGMLType::GGML_TYPE_Q5_K: FAST_COPY_WEIGHT(layer.up_proj_q5k, block_q5_K); break;
1192 case GGMLType::GGML_TYPE_Q6_K: FAST_COPY_WEIGHT(layer.up_proj_q6k, block_q6_K); break;
1193 case GGMLType::GGML_TYPE_Q8_K: FAST_COPY_WEIGHT(layer.up_proj_q8k, block_q8_K); break;
1194 case GGMLType::GGML_TYPE_BF16: FAST_COPY_WEIGHT(layer.up_proj, uint16_t); break;
1195 case GGMLType::GGML_TYPE_F32: FAST_COPY_WEIGHT(layer.up_proj_f32, float); break; // transformer_cpp fp32 exports
1196 LEGACY_OR_THROW(layer.up_proj_f32)
1197 }
1198 } else if (name.find("ffn_down.weight") != std::string::npos) {
1199 switch (info.type) {
1200 case GGMLType::GGML_TYPE_Q8_0: FAST_COPY_WEIGHT(layer.down_proj_q8_0, block_q8_0); break;
1201 case GGMLType::GGML_TYPE_Q4_K: FAST_COPY_WEIGHT(layer.down_proj_q4k, block_q4_K); break;
1202 case GGMLType::GGML_TYPE_Q5_K: FAST_COPY_WEIGHT(layer.down_proj_q5k, block_q5_K); break;
1203 case GGMLType::GGML_TYPE_Q6_K: FAST_COPY_WEIGHT(layer.down_proj_q6k, block_q6_K); break;
1204 case GGMLType::GGML_TYPE_Q8_K: FAST_COPY_WEIGHT(layer.down_proj_q8k, block_q8_K); break;
1205 case GGMLType::GGML_TYPE_BF16: FAST_COPY_WEIGHT(layer.down_proj, uint16_t); break;
1206 case GGMLType::GGML_TYPE_F32: FAST_COPY_WEIGHT(layer.down_proj_f32, float); break; // transformer_cpp fp32 exports
1207 LEGACY_OR_THROW(layer.down_proj_f32)
1208 }
1209 } else if (name.find("ffn_norm.weight") != std::string::npos) {
1210 if (info.type == GGMLType::GGML_TYPE_F32) { FAST_COPY_WEIGHT(layer.post_attention_layernorm_f32, float); }
1211 else if (!dequant_legacy_to_f32(info, tensor_data_ptr, layer.post_attention_layernorm_f32))
1212 throw std::runtime_error(name + ": unsupported tensor type " + ggml_type_name(info.type));
1213 }
1214 }
1215
1216 #undef FAST_COPY_WEIGHT
1217 #undef LEGACY_OR_THROW
1218 }
1219 processed_count++;
1220 } catch (const std::exception& e) {
1221 error_count++;
1222 std::lock_guard<std::mutex> lk(first_error_mu);
1223 if (first_error.empty()) first_error = e.what();
1224 }
1225 }
1226
1227 Logger::info("Finished mapping GGUF weights: " + std::to_string(processed_count.load()) + "/" +
1228 std::to_string(num_tensors) + " tensors processed successfully (errors: " +
1229 std::to_string(error_count.load()) + ") with ultra-optimized parallel mapping");
1230 // A tensor that failed to map would leave the model silently incomplete.
1231 if (error_count.load() > 0)
1232 throw std::runtime_error("GGUF weight mapping failed for " + std::to_string(error_count.load()) +
1233 " layer(s); first error: " + first_error);
1234}
Logger::info
static void info(const std::string &message)
Definition logger.cpp:135
Logger::error
static void error(const std::string &message)
Definition logger.cpp:143
TinyLlamaModel::embed_tokens_q6k
std::vector< block_q6_K > embed_tokens_q6k
Definition model.h:584
TinyLlamaModel::final_norm_f32
std::vector< float > final_norm_f32
Definition model.h:581
TinyLlamaModel::lm_head_q4k
std::vector< block_q4_K > lm_head_q4k
Definition model.h:582
TinyLlamaModel::lm_head_q6k
std::vector< block_q6_K > lm_head_q6k
Definition model.h:584
TinyLlamaModel::lm_head_q5k
std::vector< block_q5_K > lm_head_q5k
Definition model.h:583
TinyLlamaModel::layers
std::vector< LayerWeights > layers
Definition model.h:587
TinyLlamaModel::embed_tokens_q8_0
std::vector< block_q8_0 > embed_tokens_q8_0
Definition model.h:585
TinyLlamaModel::embed_tokens_q4k
std::vector< block_q4_K > embed_tokens_q4k
Definition model.h:582
TinyLlamaModel::lm_head_q8_0
std::vector< block_q8_0 > lm_head_q8_0
Definition model.h:585
TinyLlamaModel::embed_tokens_q5k
std::vector< block_q5_K > embed_tokens_q5k
Definition model.h:583
TinyLlamaModel::embed_tokens_q8k
std::vector< block_q8_K > embed_tokens_q8k
Definition model.h:586
TinyLlamaModel::embed_tokens_f32
std::vector< float > embed_tokens_f32
Definition model.h:581
TinyLlamaModel::lm_head_f32
std::vector< float > lm_head_f32
Definition model.h:581
TinyLlamaModel::lm_head_q8k
std::vector< block_q8_K > lm_head_q8k
Definition model.h:586
GGML_TYPE_F32
@ GGML_TYPE_F32
Definition ggml_types.h:22
GGML_TYPE_Q8_K
@ GGML_TYPE_Q8_K
Definition ggml_types.h:36
pos
int pos
Definition graph_ir_recurrence.cpp:139
GGUFTensorInfo
Information about a tensor stored in a GGUF file.
Definition gguf_structs.h:63
GGUFTensorInfo::offset
uint64_t offset
Definition gguf_structs.h:67
LayerWeights
Structure holding all weights for a single transformer layer.
Definition model.h:253
block_q8_K
8-bit K-quantized block structure with block sums
Definition quantization.h:128
FAST_COPY_WEIGHT
#define FAST_COPY_WEIGHT(target_vec, block_type)
LEGACY_OR_THROW
#define LEGACY_OR_THROW(target_f32)