tinyllama Namespace Reference

Classes
class	TinyLlamaSession
	Represents an active TinyLlama session holding the loaded model and tokenizer. More...

Functions
static void	log_vector_summary_detailed (const std::string &name, const std::vector< float > &v, int current_pos, int current_layer, int N)
static std::string	read_file_api (const std::string &path)
static int	argmax (const std::vector< float > &v)
static int	sample_top_k_top_p_temperature (const std::vector< float > &logits, float temperature, int top_k, float top_p, std::mt19937 &rng)

Function Documentation

argmax()

static int tinyllama::argmax ( const std::vector< float > &  v )

static

Definition at line 73 of file api.cpp.

                                             {
  if (v.empty()) {
    Logger::error("Cannot perform argmax on empty vector");
    return -1;
  }
 
  return std::distance(v.begin(), std::max_element(v.begin(), v.end()));
}

References Logger::error().

log_vector_summary_detailed()

static void tinyllama::log_vector_summary_detailed ( const std::string &  name, const std::vector< float > &  v, int  current_pos, int  current_layer, int  N  )

static

Definition at line 32 of file api.cpp.

                                               {
  if (v.empty()) {
    Logger::info(name + " (pos=" + std::to_string(current_pos) + ", layer=" +
                 std::to_string(current_layer) + "): EMPTY VECTOR");
    return;
  }
  std::stringstream ss;
  ss << name << " (pos=" << std::to_string(current_pos)
     << ", layer=" << std::to_string(current_layer) << "): size=" << v.size();
  ss << ", first " << N << ": [";
  for (int i = 0; i < N && i < v.size(); ++i) {
    ss << std::fixed << std::setprecision(4) << v[i]
       << (i == N - 1 || i == v.size() - 1 ? "" : ", ");
  }
  ss << "]";
  float min_val = v[0], max_val = v[0], sum = 0.0f;
  bool all_finite = true;
  for (float val : v) {
    if (val < min_val) min_val = val;
    if (val > max_val) max_val = val;
    sum += val;
    if (!std::isfinite(val)) all_finite = false;
  }
  ss << ", min=" << std::fixed << std::setprecision(4) << min_val;
  ss << ", max=" << std::fixed << std::setprecision(4) << max_val;
  ss << ", mean=" << std::fixed << std::setprecision(4) << (sum / v.size());
  ss << ", finite=" << (all_finite ? "yes" : "no");
  Logger::info(ss.str());
}

References Logger::info().

Referenced by tinyllama::TinyLlamaSession::generate().

read_file_api()

static std::string tinyllama::read_file_api ( const std::string &  path )

static

Definition at line 65 of file api.cpp.

                                                      {
  std::filesystem::path fs_path(path);
  std::ifstream file(fs_path, std::ios::binary);
  if (!file) throw std::runtime_error("Failed to open file: " + path);
  return std::string((std::istreambuf_iterator<char>(file)),
                     std::istreambuf_iterator<char>());
}

sample_top_k_top_p_temperature()

static int tinyllama::sample_top_k_top_p_temperature ( const std::vector< float > &  logits, float  temperature, int  top_k, float  top_p, std::mt19937 &  rng  )

static

Definition at line 82 of file api.cpp.

                                                                        {
  if (logits.empty()) {
    throw std::runtime_error("Cannot sample from empty logits.");
  }
 
  // If temperature is very low, fall back to greedy sampling
  if (temperature < 0.05f) {
    return std::distance(logits.begin(), std::max_element(logits.begin(), logits.end()));
  }
 
  int vocab_size = logits.size();
 
  top_k = (std::min)(top_k, vocab_size);
  if (top_k <= 0) top_k = vocab_size;
 
  std::vector<float> scaled_logits(vocab_size);
  float max_logit = -std::numeric_limits<float>::infinity();
  for (float logit : logits) max_logit = (std::max)(max_logit, logit);
 
  // Scale logits to avoid numerical instability
  const float scale = 1.0f / temperature;
  for (int i = 0; i < vocab_size; ++i) {
    scaled_logits[i] = (logits[i] - max_logit) * scale;
  }
 
  std::vector<double> probs_double(vocab_size);
  double sum_exp = 0.0;
  for (int i = 0; i < vocab_size; ++i) {
    probs_double[i] = std::exp(static_cast<double>(scaled_logits[i]));
    sum_exp += probs_double[i];
  }
 
  // Normalize probabilities
  if (sum_exp > 0.0) {
    for (int i = 0; i < vocab_size; ++i) {
      probs_double[i] /= sum_exp;
    }
  } else {
    // If all probabilities are zero, fall back to uniform distribution
    for (int i = 0; i < vocab_size; ++i) {
      probs_double[i] = 1.0 / vocab_size;
    }
  }
 
  std::vector<std::pair<float, int>> prob_idx(vocab_size);
  for (int i = 0; i < vocab_size; ++i) {
    prob_idx[i] = {static_cast<float>(probs_double[i]), i};
  }
 
  std::sort(prob_idx.begin(), prob_idx.end(),
            std::greater<std::pair<float, int>>());
 
  if (top_k < vocab_size) {
    prob_idx.resize(top_k);
  }
 
  float cumulative_prob = 0.0f;
  int last_idx = 0;
  for (int i = 0; i < prob_idx.size(); ++i) {
    cumulative_prob += prob_idx[i].first;
    last_idx = i;
    if (cumulative_prob >= top_p) {
      break;
    }
  }
  prob_idx.resize(last_idx + 1);
 
  float final_sum = 0.0f;
  for (const auto& pi : prob_idx) {
    final_sum += pi.first;
  }
 
  // Renormalize probabilities after top-k and top-p filtering
  std::vector<float> final_probs(prob_idx.size());
  if (final_sum > 0.0f) {
    for (size_t i = 0; i < prob_idx.size(); ++i) {
      final_probs[i] = prob_idx[i].first / final_sum;
    }
  } else {
    // If all probabilities are zero after filtering, use uniform distribution
    float uniform_prob = 1.0f / prob_idx.size();
    std::fill(final_probs.begin(), final_probs.end(), uniform_prob);
  }
 
  std::discrete_distribution<int> dist(final_probs.begin(), final_probs.end());
  int sampled_idx_in_filtered = dist(rng);
 
  return prob_idx[sampled_idx_in_filtered].second;
}

Referenced by tinyllama::TinyLlamaSession::generate(), and tinyllama::TinyLlamaSession::generate_batch().