main.cpp File Reference

Demonstrates various chunking strategies and operations. More...

#include "advanced_structures.hpp"
#include "chunk.hpp"
#include "chunk_compression.hpp"
#include "chunk_strategies.hpp"
#include "chunk_windows.hpp"
#include "config.hpp"
#include "data_structures.hpp"
#include "parallel_chunk.hpp"
#include "sub_chunk_strategies.hpp"
#include "utils.hpp"
#include <iomanip>
#include <iostream>
#include <memory>
#include <numeric>
#include <vector>

Include dependency graph for main.cpp:

Go to the source code of this file.

Functions
void	demonstrate_adaptive_conditional_subchunking ()
	Demonstrates adaptive conditional sub-chunking.
void	demonstrate_complex_recursive_subchunking ()
	Demonstrates complex recursive sub-chunking with multiple levels.
void	demonstrate_multi_strategy_subchunking ()
	Demonstrates hierarchical sub-chunking using multiple strategies.
int	main (int argc, char *argv[])
	Main function demonstrating various chunking strategies.
template<typename T >
void	print_chunks (const std::vector< std::vector< T > > &chunks)
	Helper function to print chunks.
template<typename T >
void	print_sub_chunks (const std::vector< std::vector< std::vector< T > > > &sub_chunks, const std::string &label, int precision=2)
	Helper function to print sub-chunks with detailed formatting.

Detailed Description

Demonstrates various chunking strategies and operations.

This file contains examples of how to use the chunking library to process data in different ways, including integer, float, and string chunking.

Definition in file main.cpp.

Function Documentation

demonstrate_adaptive_conditional_subchunking()

void demonstrate_adaptive_conditional_subchunking

(

)

Demonstrates adaptive conditional sub-chunking.

This function shows how to use conditional sub-chunking with adaptive thresholds based on chunk properties.

Definition at line 132 of file main.cpp.

                                                    {
    std::cout << "\n=== Adaptive Conditional Sub-chunking ===" << std::endl;
 
    // Example data - now using 1D vector
    std::vector<double> data = {1.0, 1.1, 1.2, 5.0, 5.1, 5.2, 2.0, 2.1, 2.2, 10.0, 10.1, 10.2};
 
    auto variance_strategy = std::make_shared<chunk_processing::VarianceStrategy<double>>(5.0);
    auto condition = [](const std::vector<double>& chunk) {
        return chunk.size() > 5; // Only sub-chunk large chunks
    };
 
    chunk_processing::ConditionalSubChunkStrategy<double> conditional_chunker(variance_strategy,
                                                                              condition, 2);
    auto conditional_result = conditional_chunker.apply(data);
 
    print_chunks(conditional_result);
}

References chunk_processing::ConditionalSubChunkStrategy< T >::apply(), and print_chunks().

Referenced by main().

demonstrate_complex_recursive_subchunking()

void demonstrate_complex_recursive_subchunking

(

)

Demonstrates complex recursive sub-chunking with multiple levels.

This function shows how to apply recursive sub-chunking strategies to data with clear patterns, using variance-based chunking at multiple levels.

Definition at line 91 of file main.cpp.

                                                 {
    std::cout << "\n=== Complex Recursive Sub-chunking ===" << std::endl;
 
    // Example data - now using 1D vector
    std::vector<double> data = {1.0, 1.1, 1.2, 5.0, 5.1, 5.2, 2.0, 2.1, 2.2, 10.0, 10.1, 10.2};
 
    auto variance_strategy = std::make_shared<chunk_processing::VarianceStrategy<double>>(3.0);
    chunk_processing::RecursiveSubChunkStrategy<double> recursive_chunker(variance_strategy, 3, 2);
    auto recursive_result = recursive_chunker.apply(data);
 
    print_chunks(recursive_result);
}

References print_chunks().

Referenced by main().

demonstrate_multi_strategy_subchunking()

void demonstrate_multi_strategy_subchunking

(

)

Demonstrates hierarchical sub-chunking using multiple strategies.

This function shows how to apply different chunking strategies in a hierarchical manner, combining variance, similarity, and entropy-based approaches.

Definition at line 110 of file main.cpp.

                                              {
    std::cout << "\n=== Multi-Strategy Sub-chunking ===" << std::endl;
 
    // Example data - now using 1D vector
    std::vector<double> data = {1.0, 1.1, 1.2, 5.0, 5.1, 5.2, 2.0, 2.1, 2.2, 10.0, 10.1, 10.2};
 
    std::vector<std::shared_ptr<chunk_processing::ChunkStrategy<double>>> strategies = {
        std::make_shared<chunk_processing::VarianceStrategy<double>>(5.0),
        std::make_shared<chunk_processing::EntropyStrategy<double>>(1.0)};
 
    chunk_processing::HierarchicalSubChunkStrategy<double> hierarchical_chunker(strategies, 2);
    auto hierarchical_result = hierarchical_chunker.apply(data);
 
    print_chunks(hierarchical_result);
}

References chunk_processing::HierarchicalSubChunkStrategy< T >::apply(), and print_chunks().

Referenced by main().

main()

int main	(	int	argc,
		char *	argv[]
	)

Main function demonstrating various chunking strategies.

Returns

0 on successful execution

Definition at line 154 of file main.cpp.

                                 {
    demonstrate_complex_recursive_subchunking();
    demonstrate_multi_strategy_subchunking();
    demonstrate_adaptive_conditional_subchunking();
 
    std::cout << "\n=== Demonstrating Advanced Chunking Structures ===\n";
 
    // Example: SemanticChunker usage
    std::cout << "\n=== SemanticChunker Example ===" << std::endl;
    SemanticChunker<std::string> text_chunker;
    std::string text = "This is the first sentence. This is the second one. And here's a third!";
    auto text_chunks = text_chunker.chunk(text);
    std::cout << "Text chunks created: " << text_chunks.size() << "\n";
 
    // Custom NLP model example
    class CustomNLPModel {
    public:
        double calculateSimilarity(const std::string& s1, const std::string& s2) {
            // Simple example: compare lengths as a similarity metric
            return std::abs(1.0 - static_cast<double>(std::abs(static_cast<int>(s1.length()) -
                                                               static_cast<int>(s2.length()))) /
                                      std::max(s1.length(), s2.length()));
        }
    };
 
    SemanticChunker<std::string, CustomNLPModel> custom_chunker;
    auto custom_chunks = custom_chunker.chunk(text);
    std::cout << "Custom model chunks created: " << custom_chunks.size() << "\n\n";
 
    return 0;
}

References advanced_structures::SemanticChunker< ContentType, ModelType >::chunk(), demonstrate_adaptive_conditional_subchunking(), demonstrate_complex_recursive_subchunking(), and demonstrate_multi_strategy_subchunking().

print_chunks()

template<typename T >

void print_chunks

(

const std::vector< std::vector< T > > &

chunks

)

Helper function to print chunks.

Template Parameters

T	The type of elements in the chunks

Parameters

chunks

The vector of chunks to print

Definition at line 50 of file main.cpp.

                                                         {
    std::cout << "Chunks: [" << std::endl;
    for (size_t i = 0; i < chunks.size(); ++i) {
        std::cout << "  " << i << ": [";
        for (const auto& value : chunks[i]) {
            std::cout << std::fixed << std::setprecision(2) << value << " ";
        }
        std::cout << "]" << std::endl;
    }
    std::cout << "]" << std::endl;
}

Referenced by demonstrate_adaptive_conditional_subchunking(), demonstrate_complex_recursive_subchunking(), and demonstrate_multi_strategy_subchunking().

print_sub_chunks()

template<typename T >

void print_sub_chunks	(	const std::vector< std::vector< std::vector< T > > > &	sub_chunks,
		const std::string &	label,
		int	precision = 2
	)

Helper function to print sub-chunks with detailed formatting.

Template Parameters

T	The type of elements in the chunks

Parameters

sub_chunks	The 3D vector containing the sub-chunks to print
label	The label to display for this set of sub-chunks
precision	The number of decimal places to show for floating-point numbers

Definition at line 70 of file main.cpp.

                                                                 {
    std::cout << "\n" << label << ":\n";
    for (size_t i = 0; i < sub_chunks.size(); ++i) {
        std::cout << "Level " << i + 1 << ":\n";
        for (size_t j = 0; j < sub_chunks[i].size(); ++j) {
            std::cout << "  Sub-chunk " << j + 1 << ": ";
            for (const auto& val : sub_chunks[i][j]) {
                std::cout << std::fixed << std::setprecision(precision) << val << " ";
            }
            std::cout << "\n";
        }
    }
}