sophisticated_chunking::MutualInformationChunking< T > Class Template Reference

Information theory based chunking using mutual information. More...

#include <sophisticated_chunking.hpp>

Public Member Functions
	MutualInformationChunking (size_t context_size=5, double mi_threshold=0.3)
	Constructor for mutual information based chunking.
std::vector< std::vector< T > >	chunk (const std::vector< T > &data) const
	Chunk data based on mutual information analysis.
size_t	get_context_size () const
	Get the size of context window.
double	get_mi_threshold () const
	Get the threshold for mutual information.
void	set_context_size (size_t size)
	Set the size of context window.
void	set_mi_threshold (double threshold)
	Set the threshold for mutual information.

Private Member Functions
double	calculateMutualInformation (const std::vector< T > &segment1, const std::vector< T > &segment2) const
	Calculate mutual information between adjacent segments.

Private Attributes
size_t	context_size_
double	mi_threshold_

Detailed Description

template<typename T>
class sophisticated_chunking::MutualInformationChunking< T >

Information theory based chunking using mutual information.

Template Parameters

T	The type of elements to be chunked

Definition at line 175 of file sophisticated_chunking.hpp.

Constructor & Destructor Documentation

MutualInformationChunking()

template<typename T >

sophisticated_chunking::MutualInformationChunking< T >::MutualInformationChunking ( size_t  context_size = 5, double  mi_threshold = 0.3  )

inline

Constructor for mutual information based chunking.

Parameters

context_size	Size of context window
mi_threshold	Threshold for mutual information

Definition at line 230 of file sophisticated_chunking.hpp.

        : context_size_(context_size), mi_threshold_(mi_threshold) {}

Member Function Documentation

calculateMutualInformation()

template<typename T >

double sophisticated_chunking::MutualInformationChunking< T >::calculateMutualInformation ( const std::vector< T > &  segment1, const std::vector< T > &  segment2  ) const

inlineprivate

Calculate mutual information between adjacent segments.

Parameters

segment1	First segment
segment2	Second segment

Returns

Mutual information value

Definition at line 186 of file sophisticated_chunking.hpp.

                                                                          {
        if (segment1.empty() || segment2.empty()) {
            return 0.0;
        }
 
        // Calculate frequency distributions
        std::map<T, double> p1, p2;
        std::map<std::pair<T, T>, double> p12;
 
        for (const auto& val : segment1) {
            p1[val] += 1.0 / segment1.size();
        }
 
        for (const auto& val : segment2) {
            p2[val] += 1.0 / segment2.size();
        }
 
        // Calculate joint distribution
        size_t min_size = std::min(segment1.size(), segment2.size());
        for (size_t i = 0; i < min_size; ++i) {
            p12[{segment1[i], segment2[i]}] += 1.0 / min_size;
        }
 
        // Calculate mutual information
        double mi = 0.0;
        for (const auto& [val1, prob1] : p1) {
            for (const auto& [val2, prob2] : p2) {
                auto joint_prob = p12[{val1, val2}];
                if (joint_prob > 0) {
                    mi += joint_prob * std::log2(joint_prob / (prob1 * prob2));
                }
            }
        }
 
        return mi;
    }

Referenced by sophisticated_chunking::MutualInformationChunking< T >::chunk().

chunk()

template<typename T >

std::vector< std::vector< T > > sophisticated_chunking::MutualInformationChunking< T >::chunk ( const std::vector< T > &  data ) const

inline

Chunk data based on mutual information analysis.

Parameters

data	Input data to be chunked

Returns

Vector of chunks

Definition at line 238 of file sophisticated_chunking.hpp.

                                                                  {
        if (data.size() < 2 * context_size_) {
            return {data};
        }
 
        std::vector<std::vector<T>> chunks;
        std::vector<T> current_chunk;
 
        for (size_t i = 0; i < data.size(); ++i) {
            current_chunk.push_back(data[i]);
 
            if (current_chunk.size() >= context_size_ && i + context_size_ < data.size()) {
                std::vector<T> next_segment(data.begin() + i + 1,
                                            data.begin() +
                                                std::min(i + 1 + context_size_, data.size()));
 
                double mi = calculateMutualInformation(current_chunk, next_segment);
 
                if (mi < mi_threshold_) {
                    chunks.push_back(current_chunk);
                    current_chunk.clear();
                }
            }
        }
 
        if (!current_chunk.empty()) {
            chunks.push_back(current_chunk);
        }
 
        return chunks;
    }

References sophisticated_chunking::MutualInformationChunking< T >::calculateMutualInformation(), sophisticated_chunking::MutualInformationChunking< T >::context_size_, and sophisticated_chunking::MutualInformationChunking< T >::mi_threshold_.

Referenced by demonstrate_mutual_information_chunking(), TEST_F(), and TEST_F().

get_context_size()

template<typename T >

size_t sophisticated_chunking::MutualInformationChunking< T >::get_context_size ( ) const

inline

Get the size of context window.

Returns

Size of context window

Definition at line 274 of file sophisticated_chunking.hpp.

                                    {
        return context_size_;
    }

References sophisticated_chunking::MutualInformationChunking< T >::context_size_.

get_mi_threshold()

template<typename T >

double sophisticated_chunking::MutualInformationChunking< T >::get_mi_threshold ( ) const

inline

Get the threshold for mutual information.

Returns

Threshold for mutual information

Definition at line 282 of file sophisticated_chunking.hpp.

                                    {
        return mi_threshold_;
    }

References sophisticated_chunking::MutualInformationChunking< T >::mi_threshold_.

set_context_size()

template<typename T >

void sophisticated_chunking::MutualInformationChunking< T >::set_context_size ( size_t  size )

inline

Set the size of context window.

Parameters

size	Size of context window

Definition at line 290 of file sophisticated_chunking.hpp.

                                       {
        if (size == 0)
            throw std::invalid_argument("Context size cannot be zero");
        context_size_ = size;
    }

References sophisticated_chunking::MutualInformationChunking< T >::context_size_.

set_mi_threshold()

template<typename T >

void sophisticated_chunking::MutualInformationChunking< T >::set_mi_threshold ( double  threshold )

inline

Set the threshold for mutual information.

Parameters

threshold

Threshold for mutual information

Definition at line 300 of file sophisticated_chunking.hpp.

                                            {
        mi_threshold_ = threshold;
    }

References sophisticated_chunking::MutualInformationChunking< T >::mi_threshold_.

Member Data Documentation

context_size_

template<typename T >

size_t sophisticated_chunking::MutualInformationChunking< T >::context_size_

private

Definition at line 177 of file sophisticated_chunking.hpp.

Referenced by sophisticated_chunking::MutualInformationChunking< T >::chunk(), sophisticated_chunking::MutualInformationChunking< T >::get_context_size(), and sophisticated_chunking::MutualInformationChunking< T >::set_context_size().

mi_threshold_

template<typename T >

double sophisticated_chunking::MutualInformationChunking< T >::mi_threshold_

private

Definition at line 178 of file sophisticated_chunking.hpp.

Referenced by sophisticated_chunking::MutualInformationChunking< T >::chunk(), sophisticated_chunking::MutualInformationChunking< T >::get_mi_threshold(), and sophisticated_chunking::MutualInformationChunking< T >::set_mi_threshold().

---

The documentation for this class was generated from the following file:

• sophisticated_chunking.hpp