Overlap Refinery

from chonkie import OverlapRefinery

# Initialize with default character-level overlap (25% context size)
overlap_refinery = OverlapRefinery()

# Initialize with a specific tokenizer and context size
overlap_refinery_token = OverlapRefinery(
    tokenizer="character",             # Default tokenizer (or use "gpt2", etc.)
    context_size=0.25,                 # The size of the context to add to the chunks.
    method="prefix",                   # Add context from the previous chunk
    merge=True                         # Merge context directly into chunk text
)

# Initialize with justified method (context from both sides)
overlap_refinery_justified = OverlapRefinery(
    tokenizer="character",
    context_size=0.25,
    method="justified",               # Add context from both previous and next chunks
    merge=True
)

# Initialize for recursive overlap based on rules
from chonkie import RecursiveRules, RecursiveLevel
rules = RecursiveRules(
    levels=[
        RecursiveLevel(delimiters=["\n\n"], include_delim="prev"),
        RecursiveLevel(delimiters=["."], include_delim="prev"),
        RecursiveLevel(whitespace=True)
    ]
)
overlap_refinery_recursive = OverlapRefinery(
    tokenizer="character",
    context_size=0.25,
    mode="recursive",
    rules=rules,
    method="suffix"
)

from chonkie import TokenChunker, OverlapRefinery

test_string = "This is the first sentence. This is the second sentence, providing context. This is the third sentence, which needs context from the second."
chunker = TokenChunker()
chunks = chunker(test_string)

# Initialize refinery to add suffix overlap
overlap_refinery = OverlapRefinery(
    tokenizer="character",
    context_size=0.5,
    method="suffix",
    merge=True
)

refined_chunks = overlap_refinery(chunks)