WordChunker

from chonkie import WordChunker

# Basic initialization with default parameters
chunker = WordChunker(
    tokenizer="gpt2",        # Supports string identifiers
    chunk_size=512,          # Maximum tokens per chunk
    chunk_overlap=128        # Overlap between chunks
)

# Using a custom tokenizer
from tokenizers import Tokenizer
custom_tokenizer = Tokenizer.from_pretrained("your-tokenizer")
chunker = WordChunker(
    tokenizer=custom_tokenizer,
    chunk_size=512,
    chunk_overlap=128
)

text = "The WordChunker splits text while preserving complete words..."
chunks = chunker.chunk(text)

for chunk in chunks:
    print(f"Chunk text: {chunk.text}")
    print(f"Token count: {chunk.token_count}")
    print(f"Start index: {chunk.start_index}")
    print(f"End index: {chunk.end_index}")

texts = [
    "First document with complete words...",
    "Second document to process..."
]
batch_chunks = chunker.chunk_batch(texts)

for doc_chunks in batch_chunks:
    for chunk in doc_chunks:
        print(f"Chunk: {chunk.text}")

# Single text
chunks = chunker("Text to chunk while preserving words...")

# Multiple texts
batch_chunks = chunker(["Text 1...", "Text 2..."])

@dataclass
class Chunk:
    text: str           # The chunk text
    start_index: int    # Starting position in original text
    end_index: int      # Ending position in original text
    token_count: int    # Number of tokens in chunk