WordChunker

from chonkie import WordChunker

# Initialize the chunker
chunker = WordChunker()

# Chunk a single text
chunks = chunker("Some long text that needs to be chunked into smaller pieces...")

# Print the chunks
for chunk in chunks:
    print(chunk.text)
    print(chunk.token_count)
    print(f"{chunk.start_index} - {chunk.end_index}")

# Import the WordChunker
from chonkie import WordChunker

# Initialize the chunker
chunker = WordChunker()

# Chunk a batch of texts
chunks = chunker.chunk_batch(["Text 1", "Text 2", "Text 3"])

# Print the chunks
for chunks in batch_chunks:
    for chunk in chunks:    
        print(chunk.text)
        print(chunk.token_count)
        print(f"{chunk.start_index} - {chunk.end_index}")

# Import the WordChunker
from chonkie import WordChunker
from chonkie import CharacterTokenizer

# Optional: Initialize the character tokenizer
tokenizer = CharacterTokenizer()

# Initialize the chunker
chunker = WordChunker(tokenizer_or_token_counter=tokenizer)
chunker = WordChunker(tokenizer_or_token_counter="character") # OR use the character tokenizer directly

# Chunk a single text
chunks = chunker("Some long text that needs to be chunked into smaller pieces...")

# Print the chunks
for chunk in chunks:
    print(chunk.text)
    print(chunk.token_count)
    print(f"{chunk.start_index} - {chunk.end_index}")

# Import the WordChunker
from chonkie import WordChunker
from chonkie import WordTokenizer

# Optional: Initialize the word tokenizer
tokenizer = WordTokenizer()

# Initialize the chunker
chunker = WordChunker(tokenizer_or_token_counter=tokenizer)
chunker = WordChunker(tokenizer_or_token_counter="word") # OR use the word tokenizer directly

# Chunk a single text
chunks = chunker("Some long text that needs to be chunked into smaller pieces...")

# Print the chunks
for chunk in chunks:
    print(chunk.text)
    print(chunk.token_count)
    print(f"{chunk.start_index} - {chunk.end_index}")

# Import the WordChunker
from chonkie import WordChunker

# Initialize the chunker
chunker = WordChunker(return_type="texts")

# Chunk a single text
chunks = chunker("Some long text that needs to be chunked into smaller pieces...")

# Print the chunks
for chunk in chunks:
    print(chunk) # This will print the chunk text (str)

# Import the WordChunker and a custom tokenizer
from chonkie import WordChunker
from tokenizers import Tokenizer

# Initialize the tokenizer
tokenizer = Tokenizer.from_pretrained("bert-base-uncased") # or any other tokenizer

# Initialize the chunker
chunker = WordChunker(tokenizer_or_token_counter=tokenizer)

# Chunk a single text
chunks = chunker("Some long text that needs to be chunked into smaller pieces...")

# Print the chunks
for chunk in chunks:
    print(chunk.text)
    print(chunk.token_count)
    print(f"{chunk.start_index} - {chunk.end_index}")

# Import the WordChunker and a custom token counter
from chonkie import WordChunker

# Define a custom token counter
def custom_token_counter(text):
    return len(text.split())

# Initialize the chunker
chunker = WordChunker(tokenizer_or_token_counter=custom_token_counter)

# Chunk a single text
chunks = chunker("Some long text that needs to be chunked into smaller pieces...")

# Print the chunks
for chunk in chunks:
    print(chunk.text)