Token Chunker

from chonkie import TokenChunker

# Create a chunker with specific parameters
chunker = TokenChunker(
    tokenizer="gpt2",
    chunk_size=1024,
    chunk_overlap=128
)

text = """Natural language processing has revolutionized how we interact with computers.
Machine learning models can now understand context, generate text, and even translate
between languages with remarkable accuracy. This transformation has enabled applications
ranging from virtual assistants to automated content generation."""

# Chunk the text
chunks = chunker.chunk(text)

# Process each chunk
for i, chunk in enumerate(chunks):
    print(f"\n--- Chunk {i+1} ---")
    print(f"Text: {chunk.text}")
    print(f"Token count: {chunk.token_count}")
    print(f"Start index: {chunk.start_index}")
    print(f"End index: {chunk.end_index}")

import { TokenChunker } from "@chonkiejs/core";

// Create a chunker with specific parameters
const chunker = await TokenChunker.create({
  chunkSize: 1024,
  chunkOverlap: 128,
});

const text = `Natural language processing has revolutionized how we interact with computers.
Machine learning models can now understand context, generate text, and even translate
between languages with remarkable accuracy. This transformation has enabled applications
ranging from virtual assistants to automated content generation.`;

// Chunk the text
const chunks = await chunker.chunk(text);

// Process each chunk
for (let i = 0; i < chunks.length; i++) {
  const chunk = chunks[i];
  console.log(`\n--- Chunk ${i + 1} ---`);
  console.log(`Text: ${chunk.text}`);
  console.log(`Token count: ${chunk.tokenCount}`);
  console.log(`Start index: ${chunk.startIndex}`);
  console.log(`End index: ${chunk.endIndex}`);
}

from chonkie import TokenChunker

# Fixed token overlap
chunker_fixed = TokenChunker(
    tokenizer="gpt2",
    chunk_size=1000,
    chunk_overlap=100  # Exactly 100 tokens overlap
)

# Percentage-based overlap
chunker_percent = TokenChunker(
    tokenizer="gpt2",
    chunk_size=1000,
    chunk_overlap=0.1  # 10% overlap (100 tokens for 1000 token chunks)
)

text = "Long document text that will be chunked with overlap..."

# Compare the results
fixed_chunks = chunker_fixed.chunk(text)
percent_chunks = chunker_percent.chunk(text)

print(f"Fixed overlap: {len(fixed_chunks)} chunks")
print(f"Percentage overlap: {len(percent_chunks)} chunks")

import { TokenChunker } from "@chonkiejs/core";

// Fixed token overlap
const chunkerFixed = await TokenChunker.create({
  chunkSize: 1000,
  chunkOverlap: 100, // Exactly 100 tokens overlap
});

const text = "Long document text that will be chunked with overlap...";

// Compare the results
const fixedChunks = await chunkerFixed.chunk(text);

console.log(`Fixed overlap (100): ${fixedChunks.length} chunks`);

from chonkie import TokenChunker

# Configure for large documents
chunker = TokenChunker(
    tokenizer="gpt2",
    chunk_size=4096,  # Larger chunks for efficiency
    chunk_overlap=512  # Maintain context between chunks
)

# Read a large document
with open("large_document.txt", "r") as f:
    large_text = f.read()

# Process efficiently
chunks = chunker.chunk(large_text)

print(f"Document statistics:")
print(f"  Original length: {len(large_text)} characters")
print(f"  Number of chunks: {len(chunks)}")
print(f"  Average chunk size: {sum(c.token_count for c in chunks) / len(chunks):.1f} tokens")

# Save chunks for further processing
for i, chunk in enumerate(chunks):
    with open(f"chunk_{i:03d}.txt", "w") as f:
        f.write(chunk.text)

import { TokenChunker } from "@chonkiejs/core";
import { readFile, writeFile } from "fs/promises";

// Configure for large documents
const chunker = await TokenChunker.create({
  chunkSize: 4096, // Larger chunks for efficiency
  chunkOverlap: 512, // Maintain context between chunks
});

// Read a large document
const largeText = await readFile("large_document.txt", "utf-8");

// Process efficiently
const chunks = await chunker.chunk(largeText);

console.log("Document statistics:");
console.log(`  Original length: ${largeText.length} characters`);
console.log(`  Number of chunks: ${chunks.length}`);

const avgTokenCount =
  chunks.reduce((sum, c) => sum + c.tokenCount, 0) / chunks.length;
console.log(`  Average chunk size: ${avgTokenCount.toFixed(1)} tokens`);

// Save chunks for further processing
for (let i = 0; i < chunks.length; i++) {
  const filename = `chunk_${i.toString().padStart(3, "0")}.txt`;
  await writeFile(filename, chunks[i].text);
}