MarkdownChef

from chonkie import MarkdownChef

# Basic initialization with default tokenizer
chef = MarkdownChef()

# Initialize with a specific tokenizer
chef = MarkdownChef(tokenizer="gpt2")

# Or use a custom tokenizer instance
from transformers import AutoTokenizer
custom_tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
chef = MarkdownChef(tokenizer=custom_tokenizer)

from chonkie import MarkdownChef

# Initialize the chef
chef = MarkdownChef()

# Process a markdown file
doc = chef.process("example.md")

# Access the extracted components
print(f"Found {len(doc.tables)} tables")
print(f"Found {len(doc.code)} code blocks")
print(f"Found {len(doc.images)} images")
print(f"Found {len(doc.chunks)} text chunks")

@dataclass
class MarkdownTable:
    content: str          # The table content
    start_index: int      # Starting position in original text
    end_index: int        # Ending position in original text

@dataclass
class MarkdownCode:
    content: str              # The code content
    language: Optional[str]   # Programming language (if specified)
    start_index: int          # Starting position in original text
    end_index: int            # Ending position in original text

@dataclass
class MarkdownImage:
    alias: str                # Alt text or filename
    content: str              # Image path or data URL
    start_index: int          # Starting position in original text
    end_index: int            # Ending position in original text
    link: Optional[str]       # Link URL (if image is clickable)

@dataclass
class MarkdownDocument(Document):
    id: str                         # Unique document ID
    content: str                    # Full markdown content
    tables: list[MarkdownTable]     # Extracted tables
    code: list[MarkdownCode]        # Extracted code blocks
    images: list[MarkdownImage]     # Extracted images
    chunks: list[Chunk]             # Remaining text chunks
    metadata: dict[str, Any]        # Additional metadata