Chomper 文档解析服务

# Clone the repository
git clone https://github.com/IcHiGo-KuRoSaKiI/Chomper.git
cd Chomper

# Create virtual environment and install
python -m venv venv
source venv/bin/activate  # or `venv\Scripts\activate` on Windows
pip install -e ".[all]"   # every format and feature; `pip install -e .` for the core

# Direct execution
python server.py

# Or via the installed command
chomper

claude mcp add -s user chomper -- /path/to/Chomper/venv/bin/python /path/to/Chomper/server.py

{
  "mcpServers": {
    "chomper": {
      "command": "/path/to/Chomper/venv/bin/python",
      "args": ["/path/to/Chomper/server.py"]
    }
  }
}

import chomper

# Parse a document
result = chomper.parse("/path/to/document.pdf")
print(result.text)        # Markdown, with ![caption](image:p1-1) figure markers
print(result.metadata)
print(f"Words: {result.word_count}, Format: {result.format}")

# Include images (decoded and downscaled only now)
result = chomper.parse("/path/to/document.pdf", include_images=True)
for img in result.images:
    print(img["id"], img["page"], img["width"], img["height"], img["mime_type"])

# Parse from bytes or base64 (cloud storage, APIs, databases); capped at 100 MB
import base64
with open("doc.pdf", "rb") as f:
    content = base64.b64encode(f.read()).decode()

result = chomper.parse_bytes(content, "doc.pdf")

# Quick metadata extraction
meta = chomper.extract_metadata("/path/to/report.pdf")
print(f"Author: {meta.author}, Pages: {meta.page_count}, Images: {meta.image_count}")

# Chunk for RAG/embeddings
chunks = chomper.chunk("/path/to/doc.pdf", chunk_size=500, keywords=True)
for chunk in chunks:
    print(f"Chunk {chunk.chunk_id}: {chunk.word_count} words, section {chunk.section_name!r}")
    print(f"Keywords: {chunk.keywords}")

# Check format support
if chomper.is_supported("report.pdf"):
    result = chomper.parse("report.pdf")

# List all formats
formats = chomper.list_formats()
for ext, info in formats.items():
    if info["available"]:
        print(f"{ext}: {info['description']}")
    else:
        print(f"{ext}: {info['reason']}")  # includes the pip install command

# ParseResult
result.text           # Extracted text (Markdown)
result.metadata       # Document metadata dict
result.format         # File format (pdf, docx, etc.)
result.word_count     # Total word count
result.char_count     # Total character count
result.images         # With include_images=True: dicts with id, page, width, height,
                      # content (base64), mime_type, position, alt
result.summary(500)   # First 500 characters

# ChunkResult (from chomper.chunk())
chunk.text            # Chunk text, an exact slice of the document text
chunk.chunk_id        # Chunk index (0-based)
chunk.start_char      # Offsets into the document text
chunk.end_char
chunk.word_count      # Words in chunk
chunk.keywords        # TF-IDF keywords (empty unless keywords=True)
chunk.section_name    # Heading, slide title or sheet the chunk belongs to
chunk.metadata        # Chunker-specific details

# MetadataResult (from chomper.extract_metadata())
meta.filename         # Base filename
meta.format           # File format
meta.file_size        # Size in bytes
meta.author           # Author (if available)
meta.title            # Title (if available)
meta.page_count       # Pages (if applicable)
meta.word_count       # Also text_length, image_count, created, modified, ...
meta.to_dict()        # Fields that are set

from src.extractors import PDFExtractor

doc = PDFExtractor(
    backend="auto",               # "layout" (pymupdf-layout model; default when installed) or "legacy"
    ocr="auto",                   # "auto" OCRs pages without a text layer if an engine is installed; "off"; "force"
    table_format="markdown",      # or "html" to keep merged cells (layout backend only)
    extract_vector_figures=True,  # mark vector charts/diagrams as figures, rendered on request
).extract("/path/to/report.pdf")
print(doc.text, doc.metadata)

# Parse and print text
chomper-parse document.pdf

# Output as JSON
chomper-parse report.docx --json

# Output in different formats (csv, markdown, xml)
chomper-parse report.pdf --format markdown
chomper-parse data.xlsx --format csv

# Show metadata only
chomper-parse data.xlsx --metadata

# Split into chunks
chomper-parse book.pdf --chunk --strategy recursive --chunk-size 500 --keywords

# Save to file
chomper-parse document.pdf -o output.txt

# List supported formats
chomper-parse --formats

# Quiet mode (no progress messages)
chomper-parse document.pdf -q

# Plain text (default)
chomper-parse document.pdf

# JSON output
chomper-parse document.pdf --format json
chomper-parse document.pdf --json  # shortcut

# CSV output
chomper-parse document.pdf --format csv

# Markdown output
chomper-parse document.pdf --format markdown

# XML output
chomper-parse document.pdf --format xml

# Custom Jinja2 template (needs the `templates` extra)
chomper-parse document.pdf --format template --template my_template.j2

# Watch a directory
chomper-parse --watch ./documents

# Watch with JSON output saved to files
chomper-parse --watch ./inbox --format json --output-dir ./parsed

# Watch only PDFs, check every 5 seconds
chomper-parse --watch ./docs --pattern "*.pdf" --interval 5

# Watch recursively (including subdirectories)
chomper-parse --watch ./project --recursive

# Watch with metadata only
chomper-parse --watch ./docs --metadata --format json

$ chomper-parse -i
Chomper Interactive Mode
Type 'help' for commands, 'exit' to quit.

chomper> parse ~/Documents/report.pdf
[Document content displayed...]

chomper> set format json
Output format set to: json

chomper> metadata ~/Documents/report.pdf
{
  "filename": "report.pdf",
  "format": "pdf",
  "page_count": 5
}

chomper> history
Files parsed this session:
  1. /Users/me/Documents/report.pdf

chomper> help
[Shows all available commands]

chomper> exit

parse_document(file_path: "/path/to/doc.pdf")
→ Returns first 5000 chars + metadata with hint to fetch more

parse_document(file_path: "/path/to/doc.pdf", output_format: "toon")
→ Returns in TOON format (~40% fewer tokens)

import base64

# Read file and encode to base64
with open("document.pdf", "rb") as f:
    content = base64.b64encode(f.read()).decode()

# Send via MCP
parse_document_bytes(
    content_base64=content,
    filename="document.pdf"
)
→ Returns extracted text + metadata (same as parse_document)

1. parse_document(file_path: "doc.pdf")
   → Returns chars 0-5000, hint: "use get_document_chunk(offset=5000)"

2. get_document_chunk(file_path: "doc.pdf", offset: 5000)
   → Returns chars 5000-10000

3. get_document_chunk(file_path: "doc.pdf", offset: 10000)
   → Returns chars 10000-15000, etc.

get_document_images(file_path: "doc.pdf", page: 1, max_images: 3)
→ Returns first 3 images from page 1 as ImageContent

get_document_images(file_path: "doc.pdf", image_id: "p3-1")
→ Returns the figure marked ![...](image:p3-1) in the text

{
  "success": true,
  "file_path": "/path/to/doc.md",
  "chunk_size": 1000,
  "overlap": 100,
  "chunking_strategy": "auto",
  "total_chunks": 25,
  "chunks": [
    {
      "chunk_id": 0,
      "text": "## Introduction\n\nChunk content...",
      "start_char": 0,
      "end_char": 1840,
      "word_count": 250,
      "metadata": {
        "section_name": "Introduction",
        "heading_path": "Report > Introduction",
        "chunk_strategy": "markdown"
      },
      "keywords": ["key", "terms"],
      "section_name": "Introduction",
      "section_type": "text"
    }
  ],
  "metadata": { "filename": "doc.md", "...": "..." },
  "statistics": {
    "total_words": 6000,
    "average_chunk_words": 240,
    "total_characters": 41000
  }
}

{
  "success": true,
  "file_path": "/path/to/report.pdf",
  "metadata": {
    "author": "John Doe",
    "title": "Document Title",
    "page_count": 10,
    "extraction_method": "pymupdf-layout"
  },
  "document_info": {
    "text_length": 35000,
    "has_structure": true,
    "page_count": 10,
    "image_count": 5
  }
}

![Figure 3: Revenue by region](image:p4-2)

Use the summarize-document prompt with file_path="/path/to/doc.pdf"

d:report.pdf|t:pdf|w:5000|c:25000|n:10
m:author=John Doe,title=Annual Report
---
0|0-2500|text|Introduction
The document begins with an overview...
k:overview,introduction,summary
---
1|2500-5000|text|Methodology
The methodology section describes...
k:methodology,approach,methods

# 1. Start with summary (default behavior)
parse_document(file_path: "large_doc.pdf")

# 2. If you need more content, paginate
get_document_chunk(file_path: "large_doc.pdf", offset: 5000)
get_document_chunk(file_path: "large_doc.pdf", offset: 10000)

# 3. Fetch a figure the text points to, or images separately when needed
get_document_images(file_path: "large_doc.pdf", image_id: "p4-2")
get_document_images(file_path: "large_doc.pdf", max_images: 3)

# 4. For RAG pipelines, chunk the document
parse_document_chunked(file_path: "doc.pdf", include_keywords: true)

# DON'T use full_text=true for large documents - will exceed token limits!
parse_document(file_path: "large_doc.pdf", full_text: true)  # Bad

┌─────────────┐    ┌─────────────┐    ┌─────────────┐    ┌─────────────┐
│  Extractors │ -> │  Chunkers   │ -> │  Enrichers  │ -> │ Formatters  │
│ (Layer 1)   │    │ (Layer 2)   │    │ (Layer 3)   │    │ (Layer 4)   │
└─────────────┘    └─────────────┘    └─────────────┘    └─────────────┘
     │                   │                  │                   │
     v                   v                  v                   v
 Markdown +         Structure-aware    Keywords +         JSON/TOON
 Structure          Chunks             Metadata            Output

pip install -e ".[all]"   # or: pip install -r requirements.txt

get_document_chunk failed (ValueError): offset must be >= 0, got -3.
parse_document failed (ValueError): File not found: /path/to/missing.pdf. Check the path and pass an absolute path.

# Install dev dependencies
pip install -e ".[dev]"

# Run tests
pytest
python src/tests/test_lightweight.py

# Format code
black .

# Lint and type check
ruff check .
mypy

# Fork and clone
git clone https://github.com/YOUR_USERNAME/Chomper.git
cd Chomper

# Setup dev environment
python -m venv venv
source venv/bin/activate
pip install -e ".[dev]"

# Run tests
pytest

# Format code
black .
ruff check .