Databricks MCP服务器

┌─────────────┐       MCP Protocol      ┌──────────────────┐        OAuth         ┌─────────────────┐
│   Claude    │ ◄─────────────────────► │  dba-mcp-proxy   │ ◄──────────────────► │ Databricks App  │
│    CLI      │     (stdio/JSON-RPC)    │ (local process)  │    (HTTPS/SSE)      │  (MCP Server)   │
└─────────────┘                         └──────────────────┘                      └─────────────────┘
                                                ▲                                           │
                                                │                                           ▼
                                                └────────── Databricks OAuth ──────► Workspace APIs

graph TB
    subgraph "MCP Client Layer"
        A[Claude CLI] 
        B[Claude Code]
        C[Other MCP Clients]
    end
    
    subgraph "MCP Proxy Layer"
        D[dba-mcp-proxy]
        E[OAuth Authentication]
        F[Token Management]
    end
    
    subgraph "Databricks Apps Layer"
        G[FastAPI App]
        H[MCP Server]
        I[React Frontend]
    end
    
    subgraph "Tools Layer"
        J[Core Tools]
        K[SQL Operations]
        L[Unity Catalog]
        M[Dashboard Tools]
        N[Widget Tools]
        O[Jobs & Pipelines]
        P[Data Management]
        Q[Governance]
    end
    
    subgraph "Databricks Services"
        R[SQL Warehouses]
        S[Unity Catalog]
        T[Jobs & DLT]
        U[DBFS & Volumes]
        V[Workspace Files]
        W[ML & Models]
    end
    
    A --> D
    B --> D
    C --> D
    D --> E
    E --> F
    F --> G
    G --> H
    H --> J
    H --> K
    H --> L
    H --> M
    H --> N
    H --> O
    H --> P
    H --> Q
    
    J --> R
    K --> R
    L --> S
    M --> S
    N --> S
    O --> T
    P --> U
    Q --> V
    
    G --> I

# Using Homebrew (recommended)
brew install databricks/tap/databricks

# Using pip
pip install databricks-cli

# Verify installation
databricks --version

# Using pip
pip install databricks-cli

# Or download from official releases
# https://github.com/databricks/databricks-cli/releases
# Download the .exe file and add to PATH

# Verify installation
databricks --version

# Using pip
pip install databricks-cli

# Using apt (Ubuntu/Debian)
curl -fsSL https://databricks.com/install-cli.sh | bash

# Using yum (RHEL/CentOS)
curl -fsSL https://databricks.com/install-cli.sh | bash

# Verify installation
databricks --version

# Generate SSH key (if you don't have one)
ssh-keygen -t ed25519 -C "your-email@example.com"

# Start ssh-agent
eval "$(ssh-agent -s)"

# Add SSH key to ssh-agent
ssh-add ~/.ssh/id_ed25519

# Copy public key to clipboard
pbcopy < ~/.ssh/id_ed25519.pub

# Add to GitHub: https://github.com/settings/keys
# Click "New SSH key" and paste the copied key

# Generate SSH key (if you don't have one)
ssh-keygen -t ed25519 -C "your-email@example.com"

# Start ssh-agent (PowerShell as Administrator)
Set-Service ssh-agent -StartupType Automatic
Start-Service ssh-agent

# Add SSH key to ssh-agent
ssh-add ~/.ssh/id_ed25519

# Copy public key to clipboard
Get-Content ~/.ssh/id_ed25519.pub | Set-Clipboard

# Add to GitHub: https://github.com/settings/keys
# Click "New SSH key" and paste the copied key

# Generate SSH key (if you don't have one)
ssh-keygen -t ed25519 -C "your-email@example.com"

# Start ssh-agent
eval "$(ssh-agent -s)"

# Add SSH key to ssh-agent
ssh-add ~/.ssh/id_ed25519

# Copy public key to clipboard (if xclip is available)
xclip -sel clip < ~/.ssh/id_ed25519.pub

# Or display the key to copy manually
cat ~/.ssh/id_ed25519.pub

# Add to GitHub: https://github.com/settings/keys
# Click "New SSH key" and paste the copied key

# Test the connection
ssh -T git@github.com

# You should see: "Hi username! You've successfully authenticated..."

# Set your Git username and email
git config --global user.name "Your Name"
git config --global user.email "your-email@example.com"

# Verify SSH is working by cloning a private repo
git clone git@github.com:username/private-repo.git

# Clone your new repository
git clone https://github.com/PulkitXChadha/awesome-databricks-mcp.git
cd awesome-databricks-mcp

# Run the interactive setup
./setup.sh

/setup-mcp

# Set your Databricks configuration
export DATABRICKS_HOST="https://your-workspace.cloud.databricks.com"
export DATABRICKS_APP_URL="https://your-app.databricksapps.com"  # Get this from ./app_status.sh
export SERVER_NAME="your-server-name"  # This comes from config.yaml (set during ./setup.sh)

# Add your MCP server to Claude (user-scoped)
claude mcp add $SERVER_NAME --scope user -- \
  uvx --from git+https://git@github.com/YOUR-USERNAME/your-repo.git dba-mcp-proxy \
  --databricks-host $DATABRICKS_HOST \
  --databricks-app-url $DATABRICKS_APP_URL

# Clone and setup
git clone <your-repo>
cd <your-repo>
./setup.sh

# Start dev server (both backend and frontend)
./watch.sh

# Set your configuration for local testing
export DATABRICKS_HOST="https://your-workspace.cloud.databricks.com"
export DATABRICKS_APP_URL="http://localhost:8000"  # Local dev server

# Add to Claude for local testing
claude mcp add databricks-mcp-local --scope local -- \
  uvx --from git+https://git@github.com/YOUR-ORG/YOUR-REPO.git dba-mcp-proxy \
  --databricks-host $DATABRICKS_HOST \
  --databricks-app-url $DATABRICKS_APP_URL

# Clone your repository
git clone https://github.com/PulkitXChadha/awesome-databricks-mcp.git
cd awesome-databricks-mcp

# Run the interactive setup script
./setup.sh

# Start both backend (FastAPI) and frontend (React) servers
./watch.sh

# Set environment variables for local testing
export DATABRICKS_HOST="https://your-workspace.cloud.databricks.com"
export DATABRICKS_APP_URL="http://localhost:8000"

# Add the local MCP server to Claude
claude mcp add databricks-mcp-local --scope local -- \
  uvx --from git+https://git@github.com/PulkitXChadha/awesome-databricks-mcp.git dba-mcp-proxy \
  --databricks-host $DATABRICKS_HOST \
  --databricks-app-url $DATABRICKS_APP_URL

# Test the connection
echo "What MCP prompts are available from databricks-mcp-local?" | claude

---
name: your_prompt_name
description: Brief description of what the prompt does
arguments:
  - name: warehouse_id
    description: SQL Warehouse ID for query execution
    required: true
    schema:
      type: string
      pattern: "^[a-f0-9]{16}$"
  
  - name: catalog
    description: Unity Catalog name
    required: false
    schema:
      type: string
      
mutually_exclusive:
  - [option1, option2]  # Optional: Define mutually exclusive arguments
---

# Your Prompt Title

## Configuration
- **Warehouse ID**: ${warehouse_id}
- **Catalog**: ${catalog}

Your prompt content here. Use ${argument_name} for placeholder substitution.

# Test local MCP server directly
./claude_scripts/test_local_mcp_curl.sh

# Test with MCP proxy
./claude_scripts/test_local_mcp_proxy.sh

# Use the web-based MCP Inspector
./claude_scripts/inspect_local_mcp.sh

# Check what's using port 8000
lsof -i :8000

# Kill process if needed
kill -9 <PID>

# Reinstall Python dependencies
uv sync

# Reinstall Node.js dependencies
cd client && bun install

# Refresh Databricks CLI credentials
databricks auth login

# Verify configuration
databricks config get

# Test MCP endpoint directly
curl http://localhost:8000/mcp/

# Check Claude MCP configuration
claude mcp list

# Deploy to Databricks Apps
./deploy.sh

# Check status and get your app URL
./app_status.sh

Human: What prompts are available?

Claude: I can see the following prompts from your Databricks MCP server:
- build_lakeview_dashboard: Build comprehensive Lakeview dashboards with data validation
- build_ldp_pipeline: Build Lakehouse Data Pipelines for data processing
- performance_optimization: Performance optimization guidance for Databricks workloads

Human: Can you execute a SQL query to show databases?

Claude: I'll execute that SQL query for you using the execute_dbsql tool.

[Executes SQL and returns results]

├── server/                    # FastAPI backend with MCP server
│   ├── app.py                # Main application + MCP server setup
│   ├── tools/                # Modular MCP tools implementation
│   │   ├── __init__.py       # Tool registration and loading
│   │   ├── core.py           # Core and health tools (✅ Active)
│   │   ├── sql_operations.py # SQL and warehouse tools (✅ Active)
│   │   ├── unity_catalog.py  # Unity Catalog tools (✅ Active)
│   │   ├── jobs_pipelines.py # Jobs and DLT pipeline tools (✅ Active)
│   │   ├── lakeview_dashboard.py # Lakeview dashboard tools (✅ Active)
│   │   ├── data_management.py # Data and DBFS tools (🚧 Available)
│   │   ├── governance.py     # Governance and compliance tools (🚧 Available)
│   │   ├── widget_specs.py   # Widget specifications and utilities
│   │   ├── layout_optimization.py # Dashboard layout optimization
│   │   └── utils.py          # Shared utility functions
│   └── routers/              # API endpoints
├── client/                   # React TypeScript frontend
│   ├── src/                  # Source code
│   │   ├── components/       # Reusable UI components
│   │   ├── pages/            # Page components
│   │   └── fastapi_client/   # Auto-generated API client
│   ├── package.json          # Node.js dependencies
│   └── tailwind.config.js    # TailwindCSS configuration
├── prompts/                  # MCP prompts (markdown files)
│   ├── build_lakeview_dashboard.md # Comprehensive dashboard building prompt
│   ├── build_ldp_pipeline.md # Lakehouse Data Pipelines building prompt
│   └── performance_optimization.md # Performance optimization guidance
├── dba_mcp_proxy/           # MCP proxy for Claude CLI
│   └── mcp_client.py        # OAuth + proxy implementation
├── claude_scripts/          # Comprehensive testing tools
│   ├── test_local_mcp_*.sh  # Local MCP testing scripts (curl & proxy)
│   ├── test_remote_mcp_*.sh # Remote MCP testing scripts (curl & proxy)
│   ├── test_mcp_tools.py    # MCP tools testing
│   ├── test_uc_tools.py     # Unity Catalog tools testing
│   ├── inspect_local_mcp.sh # Local web-based MCP Inspector
│   ├── inspect_remote_mcp.sh # Remote web-based MCP Inspector
│   └── README.md            # Testing documentation
├── docs/                     # Documentation
│   ├── databricks_apis/      # Databricks API documentation
│   ├── architecture.md       # System architecture overview
│   ├── core-tools.md         # Core tools documentation
│   ├── unity_catalog_tools.md # Unity Catalog tools documentation
│   └── api-reference.md      # Complete API reference
├── scripts/                  # Development tools
├── .github/workflows/        # CI/CD automation
│   ├── ci.yml               # Continuous integration
│   ├── deploy.yml           # Automated deployment
│   ├── security.yml         # Security scanning
│   └── test.yml             # Automated testing
└── pyproject.toml          # Python package configuration

pie title Active MCP Tools by Module
    "Core Tools" : 1
    "SQL Operations" : 15
    "Unity Catalog" : 21
    "Jobs & Pipelines" : 19
    "Lakeview Dashboard" : 3

# Using create_dashboard_file tool
create_dashboard_file(
    name="Sales Dashboard",
    warehouse_id="abc123",
    datasets=[{
        "name": "Sales Data",
        "query": "SELECT product_category, revenue, order_date FROM sales_transactions WHERE revenue > 100"
    }],
    widgets=[
        {
            "type": "bar",
            "dataset": "Sales Data",
            "config": {
                "x_field": "product_category",
                "y_field": "revenue",
                "title": "Revenue by Category"
            },
            "position": {"x": 0, "y": 0, "width": 6, "height": 4}
        },
        {
            "type": "counter",
            "dataset": "Sales Data",
            "config": {
                "value_field": "revenue",
                "title": "Total Revenue"
            },
            "position": {"x": 6, "y": 0, "width": 3, "height": 2}
        }
    ],
    file_path="sales_dashboard.lvdash.json",
    validate_sql=True
)

{
    "success": true,
    "file_path": "sales_dashboard.lvdash.json",
    "file_size": 2048,
    "message": "Dashboard file successfully created",
    "validation_results": {
        "queries_validated": [...],
        "widget_validations": [...]
    }
}

Human: Create a sales dashboard with revenue metrics and customer analysis

Claude: I'll create a Lakeview dashboard with revenue and customer metrics using the simplified tool architecture.

[Creates dashboard using manage_dashboard, then adds widgets with create_chart_widget and create_display_widget]

-- Use parameterized queries for flexibility
SELECT 
    date_trunc('month', order_date) as month,
    sum(revenue) as total_revenue,
    count(distinct customer_id) as unique_customers
FROM sales
WHERE order_date >= :start_date 
  AND order_date <= :end_date
GROUP BY 1
ORDER BY 1;

-- Include proper aliasing for widget display
SELECT 
    product_name as "Product",
    sum(quantity) as "Units Sold",
    sum(revenue) as "Revenue"
FROM sales_detail
GROUP BY 1;

# Widget positioning examples
{
    "x": 0,      # Start at left edge (0-11)
    "y": 0,      # Start at top
    "width": 6,  # Half width (1-12)
    "height": 4  # Standard height (typically 2-8)
}

# Common layout patterns:
# Full width: width=12
# Half width: width=6  
# Third width: width=4
# Quarter width: width=3

Human: Create an executive dashboard with KPIs, trends, and department breakdowns

Claude: I'll create a comprehensive executive dashboard with:
- Top KPI counters (revenue, growth, customers)
- Revenue trend line chart
- Department performance bar chart
- Regional heat map
- Key metrics table

Human: Build a customer analytics dashboard with segmentation and behavior analysis

Claude: I'll create a customer analytics dashboard featuring:
- Customer segmentation pie chart
- Cohort retention heatmap
- Purchase behavior funnel
- Customer lifetime value distribution
- Product affinity analysis

Human: Create an operations dashboard for monitoring system performance

Claude: I'll build an operational dashboard with:
- Real-time metrics counters
- Performance trend lines
- Alert status table
- Resource utilization gauges
- Error log analysis

# Find existing dashboards
echo "List all Lakeview dashboards" | claude

# Get dashboard details
echo "Show me the configuration of dashboard-id-123" | claude

# Add new widgets
echo "Add a revenue forecast widget to dashboard-id-123" | claude

# Modify existing widgets
echo "Update the sales chart to show quarterly data in dashboard-id-123" | claude

# Share with users
echo "Share dashboard-id-123 with user@company.com with READ access" | claude

# Share with groups
echo "Grant WRITE access to dashboard-id-123 for the analytics team" | claude

Human: Create a comprehensive sales dashboard with revenue trends, product performance, and regional analysis

Claude: I'll create a complete sales dashboard with multiple sections:

1. **KPI Section** (top row):
   - Total Revenue counter
   - Growth Rate counter  
   - Active Customers counter
   - Average Order Value counter

2. **Trends Section** (second row):
   - Monthly Revenue line chart
   - Product Category performance bar chart

3. **Analysis Section** (third row):
   - Regional sales heatmap
   - Top Products table
   - Customer Segment pie chart

4. **Details Section** (bottom):
   - Detailed transactions table with filters
   - Markdown widget with data definitions

graph LR
    subgraph "Test Types"
        A[Command Line Tests]
        B[Web UI Tests]
        C[Integration Tests]
        D[Performance Tests]
    end
    
    subgraph "Test Coverage"
        E[Local MCP Server]
        F[Remote MCP Server]
        G[OAuth Authentication]
        H[MCP Protocol]
        I[Tool Discovery]
        J[Tool Execution]
    end
    
    subgraph "Test Tools"
        K[test_local_mcp_curl.sh]
        L[test_local_mcp_proxy.sh]
        M[test_remote_mcp_curl.sh]
        N[test_remote_mcp_proxy.sh]
        O[inspect_local_mcp.sh]
        P[inspect_remote_mcp.sh]
    end
    
    A --> E
    A --> F
    B --> E
    B --> F
    C --> G
    C --> H
    D --> I
    D --> J
    
    K --> E
    L --> E
    M --> F
    N --> F
    O --> E
    P --> F

# List available prompts and tools
echo "What MCP prompts are available from databricks-mcp?" | claude

# Test a specific prompt
echo "Use the build_lakeview_dashboard prompt from databricks-mcp" | claude

# Test local MCP server (requires ./watch.sh to be running)
./claude_scripts/test_local_mcp_curl.sh      # Direct HTTP/curl tests
./claude_scripts/test_local_mcp_proxy.sh     # MCP proxy client tests

# Test remote MCP server (requires Databricks auth and deployment)
./claude_scripts/test_remote_mcp_curl.sh     # OAuth + HTTP tests
./claude_scripts/test_remote_mcp_proxy.sh    # Full end-to-end MCP proxy tests

# Specialized tool testing
./claude_scripts/test_mcp_tools.py           # MCP tools validation
./claude_scripts/test_uc_tools.py            # Unity Catalog tools testing

# Launch MCP Inspector for visual testing
./claude_scripts/inspect_local_mcp.sh        # Local server web interface
./claude_scripts/inspect_remote_mcp.sh       # Remote server web interface

graph LR
    A[Code Push] --> B[Automated Testing]
    B --> C[Code Quality Checks]
    C --> D[Security Scanning]
    D --> E[Build Validation]
    E --> F[Test Results]
    
    subgraph "Quality Gates"
        G[Python Linting]
        H[Type Checking]
        I[Unit Tests]
        J[Integration Tests]
        K[Frontend Build]
        L[Security Scan]
    end
    
    B --> G
    B --> H
    B --> I
    B --> J
    B --> K
    D --> L

DATABRICKS_HOST=https://your-workspace.cloud.databricks.com
DATABRICKS_TOKEN=your-token  # For local development
DATABRICKS_SQL_WAREHOUSE_ID=your-warehouse-id  # For SQL tools

@mcp_server.tool
def create_job(name: str, notebook_path: str, cluster_id: str) -> dict:
    """Create a Databricks job."""
    w = get_workspace_client()
    job = w.jobs.create(
        name=name,
        tasks=[{
            "task_key": "main",
            "notebook_task": {"notebook_path": notebook_path},
            "existing_cluster_id": cluster_id
        }]
    )
    return {"job_id": job.job_id, "run_now_url": f"{DATABRICKS_HOST}/#job/{job.job_id}"}