AI评估平台

# 🎯 One-command setup
pip install -e ".[dev]"

# 🔥 Launch MCP server for Claude Desktop, MCP clients
python -m mcp_eval_server.server
# or
make dev

# 🏥 Health check (automatic on port 8080)
curl http://localhost:8080/health   # ✅ Liveness probe
curl http://localhost:8080/ready    # 🎯 Readiness probe
curl http://localhost:8080/metrics  # 📊 Performance metrics

# 🚀 Launch REST API server with FastAPI
python -m mcp_eval_server.rest_server --port 8080 --host 0.0.0.0
# or
make serve-rest

# 📚 Interactive API documentation
open http://localhost:8080/docs

# 🧪 Quick API test
curl http://localhost:8080/health
curl http://localhost:8080/tools/categories

# 🌍 MCP protocol over HTTP with Server-Sent Events
make serve-http

# 📡 Access via JSON-RPC over HTTP on port 9000
curl -X POST -H 'Content-Type: application/json' \
     -d '{"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}' \
     http://localhost:9000/

# Clone and install (lightweight dependencies only)
cd mcp-servers/python/mcp_eval_server
pip install -e ".[dev]"

# Set up API keys (optional - rule-based judge works without them)
export OPENAI_API_KEY="sk-your-key-here"
export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
export AZURE_OPENAI_API_KEY="your-azure-api-key"

# Configure health check endpoints (optional)
export HEALTH_CHECK_PORT=8080        # Default: 8080
export HEALTH_CHECK_HOST=0.0.0.0     # Default: 0.0.0.0

# Note: No heavy ML dependencies required!
# Uses efficient TF-IDF + scikit-learn instead of transformers

{
  "command": "python",
  "args": ["-m", "mcp_eval_server.server"],
  "cwd": "/path/to/mcp-servers/python/mcp_eval_server"
}

# Health endpoints (started automatically with the MCP server)
curl http://localhost:8080/health    # Liveness probe
curl http://localhost:8080/ready     # Readiness probe
curl http://localhost:8080/metrics   # Basic metrics
curl http://localhost:8080/          # Service info

# Kubernetes-style endpoints
curl http://localhost:8080/healthz   # Alternative health
curl http://localhost:8080/readyz    # Alternative readiness

{
  "status": "healthy",
  "timestamp": 1698765432.123,
  "uptime_seconds": 45.67,
  "service": "mcp-eval-server",
  "version": "0.1.0",
  "checks": {
    "server_running": true,
    "uptime_ok": true
  }
}

{
  "status": "ready",
  "timestamp": 1698765432.123,
  "service": "mcp-eval-server",
  "version": "0.1.0",
  "checks": {
    "server_initialized": true,
    "judge_tools_loaded": true,
    "storage_initialized": true
  }
}

# Build container
make build

# Run with environment
make run

# Or use docker-compose
make compose-up

# Install development dependencies
make dev-install

# Run development server
make dev

# Run tests
make test

# Check code quality
make lint

# Multi-criteria evaluation with MCP client
result = await mcp_client.call_tool("judge.evaluate_response", {
    "response": "Detailed technical explanation...",
    "criteria": [
        {"name": "technical_accuracy", "description": "Correctness of technical details", "scale": "1-5", "weight": 0.4},
        {"name": "clarity", "description": "Explanation clarity", "scale": "1-5", "weight": 0.3},
        {"name": "completeness", "description": "Coverage of key points", "scale": "1-5", "weight": 0.3}
    ],
    "rubric": {
        "criteria": [],
        "scale_description": {
            "1": "Severely lacking",
            "2": "Below expectations",
            "3": "Meets basic requirements",
            "4": "Exceeds expectations",
            "5": "Outstanding quality"
        }
    },
    "judge_model": "gpt-4",
    "use_cot": True
})

# Evaluate response via REST API
curl -X POST http://localhost:8080/judge/evaluate \
  -H "Content-Type: application/json" \
  -d '{
    "response": "Paris is the capital of France",
    "criteria": [
      {
        "name": "accuracy",
        "description": "Factual accuracy",
        "scale": "1-5",
        "weight": 1.0
      }
    ],
    "rubric": {
      "criteria": [],
      "scale_description": {
        "1": "Wrong",
        "5": "Correct"
      }
    },
    "judge_model": "gpt-4o-mini"
  }'

# Python REST API client
import httpx
import asyncio

async def evaluate_via_rest():
    async with httpx.AsyncClient() as client:
        response = await client.post("http://localhost:8080/judge/evaluate", json={
            "response": "Technical explanation...",
            "criteria": [
                {"name": "quality", "description": "Overall quality", "scale": "1-5", "weight": 1.0}
            ],
            "rubric": {
                "criteria": [],
                "scale_description": {"1": "Poor", "5": "Excellent"}
            },
            "judge_model": "gpt-4o-mini"
        })
        result = response.json()
        return result

# Run evaluation
result = asyncio.run(evaluate_via_rest())
print(f"Overall score: {result['overall_score']}")

# Head-to-head comparison with bias mitigation
comparison = await mcp_client.call_tool("judge.pairwise_comparison", {
    "response_a": "Technical solution A with implementation details...",
    "response_b": "Alternative solution B with different approach...",
    "criteria": [
        {"name": "innovation", "description": "Novelty and creativity", "scale": "1-5", "weight": 0.4},
        {"name": "feasibility", "description": "Implementation practicality", "scale": "1-5", "weight": 0.3},
        {"name": "efficiency", "description": "Resource optimization", "scale": "1-5", "weight": 0.3}
    ],
    "context": "Solutions for enterprise-scale data processing challenge",
    "position_bias_mitigation": True,
    "judge_model": "gpt-4-turbo"
})

# Full agent performance assessment
benchmark_result = await mcp_client.call_tool("agent.benchmark_performance", {
    "benchmark_suite": "advanced_skills",
    "agent_config": {
        "model": "gpt-4",
        "temperature": 0.7,
        "tools_enabled": ["search", "calculator", "code_executor"]
    },
    "baseline_comparison": {
        "name": "GPT-3.5 Baseline",
        "scores": {"accuracy": 0.75, "efficiency": 0.68, "reliability": 0.72}
    },
    "metrics_focus": ["accuracy", "efficiency", "reliability", "creativity"]
})

# Create sophisticated evaluation pipeline
suite = await mcp_client.call_tool("workflow.create_evaluation_suite", {
    "suite_name": "comprehensive_ai_assessment",
    "description": "Full-spectrum AI capability evaluation",
    "evaluation_steps": [
        {
            "tool": "prompt.evaluate_clarity",
            "weight": 0.15,
            "parameters": {"target_model": "gpt-4", "domain_context": "technical"}
        },
        {
            "tool": "judge.evaluate_response",
            "weight": 0.25,
            "parameters": {
                "criteria": [
                    {"name": "technical_depth", "description": "Technical sophistication", "scale": "1-5", "weight": 0.4},
                    {"name": "practical_utility", "description": "Real-world applicability", "scale": "1-5", "weight": 0.6}
                ],
                "judge_model": "gpt-4"
            }
        },
        {
            "tool": "quality.evaluate_factuality",
            "weight": 0.20
        },
        {
            "tool": "quality.measure_coherence",
            "weight": 0.15
        },
        {
            "tool": "quality.assess_toxicity",
            "weight": 0.10
        },
        {
            "tool": "agent.analyze_reasoning",
            "weight": 0.15,
            "parameters": {"judge_model": "gpt-4-turbo"}
        }
    ],
    "success_thresholds": {
        "overall": 0.85,
        "quality.evaluate_factuality": 0.90,
        "quality.assess_toxicity": 0.95
    },
    "weights": {
        "accuracy": 0.4,
        "safety": 0.3,
        "utility": 0.3
    }
})

# Execute comprehensive evaluation
results = await mcp_client.call_tool("workflow.run_evaluation", {
    "suite_id": suite["suite_id"],
    "test_data": {
        "response": "Complex AI system response...",
        "context": "Enterprise deployment scenario...",
        "reasoning_trace": [...],
        "agent_trace": {...}
    },
    "parallel_execution": True,
    "max_concurrent": 5
})

# Use custom model configuration
export MCP_EVAL_MODELS_CONFIG="./my-custom-models.yaml"
export DEFAULT_JUDGE_MODEL="my-custom-judge"

# Copy default config for customization
make copy-config                    # Copies to ./custom-config/
make show-config                    # Show current configuration status
make validate-config                # Validate configuration syntax

models:
  azure:
    my-enterprise-gpt4:
      provider: "azure"
      deployment_name: "my-gpt4-deployment"
      model_name: "gpt-4"
      api_base_env: "AZURE_OPENAI_ENDPOINT"
      api_key_env: "AZURE_OPENAI_API_KEY"
      api_version_env: "AZURE_OPENAI_API_VERSION"
      deployment_name_env: "AZURE_DEPLOYMENT_NAME"
      default_temperature: 0.1  # Custom temperature
      max_tokens: 3000           # Custom token limit
      capabilities:
        supports_cot: true
        supports_pairwise: true
        supports_ranking: true
        supports_reference: true
        max_context_length: 8192
        optimal_temperature: 0.1
        consistency_level: "very_high"
      metadata:
        purpose: "production_evaluation"
        cost_tier: "premium"

  ollama:
    my-local-llama:
      provider: "ollama"
      model_name: "llama3:70b"
      base_url_env: "OLLAMA_BASE_URL"
      default_temperature: 0.3
      max_tokens: 2000
      request_timeout: 120  # Longer timeout for large models

# Custom defaults
defaults:
  primary_judge: "my-enterprise-gpt4"
  fallback_judge: "my-local-llama"

# Custom recommendations
recommendations:
  production: ["my-enterprise-gpt4"]
  development: ["my-local-llama"]

rubrics:
  technical_excellence:
    name: "Technical Excellence Assessment"
    criteria:
      - name: "code_quality"
        description: "Code structure, efficiency, and best practices"
        scale: "1-10"
        weight: 0.3
      - name: "innovation"
        description: "Novel approaches and creative solutions"
        scale: "1-10"
        weight: 0.25
      - name: "scalability"
        description: "System scalability and performance considerations"
        scale: "1-10"
        weight: 0.25
      - name: "maintainability"
        description: "Code maintainability and documentation quality"
        scale: "1-10"
        weight: 0.2
    scale_description:
      "1-2": "Severely deficient, requires major rework"
      "3-4": "Below standards, significant improvements needed"
      "5-6": "Meets basic requirements, minor improvements possible"
      "7-8": "Exceeds expectations, high quality work"
      "9-10": "Exceptional excellence, industry-leading quality"

benchmarks:
  enterprise_readiness:
    name: "Enterprise Readiness Assessment"
    category: "production"
    tasks:
      - name: "security_analysis"
        description: "Security vulnerability assessment and mitigation"
        difficulty: "advanced"
        expected_tools: ["security_scanner", "vulnerability_analyzer", "mitigation_planner"]
        evaluation_metrics: ["threat_identification", "risk_assessment", "solution_quality"]
      - name: "performance_optimization"
        description: "System performance analysis and optimization"
        difficulty: "advanced"
        expected_tools: ["profiler", "optimizer", "benchmarker"]
        evaluation_metrics: ["performance_gain", "resource_efficiency", "scalability_impact"]

🏆 MCP EVALUATION SERVER - 63 SPECIALIZED TOOLS 🏆
═══════════════════════════════════════════════════════════

📊 CORE EVALUATION SUITE (15 tools)
├── 🤖 Judge Tools (4) ────── LLM-as-a-judge evaluation
├── 📝 Prompt Tools (4) ───── Clarity, consistency, optimization
├── 🛠️ Agent Tools (4) ────── Performance, reasoning, benchmarking
└── 🔍 Quality Tools (3) ──── Factuality, coherence, toxicity

🔬 ADVANCED ASSESSMENT SUITE (39 tools)
├── 🔗 RAG Tools (8) ──────── Retrieval relevance, grounding, citations
├── ⚖️ Bias & Fairness (6) ── Demographic bias, intersectional analysis
├── 🛡️ Robustness (5) ──────── Adversarial testing, injection resistance
├── 🔒 Safety & Alignment (4) Harmful content, value alignment
├── 🌍 Multilingual (4) ────── Translation, cultural adaptation
├── ⚡ Performance (4) ──────── Latency, efficiency, scaling
└── 🔐 Privacy (8) ───────── PII detection, compliance, anonymization

🔧 SYSTEM MANAGEMENT (9 tools)
├── 🔄 Workflow Tools (3) ─── Evaluation suites, parallel execution
├── 📊 Calibration (2) ────── Judge agreement, rubric optimization
└── 🏥 Server Tools (4) ───── Health monitoring, system management

🎯 TOTAL: 63 TOOLS ACROSS 14 CATEGORIES 🎯

# 1. Run MCP server (for Claude Desktop, etc.)
make dev                    # Shows connection info + starts server

# 2. Test basic functionality
make example               # Run evaluation example
make test-mcp             # Test MCP protocol

# 1. Run native REST API server
make serve-rest          # Starts on http://localhost:8080

# 2. Test REST API endpoints
make test-rest           # Test all REST endpoints

# 3. View interactive documentation
open http://localhost:8080/docs    # Swagger UI
open http://localhost:8080/redoc   # ReDoc

# 4. Get connection info
make rest-info           # Show complete REST API guide

# 1. Run MCP protocol over HTTP
make serve-http          # Starts on http://localhost:9000

# 2. Test HTTP endpoints
make test-http           # Test MCP JSON-RPC endpoints

# 3. Get connection info
make http-info           # Show complete HTTP bridge guide

# Build and deploy
make build && make run

# Basic MCP integration
from mcp import Client
client = Client("mcp-eval-server")

# Evaluate any AI output
result = await client.call_tool("judge.evaluate_response", {
    "response": "Your AI output here",
    "criteria": [{"name": "quality", "description": "Overall quality", "scale": "1-5", "weight": 1.0}],
    "rubric": {"criteria": [], "scale_description": {"1": "Poor", "5": "Excellent"}}
})

# Start REST API server
make serve-rest

# Check server health
curl http://localhost:8080/health

# List tool categories
curl http://localhost:8080/tools/categories

# Evaluate response directly via REST
curl -X POST http://localhost:8080/judge/evaluate \
  -H "Content-Type: application/json" \
  -d '{
    "response": "Paris is the capital of France.",
    "criteria": [
      {
        "name": "accuracy",
        "description": "Factual accuracy",
        "scale": "1-5",
        "weight": 1.0
      }
    ],
    "rubric": {
      "criteria": [],
      "scale_description": {"1": "Wrong", "5": "Correct"}
    },
    "judge_model": "rule-based"
  }'

# Start HTTP bridge server
make serve-http

# List available tools (JSON-RPC)
curl -X POST \
     -H "Content-Type: application/json" \
     -d '{"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}' \
     http://localhost:9000/

# Evaluate response via HTTP bridge (JSON-RPC)
curl -X POST \
     -H "Content-Type: application/json" \
     -d '{
       "jsonrpc": "2.0",
       "id": 2,
       "method": "tools/call",
       "params": {
         "name": "judge.evaluate_response",
         "arguments": {
           "response": "Paris is the capital of France.",
           "criteria": [{"name": "accuracy", "description": "Factual accuracy", "scale": "1-5", "weight": 1.0}],
           "rubric": {"criteria": [], "scale_description": {"1": "Wrong", "5": "Correct"}},
           "judge_model": "rule-based"
         }
       }
     }' \
     http://localhost:9000/

import httpx
import asyncio

async def evaluate_via_rest_api():
    """Example using native REST API endpoints."""
    async with httpx.AsyncClient() as client:
        base_url = "http://localhost:8080"

        # Check health
        health = await client.get(f"{base_url}/health")
        print(f"Server status: {health.json()['status']}")

        # List tool categories
        categories = await client.get(f"{base_url}/tools/categories")
        print(f"Available categories: {len(categories.json()['categories'])}")

        # Evaluate response using REST endpoint
        evaluation = await client.post(f"{base_url}/judge/evaluate", json={
            "response": "Your AI response here",
            "criteria": [
                {"name": "quality", "description": "Overall quality", "scale": "1-5", "weight": 1.0}
            ],
            "rubric": {
                "criteria": [],
                "scale_description": {"1": "Poor", "5": "Excellent"}
            },
            "judge_model": "rule-based"
        })
        result = evaluation.json()
        print(f"Evaluation score: {result['overall_score']}")

        # Check content toxicity
        toxicity = await client.post(f"{base_url}/quality/toxicity", json={
            "content": "This is a test message",
            "toxicity_categories": ["profanity", "hate_speech"],
            "sensitivity_level": "moderate",
            "judge_model": "rule-based"
        })
        result = toxicity.json()
        print(f"Toxicity detected: {result['toxicity_detected']}")

# Run evaluation
asyncio.run(evaluate_via_rest_api())

import httpx
import asyncio

async def evaluate_via_http_bridge():
    """Example using MCP over HTTP bridge."""
    async with httpx.AsyncClient() as client:
        base_url = "http://localhost:9000"

        # List tools via JSON-RPC
        tools_request = {
            "jsonrpc": "2.0",
            "id": 1,
            "method": "tools/list",
            "params": {}
        }

        response = await client.post(base_url, json=tools_request)
        result = response.json()
        tools = result.get("result", [])
        print(f"Available tools: {len(tools)}")

        # Evaluate response via JSON-RPC
        eval_request = {
            "jsonrpc": "2.0",
            "id": 2,
            "method": "tools/call",
            "params": {
                "name": "judge.evaluate_response",
                "arguments": {
                    "response": "Your AI response here",
                    "criteria": [{"name": "quality", "description": "Overall quality", "scale": "1-5", "weight": 1.0}],
                    "rubric": {"criteria": [], "scale_description": {"1": "Poor", "5": "Excellent"}},
                    "judge_model": "rule-based"
                }
            }
        }

        response = await client.post(base_url, json=eval_request)
        result = response.json()
        print(f"Evaluation result: {result}")

# Run evaluation
asyncio.run(evaluate_via_http_bridge())