网页抓取MCP服务器

{
  "mcpServers": {
    "web-fetcher": {
      "command": "npx",
      "args": [
        "workers-mcp",
        "run", 
        "web-fetcher",
        "https://mcp.llmbase.ai/mcp/web-fetch"
      ]
    }
  }
}

npm install @llmbase/mcp-web-fetch

{
  "mcpServers": {
    "web-fetcher": {
      "command": "npx",
      "args": ["@llmbase/mcp-web-fetch"]
    }
  }
}

git clone https://github.com/llmbaseai/mcp-servers
cd mcp-servers/templates

# Copy template files
cp package.example.json ../my-mcp-project/package.json
cp wrangler.example.jsonc ../my-mcp-project/wrangler.jsonc
cp index.example.ts ../my-mcp-project/src/index.ts
cp tsconfig.example.json ../my-mcp-project/tsconfig.json

cd ../my-mcp-project
npm install

npx wrangler login
# Edit wrangler.jsonc with your settings
npm run deploy

{
  "mcpServers": {
    "web-fetcher": {
      "command": "npx",
      "args": [
        "workers-mcp", 
        "run", 
        "web-fetcher",
        "https://your-worker.your-subdomain.workers.dev/mcp/web-fetch"
      ]
    }
  }
}

{
  "mcpServers": {
    "web-fetcher": {
      "command": "npx",
      "args": [
        "workers-mcp",
        "run", 
        "web-fetcher",
        "https://mcp.llmbase.ai/mcp/web-fetch"
      ]
    }
  }
}

# Fetch single website
curl -X POST https://mcp.llmbase.ai/api/fetch \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com", "format": "markdown"}'

# Batch processing with streaming
curl -X POST https://mcp.llmbase.ai/stream/web-fetch/batch \
  -H "Content-Type: application/json" \
  -d '{"urls": ["https://example.com", "https://github.com"]}' \
  --no-buffer

# Clone repository
git clone https://github.com/llmbaseai/mcp-servers
cd mcp-servers

# Install dependencies
bun install

# Start development server
bun run dev

# Build for production
bun run build

# Deploy to Cloudflare
bun run deploy

src/
├── index.ts                    # Worker entry point
├── router.ts                   # Hono.js routing
├── types.ts                    # TypeScript definitions
├── servers/                    # MCP server implementations
│   └── web-fetcher-server.ts
├── services/                   # Business logic
│   ├── web-fetcher.ts
│   └── sse-service.ts
└── utils/                      # Utility functions
    └── html-processor.ts

// src/servers/my-server.ts
import { WorkerEntrypoint } from 'cloudflare:workers';
import type { Env } from '../types';

export class MyMCPServer extends WorkerEntrypoint<Env> {
  /**
   * Description of what this method does
   * @param param1 Parameter description
   * @returns What it returns
   */
  async myTool(param1: string) {
    return { result: `Hello ${param1}` };
  }
}

// src/router.ts
app.all('/mcp/my-server/*', async (c) => {
  const server = new MyMCPServer(c.executionCtx, c.env);
  const proxy = new ProxyToSelf(server);
  return proxy.fetch(c.req.raw);
});

// Add to servers array in router.ts
{
  name: 'my-server',
  description: 'My custom MCP server',
  endpoint: '/mcp/my-server',
  tools: ['myTool']
}

{
  "success": true,
  "data": {
    "content": "Website content...",
    "title": "Page Title",
    "url": "https://example.com",
    "contentType": "text/html",
    "statusCode": 200
  }
}

{
  "success": false,
  "error": "Error description",
  "url": "https://example.com"
}

data: {"type": "start", "totalUrls": 5}

data: {"type": "result", "url": "...", "success": true, "data": {...}}

data: {"type": "complete", "totalCompleted": 5}

{
  "vars": {
    "ENVIRONMENT": "production"
  }
}

{
  "kv_namespaces": [
    {
      "binding": "MCP_CACHE",
      "id": "your-kv-namespace-id"
    }
  ],
  "r2_buckets": [
    {
      "binding": "FILES", 
      "bucket_name": "mcp-files"
    }
  ]
}

// Format options
"raw"      // Original HTML
"markdown" // Clean Markdown (recommended)
"text"     // Plain text only

# Login to Cloudflare
npx wrangler login

# Deploy to production
bun run deploy

# Deploy with custom domain
# Configure DNS: CNAME mcp.llmbase.ai → your-worker.workers.dev

{
  "routes": [
    {
      "pattern": "your-domain.com/*",
      "custom_domain": true
    }
  ]
}

# Health check
curl https://mcp.llmbase.ai/

# Test web fetching
curl -X POST https://mcp.llmbase.ai/api/fetch \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com"}'