文档搜索索引平台

┌─────────────────────────────────────────────────────────────────┐
│                         Docs Hound                               │
├─────────────────────────────────────────────────────────────────┤
│                                                                  │
│  ┌─────────────┐    ┌─────────────┐    ┌─────────────┐          │
│  │   Web UI    │    │ MCP Server  │    │  Chat API   │          │
│  │  (Next.js)  │    │   (stdio)   │    │ (streaming) │          │
│  └──────┬──────┘    └──────┬──────┘    └──────┬──────┘          │
│         │                  │                  │                  │
│         └──────────────────┼──────────────────┘                  │
│                            │                                     │
│                   ┌────────┴────────┐                            │
│                   │   Search Tool   │                            │
│                   └────────┬────────┘                            │
│                            │                                     │
│         ┌──────────────────┼──────────────────┐                  │
│         │                  │                  │                  │
│  ┌──────┴──────┐    ┌──────┴──────┐    ┌──────┴──────┐          │
│  │   Crawler   │    │  Upstash    │    │  Upstash    │          │
│  │  (scraper)  │    │   Vector    │    │   Redis     │          │
│  └─────────────┘    └─────────────┘    └─────────────┘          │
│                                                                  │
└─────────────────────────────────────────────────────────────────┘

User adds URL → Discovery Crawl → URL Review → Full Index → Searchable
     │                │               │              │            │
     ▼                ▼               ▼              ▼            ▼
  pending →      discovering →   discovered →   indexing →    indexed

docs-hound/
├── apps/
│   └── web/                        # Next.js 16 + React 19 web application
│       ├── src/
│       │   ├── app/
│       │   │   ├── page.tsx        # Dashboard - list all sites
│       │   │   ├── chat/           # Chat interface
│       │   │   ├── settings/       # MCP configuration
│       │   │   ├── sites/
│       │   │   │   ├── new/        # Add new site form
│       │   │   │   └── [domain]/   # Site detail with workflow
│       │   │   └── api/
│       │   │       ├── chat/       # Streaming chat endpoint
│       │   │       ├── discover/   # Start preview crawl
│       │   │       ├── index/      # Start full indexing
│       │   │       └── sites/      # Site CRUD operations
│       │   └── components/
│       │       ├── ChatInterface.tsx
│       │       ├── DeleteSiteButton.tsx
│       │       ├── DiscoveredUrlsList.tsx
│       │       ├── IndexedPagesList.tsx
│       │       └── SiteStatusPoller.tsx
│       └── package.json
│
├── packages/
│   ├── agents/
│   │   └── chat/                   # Chat agent with tool integration
│   │       └── src/index.ts        # runChatAgent() with searchDocsTool
│   │
│   ├── doc-scraper/                # Web crawling and content extraction
│   │   └── src/
│   │       ├── lib/
│   │       │   ├── crawler.ts      # URL discovery with depth limiting
│   │       │   └── smart-scraper.ts # Content extraction (Readability + Turndown)
│   │       └── storage/
│   │           └── upstash.ts      # Vector storage with embeddings
│   │
│   ├── shared/
│   │   └── db/                     # Site registry (Upstash Redis)
│   │       └── src/
│   │           ├── redis-client.ts # SiteRegistry class
│   │           └── types.ts        # Site, SiteMetadata types
│   │
│   └── tools/
│       └── docs-search/            # Semantic search tool
│           └── src/
│               ├── index.ts        # searchDocsTool definition
│               └── lib/
│                   └── search-client.ts # Upstash Vector query
│
├── mcp-server/                     # MCP server for Cursor integration
│   └── src/index.ts                # search_docs, list_sources, get_source_info
│
├── turbo.json                      # Turborepo task configuration
├── pnpm-workspace.yaml             # pnpm workspace definition
└── package.json                    # Root scripts and dependencies

# Clone the repository
git clone <repository-url>
cd docs-hound

# Install dependencies
pnpm install

# Set up environment variables (choose one method):

# Method 1: Pull from Vercel (if you have a linked Vercel project)
pnpm env:pull

# Method 2: Copy from template and edit manually
cp env.template apps/web/.env.local
# Edit apps/web/.env.local with your credentials

# Start development server
pnpm dev

# OpenAI - Required for embeddings and chat
OPENAI_API_KEY=sk-...

# Upstash Vector - Single index for all documentation
UPSTASH_VECTOR_REST_URL=https://...upstash.io
UPSTASH_VECTOR_REST_TOKEN=...

# Upstash Redis - Site registry and metadata
# For local dev, use UPSTASH_REDIS_* naming
# Vercel uses KV_REST_API_URL and KV_REST_API_TOKEN (code supports both)
UPSTASH_REDIS_REST_URL=https://...upstash.io
UPSTASH_REDIS_REST_TOKEN=...

# QStash - Optional, for background job processing
QSTASH_TOKEN=...

# Security - Optional
CRON_SECRET=...
UI_PASSWORD=...
MCP_API_KEY=...

# From the project root
pnpm env:pull

{
  "mcpServers": {
    "docs-hound-local": {
      "command": "npx",
      "args": ["tsx", "/absolute/path/to/docs-hound/mcp-server/src/index.ts"],
      "cwd": "/absolute/path/to/docs-hound",
      "env": {
        "OPENAI_API_KEY": "your-key",
        "KV_REST_API_URL": "your-redis-url",
        "KV_REST_API_TOKEN": "your-redis-token",
        "UPSTASH_VECTOR_REST_URL": "your-vector-url",
        "UPSTASH_VECTOR_REST_TOKEN": "your-vector-token"
      }
    }
  }
}

openssl rand -base64 32

{
  "mcpServers": {
    "docs-hound-hosted": {
      "url": "https://your-app.vercel.app/api/mcp",
      "headers": {
        "Authorization": "Bearer your-mcp-api-key-here"
      }
    }
  }
}

GET    /api/sites              # List all sites
POST   /api/sites              # Add new site
GET    /api/sites/[domain]     # Get site details
DELETE /api/sites/[domain]     # Delete site and its data

POST   /api/discover           # Start discovery crawl
       Body: { domain: string }

POST   /api/index              # Start full indexing
       Body: { domain: string }

POST   /api/chat               # Streaming chat
       Body: {
         messages: Array<{role: 'user'|'assistant', content: string}>,
         selectedDomain?: string
       }

POST   /api/mcp                # Hosted MCP server endpoint
       Headers: Authorization: Bearer <MCP_API_KEY>
       Body: JSON-RPC message (standard MCP protocol)

import {
  getSiteRegistry,
  type Site,
  type SiteMetadata,
} from '@docs-hound/shared-db'

const registry = getSiteRegistry()

// Site operations
await registry.addSite(domain, metadata)
await registry.getSite(domain)
await registry.updateSite(domain, updates)
await registry.deleteSite(domain)
await registry.listSites()

// Discovery operations
await registry.setDiscoveredUrls(domain, urls)
await registry.getDiscoveredUrls(domain)

// Indexing operations
await registry.addIndexedPage(domain, url)
await registry.getIndexedPages(domain)

import {
  Crawler,
  scrapePageToMarkdown,
  UpstashDocStorage,
} from '@docs-hound/doc-scraper'

// Discover URLs
const crawler = new Crawler({ maxPages: 1000, maxDepth: 2 })
const urls = await crawler.discoverUrls('https://docs.example.com')

// Scrape content
const { markdown, title } = await scrapePageToMarkdown(url)

// Store in vector database
const storage = new UpstashDocStorage()
await storage.upsertDocument({
  url,
  content: markdown,
  title,
  source: 'docs.example.com',
})

import { searchDocsTool, searchUpstash } from '@docs-hound/tool-docs-search'

// Direct search
const results = await searchUpstash({
  query: 'authentication setup',
  source: 'docs.example.com', // optional
  topK: 5,
})

// As AI SDK tool
import { generateText } from 'ai'
const result = await generateText({
  model: openai('gpt-4o'),
  tools: { searchDocs: searchDocsTool },
  prompt: 'Find information about authentication',
})

import { runChatAgent, type ChatMessage } from '@docs-hound/agent-chat'

const response = await runChatAgent({
  messages: [{ role: 'user', content: 'How do I authenticate?' }],
  source: 'docs.example.com', // optional filter
})
// Returns a streaming Response

pnpm typecheck && pnpm lint && pnpm format:check

feat(web): add site deletion confirmation dialog
fix(scraper): handle timeout errors gracefully
docs: update MCP configuration instructions
chore(deps): update ai-sdk to v6.0.4

DEBUG=docs-hound:*