AI代理Web交互工具

npm install spectrawl

npm install spectrawl
export GEMINI_API_KEY=your-free-key  # Get one at aistudio.google.com

const { Spectrawl } = require('spectrawl')
const web = new Spectrawl()

// Deep search — returns sources for your agent/LLM to process
const result = await web.deepSearch('how to build an MCP server in Node.js')
console.log(result.sources)   // [{ title, url, content, score }]

// With AI summary (opt-in — uses extra Gemini call)
const withAnswer = await web.deepSearch('query', { summarize: true })
console.log(withAnswer.answer)  // AI-generated answer with [1] [2] citations

// Fast mode — snippets only, skip scraping
const fast = await web.deepSearch('query', { mode: 'fast' })

// Basic search — raw results
const basic = await web.search('query')

const results = await web.search('query')

const results = await web.deepSearch('query', { summarize: true })

Query → Gemini Grounded + DDG (parallel)
  → Merge & deduplicate (12-19 results)
  → Source quality ranking (boost GitHub/SO/Reddit, penalize SEO spam)
  → Parallel scraping (Jina → readability → Playwright fallback)
  → Returns sources to your agent (AI summary opt-in with summarize: true)

const page = await web.browse('https://example.com', {
  screenshot: true,    // Take a PNG screenshot
  fullPage: true,      // Full page screenshot (not just viewport)
  html: true,          // Return raw HTML alongside markdown
  stealth: true,       // Force stealth mode
  camoufox: true,      // Force Camoufox engine
  noCache: true,       // Bypass cache
  auth: 'reddit'       // Use stored auth cookies for this platform
})

{
  content: "# Page Title\n\nExtracted markdown content...",
  url: "https://example.com",
  title: "Page Title",
  statusCode: 200,
  cached: false,
  engine: "camoufox",            // which engine was used
  screenshot: Buffer<png>,        // PNG buffer (JS) or base64 (HTTP)
  html: "<html>...</html>",       // raw HTML (if html: true)
  blocked: false,                 // true if block page detected
  blockInfo: null                 // { type: 'cloudflare', detail: '...' }
}

# Bright Data (recommended)
npx spectrawl config set proxy '{"host":"brd.superproxy.io","port":22225,"username":"YOUR_ZONE_USER","password":"YOUR_PASS"}'

# Smartproxy (budget alternative)
npx spectrawl config set proxy '{"host":"gate.smartproxy.com","port":10001,"username":"YOUR_USER","password":"YOUR_PASS"}'

# Store your LinkedIn cookies (export from browser)
npx spectrawl login linkedin --account yourname --cookies ./linkedin-cookies.json

# Now browse LinkedIn normally
curl localhost:3900/browse -d '{"url":"https://www.linkedin.com/in/someone"}'

const result = await web.extract('https://news.ycombinator.com', {
  instruction: 'Extract the top 3 story titles and their point counts',
  schema: {
    type: 'object',
    properties: {
      stories: {
        type: 'array',
        items: {
          type: 'object',
          properties: {
            title: { type: 'string' },
            points: { type: 'number' }
          }
        }
      }
    }
  }
})
// result.data = { stories: [{ title: "...", points: 210 }, ...] }

curl -X POST http://localhost:3900/extract \
  -H 'Content-Type: application/json' \
  -d '{
    "url": "https://example.com",
    "instruction": "Extract the page title and main heading",
    "schema": {"type": "object", "properties": {"title": {"type": "string"}, "heading": {"type": "string"}}}
  }'

{
  "data": { "title": "Example Domain", "heading": "Example Domain" },
  "url": "https://example.com",
  "title": "Example Domain",
  "contentLength": 129,
  "duration": 679
}

const result = await web.extract('https://news.ycombinator.com', {
  instruction: 'Extract all story titles',
  selector: '.titleline',  // CSS selector
  // or: selector: 'xpath=//table[@class="itemlist"]'
  schema: { type: 'object', properties: { titles: { type: 'array', items: { type: 'string' } } } }
})

const result = await web.extract('https://en.wikipedia.org/wiki/Node.js', {
  instruction: 'Extract the creator and release date',
  relevanceFilter: true   // BM25 scoring keeps only relevant sections
})
// Content reduced from 50K+ chars to ~2K relevant chars

const result = await web.extractFromContent(markdownContent, {
  instruction: 'Extract all email addresses',
  schema: { type: 'object', properties: { emails: { type: 'array', items: { type: 'string' } } } }
})

const result = await web.agent('https://example.com', 'click the More Information link', {
  maxSteps: 5,       // max actions to take
  screenshot: true   // screenshot after completion
})
// result.success = true
// result.url = "https://www.iana.org/domains/reserved" (navigated!)
// result.steps = [{ step: 1, action: "click", elementIdx: 0, result: "clicked" }, ...]

curl -X POST http://localhost:3900/agent \
  -H 'Content-Type: application/json' \
  -d '{"url": "https://example.com", "instruction": "click the More Information link", "maxSteps": 3}'

{
  "success": true,
  "url": "https://www.iana.org/domains/reserved",
  "title": "IANA — Reserved Domains",
  "steps": [
    { "step": 1, "action": "click", "elementIdx": 0, "reason": "clicking the More Information link", "result": "clicked" }
  ],
  "content": "...",
  "duration": 5200
}

const result = await web.browse('https://example.com', {
  captureNetwork: true,
  captureNetworkHeaders: true,  // include request headers
  captureNetworkBody: true      // include response bodies (<50KB)
})
// result.networkRequests = [
//   { url: "https://api.example.com/data", method: "GET", status: 200, contentType: "application/json", body: "..." }
// ]

curl -X POST http://localhost:3900/browse \
  -d '{"url": "https://example.com", "captureNetwork": true, "captureNetworkBody": true}'

const result = await web.browse('https://example.com', {
  screenshot: true,
  fullPage: true       // optional: capture entire page, not just viewport
})
// result.screenshot is a PNG Buffer
fs.writeFileSync('screenshot.png', result.screenshot)

curl -X POST http://localhost:3900/browse \
  -H 'Content-Type: application/json' \
  -d '{"url": "https://example.com", "screenshot": true, "fullPage": true}'

{
  "content": "# Page Title\n\nExtracted markdown...",
  "url": "https://example.com",
  "title": "Page Title",
  "screenshot": "iVBORw0KGgo...base64-encoded-png...",
  "cached": false
}

const result = await web.crawl('https://docs.example.com', {
  depth: 2,              // how many link levels to follow
  maxPages: 50,          // stop after N pages
  format: 'markdown',    // 'markdown' or 'html'
  scope: 'domain',       // 'domain' | 'subdomain' | 'path'
  concurrency: 'auto',   // auto-detect from available RAM, or set a number
  merge: true,           // merge all pages into one document
  includePatterns: [],   // regex patterns to include
  excludePatterns: [],   // regex patterns to skip
  delay: 300,            // ms between batch launches (politeness)
  stealth: true          // use anti-detect browsing
})

{
  pages: [
    { url: 'https://docs.example.com/', content: '...', title: '...', statusCode: 200 },
    { url: 'https://docs.example.com/guide', content: '...', title: '...', statusCode: 200 },
    // ...
  ],
  stats: {
    pagesScraped: 23,
    duration: 45000,
    concurrency: 4
  }
}

const result = await web.crawl('https://docs.example.com', {
  useSitemap: true,  // enabled by default
  maxPages: 20
})
// [crawl] Found sitemap at https://docs.example.com/sitemap.xml with 82 URLs
// [crawl] Pre-seeded 20 URLs from sitemap

curl -X POST http://localhost:3900/crawl \
  -d '{"url": "https://docs.example.com", "maxPages": 50, "webhook": "https://your-server.com/webhook"}'

# Start a crawl job (returns immediately)
curl -X POST http://localhost:3900/crawl \
  -d '{"url": "https://docs.example.com", "depth": 3, "maxPages": 100, "async": true}'
# Response: { "jobId": "abc123", "status": "running" }

# Check job status
curl http://localhost:3900/crawl/abc123

# List all jobs
curl http://localhost:3900/crawl/jobs

# Check system capacity
curl http://localhost:3900/crawl/capacity

// Add account
await web.auth.add('x', { account: '@myhandle', method: 'cookie', cookies })

// Check health
const accounts = await web.auth.getStatus()
// [{ platform: 'x', account: '@myhandle', status: 'valid', expiresAt: '...' }]

const { EVENTS } = require('spectrawl')

web.on(EVENTS.COOKIE_EXPIRING, (data) => {
  console.log(`Cookie expiring for ${data.platform}:${data.account}`)
})

web.on(EVENTS.RATE_LIMITED, (data) => {
  console.log(`Rate limited on ${data.platform}`)
})

// Wildcard — catch everything
web.on('*', ({ event, ...data }) => {
  console.log(`Event: ${event}`, data)
})

await web.act('github', 'create-issue', { repo: 'user/repo', title: 'Bug report', body: '...' })
await web.act('reddit', 'post', { subreddit: 'node', title: '...', text: '...' })
await web.act('devto', 'post', { title: '...', body: '...', tags: ['ai'] })
await web.act('huggingface', 'create-repo', { name: 'my-model', type: 'model' })

const web = new Spectrawl({
  sourceRanker: {
    boost: ['github.com', 'news.ycombinator.com'],
    block: ['spamsite.com']
  }
})

npx spectrawl serve --port 3900

curl -X POST http://localhost:3900/search \
  -H 'Content-Type: application/json' \
  -d '{"query": "best headless browsers 2026", "summarize": true}'

{
  "sources": [
    {
      "title": "Top Headless Browsers in 2026",
      "url": "https://example.com/article",
      "snippet": "Short snippet from search...",
      "content": "Full page markdown content (if scraped)...",
      "source": "gemini-grounded",
      "confidence": 0.95
    }
  ],
  "answer": "AI-generated summary with [1] citations... (only if summarize: true)",
  "cached": false
}

curl -X POST http://localhost:3900/browse \
  -H 'Content-Type: application/json' \
  -d '{"url": "https://example.com", "screenshot": true, "fullPage": true}'

{
  "content": "# Example Domain\n\nThis domain is for use in illustrative examples...",
  "url": "https://example.com",
  "title": "Example Domain",
  "statusCode": 200,
  "screenshot": "iVBORw0KGgoAAAANSUhEUg...base64...",
  "cached": false,
  "engine": "playwright"
}

curl -X POST http://localhost:3900/crawl \
  -H 'Content-Type: application/json' \
  -d '{"url": "https://docs.example.com", "depth": 2, "maxPages": 10}'

{
  "pages": [
    {
      "url": "https://docs.example.com/",
      "content": "# Docs Home\n\n...",
      "title": "Documentation",
      "statusCode": 200
    }
  ],
  "stats": {
    "pagesScraped": 8,
    "duration": 12000,
    "concurrency": 4
  }
}

curl -X POST http://localhost:3900/act \
  -H 'Content-Type: application/json' \
  -d '{"platform": "github", "action": "create-issue", "repo": "user/repo", "title": "Bug", "body": "Details..."}'

{
  "type": "https://spectrawl.dev/errors/rate-limited",
  "status": 429,
  "title": "rate limited",
  "detail": "Reddit rate limit: max 3 posts per hour",
  "retryable": true
}

{
  "browse": {
    "proxy": {
      "host": "gate.smartproxy.com",
      "port": 10001,
      "username": "YOUR_USER",
      "password": "YOUR_PASS"
    }
  }
}

npx spectrawl proxy --port 8080

npx spectrawl mcp

npx spectrawl init              # create spectrawl.json
npx spectrawl search "query"    # search from terminal
npx spectrawl status            # check auth health
npx spectrawl serve             # start HTTP server
npx spectrawl mcp               # start MCP server
npx spectrawl proxy             # start rotating proxy server
npx spectrawl install-stealth   # download Camoufox browser
npx spectrawl version           # show version

{
  "port": 3900,
  "concurrency": 3,
  "search": {
    "cascade": ["gemini-grounded", "tavily", "brave"],
    "scrapeTop": 5
  },
  "browse": {
    "defaultEngine": "playwright",
    "proxy": null,
    "humanlike": {
      "minDelay": 500,
      "maxDelay": 2000,
      "scrollBehavior": true
    }
  },
  "auth": {
    "refreshInterval": "4h",
    "cookieStore": "./data/cookies.db"
  },
  "cache": {
    "path": "./data/cache.db",
    "searchTtl": 3600,
    "scrapeTtl": 86400,
    "screenshotTtl": 3600
  },
  "rateLimit": {
    "x": { "postsPerHour": 5, "minDelayMs": 30000 },
    "reddit": { "postsPerHour": 3, "minDelayMs": 600000 }
  }
}

GEMINI_API_KEY      Free — primary search + summarization (aistudio.google.com)
BRAVE_API_KEY       Brave Search (2,000 free/month)
TAVILY_API_KEY      Tavily Search (1,000 free/month)
SERPER_API_KEY      Serper.dev (2,500 trial queries)
GITHUB_TOKEN        For GitHub adapter
DEVTO_API_KEY       For Dev.to adapter
HF_TOKEN            For HuggingFace adapter
OPENAI_API_KEY      Alternative LLM for summarization
ANTHROPIC_API_KEY   Alternative LLM for summarization