网页数据采集工具包

export BRIGHTDATA_API_KEY="your_api_key_here"

# Zone names — find yours at brightdata.com/cp/zones
export BRIGHTDATA_UNLOCKER_ZONE="web_unlocker1"     # Web Unlocker zone
export BRIGHTDATA_SERP_ZONE="serp_api1"              # SERP API zone

import requests

response = requests.post(
    "https://api.brightdata.com/request",
    headers={
        "Authorization": f"Bearer {YOUR_API_KEY}",
        "Content-Type": "application/json",
    },
    json={
        "zone": "web_unlocker1",
        "url": "https://www.example.com",
        "format": "raw",
    },
)
print(response.text)  # HTML of the page

const response = await fetch("https://api.brightdata.com/request", {
  method: "POST",
  headers: {
    Authorization: `Bearer ${YOUR_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    zone: "web_unlocker1",
    url: "https://www.example.com",
    format: "raw",
  }),
});
console.log(await response.text()); // HTML of the page

curl -X POST "https://api.brightdata.com/request" \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"zone": "web_unlocker1", "url": "https://www.example.com", "format": "raw"}'

https://mcp.brightdata.com/mcp?token=YOUR_API_TOKEN

{
  "mcpServers": {
    "Bright Data": {
      "command": "npx",
      "args": ["@brightdata/mcp"],
      "env": {
        "API_TOKEN": "YOUR_API_TOKEN"
      }
    }
  }
}

from llama_index.tools.brightdata import BrightDataToolSpec
from llama_index.llms.openai import OpenAI
from llama_index.core.agent.workflow import FunctionAgent

agent = FunctionAgent(
    tools=BrightDataToolSpec(api_key=API_KEY, zone="mcp_unlocker").to_tool_list(),
    llm=OpenAI(model="gpt-4o-mini"),
)
response = await agent.run("Find the top AI trends this week and summarize them")

cd examples/python
pip install -r requirements.txt    # Installs all deps (including MCP agent)
export BRIGHTDATA_API_KEY="your_key"
export BRIGHTDATA_UNLOCKER_ZONE="web_unlocker1"
python 04_web_unlocker.py          # Try the simplest example first

# For the MCP agent (05), you also need:
export OPENAI_API_KEY="your_openai_key"
python 05_mcp_agent.py

cd examples/javascript
npm install
export BRIGHTDATA_API_KEY="your_key"
export BRIGHTDATA_UNLOCKER_ZONE="web_unlocker1"
node 04_web_unlocker.js             # Try the simplest example first

pip install brightdata

from brightdata import BrightDataClient

async with BrightDataClient() as client:
    result = await client.scrape_url("https://example.com")
    search = await client.search.google(query="python scraping")

cd starter-template
npm install
cp .env.example .env     # Add your API key
npm run dev               # Open http://localhost:3000