网页内容清理工具

Question
  |
  +- (optional) LLM query expansion -> multiple search variants
  |
  +- Search via backend (SearXNG, Brave, Tavily, Exa, SerpAPI, Google, Bing, HTTP)
  |
  +- Deduplicate + filter binary URLs
  |
  +- Streaming fetch with per-page size cap
  |
  +- HTML cleaning -> plain text (noise elements, scripts, nav removed)
  |
  +- Unicode/BiDi sterilization
  |
  +- BM25 deterministic reranking
  |   +- (optional) LLM-assisted tier-2 reranking
  |
  +- Budget-aware truncation across all sources
  |
  +- (optional) LLM Markdown summary with inline citations
  |
  +- Structured JSON output

cargo install webshift-mcp

cargo install --path crates/webshift-mcp

# Full pipeline (search + fetch + clean + rerank)
webshift = "0.2"

# Cleaner + fetcher only (no search backends)
webshift = { version = "0.2", default-features = false }

# Text-map only (extract/replace text nodes in HTML)
webshift = { version = "0.2", default-features = false, features = ["text-map"] }

# Everything including LLM features
webshift = { version = "0.2", features = ["llm"] }

docker run -d -p 8080:8080 searxng/searxng

{
  "mcpServers": {
    "webshift": {
      "command": "mcp-webshift",
      "args": ["--default-backend", "searxng"]
    }
  }
}

[server]
max_query_budget    = 32000   # total char budget across all sources
max_result_length   = 8000    # per-page char cap
max_total_results   = 20      # hard cap on results per call
max_download_mb     = 1       # streaming cap per page (MB)
search_timeout      = 8       # seconds
results_per_query   = 5
oversampling_factor = 2
adaptive_budget     = "auto"  # "auto" | "on" | "off" — budget allocation mode

[backends]
default = "searxng"

[backends.searxng]
url = "http://localhost:8080"

[backends.brave]
api_key = "BSA-..."

[backends.tavily]
api_key = "tvly-..."

[backends.exa]
api_key = "..."

[backends.serpapi]
api_key = "..."
engine  = "google"    # google | bing | duckduckgo | yandex

[backends.google]
api_key = "..."
cx      = "..."       # Custom Search Engine ID

[backends.bing]
api_key = "..."
market  = "en-US"

[backends.http]
url           = "https://my-search.example.com/api/search"
query_param   = "q"
count_param   = "limit"
results_path  = "data.items"     # dot-path to results array in JSON response
title_field   = "title"
url_field     = "link"
snippet_field = "description"

[backends.http.headers]
"Authorization" = "Bearer my-token"

[llm]
enabled               = false
base_url              = "http://localhost:11434/v1"   # OpenAI-compatible
api_key               = ""
model                 = "gemma3:27b"
timeout               = 60
expansion_enabled     = true
summarization_enabled = true
llm_rerank_enabled    = false

WEBSHIFT_DEFAULT_BACKEND=searxng
WEBSHIFT_SEARXNG_URL=http://localhost:8080
WEBSHIFT_BRAVE_API_KEY=BSA-xxx
WEBSHIFT_GOOGLE_API_KEY=xxx
WEBSHIFT_GOOGLE_CX=xxx
WEBSHIFT_BING_API_KEY=xxx
WEBSHIFT_LLM_ENABLED=true
WEBSHIFT_LLM_BASE_URL=http://localhost:11434/v1
WEBSHIFT_LLM_MODEL=gemma3:27b

[llm]
enabled  = true
base_url = "http://localhost:11434/v1"
model    = "gemma3:27b"

use webshift::{Config, clean, fetch, query};

// Clean raw HTML — cap output at 8000 chars
let result = clean("<html><body><p>Hello world</p></body></html>", 8000);
println!("{}", result.text);

// Pass 0 to disable the per-page cap entirely (no truncation)
let full = clean("<html><body><p>Hello world</p></body></html>", 0);
assert!(!full.truncated);

// Fetch and clean a single page
let config = Config::default();
let page = fetch("https://example.com", &config).await?;

// Full search pipeline
let results = query(&["rust async programming"], &config).await?;
for source in &results.sources {
    println!("[{}] {} — {} chars", source.id, source.title, source.content.len());
}

// Backend partial failures (CAPTCHA, rate-limits, engine outages) are surfaced
// in `warnings` rather than as Err. Empty sources + non-empty warnings means
// "all backends blocked" — distinct from a legitimate "no matches found".
if results.sources.is_empty() && !results.warnings.is_empty() {
    eprintln!("all backends failed: {:?}", results.warnings);
}

use webshift::{extract_text_nodes, replace_text_nodes, TextReplacement};

let html = r#"<p>Hello <a href="https://example.com">world</a></p>"#;
let map = extract_text_nodes(html);
// map.nodes = [(0, "Hello"), (1, "world")]

let replacements = vec![
    TextReplacement { id: 0, text: "Ciao".into() },
    TextReplacement { id: 1, text: "mondo".into() },
];
let result = replace_text_nodes(html, &replacements).unwrap();
// → <p>Ciao <a href="https://example.com">mondo</a></p>
// href untouched, tags intact, only text changed.