AI多代理二进制分析与修补系统

$env:BACKEND_URL="https://jaegis-raverse.onrender.com";
$env:PROXY_URL="https://raverse-mcp-proxy.use-manus-ai.workers.dev";
$env:DATABASE_URL="postgres://avnadmin:***@raverse-pg-db-raverse-pg-db.i.aivencloud.com:23055/defaultdb?sslmode=require";
$env:REDIS_URL="rediss://default:***@raverse-valkey-cache-raverse-pg-db.g.aivencloud.com:23056";
$env:POSTGRES_CA_CERT="-----BEGIN CERTIFICATE-----`n...your Aiven CA pem lines...`n-----END CERTIFICATE-----";
$env:LOG_LEVEL="INFO"; $env:SERVER_VERSION="1.0.11";

npx -y raverse-mcp-server@latest

{
  "target_url": "https://your-shop.example.com",
  "depth": 1,
  "include_js": true,
  "include_sitemaps": true
}

{
  "url": "https://your-shop.example.com/_next/static/chunks/auth.abcdef.js",
  "timeout": 20,
  "retries": 1
}

{
  "js_code": "<paste JS content here>",
  "deobfuscate": true,
  "signals": ["HMAC","JWT","nonce","timestamp","crypto","subtle"]
}

{
  "mode": "har",
  "har": { "log": { "entries": [/* pasted HAR entries */] } }
}

{
  "traffic_data": { "har": { "log": { "entries": [/* … */] } } },
  "js_analysis": { "functions": ["sign","getToken"], "algos": ["HMAC-SHA256"] },
  "emit_openapi": true
}

{
  "target_url": "https://your-shop.example.com",
  "depth": 1,
  "include_js": true,
  "include_sitemaps": true
}

{
  "url": "https://your-shop.example.com/_next/static/chunks/auth.abcdef.js",
  "timeout": 20,
  "retries": 1
}

{
  "js_code": "<paste JS content here>",
  "deobfuscate": true,
  "signals": ["HMAC","JWT","nonce","timestamp","crypto","subtle"]
}

{
  "content": "…short snippet with fetch('/api/auth/token', {…})…",
  "hints": ["Authorization", "x-api-key", "x-signature", "bearer"]
}

{
  "mode": "har",
  "har": { "log": { "entries": [/* pasted HAR entries */] } }
}

{
  "traffic_data": { "har": { "log": { "entries": [/* … */] } } },
  "js_analysis": { "functions": ["sign","getToken"], "algos": ["HMAC-SHA256"] },
  "emit_openapi": true
}

{
  "samples": [{
    "url": "https://your-shop.example.com/api/orders",
    "status": 403,
    "headers": {"x-error-code":"sig_invalid"}
  }]
}

{
  "content": "Auth uses HMAC-SHA256(ts|method|path|body)…",
  "metadata": { "project":"shop", "type":"auth_notes", "env":"prod" }
}

graph LR
    A[reconnaissance] --> B[fetch_content]
    B --> C[analyze_javascript]
    C --> D[traffic_interception]
    D --> E[reverse_engineer_api]

{ "binary_path": "./bin/target.exe", "architecture": "x86_64" }

{ "binary_path": "./bin/target.exe", "patches": [{"offset":"0x401000","bytes":"90 90"}] }

{ "binary_path": "./bin/target.exe", "checks": ["integrity","entrypoint"] }

{ "query": "HMAC timestamp signature format", "top_k": 5 }

{
  "binary_path": "./bin/app_with_wasm.exe",
  "architecture": "x86_64"
}

{
  "wasm_data": "AGFzbQEAAAABAAA..."
}

{
  "binary_path": "./bin/app_with_wasm.exe",
  "backup": true,
  "patches": [
    {"offset": "0x40123A", "bytes": "90 90"},
    {"offset": "0x402010", "bytes": "EB 0A"}
  ]
}

{
  "binary_path": "./bin/module.wasm",
  "backup": true,
  "patches": [
    {"offset": "0x0000010", "bytes": "00"},
    {"offset": "0x00000A2", "bytes": "01"}
  ]
}

{
  "original_binary": "./bin/app_with_wasm.exe.bak",
  "patched_binary": "./bin/app_with_wasm.exe"
}

{
  "original_binary": "./bin/module.wasm.bak",
  "patched_binary": "./bin/module.wasm"
}

# Run the latest version without installation
npx raverse-mcp-server@latest

# Or with specific version
npx raverse-mcp-server@1.0.2

# Install globally
npm install -g raverse-mcp-server

# Run the server
raverse-mcp-server

# Install from PyPI
pip install jaegis-raverse-mcp-server

# Run the server
python -m jaegis_raverse_mcp_server.server

git clone https://github.com/usemanusai/RAVERSE.git
cd RAVERSE

python -m venv venv
source venv/bin/activate  # On Windows: venv\Scripts\activate

pip install -r requirements.txt

cp .env.example .env
# Edit .env with your API keys and database credentials

# Offline binary analysis
python src/main.py path/to/binary.exe

# Online target analysis
python src/raverse_online_cli.py --scope examples/scope_example.json --options examples/options_example.json

┌─────────────────────────────────────────────────────────────────┐
│                     RAVERSE 2.0 SYSTEM                          │
│                                                                 │
│  ┌──────────────────────────────────────────────────────────┐   │
│  │         Orchestrator (Offline & Online)                  │   │
│  │  Coordinates agents, manages workflow, handles I/O       │   │
│  └──────────────────────────────────────────────────────────┘   │
│                            │                                    │
│          ┌─────────────────┼─────────────────┐                  │
│          │                 │                 │                  │
│     ┌────▼────┐      ┌─────▼─────┐     ┌─────▼─────┐            │
│     │Offline  │      │  Online   │     │ Advanced  │            │
│     │Pipeline │      │ Pipeline  │     │  Agents   │            │
│     │(DAA→    │      │ (Recon→   │     │ (RAG, KB, │            │
│     │LIMA→PEA │      │ Traffic→  │     │  Quality) │            │
│     │→VA)     │      │ JS→API→   │     │           │            │
│     │         │      │ WASM→Sec) │     │           │            │
│     └────┬────┘      └─────┬─────┘     └─────┬─────┘            │
│          │                 │                 │                  │
│          └─────────────────┼─────────────────┘                  │
│                            │                                    │
│          ┌─────────────────┼─────────────────┐                  │
│          │                 │                 │                  │
│    ┌─────▼────┐      ┌─────▼─────┐     ┌─────▼─────┐            │
│    │PostgreSQL│      │   Redis   │     │Prometheus │            │
│    │ +pgvector│      │   Cache   │     │  Metrics  │            │
│    │ (Persist)│      │(Fast I/O) │     │(Observ.)  │            │
│    └──────────┘      └───────────┘     └───────────┘            │
└─────────────────────────────────────────────────────────────────┘

Binary Input → Disassembly Analysis (DAA) → Logic Identification (LIMA)
→ Patching Execution (PEA) → Verification (VA) → Patched Binary Output

Target URL → Reconnaissance → Traffic Interception → JavaScript Analysis
→ API Reverse Engineering → WebAssembly Analysis → Security Analysis
→ Validation → Reporting

RAVERSE/
├── src/                          # Source code
│   ├── agents/                   # 21+ AI agent implementations
│   │   ├── orchestrator.py       # Main orchestration agent
│   │   ├── online_*.py           # Online analysis agents
│   │   └── ...
│   ├── utils/                    # Utility modules
│   │   ├── database.py           # PostgreSQL integration
│   │   ├── cache.py              # Redis caching
│   │   ├── embeddings.py         # Vector embeddings
│   │   └── ...
│   ├── config/                   # Configuration management
│   │   ├── settings.py           # Main settings
│   │   ├── agent_memory_config.py # Memory strategies
│   │   └── ...
│   ├── main.py                   # Offline analysis entry point
│   └── raverse_online_cli.py     # Online analysis CLI
├── tests/                        # Test suite (81+ tests)
│   ├── unit/                     # Unit tests
│   ├── integration/              # Integration tests
│   ├── deepcrawler/              # DeepCrawler tests
│   └── memory/                   # Memory integration tests
├── docs/                         # Documentation
│   ├── ARCHITECTURE.md           # System architecture
│   ├── PRODUCTION_DEPLOYMENT_GUIDE.md
│   ├── QUICK_START_AI_FEATURES.md
│   └── archive/                  # Historical documentation
├── examples/                     # Configuration examples
│   ├── scope_example.json        # Analysis scope config
│   ├── options_example.json      # Execution options
│   └── comprehensive_demo.py     # Demo script
├── scripts/                      # Automation scripts
│   ├── run_tests.ps1             # Test runner (PowerShell)
│   ├── run_tests.sh              # Test runner (Bash)
│   └── migrations/               # Database migrations
├── docker/                       # Docker infrastructure
│   ├── postgres/                 # PostgreSQL config
│   ├── redis/                    # Redis config
│   ├── prometheus/               # Prometheus config
│   └── grafana/                  # Grafana config
├── requirements.txt              # Python dependencies
├── Dockerfile                    # Container image
├── docker-compose.yml            # Multi-container setup
├── .env.example                  # Environment template
├── .gitignore                    # Git ignore rules
└── README.md                     # This file

-- Find similar instructions using cosine similarity
SELECT
    address, instruction, opcode,
    1 - (embedding <=> %s::vector) AS similarity
FROM disassembly_cache
WHERE 1 - (embedding <=> %s::vector) >= 0.7
ORDER BY embedding <=> %s::vector
LIMIT 10;

Binary File
    ↓
[DAA] Disassembly Analysis Agent
  • Extracts binary metadata (PE/ELF format, architecture)
  • Disassembles code using Capstone engine
  • Identifies functions and code sections
  • Generates instruction embeddings
    ↓
[LIMA] Logic Identification & Mapping Agent
  • Analyzes control flow (branches, loops, calls)
  • Analyzes data flow (register/memory operations)
  • Identifies algorithms and patterns
  • Generates flowcharts and logic maps
    ↓
[PEA] Patching Execution Agent
  • Converts virtual addresses to file offsets
  • Generates patch opcodes
  • Writes patches to binary
  • Creates backup before modification
    ↓
[VA] Verification Agent
  • Validates patch integrity
  • Verifies binary structure
  • Tests patched functionality
  • Generates verification report
    ↓
Patched Binary Output

Target URL + Scope + Options
    ↓
[Phase 1] Reconnaissance Agent
  • Identifies technology stack
  • Discovers endpoints and services
  • Maps network topology
    ↓
[Phase 2] Traffic Interception Agent
  • Captures HTTP(S) traffic
  • Analyzes request/response patterns
  • Extracts API calls
    ↓
[Phase 3] JavaScript Analysis Agent
  • Deobfuscates JavaScript code
  • Extracts API calls from JS
  • Identifies client-side logic
    ↓
[Phase 4] API Reverse Engineering Agent
  • Maps API endpoints
  • Generates OpenAPI documentation
  • Identifies authentication methods
    ↓
[Phase 5] WebAssembly Analysis Agent
  • Decompiles WASM modules
  • Analyzes compiled code
  • Extracts functionality
    ↓
[Phase 6] Security Analysis Agent
  • Identifies vulnerabilities
  • Analyzes security headers
  • Checks for common weaknesses
    ↓
[Phase 7] Validation Agent
  • Generates proof-of-concept exploits
  • Validates findings
  • Captures evidence
    ↓
[Phase 8] Reporting Agent
  • Generates comprehensive reports
  • Formats findings (JSON, HTML, PDF)
  • Creates executive summary
    ↓
Analysis Report Output

from src.agents.orchestrator import OrchestratingAgent

# Initialize orchestrator with OpenRouter API
oa = OrchestratingAgent(
    openrouter_api_key="sk-or-v1-your-key",
    model="meta-llama/llama-3.3-70b-instruct:free",
    use_database=True
)

# Analyze binary file
result = oa.run("path/to/binary.exe")

# Result structure
print(f"Success: {result.get('success')}")
print(f"Patches Applied: {result.get('patches_applied')}")
print(f"Verification: {result.get('verification_status')}")

from src.agents.orchestrator import OrchestratingAgent
from src.utils.database import DatabaseManager
from src.utils.cache import CacheManager

# Initialize with database and cache
oa = OrchestratingAgent(use_database=True)

# Analyze binary
result = oa.run("path/to/binary.exe")

# Query analysis results from database
db = DatabaseManager()
binary_records = db.execute_query(
    "SELECT * FROM raverse.binaries WHERE file_hash = %s",
    (result['binary_hash'],)
)

# Retrieve cached results
cache = CacheManager()
cached_result = cache.get_cached_analysis(result['binary_hash'])

from src.utils.semantic_search import SemanticSearchEngine
from src.utils.database import DatabaseManager
from src.utils.cache import CacheManager

# Initialize search engine
db = DatabaseManager()
cache = CacheManager()
search_engine = SemanticSearchEngine(db, cache)

# Store code snippet with embedding
search_engine.store_code_embedding(
    binary_hash="abc123def456",
    code_snippet="cmp eax, 0x0; je 0x401000",
    metadata={"function": "main", "offset": "0x401000"}
)

# Find similar code
results = search_engine.find_similar_code(
    query="compare eax with zero and jump if equal",
    limit=10,
    similarity_threshold=0.7
)

for result in results:
    print(f"Similarity: {result['similarity']:.2%}")
    print(f"Code: {result['code_snippet']}")
    print(f"Binary: {result['binary_hash'][:8]}...")

python src/raverse_online_cli.py \
  --target https://api.example.com \
  --scope examples/scope_example.json \
  --options examples/options_example.json \
  --output results/

{
  "target_url": "https://api.example.com",
  "allowed_domains": ["api.example.com", "*.example.com"],
  "excluded_paths": ["/admin", "/internal"],
  "max_depth": 3,
  "max_urls": 1000
}

{
  "recon": {
    "detect_technologies": true,
    "detect_endpoints": true
  },
  "traffic": {
    "duration_seconds": 60,
    "capture_ssl": true
  },
  "api_discovery": {
    "detect_rest": true,
    "detect_graphql": true,
    "detect_websockets": true
  },
  "security": {
    "check_vulnerabilities": true,
    "generate_poc": true
  }
}

from src.agents.online_orchestrator import OnlineOrchestrationAgent

# Initialize online orchestrator
oa = OnlineOrchestrationAgent(
    api_key="sk-or-v1-your-key",
    model="meta-llama/llama-3.3-70b-instruct:free"
)

# Execute online analysis
result = oa.execute(
    target_url="https://api.example.com",
    scope={
        "target_url": "https://api.example.com",
        "allowed_domains": ["api.example.com"],
        "max_depth": 3
    },
    options={
        "recon": {"detect_technologies": True},
        "traffic": {"duration_seconds": 60},
        "api_discovery": {"detect_rest": True}
    }
)

# Access results
print(f"Reconnaissance: {result['recon']}")
print(f"APIs Discovered: {result['api_reeng']}")
print(f"Vulnerabilities: {result['security']}")

from src.agents.online_rag_orchestrator_agent import RAGOrchestratorAgent
from src.utils.semantic_search import SemanticSearchEngine

# Initialize RAG orchestrator
rag = RAGOrchestratorAgent(
    api_key="sk-or-v1-your-key",
    model="meta-llama/llama-3.3-70b-instruct:free"
)

# Execute RAG query
result = rag.execute({
    "query": "What are common binary patching techniques?",
    "context": "Binary analysis and security patching"
})

# Result includes retrieved knowledge + generated response
print(f"Retrieved Knowledge: {result['retrieved_knowledge']}")
print(f"Generated Response: {result['generated_response']}")
print(f"Confidence: {result['confidence']}")

from src.config.agent_memory_config import AGENT_MEMORY_CONFIG, MEMORY_PRESETS

# Get memory configuration for specific agent
kb_config = AGENT_MEMORY_CONFIG['knowledge_base']
print(f"Strategy: {kb_config['strategy']}")
print(f"Preset: {kb_config['preset']}")
print(f"Reason: {kb_config['reason']}")

# Get preset details
heavy_preset = MEMORY_PRESETS['heavy']
print(f"Description: {heavy_preset['description']}")
print(f"RAM: {heavy_preset['ram_mb']} MB")
print(f"CPU: {heavy_preset['cpu_percent']}%")

pytest tests/ -v --cov=src --cov-report=html

# Unit tests
pytest tests/unit/ -v

# Integration tests
pytest tests/integration/ -v

# DeepCrawler tests
pytest tests/deepcrawler/ -v

# Memory integration tests
pytest tests/memory/ -v

# Complete architecture tests
pytest tests/test_complete_architecture.py -v

# Run only fast tests
pytest tests/ -m "not slow" -v

# Run only integration tests
pytest tests/ -m "integration" -v

# Run with specific keyword
pytest tests/ -k "orchestrator" -v

# Generate HTML coverage report
pytest tests/ --cov=src --cov-report=html

# View report
open htmlcov/index.html

OPENROUTER_API_KEY=sk-or-v1-your-key-here
OPENROUTER_MODEL=meta-llama/llama-3.3-70b-instruct:free

DB_HOST=localhost
DB_PORT=5432
DB_USER=raverse
DB_PASSWORD=your_password
DB_NAME=raverse_db

REDIS_HOST=localhost
REDIS_PORT=6379
REDIS_DB=0

LOG_LEVEL=INFO
LOG_FILE=logs/raverse.log

DEEPCRAWLER_MAX_DEPTH=3
DEEPCRAWLER_MAX_URLS=10000
DEEPCRAWLER_MAX_CONCURRENT=5
DEEPCRAWLER_TIMEOUT=30
DEEPCRAWLER_RATE_LIMIT=20.0

MEMORY_PRESET=medium  # none, light, medium, heavy

Query Input
    ↓
[Embedding Generation]
  Generate query embedding (384-dim)
    ↓
[Vector Similarity Search]
  Search knowledge_base table using pgvector
  Cosine similarity with threshold (0.7)
    ↓
[Retrieved Context]
  Top-k results (k=5 default)
  Ranked by similarity score
    ↓
[Prompt Augmentation]
  Combine query + retrieved context
  Maintain token budget
    ↓
[LLM Generation]
  OpenRouter API call
  Generate response with context
    ↓
[Response Output]
  Formatted result with sources

-- Find top-k similar knowledge
SELECT
    knowledge_id, content, metadata,
    1 - (embedding <=> query_embedding::vector) AS similarity
FROM knowledge_base
WHERE 1 - (embedding <=> query_embedding::vector) >= 0.7
ORDER BY embedding <=> query_embedding::vector
LIMIT 5;

Target Application
    ↓
[Browser Automation] (Playwright)
  • Headless browser navigation
  • JavaScript execution
  • Form interaction
    ↓
[Traffic Interception] (mitmproxy)
  • HTTP(S) traffic capture
  • Request/response analysis
  • API call extraction
    ↓
[API Pattern Detection]
  • REST endpoint identification
  • GraphQL query detection
  • WebSocket connection tracking
    ↓
[Endpoint Analysis]
  • HTTP method detection
  • Parameter extraction
  • Authentication analysis
    ↓
[Documentation Generation]
  • OpenAPI spec creation
  • Endpoint cataloging
  • Example generation
    ↓
API Documentation Output

# Crawling parameters
max_depth: int = 3              # Maximum crawl depth
max_urls: int = 10000           # Maximum URLs to crawl
max_concurrent: int = 5         # Concurrent requests
timeout: int = 30               # Request timeout (seconds)
rate_limit: float = 20.0        # Requests per minute

# API detection
detect_rest_apis: bool = True
detect_graphql: bool = True
detect_websockets: bool = True
min_confidence_score: float = 0.6

# Output
output_format: str = 'openapi'  # openapi, json, yaml
output_dir: str = './crawl_results'

python src/raverse_online_cli.py \
  --target https://api.example.com \
  --scope examples/scope_example.json \
  --options examples/options_example.json \
  --output results/

{
  "openapi": "3.0.0",
  "info": {
    "title": "Discovered API",
    "version": "1.0.0"
  },
  "paths": {
    "/api/users": {
      "get": {
        "summary": "List users",
        "parameters": [...],
        "responses": {...}
      }
    }
  }
}

{
  "endpoints": [
    {
      "url": "/api/users",
      "method": "GET",
      "parameters": [...],
      "authentication": "Bearer token",
      "confidence": 0.95
    }
  ]
}

# Build and run with Docker Compose
docker-compose up -d

# View logs
docker-compose logs -f

# Stop services
docker-compose down

{
  "timestamp": "2025-10-26T10:30:00Z",
  "level": "INFO",
  "logger": "orchestrator",
  "message": "Starting DAA.disassemble",
  "binary_id": "abc123",
  "execution_time_ms": 1234,
  "tags": ["offline", "analysis"]
}

http://localhost:9090

http://localhost:3000
Default credentials: admin / admin

# View live logs
docker-compose logs -f raverse

# View specific service logs
docker-compose logs -f postgres
docker-compose logs -f redis

# PostgreSQL connection pooling
pool_size = 10
max_overflow = 20
pool_recycle = 3600

# Batch embedding generation
batch_size = 32
# Batch database inserts
insert_batch_size = 1000

from src.utils.database import DatabaseManager

# Initialize with custom pool settings
db = DatabaseManager(
    pool_size=10,           # Minimum connections
    max_overflow=20,        # Maximum overflow connections
    pool_recycle=3600,      # Recycle connections after 1 hour
    pool_pre_ping=True      # Test connections before use
)

from src.utils.cache import CacheManager

# Initialize with Redis cluster
cache = CacheManager(
    redis_nodes=[
        ('redis-node-1', 6379),
        ('redis-node-2', 6379),
        ('redis-node-3', 6379)
    ],
    cluster_mode=True,
    skip_full_coverage_check=True
)

# Free models (rate-limited)
models = [
    "meta-llama/llama-3.3-70b-instruct:free",
    "meta-llama/llama-3.2-3b-instruct:free",
    "mistralai/mistral-7b-instruct:free"
]

# Premium models (faster, better quality)
premium_models = [
    "openai/gpt-4-turbo",
    "anthropic/claude-3-opus",
    "google/gemini-pro"
]

# Select model based on use case
from src.agents.orchestrator import OrchestratingAgent

oa = OrchestratingAgent(model="openai/gpt-4-turbo")

from src.utils.embeddings_v2 import EmbeddingGenerator

# Initialize with specific model
embedding_gen = EmbeddingGenerator(
    model_name="all-MiniLM-L6-v2",  # 384 dimensions
    batch_size=32,
    cache_manager=cache_manager
)

# Generate embeddings
embeddings = embedding_gen.batch_encode(
    texts=["code snippet 1", "code snippet 2"],
    show_progress_bar=True
)

from src.agents.online_base_agent import OnlineBaseAgent

class CustomAnalysisAgent(OnlineBaseAgent):
    """Custom agent for specialized analysis."""

    def __init__(self, orchestrator, api_key, model):
        super().__init__(
            name="CustomAnalysis",
            orchestrator=orchestrator,
            api_key=api_key,
            model=model
        )

    def _execute_impl(self, task):
        """Implement custom analysis logic."""
        # Your implementation here
        return {
            "status": "success",
            "results": {...}
        }

# Adjust HNSW index parameters
# In PostgreSQL:
CREATE INDEX idx_embeddings_hnsw ON embeddings
USING hnsw (embedding vector_cosine_ops)
WITH (m = 32, ef_construction = 128);  # Higher values = better quality, slower

# Query optimization
SET hnsw.ef_search = 200;  # Higher = more accurate, slower

from src.utils.database import DatabaseManager

db = DatabaseManager()

# Batch insert embeddings
embeddings_batch = [
    (binary_hash, code_snippet, embedding, metadata)
    for binary_hash, code_snippet, embedding, metadata in data
]

db.batch_insert_embeddings(embeddings_batch, batch_size=1000)

from src.utils.multi_level_cache import MultiLevelCache

cache = MultiLevelCache(
    l1_size=1000,           # In-memory cache size
    l2_ttl=3600,            # Redis TTL (1 hour)
    l3_ttl=86400            # PostgreSQL TTL (1 day)
)

# Warm up cache
cache.warm_up(frequently_accessed_keys)

export OPENROUTER_API_KEY=sk-or-v1-your-key
# or
python -c "from src.agents.orchestrator import OrchestratingAgent; oa = OrchestratingAgent(openrouter_api_key='sk-or-v1-your-key')"

# Check PostgreSQL status
docker-compose ps postgres

# Verify connection
psql -h localhost -U raverse -d raverse_db -c "SELECT 1"

# Check logs
docker-compose logs postgres

# Check Redis status
docker-compose ps redis

# Test connection
redis-cli -h localhost -p 6379 ping

# Check logs
docker-compose logs redis

-- Check embedding count
SELECT COUNT(*) FROM code_embeddings WHERE embedding IS NOT NULL;

-- Check index status
SELECT * FROM pg_indexes WHERE tablename = 'code_embeddings';

-- Verify similarity threshold
SELECT 1 - (embedding <=> query_embedding::vector) AS similarity
FROM code_embeddings
LIMIT 1;

# Increase timeout
oa = OrchestratingAgent(timeout=60)  # 60 seconds

# Or optimize agent logic
# - Reduce binary size
# - Use cached results
# - Increase LLM timeout

# Use lighter memory preset
from src.config.agent_memory_config import MEMORY_PRESETS
preset = MEMORY_PRESETS['light']

# Or reduce batch sizes
embedding_gen = EmbeddingGenerator(batch_size=8)  # Reduce from 32

# Or use streaming for large files

import logging

# Set debug level
logging.basicConfig(level=logging.DEBUG)

# Or for specific module
logger = logging.getLogger('src.agents.orchestrator')
logger.setLevel(logging.DEBUG)

import cProfile
import pstats
from src.agents.orchestrator import OrchestratingAgent

# Profile binary analysis
profiler = cProfile.Profile()
profiler.enable()

oa = OrchestratingAgent()
result = oa.run("path/to/binary.exe")

profiler.disable()
stats = pstats.Stats(profiler)
stats.sort_stats('cumulative')
stats.print_stats(20)  # Top 20 functions

-- Check binary analysis status
SELECT id, file_name, status, created_at FROM raverse.binaries
ORDER BY created_at DESC LIMIT 10;

-- Check analysis results
SELECT binary_id, result_data FROM raverse.analysis_results
WHERE binary_id = 123;

-- Check vector search index health
SELECT schemaname, tablename, indexname, idx_scan, idx_tup_read, idx_tup_fetch
FROM pg_stat_user_indexes
WHERE tablename LIKE '%embedding%';

-- Check cache efficiency
SELECT COUNT(*) as total_queries,
       SUM(CASE WHEN cached THEN 1 ELSE 0 END) as cached_queries,
       ROUND(100.0 * SUM(CASE WHEN cached THEN 1 ELSE 0 END) / COUNT(*), 2) as cache_hit_ratio
FROM query_log;

# PowerShell
.\scripts\run_tests.ps1 -Verbose -Coverage

# Bash
bash scripts/run_tests.sh --verbose --coverage

# Format code
black src/ tests/

# Type checking
mypy src/

# Linting
ruff check src/

class OrchestratingAgent:
    def __init__(self, openrouter_api_key=None, model=None, use_database=True)
    def run(self, binary_path: str) -> Dict
    def call_openrouter(self, prompt: str, max_tokens: int = 500) -> Dict

class OnlineOrchestrationAgent:
    def __init__(self, api_key: str, model: str)
    def execute(self, target_url: str, scope: Dict, options: Dict) -> Dict
    def _execute_pipeline(self, target_url: str, scope: Dict, options: Dict) -> Dict

class DatabaseManager:
    def __init__(self, host='localhost', port=5432, user='raverse', password='', database='raverse_db')
    def get_connection(self) -> Connection
    def create_binary_record(self, file_name, file_path, file_hash, file_size, file_type, architecture, metadata) -> int
    def search_similar_instructions(self, embedding: List[float], limit: int = 10) -> List[Dict]
    def execute_query(self, query: str, params: Tuple = ()) -> List[Dict]

class CacheManager:
    def __init__(self, redis_host='localhost', redis_port=6379)
    def cache_analysis(self, binary_hash: str, analysis_type: str, result: Dict) -> None
    def get_cached_analysis(self, binary_hash: str, analysis_type: str = 'full_analysis') -> Optional[Dict]
    def cache_llm_response(self, prompt: str, model: str, response: Dict) -> None
    def get_cached_llm_response(self, prompt: str, model: str) -> Optional[Dict]

class EmbeddingGenerator:
    def __init__(self, model_name='all-MiniLM-L6-v2', batch_size=32, cache_manager=None)
    def generate_embedding(self, text: str) -> np.ndarray
    def generate_code_embedding(self, code: str) -> np.ndarray
    def batch_encode(self, texts: List[str], show_progress_bar=False) -> np.ndarray

class SemanticSearchEngine:
    def __init__(self, db_manager: DatabaseManager, cache_manager: CacheManager)
    def store_code_embedding(self, binary_hash: str, code_snippet: str, metadata: Dict = None) -> int
    def find_similar_code(self, query: str, limit: int = 10, similarity_threshold: float = 0.7) -> List[Dict]
    def search_by_pattern(self, pattern: str, limit: int = 10) -> List[Dict]

from flask import Flask, request
from src.agents.orchestrator import OrchestratingAgent

app = Flask(__name__)
oa = OrchestratingAgent()

@app.route('/analyze', methods=['POST'])
def analyze_binary():
    """Webhook endpoint for binary analysis."""
    binary_path = request.json.get('binary_path')

    try:
        result = oa.run(binary_path)
        return {
            'status': 'success',
            'result': result
        }, 200
    except Exception as e:
        return {
            'status': 'error',
            'message': str(e)
        }, 500

import pika
import json
from src.agents.orchestrator import OrchestratingAgent

# Connect to RabbitMQ
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue='binary_analysis')

oa = OrchestratingAgent()

def callback(ch, method, properties, body):
    """Process binary analysis from queue."""
    message = json.loads(body)
    binary_path = message['binary_path']

    result = oa.run(binary_path)

    # Publish result
    channel.basic_publish(
        exchange='',
        routing_key='analysis_results',
        body=json.dumps(result)
    )

    ch.basic_ack(delivery_tag=method.delivery_tag)

channel.basic_consume(queue='binary_analysis', on_message_callback=callback)
channel.start_consuming()

from fastapi import FastAPI, File, UploadFile
from src.agents.orchestrator import OrchestratingAgent
import tempfile
import os

app = FastAPI()
oa = OrchestratingAgent()

@app.post("/api/v1/analyze")
async def analyze_binary(file: UploadFile = File(...)):
    """REST API endpoint for binary analysis."""

    # Save uploaded file
    with tempfile.NamedTemporaryFile(delete=False) as tmp:
        contents = await file.read()
        tmp.write(contents)
        tmp_path = tmp.name

    try:
        # Analyze binary
        result = oa.run(tmp_path)
        return {
            'status': 'success',
            'analysis': result
        }
    finally:
        # Clean up
        os.unlink(tmp_path)

@app.get("/api/v1/status/{analysis_id}")
async def get_analysis_status(analysis_id: str):
    """Get analysis status."""
    from src.utils.database import DatabaseManager

    db = DatabaseManager()
    result = db.execute_query(
        "SELECT status FROM raverse.binaries WHERE id = %s",
        (analysis_id,)
    )

    if result:
        return {'status': result[0]['status']}
    return {'error': 'Analysis not found'}, 404

apiVersion: apps/v1
kind: Deployment
metadata:
  name: raverse-analyzer
spec:
  replicas: 3
  selector:
    matchLabels:
      app: raverse-analyzer
  template:
    metadata:
      labels:
        app: raverse-analyzer
    spec:
      containers:
      - name: raverse
        image: raverse:latest
        env:
        - name: OPENROUTER_API_KEY
          valueFrom:
            secretKeyRef:
              name: raverse-secrets
              key: api-key
        - name: DB_HOST
          value: postgres-service
        - name: REDIS_HOST
          value: redis-service
        resources:
          requests:
            memory: "4Gi"
            cpu: "2"
          limits:
            memory: "8Gi"
            cpu: "4"
        ports:
        - containerPort: 8000

class DisassemblyAnalysisAgent:
    def __init__(self, openrouter_agent):
        self.openrouter_agent = openrouter_agent
        self.analyzer = BinaryAnalyzer()

    def disassemble(self, binary_path: str) -> Dict:
        """Disassemble binary and extract functions."""
        # Extract metadata
        metadata = self.analyzer.extract_metadata(binary_path)

        # Disassemble using Capstone
        disassembly = self.analyzer.disassemble(binary_path)

        # Identify functions
        functions = self.analyzer.identify_functions(disassembly)

        # Generate embeddings for semantic search
        embeddings = self._generate_embeddings(disassembly)

        return {
            'metadata': metadata,
            'disassembly': disassembly,
            'functions': functions,
            'embeddings': embeddings
        }

class LogicIdentificationMappingAgent:
    def __init__(self, openrouter_agent):
        self.openrouter_agent = openrouter_agent

    def identify_logic(self, daa_output: Dict) -> Dict:
        """Identify logic and generate mapping."""
        # Analyze control flow
        control_flow = self._analyze_control_flow(daa_output)

        # Analyze data flow
        data_flow = self._analyze_data_flow(daa_output)

        # Identify algorithms
        algorithms = self._identify_algorithms(control_flow, data_flow)

        # Generate flowchart
        flowchart = self._generate_flowchart(control_flow)

        # Use LLM for semantic analysis
        llm_analysis = self.openrouter_agent.call_openrouter(
            f"Analyze this binary logic: {control_flow}"
        )

        return {
            'control_flow': control_flow,
            'data_flow': data_flow,
            'algorithms': algorithms,
            'flowchart': flowchart,
            'llm_analysis': llm_analysis
        }

class PatchingExecutionAgent:
    def __init__(self, openrouter_agent):
        self.openrouter_agent = openrouter_agent

    def patch_binary(self, lima_output: Dict, binary_path: str) -> str:
        """Apply patches to binary."""
        # Extract patch information
        jump_addr = lima_output.get('jump_addr')
        opcode = lima_output.get('opcode')

        # Create backup
        backup_path = f"{binary_path}.backup"
        shutil.copy2(binary_path, backup_path)

        # Convert virtual address to file offset
        file_offset = self._va_to_file_offset(binary_path, jump_addr)

        # Apply patch
        with open(binary_path, 'r+b') as f:
            f.seek(file_offset)
            f.write(bytes.fromhex(opcode))

        return binary_path

class VerificationAgent:
    def __init__(self, openrouter_agent):
        self.openrouter_agent = openrouter_agent

    def verify_patch(self, pea_output: str, original_binary: str) -> Dict:
        """Verify patch integrity."""
        # Verify binary structure
        structure_valid = self._verify_structure(pea_output)

        # Verify patch was applied
        patch_applied = self._verify_patch_applied(pea_output, original_binary)

        # Test functionality
        functionality_ok = self._test_functionality(pea_output)

        # Generate verification report
        report = {
            'structure_valid': structure_valid,
            'patch_applied': patch_applied,
            'functionality_ok': functionality_ok,
            'success': all([structure_valid, patch_applied, functionality_ok])
        }

        return report

class ReconnaissanceAgent(OnlineBaseAgent):
    def _execute_impl(self, task: Dict) -> Dict:
        target_url = task.get('target_url')

        # Detect technologies
        tech_stack = self._detect_technologies(target_url)

        # Discover endpoints
        endpoints = self._discover_endpoints(target_url)

        # Gather server info
        server_info = self._gather_server_info(target_url)

        return {
            'technologies': tech_stack,
            'endpoints': endpoints,
            'server_info': server_info
        }

class TrafficInterceptionAgent(OnlineBaseAgent):
    def _execute_impl(self, task: Dict) -> Dict:
        target_url = task.get('target_url')
        duration = task.get('duration_seconds', 60)

        # Start traffic capture
        captured_traffic = self._capture_traffic(target_url, duration)

        # Analyze traffic
        api_calls = self._extract_api_calls(captured_traffic)
        patterns = self._detect_patterns(captured_traffic)

        return {
            'traffic': captured_traffic,
            'api_calls': api_calls,
            'patterns': patterns
        }

class JavaScriptAnalysisAgent(OnlineBaseAgent):
    def _execute_impl(self, task: Dict) -> Dict:
        js_code = task.get('javascript_code')

        # Deobfuscate JavaScript
        deobfuscated = self._deobfuscate(js_code)

        # Extract API calls
        api_calls = self._extract_api_calls(deobfuscated)

        # Analyze logic
        logic_analysis = self.orchestrator.call_openrouter(
            f"Analyze this JavaScript: {deobfuscated}"
        )

        return {
            'deobfuscated_code': deobfuscated,
            'api_calls': api_calls,
            'logic_analysis': logic_analysis
        }

class APIReverseEngineeringAgent(OnlineBaseAgent):
    def _execute_impl(self, task: Dict) -> Dict:
        traffic_data = task.get('traffic_data')

        # Extract endpoints
        endpoints = self._extract_endpoints(traffic_data)

        # Generate OpenAPI spec
        openapi_spec = self._generate_openapi_spec(endpoints)

        # Detect authentication
        auth_methods = self._detect_authentication(traffic_data)

        return {
            'endpoints': endpoints,
            'openapi_spec': openapi_spec,
            'authentication': auth_methods
        }

class RAGOrchestratorAgent(OnlineBaseAgent):
    def _execute_impl(self, task: Dict) -> Dict:
        query = task.get('query')
        context = task.get('context')

        # Generate query embedding
        query_embedding = self._generate_embedding(query)

        # Retrieve relevant knowledge
        retrieved_knowledge = self._retrieve_knowledge(
            query_embedding,
            limit=5,
            threshold=0.7
        )

        # Augment prompt with retrieved knowledge
        augmented_prompt = self._augment_prompt(query, retrieved_knowledge)

        # Generate response
        response = self.orchestrator.call_openrouter(augmented_prompt)

        return {
            'query': query,
            'retrieved_knowledge': retrieved_knowledge,
            'generated_response': response,
            'confidence': self._calculate_confidence(retrieved_knowledge)
        }

class KnowledgeBaseAgent(OnlineBaseAgent):
    def _execute_impl(self, task: Dict) -> Dict:
        action = task.get('action')  # 'store', 'retrieve', 'search'

        if action == 'store':
            # Store knowledge with embedding
            knowledge_id = self._store_knowledge(
                content=task.get('content'),
                metadata=task.get('metadata')
            )
            return {'knowledge_id': knowledge_id}

        elif action == 'retrieve':
            # Retrieve knowledge by ID
            knowledge = self._retrieve_knowledge_by_id(task.get('knowledge_id'))
            return {'knowledge': knowledge}

        elif action == 'search':
            # Search knowledge by similarity
            results = self._search_knowledge(
                query=task.get('query'),
                limit=task.get('limit', 10)
            )
            return {'results': results}

version: '3.8'
services:
  postgres:
    image: pgvector/pgvector:pg17-latest
    environment:
      POSTGRES_USER: raverse
      POSTGRES_PASSWORD: ${DB_PASSWORD}
      POSTGRES_DB: raverse_db
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - ./docker/postgres/init:/docker-entrypoint-initdb.d
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U raverse"]
      interval: 10s
      timeout: 5s
      retries: 5
    restart: unless-stopped

  redis:
    image: redis:8.2-alpine
    command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD}
    volumes:
      - redis_data:/data
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 5s
      retries: 5
    restart: unless-stopped

  raverse:
    build:
      context: .
      dockerfile: Dockerfile
    environment:
      OPENROUTER_API_KEY: ${OPENROUTER_API_KEY}
      DB_HOST: postgres
      DB_PORT: 5432
      DB_USER: raverse
      DB_PASSWORD: ${DB_PASSWORD}
      DB_NAME: raverse_db
      REDIS_HOST: redis
      REDIS_PORT: 6379
      REDIS_PASSWORD: ${REDIS_PASSWORD}
      LOG_LEVEL: INFO
    depends_on:
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
    volumes:
      - ./logs:/app/logs
      - ./results:/app/results
    restart: unless-stopped

  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    ports:
      - "9090:9090"
    restart: unless-stopped

  grafana:
    image: grafana/grafana:latest
    environment:
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD}
    volumes:
      - grafana_data:/var/lib/grafana
      - ./docker/grafana/provisioning:/etc/grafana/provisioning
    ports:
      - "3000:3000"
    restart: unless-stopped

volumes:
  postgres_data:
  redis_data:
  prometheus_data:
  grafana_data:

#!/bin/bash
# backup.sh - Daily PostgreSQL backup

BACKUP_DIR="/backups/raverse"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
BACKUP_FILE="$BACKUP_DIR/raverse_db_$TIMESTAMP.sql.gz"

# Create backup
pg_dump -h $DB_HOST -U $DB_USER -d $DB_NAME | gzip > $BACKUP_FILE

# Keep only last 30 days
find $BACKUP_DIR -name "raverse_db_*.sql.gz" -mtime +30 -delete

# Upload to S3
aws s3 cp $BACKUP_FILE s3://raverse-backups/

apiVersion: apps/v1
kind: Deployment
metadata:
  name: raverse
spec:
  replicas: 3
  selector:
    matchLabels:
      app: raverse
  template:
    metadata:
      labels:
        app: raverse
    spec:
      containers:
      - name: raverse
        image: raverse:latest
        env:
        - name: OPENROUTER_API_KEY
          valueFrom:
            secretKeyRef:
              name: raverse-secrets
              key: api-key
        - name: DB_HOST
          value: postgres-service
        - name: REDIS_HOST
          value: redis-service
        resources:
          requests:
            memory: "4Gi"
            cpu: "2"
          limits:
            memory: "8Gi"
            cpu: "4"
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /ready
            port: 8000
          initialDelaySeconds: 10
          periodSeconds: 5

# ✓ GOOD: Clear separation of concerns
class MyAgent(OnlineBaseAgent):
    def __init__(self, orchestrator, api_key, model):
        super().__init__(name="MyAgent", orchestrator=orchestrator,
                        api_key=api_key, model=model)
        self.db = DatabaseManager()
        self.cache = CacheManager()

    def _execute_impl(self, task: Dict) -> Dict:
        """Implement agent logic."""
        # Validate input
        if not self._validate_input(task):
            return {'error': 'Invalid input'}

        # Check cache
        cached = self.cache.get(task['id'])
        if cached:
            return cached

        # Execute logic
        result = self._process(task)

        # Cache result
        self.cache.set(task['id'], result, ttl=3600)

        return result

    def _validate_input(self, task: Dict) -> bool:
        """Validate input parameters."""
        required_fields = ['id', 'data']
        return all(field in task for field in required_fields)

    def _process(self, task: Dict) -> Dict:
        """Process task logic."""
        # Implementation here
        pass

# ✓ GOOD: Comprehensive error handling
try:
    result = oa.run(binary_path)
except FileNotFoundError:
    logger.error(f"Binary not found: {binary_path}")
    return {'error': 'Binary not found'}
except ValueError as e:
    logger.error(f"Invalid input: {e}")
    return {'error': str(e)}
except Exception as e:
    logger.exception(f"Unexpected error: {e}")
    return {'error': 'Internal server error'}

# ✓ GOOD: Proper connection pooling
from sqlalchemy import create_engine

engine = create_engine(
    f"postgresql://{user}:{password}@{host}:{port}/{database}",
    pool_size=10,
    max_overflow=20,
    pool_recycle=3600,
    pool_pre_ping=True
)

# ✓ GOOD: Efficient queries with indexes
# Create indexes for frequently searched columns
CREATE INDEX idx_binary_hash ON raverse.binaries(file_hash);
CREATE INDEX idx_embedding_hnsw ON raverse.code_embeddings
  USING hnsw (embedding vector_cosine_ops);

# Use EXPLAIN to analyze queries
EXPLAIN ANALYZE
SELECT * FROM code_embeddings
WHERE 1 - (embedding <=> query_embedding::vector) >= 0.7
ORDER BY embedding <=> query_embedding::vector
LIMIT 10;

# ✓ GOOD: Batch inserts for performance
def batch_insert_embeddings(embeddings_list, batch_size=1000):
    """Insert embeddings in batches."""
    for i in range(0, len(embeddings_list), batch_size):
        batch = embeddings_list[i:i+batch_size]
        db.execute_many(
            "INSERT INTO code_embeddings (binary_hash, code, embedding) VALUES (%s, %s, %s)",
            batch
        )

# ✓ GOOD: Multi-level cache hierarchy
class MultiLevelCache:
    def __init__(self):
        self.l1_cache = {}  # In-memory (fast, limited)
        self.l2_cache = redis_client  # Redis (medium, distributed)
        self.l3_cache = db  # PostgreSQL (slow, persistent)

    def get(self, key):
        # Try L1 first
        if key in self.l1_cache:
            return self.l1_cache[key]

        # Try L2
        value = self.l2_cache.get(key)
        if value:
            self.l1_cache[key] = value
            return value

        # Try L3
        value = self.l3_cache.get(key)
        if value:
            self.l2_cache.set(key, value, ttl=3600)
            self.l1_cache[key] = value
            return value

        return None

# ✓ GOOD: Proper cache invalidation
def update_binary_analysis(binary_id, new_result):
    """Update analysis and invalidate cache."""
    # Update database
    db.update_analysis(binary_id, new_result)

    # Invalidate caches
    cache_key = f"analysis:{binary_id}"
    redis_client.delete(cache_key)

    # Notify other services
    publish_event('analysis_updated', {'binary_id': binary_id})

# ✓ GOOD: Async operations for I/O
import asyncio

async def analyze_multiple_binaries(binary_paths):
    """Analyze multiple binaries concurrently."""
    tasks = [
        asyncio.create_task(analyze_binary_async(path))
        for path in binary_paths
    ]
    results = await asyncio.gather(*tasks)
    return results

async def analyze_binary_async(binary_path):
    """Async binary analysis."""
    loop = asyncio.get_event_loop()
    return await loop.run_in_executor(None, oa.run, binary_path)

# ✓ GOOD: Batch processing for efficiency
def process_embeddings_batch(texts, batch_size=32):
    """Process embeddings in batches."""
    embeddings = []
    for i in range(0, len(texts), batch_size):
        batch = texts[i:i+batch_size]
        batch_embeddings = embedding_gen.batch_encode(batch)
        embeddings.extend(batch_embeddings)
    return embeddings

# ✓ GOOD: Structured logging
import logging
import json

logger = logging.getLogger(__name__)

def log_analysis(binary_id, status, duration_ms):
    """Log analysis with structured format."""
    logger.info(json.dumps({
        'event': 'analysis_complete',
        'binary_id': binary_id,
        'status': status,
        'duration_ms': duration_ms,
        'timestamp': datetime.utcnow().isoformat()
    }))

# ✓ GOOD: Prometheus metrics
from prometheus_client import Counter, Histogram, Gauge

analysis_duration = Histogram(
    'analysis_duration_seconds',
    'Time to complete analysis',
    buckets=(1, 2, 5, 10, 30, 60)
)

analysis_errors = Counter(
    'analysis_errors_total',
    'Total analysis errors'
)

cache_hit_ratio = Gauge(
    'cache_hit_ratio',
    'Cache hit ratio'
)

# Use in code
with analysis_duration.time():
    result = oa.run(binary_path)

# ✓ GOOD: Comprehensive input validation
def validate_binary_path(path):
    """Validate binary path."""
    # Check path exists
    if not os.path.exists(path):
        raise FileNotFoundError(f"Binary not found: {path}")

    # Check path is file
    if not os.path.isfile(path):
        raise ValueError(f"Path is not a file: {path}")

    # Check path is within allowed directory
    allowed_dir = os.path.abspath('/binaries')
    real_path = os.path.abspath(path)
    if not real_path.startswith(allowed_dir):
        raise ValueError(f"Path outside allowed directory: {path}")

    return real_path

# ✓ GOOD: Secure secrets handling
import os
from dotenv import load_dotenv

# Load from .env file
load_dotenv()

# Get secrets from environment
api_key = os.getenv('OPENROUTER_API_KEY')
if not api_key:
    raise ValueError("OPENROUTER_API_KEY not set")

# Never log secrets
logger.info(f"Using API key: {api_key[:10]}...")  # Only show prefix

# ✓ GOOD: Comprehensive unit tests
import pytest

class TestOrchestrator:
    @pytest.fixture
    def orchestrator(self):
        return OrchestratingAgent(use_database=False)

    def test_run_success(self, orchestrator, tmp_path):
        """Test successful binary analysis."""
        # Create test binary
        binary_path = tmp_path / "test.bin"
        binary_path.write_bytes(b"test")

        # Run analysis
        result = orchestrator.run(str(binary_path))

        # Assert success
        assert result['success']
        assert 'patches' in result

    def test_run_invalid_path(self, orchestrator):
        """Test with invalid binary path."""
        with pytest.raises(FileNotFoundError):
            orchestrator.run("/nonexistent/binary")

# ✓ GOOD: Integration tests with fixtures
@pytest.fixture
def db_session():
    """Create test database session."""
    db = DatabaseManager(database='raverse_test')
    db.create_tables()
    yield db
    db.drop_tables()

def test_end_to_end_analysis(db_session):
    """Test complete analysis pipeline."""
    # Setup
    binary_path = "tests/fixtures/test_binary.exe"

    # Execute
    oa = OrchestratingAgent(use_database=True)
    result = oa.run(binary_path)

    # Verify
    assert result['success']

    # Check database
    records = db_session.execute_query(
        "SELECT * FROM raverse.binaries WHERE file_hash = %s",
        (result['binary_hash'],)
    )
    assert len(records) > 0

-- Create HNSW index with optimal parameters
CREATE INDEX idx_embeddings_hnsw ON code_embeddings
USING hnsw (embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);

-- Parameters explanation:
-- m = 16: Number of connections per node (higher = better quality, slower)
-- ef_construction = 64: Size of dynamic candidate list (higher = better quality, slower)

-- For production with large datasets:
CREATE INDEX idx_embeddings_hnsw_prod ON code_embeddings
USING hnsw (embedding vector_cosine_ops)
WITH (m = 32, ef_construction = 128);

-- Query-time parameter
SET hnsw.ef_search = 200;  -- Higher = more accurate, slower

-- Cosine distance formula: 1 - (dot_product / (norm_a * norm_b))
-- In pgvector: 1 - (embedding <=> query_embedding::vector)

-- Example: Find top-10 similar code snippets
SELECT
    id, code_snippet, metadata,
    1 - (embedding <=> query_embedding::vector) AS similarity_score
FROM code_embeddings
WHERE 1 - (embedding <=> query_embedding::vector) >= 0.7
ORDER BY embedding <=> query_embedding::vector
LIMIT 10;

-- Similarity score interpretation:
-- 1.0 = Identical
-- 0.8-1.0 = Very similar
-- 0.6-0.8 = Similar
-- 0.4-0.6 = Somewhat similar
-- <0.4 = Dissimilar

# Step 1: Load binary and extract metadata
binary = BinaryAnalyzer.load_binary(binary_path)
metadata = {
    'architecture': binary.arch,  # x86, x64, ARM, etc.
    'file_type': binary.file_type,  # ELF, PE, Mach-O
    'entry_point': binary.entry_point,
    'sections': binary.sections
}

# Step 2: Disassemble using Capstone
from capstone import *
md = Cs(CS_ARCH_X86, CS_MODE_64)
instructions = list(md.disasm(binary_code, binary.entry_point))

# Step 3: Identify functions
functions = BinaryAnalyzer.identify_functions(instructions)

# Step 4: Extract control flow
cfg = BinaryAnalyzer.build_control_flow_graph(functions)

# Step 5: Generate embeddings for semantic search
embeddings = embedding_gen.batch_encode([
    instr.mnemonic + ' ' + instr.op_str
    for instr in instructions
])

# Patch format: (address, original_bytes, new_bytes)
patches = [
    (0x401000, b'\x90\x90\x90\x90', b'\xc3\x00\x00\x00'),  # NOP to RET
    (0x401010, b'\x74\x05', b'\xeb\x05'),  # JE to JMP
]

# Apply patches
for address, original, new in patches:
    file_offset = va_to_file_offset(binary_path, address)
    with open(binary_path, 'r+b') as f:
        f.seek(file_offset)
        current = f.read(len(original))
        if current == original:
            f.seek(file_offset)
            f.write(new)
        else:
            raise ValueError(f"Patch mismatch at {hex(address)}")

from mitmproxy import http
from mitmproxy.tools.main import mitmdump

class APIInterceptor:
    def __init__(self):
        self.captured_requests = []
        self.captured_responses = []

    def request(self, flow: http.HTTPFlow) -> None:
        """Intercept HTTP request."""
        self.captured_requests.append({
            'method': flow.request.method,
            'url': flow.request.url,
            'headers': dict(flow.request.headers),
            'body': flow.request.content,
            'timestamp': time.time()
        })

    def response(self, flow: http.HTTPFlow) -> None:
        """Intercept HTTP response."""
        self.captured_responses.append({
            'status_code': flow.response.status_code,
            'headers': dict(flow.response.headers),
            'body': flow.response.content,
            'timestamp': time.time()
        })

# Start interception
interceptor = APIInterceptor()
mitmdump(['-s', 'interceptor.py', '--mode', 'transparent'])

# Deobfuscate JavaScript using js-beautify
import jsbeautifier

obfuscated_js = """
var _0x4e2a=['log','Hello'];
(function(_0x2d3a1c){
    var _0x4e2a1f=function(_0x2d3a1c){
        while(--_0x2d3a1c){
            _0x2d3a1c['push'](_0x2d3a1c['shift']());
        }
    };
    _0x4e2a1f(++_0x2d3a1c);
}(_0x4e2a,0x1a7));

var _0x4e2a=function(_0x2d3a1c,_0x4e2a1f){
    _0x2d3a1c=_0x2d3a1c-0x0;
    var _0x4e2a1f=_0x4e2a[_0x2d3a1c];
    return _0x4e2a1f;
};

console[_0x4e2a('0x0')](_0x4e2a('0x1'));
"""

# Beautify
beautified = jsbeautifier.beautify(obfuscated_js)
print(beautified)

# Step 1: Generate query embedding
query = "How to bypass authentication?"
query_embedding = embedding_gen.generate_embedding(query)

# Step 2: Search knowledge base
results = db.search_similar_instructions(
    embedding=query_embedding,
    limit=5
)

# Step 3: Rank results by relevance
ranked_results = sorted(
    results,
    key=lambda x: x['similarity'],
    reverse=True
)

# Step 4: Filter by threshold
filtered_results = [
    r for r in ranked_results
    if r['similarity'] >= 0.7
]

# Step 5: Augment prompt
context = "\n".join([
    f"- {r['content']} (similarity: {r['similarity']:.2%})"
    for r in filtered_results
])

augmented_prompt = f"""
Based on the following knowledge:
{context}

Answer this question: {query}
"""

# Step 6: Generate response
response = llm.generate(augmented_prompt)

# Store knowledge with metadata
knowledge_entry = {
    'content': 'Binary patching technique using NOP instructions',
    'metadata': {
        'category': 'patching',
        'difficulty': 'beginner',
        'tags': ['binary', 'patch', 'nop'],
        'source': 'documentation',
        'created_at': datetime.utcnow()
    },
    'embedding': embedding_gen.generate_embedding(content)
}

# Insert into database
db.execute_query("""
    INSERT INTO knowledge_base (content, metadata, embedding)
    VALUES (%s, %s, %s)
""", (
    knowledge_entry['content'],
    json.dumps(knowledge_entry['metadata']),
    knowledge_entry['embedding']
))

class HierarchicalMemory:
    def __init__(self, window_size=3):
        self.recent = []  # Recent messages
        self.important = []  # Important messages
        self.archived = []  # Archived messages
        self.window_size = window_size

    def add_message(self, message, importance=0.5):
        """Add message to memory."""
        self.recent.append({
            'content': message,
            'importance': importance,
            'timestamp': time.time()
        })

        # Promote important messages
        if importance > 0.8:
            self.important.append(self.recent.pop())

        # Archive old messages
        if len(self.recent) > self.window_size:
            self.archived.append(self.recent.pop(0))

    def get_context(self):
        """Get context for LLM."""
        context = []
        context.extend(self.recent)
        context.extend(self.important[:5])
        return context

class MemoryAugmentedAgent:
    def __init__(self):
        self.memory = HierarchicalMemory()
        self.llm = OpenRouterLLM()

    def execute(self, task):
        """Execute task with memory augmentation."""
        # Get memory context
        memory_context = self.memory.get_context()

        # Augment prompt with memory
        augmented_prompt = self._augment_prompt(task, memory_context)

        # Generate response
        response = self.llm.generate(augmented_prompt)

        # Store in memory
        self.memory.add_message(
            f"Task: {task}\nResponse: {response}",
            importance=0.7
        )

        return response

import cProfile
import pstats

# Profile binary analysis
profiler = cProfile.Profile()
profiler.enable()

oa = OrchestratingAgent()
result = oa.run("test_binary.exe")

profiler.disable()

# Print statistics
stats = pstats.Stats(profiler)
stats.sort_stats('cumulative')
stats.print_stats(20)  # Top 20 functions

from memory_profiler import profile

@profile
def analyze_large_binary(binary_path):
    """Profile memory usage."""
    oa = OrchestratingAgent()
    result = oa.run(binary_path)
    return result

# Run with memory profiler
# python -m memory_profiler script.py

-- Enable query logging
SET log_statement = 'all';
SET log_duration = on;
SET log_min_duration_statement = 100;  -- Log queries > 100ms

-- Analyze query plan
EXPLAIN ANALYZE
SELECT * FROM code_embeddings
WHERE 1 - (embedding <=> query_embedding::vector) >= 0.7
ORDER BY embedding <=> query_embedding::vector
LIMIT 10;

-- Check index usage
SELECT schemaname, tablename, indexname, idx_scan, idx_tup_read, idx_tup_fetch
FROM pg_stat_user_indexes
ORDER BY idx_scan DESC;

def run(self, binary_path: str) -> Dict[str, Any]:
    """
    Execute complete offline binary analysis pipeline.

    Args:
        binary_path (str): Path to binary file to analyze

    Returns:
        Dict with keys:
            - success (bool): Whether analysis succeeded
            - binary_id (int): Database ID of binary record
            - binary_hash (str): SHA256 hash of binary
            - metadata (Dict): Binary metadata (arch, type, size, etc.)
            - disassembly (Dict): Disassembly output from DAA
            - logic_map (Dict): Logic mapping from LIMA
            - patches (List): Generated patches from PEA
            - verification (Dict): Verification results from VA
            - execution_time_ms (int): Total execution time

    Raises:
        FileNotFoundError: If binary_path doesn't exist
        ValueError: If API key not configured
        RuntimeError: If analysis fails

    Example:
        >>> oa = OrchestratingAgent()
        >>> result = oa.run('/path/to/binary.exe')
        >>> print(f"Success: {result['success']}")
        >>> print(f"Patches: {len(result['patches'])}")
    """

def call_openrouter(self, prompt: str, max_tokens: int = 500) -> Dict[str, Any]:
    """
    Call OpenRouter LLM API with caching.

    Args:
        prompt (str): Prompt to send to LLM
        max_tokens (int): Maximum tokens in response (default: 500)

    Returns:
        Dict with keys:
            - content (str): LLM response text
            - model (str): Model used
            - tokens_used (int): Tokens consumed
            - cached (bool): Whether response was cached

    Raises:
        ValueError: If API key not set
        RuntimeError: If API call fails

    Example:
        >>> response = oa.call_openrouter("Analyze this code: ...")
        >>> print(response['content'])
    """

def search_similar_instructions(
    self,
    embedding: List[float],
    limit: int = 10,
    threshold: float = 0.7
) -> List[Dict[str, Any]]:
    """
    Search for similar instructions using vector similarity.

    Args:
        embedding (List[float]): Query embedding (384 dimensions)
        limit (int): Maximum results to return (default: 10)
        threshold (float): Minimum similarity score (default: 0.7)

    Returns:
        List of dicts with keys:
            - address (str): Instruction address
            - instruction (str): Instruction text
            - opcode (str): Opcode bytes
            - operands (str): Operand text
            - similarity (float): Similarity score (0-1)

    Example:
        >>> embedding = embedding_gen.generate_embedding("cmp eax, 0")
        >>> results = db.search_similar_instructions(embedding, limit=5)
        >>> for r in results:
        ...     print(f"{r['instruction']} ({r['similarity']:.2%})")
    """

def create_binary_record(
    self,
    file_name: str,
    file_path: str,
    file_hash: str,
    file_size: int,
    file_type: str,
    architecture: str,
    metadata: Dict[str, Any] = None
) -> int:
    """
    Create binary record in database.

    Args:
        file_name (str): Binary filename
        file_path (str): Full path to binary
        file_hash (str): SHA256 hash
        file_size (int): File size in bytes
        file_type (str): File type (ELF, PE, Mach-O)
        architecture (str): Architecture (x86, x64, ARM)
        metadata (Dict): Additional metadata

    Returns:
        int: Binary ID in database

    Raises:
        IntegrityError: If binary already exists

    Example:
        >>> binary_id = db.create_binary_record(
        ...     file_name="app.exe",
        ...     file_path="/binaries/app.exe",
        ...     file_hash="abc123...",
        ...     file_size=1024000,
        ...     file_type="PE",
        ...     architecture="x64"
        ... )
    """

def cache_analysis(
    self,
    binary_hash: str,
    analysis_type: str,
    result: Dict[str, Any],
    ttl: int = 604800
) -> None:
    """
    Cache analysis result.

    Args:
        binary_hash (str): Binary SHA256 hash
        analysis_type (str): Type of analysis (full_analysis, quick_scan)
        result (Dict): Analysis result to cache
        ttl (int): Time to live in seconds (default: 7 days)

    Example:
        >>> cache.cache_analysis(
        ...     binary_hash="abc123...",
        ...     analysis_type="full_analysis",
        ...     result=analysis_result,
        ...     ttl=86400  # 1 day
        ... )
    """

def get_cached_analysis(
    self,
    binary_hash: str,
    analysis_type: str = 'full_analysis'
) -> Optional[Dict[str, Any]]:
    """
    Retrieve cached analysis result.

    Args:
        binary_hash (str): Binary SHA256 hash
        analysis_type (str): Type of analysis

    Returns:
        Dict with cached result, or None if not found/expired

    Example:
        >>> cached = cache.get_cached_analysis("abc123...")
        >>> if cached:
        ...     print("Using cached result")
        ... else:
        ...     print("Cache miss, running analysis")
    """

# OpenRouter API
OPENROUTER_API_KEY=sk-or-v1-...          # Required: OpenRouter API key
OPENROUTER_MODEL=meta-llama/llama-3.3-70b-instruct:free  # LLM model to use
OPENROUTER_TIMEOUT=30                    # API timeout in seconds
OPENROUTER_MAX_RETRIES=3                 # Max retry attempts

# PostgreSQL
DB_HOST=localhost                        # Database host
DB_PORT=5432                             # Database port
DB_USER=raverse                          # Database user
DB_PASSWORD=your_password                # Database password
DB_NAME=raverse_db                       # Database name
DB_POOL_SIZE=10                          # Connection pool size
DB_MAX_OVERFLOW=20                       # Max overflow connections
DB_POOL_RECYCLE=3600                     # Recycle connections after (seconds)

# Redis
REDIS_HOST=localhost                     # Redis host
REDIS_PORT=6379                          # Redis port
REDIS_DB=0                               # Redis database number
REDIS_PASSWORD=                          # Redis password (if required)
REDIS_CLUSTER_MODE=false                 # Enable cluster mode
REDIS_CACHE_TTL=604800                   # Cache TTL in seconds (7 days)

# Logging
LOG_LEVEL=INFO                           # Log level (DEBUG, INFO, WARNING, ERROR)
LOG_FILE=logs/raverse.log                # Log file path
LOG_FORMAT=json                          # Log format (json, text)
LOG_MAX_SIZE=104857600                   # Max log file size (100MB)
LOG_BACKUP_COUNT=10                      # Number of backup log files

# Features
ENABLE_VECTOR_SEARCH=true                # Enable vector similarity search
ENABLE_RAG=true                          # Enable RAG augmentation
ENABLE_CACHING=true                      # Enable result caching
ENABLE_MONITORING=true                   # Enable Prometheus metrics
ENABLE_PROFILING=false                   # Enable performance profiling

# Performance
BATCH_SIZE=32                            # Embedding batch size
MAX_CONCURRENT_ANALYSES=5                # Max concurrent analyses
EMBEDDING_CACHE_SIZE=10000               # In-memory embedding cache size
VECTOR_SEARCH_LIMIT=10                   # Default vector search limit
VECTOR_SEARCH_THRESHOLD=0.7              # Vector similarity threshold

# Main application settings
DEBUG = False
ENVIRONMENT = 'production'
LOG_LEVEL = 'INFO'

# Database settings
DATABASE = {
    'host': 'localhost',
    'port': 5432,
    'user': 'raverse',
    'password': '',
    'database': 'raverse_db'
}

# Cache settings
CACHE = {
    'backend': 'redis',
    'host': 'localhost',
    'port': 6379,
    'ttl': 604800  # 7 days
}

# LLM settings
LLM = {
    'provider': 'openrouter',
    'model': 'meta-llama/llama-3.3-70b-instruct:free',
    'timeout': 30,
    'max_retries': 3
}

# Agent memory configurations
AGENT_MEMORY_CONFIG = {
    'version_manager': {
        'strategy': 'hierarchical',
        'preset': 'medium',
        'reason': 'Critical version info retention'
    },
    'knowledge_base': {
        'strategy': 'retrieval',
        'preset': 'heavy',
        'reason': 'Semantic search for knowledge'
    },
    'quality_gate': {
        'strategy': 'memory_augmented',
        'preset': 'medium',
        'reason': 'Critical metrics + context'
    },
    # ... more agents
}

# Memory presets
MEMORY_PRESETS = {
    'none': {
        'description': 'No memory',
        'ram_mb': 0,
        'cpu_percent': 0
    },
    'light': {
        'description': 'Sliding window memory',
        'ram_mb': 5,
        'cpu_percent': 1
    },
    'medium': {
        'description': 'Hierarchical memory',
        'ram_mb': 20,
        'cpu_percent': 3
    },
    'heavy': {
        'description': 'Retrieval + RAG',
        'ram_mb': 100,
        'cpu_percent': 5
    }
}

# DeepCrawler configuration
DEEPCRAWLER_CONFIG = {
    'max_depth': 3,
    'max_urls': 10000,
    'max_concurrent': 5,
    'timeout': 30,
    'rate_limit': 20.0,  # requests per minute
    'detect_rest_apis': True,
    'detect_graphql': True,
    'detect_websockets': True,
    'min_confidence_score': 0.6,
    'output_format': 'openapi',
    'output_dir': './crawl_results'
}

# Increase pool size
db = DatabaseManager(
    pool_size=20,
    max_overflow=30
)

# Or check active connections
SELECT count(*) FROM pg_stat_activity;

# Kill idle connections
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE state = 'idle' AND query_start < now() - interval '1 hour';

-- Check index exists
SELECT * FROM pg_indexes WHERE tablename = 'code_embeddings';

-- Rebuild index if needed
REINDEX INDEX idx_embeddings_hnsw;

-- Lower threshold
SELECT * FROM code_embeddings
WHERE 1 - (embedding <=> query_embedding::vector) >= 0.5
LIMIT 10;

# Check Redis status
redis-cli ping

# Start Redis
docker-compose up -d redis

# Test connection
redis-cli -h localhost -p 6379 ping

# Increase TTL
cache.cache_analysis(
    binary_hash="abc123",
    analysis_type="full_analysis",
    result=result,
    ttl=2592000  # 30 days instead of 7
)

# Increase cache size
cache = CacheManager(
    redis_host='localhost',
    redis_port=6379,
    max_memory='2gb'  # Increase from 1gb
)

# Use faster model
oa = OrchestratingAgent(
    model="meta-llama/llama-3.2-3b-instruct:free"  # Faster
)

# Or reduce binary size
# Split large binary into chunks
chunks = split_binary(binary_path, chunk_size=1000000)
for chunk in chunks:
    result = oa.run(chunk)

# Use lighter memory preset
from src.config.agent_memory_config import MEMORY_PRESETS
preset = MEMORY_PRESETS['light']

# Or reduce batch size
embedding_gen = EmbeddingGenerator(batch_size=8)

# Or use streaming
for chunk in stream_embeddings(texts, batch_size=16):
    process_chunk(chunk)

# Implement rate limiting
from ratelimit import limits, sleep_and_retry

@sleep_and_retry
@limits(calls=10, period=60)  # 10 calls per minute
def call_openrouter(prompt):
    return oa.call_openrouter(prompt)

# Or use queue
from queue import Queue
request_queue = Queue(maxsize=10)

# Check environment variable
echo $OPENROUTER_API_KEY

# Set if missing
export OPENROUTER_API_KEY=sk-or-v1-your-key

# Verify key format
# Should start with: sk-or-v1-

class Instruction:
    """Represents a single CPU instruction."""

    def __init__(self, address, opcode, mnemonic, operands):
        self.address = address  # Virtual address
        self.opcode = opcode    # Raw bytes
        self.mnemonic = mnemonic  # e.g., "mov", "jmp"
        self.operands = operands  # e.g., ["rax", "rbx"]
        self.size = len(opcode)
        self.embedding = None   # Vector embedding

    def __repr__(self):
        return f"{hex(self.address)}: {self.mnemonic} {', '.join(self.operands)}"

class Function:
    """Represents a function in binary."""

    def __init__(self, address, name=None):
        self.address = address
        self.name = name or f"func_{hex(address)}"
        self.instructions = []  # List of Instruction objects
        self.basic_blocks = []  # List of BasicBlock objects
        self.calls = []  # Functions this calls
        self.callers = []  # Functions that call this
        self.size = 0

    def add_instruction(self, instruction):
        """Add instruction to function."""
        self.instructions.append(instruction)
        self.size += instruction.size

    def get_control_flow_graph(self):
        """Build control flow graph."""
        cfg = {}
        for bb in self.basic_blocks:
            cfg[bb.address] = bb.successors
        return cfg

class BasicBlock:
    """Represents a basic block (straight-line code)."""

    def __init__(self, address):
        self.address = address
        self.instructions = []
        self.successors = []  # Next basic blocks
        self.predecessors = []  # Previous basic blocks

    def is_terminator(self, instruction):
        """Check if instruction terminates block."""
        terminators = ['jmp', 'je', 'jne', 'ret', 'call']
        return instruction.mnemonic in terminators

class HNSWIndex:
    """HNSW index for approximate nearest neighbor search."""

    def __init__(self, m=16, ef_construction=64, ef_search=200):
        self.m = m  # Number of connections per node
        self.ef_construction = ef_construction  # Construction parameter
        self.ef_search = ef_search  # Search parameter
        self.graph = {}  # Adjacency list
        self.data = {}  # Vector data

    def insert(self, vector_id, vector):
        """Insert vector into index."""
        # Find nearest neighbors
        neighbors = self._find_neighbors(vector, self.ef_construction)

        # Add to graph
        self.graph[vector_id] = neighbors[:self.m]
        self.data[vector_id] = vector

    def search(self, query_vector, k=10):
        """Search for k nearest neighbors."""
        # Start from random node
        candidates = self._find_neighbors(query_vector, self.ef_search)

        # Return top-k
        return sorted(
            candidates,
            key=lambda x: self._distance(query_vector, self.data[x])
        )[:k]

    def _find_neighbors(self, vector, ef):
        """Find approximate neighbors."""
        # Implementation of HNSW search algorithm
        pass

    def _distance(self, v1, v2):
        """Compute cosine distance."""
        return 1 - (np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2)))

def cosine_similarity(v1, v2):
    """Compute cosine similarity between vectors."""
    dot_product = np.dot(v1, v2)
    norm_v1 = np.linalg.norm(v1)
    norm_v2 = np.linalg.norm(v2)

    if norm_v1 == 0 or norm_v2 == 0:
        return 0

    return dot_product / (norm_v1 * norm_v2)

def cosine_distance(v1, v2):
    """Compute cosine distance (1 - similarity)."""
    return 1 - cosine_similarity(v1, v2)

def dfs_traverse(start_block, graph):
    """Traverse control flow graph using DFS."""
    visited = set()
    stack = [start_block]
    order = []

    while stack:
        block = stack.pop()
        if block in visited:
            continue

        visited.add(block)
        order.append(block)

        # Add successors to stack
        for successor in graph.get(block, []):
            if successor not in visited:
                stack.append(successor)

    return order

def compute_dominators(start_block, graph):
    """Compute dominator tree."""
    blocks = set(graph.keys())
    dominators = {block: blocks.copy() for block in blocks}
    dominators[start_block] = {start_block}

    changed = True
    while changed:
        changed = False
        for block in blocks:
            if block == start_block:
                continue

            # Intersection of dominators of predecessors
            new_dom = blocks.copy()
            for pred in get_predecessors(block, graph):
                new_dom &= dominators[pred]

            new_dom.add(block)

            if new_dom != dominators[block]:
                dominators[block] = new_dom
                changed = True

    return dominators

def generate_embeddings(texts, model_name='all-MiniLM-L6-v2'):
    """Generate embeddings using sentence transformers."""
    from sentence_transformers import SentenceTransformer

    model = SentenceTransformer(model_name)
    embeddings = model.encode(
        texts,
        batch_size=32,
        show_progress_bar=True,
        convert_to_numpy=True
    )

    return embeddings  # Shape: (n_texts, 384)

def batch_encode_with_cache(texts, cache, model):
    """Encode texts with caching."""
    embeddings = []
    uncached_texts = []
    uncached_indices = []

    # Check cache
    for i, text in enumerate(texts):
        text_hash = hashlib.sha256(text.encode()).hexdigest()
        cached = cache.get(f"embedding:{text_hash}")

        if cached:
            embeddings.append(cached)
        else:
            uncached_texts.append(text)
            uncached_indices.append(i)

    # Encode uncached
    if uncached_texts:
        new_embeddings = model.encode(uncached_texts, batch_size=32)

        # Cache and add to results
        for i, (text, embedding) in enumerate(zip(uncached_texts, new_embeddings)):
            text_hash = hashlib.sha256(text.encode()).hexdigest()
            cache.set(f"embedding:{text_hash}", embedding, ttl=604800)
            embeddings.insert(uncached_indices[i], embedding)

    return np.array(embeddings)

class SlidingWindowMemory:
    """Sliding window memory with fixed size."""

    def __init__(self, window_size=3):
        self.window_size = window_size
        self.messages = []

    def add_message(self, message):
        """Add message to window."""
        self.messages.append(message)

        # Remove oldest if exceeds window size
        if len(self.messages) > self.window_size:
            self.messages.pop(0)

    def get_context(self):
        """Get current context."""
        return self.messages

class HierarchicalMemory:
    """Hierarchical memory with importance-based promotion."""

    def __init__(self, recent_size=3, important_size=5):
        self.recent = []
        self.important = []
        self.archived = []
        self.recent_size = recent_size
        self.important_size = important_size

    def add_message(self, message, importance=0.5):
        """Add message with importance score."""
        msg_obj = {
            'content': message,
            'importance': importance,
            'timestamp': time.time()
        }

        if importance > 0.8:
            # High importance: add to important
            self.important.append(msg_obj)
            if len(self.important) > self.important_size:
                self.archived.append(self.important.pop(0))
        else:
            # Normal: add to recent
            self.recent.append(msg_obj)
            if len(self.recent) > self.recent_size:
                self.archived.append(self.recent.pop(0))

    def get_context(self, max_messages=10):
        """Get context for LLM."""
        context = []
        context.extend(self.recent)
        context.extend(self.important)

        # Sort by timestamp (most recent first)
        context.sort(key=lambda x: x['timestamp'], reverse=True)

        return context[:max_messages]

from collections import OrderedDict

class LRUCache:
    """Least Recently Used cache."""

    def __init__(self, capacity=1000):
        self.cache = OrderedDict()
        self.capacity = capacity

    def get(self, key):
        """Get value from cache."""
        if key not in self.cache:
            return None

        # Move to end (most recently used)
        self.cache.move_to_end(key)
        return self.cache[key]

    def put(self, key, value):
        """Put value in cache."""
        if key in self.cache:
            self.cache.move_to_end(key)

        self.cache[key] = value

        # Remove least recently used if exceeds capacity
        if len(self.cache) > self.capacity:
            self.cache.popitem(last=False)

class MultiLevelCache:
    """Multi-level cache: L1 (memory) -> L2 (Redis) -> L3 (DB)."""

    def __init__(self, l1_size=1000, l2_ttl=3600, l3_ttl=86400):
        self.l1 = LRUCache(capacity=l1_size)
        self.l2_ttl = l2_ttl
        self.l3_ttl = l3_ttl
        self.redis = redis.Redis()
        self.db = DatabaseManager()

    def get(self, key):
        """Get from cache hierarchy."""
        # Try L1
        value = self.l1.get(key)
        if value:
            return value

        # Try L2
        value = self.redis.get(key)
        if value:
            self.l1.put(key, value)
            return value

        # Try L3
        value = self.db.get(key)
        if value:
            self.redis.set(key, value, ex=self.l2_ttl)
            self.l1.put(key, value)
            return value

        return None

    def put(self, key, value):
        """Put in all cache levels."""
        self.l1.put(key, value)
        self.redis.set(key, value, ex=self.l2_ttl)
        self.db.put(key, value, ttl=self.l3_ttl)

class BaseAgent:
    """Base class for all agents."""

    def __init__(self, name, orchestrator, api_key, model):
        self.name = name
        self.orchestrator = orchestrator
        self.api_key = api_key
        self.model = model
        self.logger = logging.getLogger(self.name)

    def execute(self, task):
        """Execute agent task."""
        try:
            self.logger.info(f"Starting {self.name}")
            result = self._execute_impl(task)
            self.logger.info(f"Completed {self.name}")
            return result
        except Exception as e:
            self.logger.exception(f"Error in {self.name}: {e}")
            raise

    def _execute_impl(self, task):
        """Implement agent logic (override in subclass)."""
        raise NotImplementedError

class Pipeline:
    """Execute agents in sequence."""

    def __init__(self, agents):
        self.agents = agents

    def execute(self, input_data):
        """Execute pipeline."""
        result = input_data

        for agent in self.agents:
            result = agent.execute(result)

        return result

class AgentFactory:
    """Factory for creating agents."""

    _agents = {}

    @classmethod
    def register(cls, name, agent_class):
        """Register agent class."""
        cls._agents[name] = agent_class

    @classmethod
    def create(cls, name, **kwargs):
        """Create agent instance."""
        if name not in cls._agents:
            raise ValueError(f"Unknown agent: {name}")

        return cls._agents[name](**kwargs)

# Register agents
AgentFactory.register('daa', DisassemblyAnalysisAgent)
AgentFactory.register('lima', LogicIdentificationMappingAgent)
AgentFactory.register('pea', PatchingExecutionAgent)
AgentFactory.register('va', VerificationAgent)

# tests/unit/test_orchestrator.py
import pytest
from src.agents.orchestrator import OrchestratingAgent

class TestOrchestratingAgent:
    """Test suite for OrchestratingAgent."""

    @pytest.fixture
    def orchestrator(self):
        """Create test orchestrator."""
        return OrchestratingAgent(use_database=False)

    @pytest.fixture
    def sample_binary(self, tmp_path):
        """Create sample binary for testing."""
        binary_path = tmp_path / "test.bin"
        binary_path.write_bytes(b"\x55\x89\xe5\x83\xec\x10")  # x86 prologue
        return str(binary_path)

    def test_run_success(self, orchestrator, sample_binary):
        """Test successful binary analysis."""
        result = orchestrator.run(sample_binary)

        assert result['success']
        assert 'binary_hash' in result
        assert 'disassembly' in result

    def test_run_invalid_path(self, orchestrator):
        """Test with invalid binary path."""
        with pytest.raises(FileNotFoundError):
            orchestrator.run("/nonexistent/binary")

    def test_call_openrouter_success(self, orchestrator):
        """Test OpenRouter API call."""
        response = orchestrator.call_openrouter("Test prompt")

        assert 'content' in response
        assert response['model'] == orchestrator.model

# tests/conftest.py
import pytest
from src.utils.database import DatabaseManager
from src.utils.cache import CacheManager

@pytest.fixture(scope='session')
def test_db():
    """Create test database."""
    db = DatabaseManager(database='raverse_test')
    db.create_tables()
    yield db
    db.drop_tables()

@pytest.fixture(scope='session')
def test_cache():
    """Create test cache."""
    cache = CacheManager(redis_db=15)  # Use separate Redis DB
    yield cache
    cache.flush()

@pytest.fixture
def sample_embedding():
    """Create sample embedding."""
    return [0.1] * 384  # 384-dimensional vector

# tests/integration/test_end_to_end.py
import pytest

class TestEndToEnd:
    """End-to-end integration tests."""

    def test_complete_analysis_pipeline(self, test_db, test_cache):
        """Test complete offline pipeline."""
        from src.agents.orchestrator import OrchestratingAgent

        # Setup
        oa = OrchestratingAgent(use_database=True)
        binary_path = "tests/fixtures/test_binary.exe"

        # Execute
        result = oa.run(binary_path)

        # Verify
        assert result['success']

        # Check database
        records = test_db.execute_query(
            "SELECT * FROM raverse.binaries WHERE file_hash = %s",
            (result['binary_hash'],)
        )
        assert len(records) > 0

        # Check cache
        cached = test_cache.get_cached_analysis(result['binary_hash'])
        assert cached is not None

    def test_vector_search_integration(self, test_db):
        """Test vector search integration."""
        from src.utils.semantic_search import SemanticSearchEngine
        from src.utils.embeddings_v2 import EmbeddingGenerator

        # Setup
        embedding_gen = EmbeddingGenerator()
        search_engine = SemanticSearchEngine(test_db, None)

        # Store code
        code = "cmp eax, 0x0; je 0x401000"
        embedding = embedding_gen.generate_embedding(code)
        search_engine.store_code_embedding(
            binary_hash="test123",
            code_snippet=code,
            metadata={'function': 'main'}
        )

        # Search
        results = search_engine.find_similar_code(
            query="compare eax with zero",
            limit=5
        )

        assert len(results) > 0
        assert results[0]['similarity'] > 0.7

# tests/performance/test_benchmarks.py
import pytest
import time

class TestPerformance:
    """Performance benchmark tests."""

    @pytest.mark.benchmark
    def test_binary_analysis_performance(self, benchmark):
        """Benchmark binary analysis."""
        from src.agents.orchestrator import OrchestratingAgent

        oa = OrchestratingAgent()
        binary_path = "tests/fixtures/test_binary.exe"

        # Run benchmark
        result = benchmark(oa.run, binary_path)

        # Assert performance
        assert result['execution_time_ms'] < 5000  # < 5 seconds

    @pytest.mark.benchmark
    def test_vector_search_performance(self, benchmark):
        """Benchmark vector search."""
        from src.utils.semantic_search import SemanticSearchEngine

        search_engine = SemanticSearchEngine(None, None)
        query_embedding = [0.1] * 384

        # Run benchmark
        result = benchmark(
            search_engine.find_similar_code,
            query="test",
            limit=10
        )

        # Assert performance
        assert result['execution_time_ms'] < 100  # < 100ms

# .coveragerc
[run]
source = src
omit =
    */tests/*
    */venv/*
    setup.py

[report]
exclude_lines =
    pragma: no cover
    def __repr__
    raise AssertionError
    raise NotImplementedError
    if __name__ == .__main__.:
    if TYPE_CHECKING:
    @abstractmethod

[html]
directory = htmlcov

# Generate coverage report
pytest tests/ --cov=src --cov-report=html --cov-report=term

# View report
open htmlcov/index.html

# Check coverage threshold
pytest tests/ --cov=src --cov-fail-under=80

# .github/workflows/ci.yml
name: CI/CD Pipeline

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main, develop]

jobs:
  test:
    runs-on: ubuntu-latest

    services:
      postgres:
        image: pgvector/pgvector:pg17-latest
        env:
          POSTGRES_USER: raverse
          POSTGRES_PASSWORD: test
          POSTGRES_DB: raverse_test
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432

      redis:
        image: redis:8.2-alpine
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 6379:6379

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.13'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install pytest pytest-cov pytest-xdist

      - name: Lint with ruff
        run: ruff check src/ tests/

      - name: Type check with mypy
        run: mypy src/

      - name: Run tests
        run: |
          pytest tests/ -v --cov=src --cov-report=xml
        env:
          DB_HOST: localhost
          DB_PORT: 5432
          DB_USER: raverse
          DB_PASSWORD: test
          DB_NAME: raverse_test
          REDIS_HOST: localhost
          REDIS_PORT: 6379

      - name: Upload coverage
        uses: codecov/codecov-action@v3
        with:
          files: ./coverage.xml
          flags: unittests
          name: codecov-umbrella

# .github/workflows/docker.yml
name: Docker Build & Push

on:
  push:
    branches: [main]
    tags: ['v*']

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Build and push
        uses: docker/build-push-action@v4
        with:
          context: .
          push: true
          tags: |
            ${{ secrets.DOCKER_USERNAME }}/raverse:latest
            ${{ secrets.DOCKER_USERNAME }}/raverse:${{ github.sha }}
          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/raverse:buildcache
          cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/raverse:buildcache,mode=max

1. INPUT: Binary file path
   ↓
2. METADATA EXTRACTION
   - File type detection (ELF, PE, Mach-O)
   - Architecture detection (x86, x64, ARM)
   - Size and hash calculation
   ↓
3. DISASSEMBLY (DAA)
   - Load binary with Capstone
   - Disassemble all code sections
   - Identify functions
   - Generate embeddings
   ↓
4. LOGIC ANALYSIS (LIMA)
   - Build control flow graph
   - Analyze data flow
   - Identify algorithms
   - LLM semantic analysis
   ↓
5. PATCH GENERATION (PEA)
   - Identify vulnerable patterns
   - Generate patches
   - Apply patches to binary
   - Create backup
   ↓
6. VERIFICATION (VA)
   - Verify binary structure
   - Verify patches applied
   - Test functionality
   - Generate report
   ↓
7. OUTPUT: Analysis result with patches

1. INPUT: Target URL
   ↓
2. RECONNAISSANCE
   - Technology stack detection
   - Endpoint discovery
   - Server information gathering
   ↓
3. TRAFFIC INTERCEPTION
   - Start mitmproxy
   - Navigate application
   - Capture HTTP(S) traffic
   - Extract API calls
   ↓
4. JAVASCRIPT ANALYSIS
   - Extract JavaScript code
   - Deobfuscate
   - Analyze client-side logic
   - Extract API calls
   ↓
5. API REVERSE ENGINEERING
   - Map endpoints
   - Extract parameters
   - Detect authentication
   - Generate OpenAPI spec
   ↓
6. SECURITY ANALYSIS
   - Identify vulnerabilities
   - Generate POCs
   - Assess risk
   ↓
7. VALIDATION & REPORTING
   - Validate findings
   - Generate report
   - Export results
   ↓
8. OUTPUT: API documentation + security report

1. INPUT: User query
   ↓
2. EMBEDDING GENERATION
   - Convert query to embedding
   - Use sentence-transformers
   ↓
3. KNOWLEDGE RETRIEVAL
   - Search knowledge base
   - Use vector similarity
   - Filter by threshold
   - Rank by relevance
   ↓
4. CONTEXT AUGMENTATION
   - Combine query + retrieved knowledge
   - Maintain token budget
   - Format for LLM
   ↓
5. LLM GENERATION
   - Call OpenRouter API
   - Generate response
   - Include sources
   ↓
6. OUTPUT: Response with sources

# 1. Clone repository
git clone https://github.com/usemanusai/RAVERSE.git
cd RAVERSE

# 2. Create virtual environment
python -m venv venv
source venv/bin/activate  # On Windows: venv\Scripts\activate

# 3. Install dependencies
pip install -r requirements.txt

# 4. Configure environment
cp .env.example .env
# Edit .env with your settings

# 5. Initialize database
python -m src.utils.database --init

# 6. Run application
python src/main.py

# 1. Build Docker image
docker build -t raverse:latest .

# 2. Push to registry
docker push your-registry/raverse:latest

# 3. Deploy with Docker Compose
docker-compose -f docker-compose.prod.yml up -d

# 4. Verify deployment
docker-compose ps
docker-compose logs -f raverse

# 5. Run health checks
curl http://localhost:8000/health

# 1. Create namespace
kubectl create namespace raverse

# 2. Create secrets
kubectl create secret generic raverse-secrets \
  --from-literal=api-key=$OPENROUTER_API_KEY \
  -n raverse

# 3. Deploy Helm chart
helm install raverse ./helm/raverse \
  -n raverse \
  -f helm/values-prod.yaml

# 4. Verify deployment
kubectl get pods -n raverse
kubectl logs -f deployment/raverse -n raverse

# 5. Access application
kubectl port-forward svc/raverse 8000:8000 -n raverse

# 1. Navigate to workflows directory
cd workflows-starter

# 2. Install dependencies
npm install

# 3. Authenticate with Cloudflare
npx wrangler login

# 4. Setup infrastructure (KV namespaces and D1 database)
npm run setup

# 5. Set secrets
npx wrangler secret put OPENROUTER_API_KEY

# 6. Deploy to Cloudflare
npm run deploy

# 7. Verify deployment
curl https://raverse-workflows.use-manus-ai.workers.dev/health

Client → Cloudflare Workers (Edge) → Cloudflare Workflows → Render (RAVERSE API)
                    ↓
            KV Cache (RAVERSE_CACHE)
            D1 Database (raverse-workflows)

Layer 1: Network Security
├── TLS/SSL encryption for all connections
├── Network policies for pod-to-pod communication
├── Firewall rules for ingress/egress
└── DDoS protection

Layer 2: Application Security
├── Input validation and sanitization
├── SQL injection prevention (parameterized queries)
├── XSS protection
├── CSRF tokens
└── Rate limiting

Layer 3: Data Security
├── Encryption at rest (AES-256)
├── Encryption in transit (TLS 1.3)
├── Database encryption
├── Secrets management (Vault/K8s Secrets)
└── Data masking for sensitive fields

Layer 4: Access Control
├── Authentication (API keys, OAuth)
├── Authorization (RBAC)
├── Audit logging
├── Session management
└── Multi-factor authentication

# ✓ GOOD: Secure secrets handling
import os
from dotenv import load_dotenv

# Load from .env (never commit to git)
load_dotenv()

# Get secrets from environment
api_key = os.getenv('OPENROUTER_API_KEY')
db_password = os.getenv('DB_PASSWORD')

# Validate secrets are set
if not api_key:
    raise ValueError("OPENROUTER_API_KEY not configured")

# Never log secrets
logger.info(f"Using API key: {api_key[:10]}...")  # Only show prefix

# Use Kubernetes Secrets in production
# kubectl create secret generic raverse-secrets \
#   --from-literal=api-key=$OPENROUTER_API_KEY

# ✓ GOOD: Comprehensive input validation
import os
from pathlib import Path

def validate_binary_path(path: str) -> str:
    """Validate binary path for security."""
    # Check path exists
    if not os.path.exists(path):
        raise FileNotFoundError(f"Binary not found: {path}")

    # Check path is file
    if not os.path.isfile(path):
        raise ValueError(f"Path is not a file: {path}")

    # Check path is within allowed directory
    allowed_dir = os.path.abspath('/binaries')
    real_path = os.path.abspath(path)

    if not real_path.startswith(allowed_dir):
        raise ValueError(f"Path outside allowed directory: {path}")

    # Check file size (prevent DoS)
    max_size = 1024 * 1024 * 100  # 100 MB
    if os.path.getsize(real_path) > max_size:
        raise ValueError(f"Binary too large: {os.path.getsize(real_path)} bytes")

    return real_path

# ✓ GOOD: Parameterized queries
from src.utils.database import DatabaseManager

db = DatabaseManager()

# GOOD: Parameterized query
result = db.execute_query(
    "SELECT * FROM binaries WHERE file_hash = %s",
    (user_input,)  # Parameters passed separately
)

# BAD: String concatenation (vulnerable)
# result = db.execute_query(f"SELECT * FROM binaries WHERE file_hash = '{user_input}'")

# Data retention policy
DATA_RETENTION_POLICY = {
    'analysis_results': 90,  # days
    'user_data': 365,
    'logs': 30,
    'backups': 90
}

# Right to be forgotten
def delete_user_data(user_id):
    """Delete all user data (GDPR right to be forgotten)."""
    db = DatabaseManager()

    # Delete analysis results
    db.execute_query(
        "DELETE FROM analysis_results WHERE user_id = %s",
        (user_id,)
    )

    # Delete user record
    db.execute_query(
        "DELETE FROM users WHERE id = %s",
        (user_id,)
    )

    # Delete from cache
    cache = CacheManager()
    cache.delete_user_cache(user_id)

    # Log deletion
    logger.info(f"User data deleted: {user_id}")

# Audit logging for HIPAA
def log_access(user_id, resource_id, action):
    """Log access for audit trail."""
    audit_log = {
        'timestamp': datetime.utcnow(),
        'user_id': user_id,
        'resource_id': resource_id,
        'action': action,
        'ip_address': get_client_ip(),
        'user_agent': get_user_agent()
    }

    db = DatabaseManager()
    db.execute_query(
        "INSERT INTO audit_log (timestamp, user_id, resource_id, action, ip_address, user_agent) "
        "VALUES (%s, %s, %s, %s, %s, %s)",
        (audit_log['timestamp'], audit_log['user_id'], audit_log['resource_id'],
         audit_log['action'], audit_log['ip_address'], audit_log['user_agent'])
    )

# SOC 2 requirements
SOC2_REQUIREMENTS = {
    'CC6.1': 'Logical access controls',
    'CC6.2': 'Prior to issuing system credentials',
    'CC7.1': 'System monitoring and alerting',
    'CC7.2': 'System monitoring tools',
    'CC7.3': 'Unauthorized activities detection',
    'CC7.4': 'Identified security incidents response',
    'CC8.1': 'Incident response procedures',
    'CC9.1': 'Change management procedures'
}

# Implement monitoring
def setup_monitoring():
    """Setup SOC 2 monitoring."""
    # Enable audit logging
    enable_audit_logging()

    # Setup alerting
    setup_alerts()

    # Enable encryption
    enable_encryption()

    # Setup access controls
    setup_rbac()

# Detect common vulnerability patterns
VULNERABILITY_PATTERNS = {
    'buffer_overflow': {
        'pattern': r'mov.*\[.*\+.*\]',
        'description': 'Potential buffer overflow',
        'severity': 'high'
    },
    'use_after_free': {
        'pattern': r'mov.*\[.*\].*free',
        'description': 'Potential use-after-free',
        'severity': 'high'
    },
    'integer_overflow': {
        'pattern': r'add.*jno',
        'description': 'Potential integer overflow',
        'severity': 'medium'
    },
    'format_string': {
        'pattern': r'printf.*%x',
        'description': 'Potential format string vulnerability',
        'severity': 'high'
    }
}

def detect_vulnerabilities(disassembly):
    """Detect vulnerabilities in disassembly."""
    vulnerabilities = []

    for vuln_name, vuln_info in VULNERABILITY_PATTERNS.items():
        pattern = vuln_info['pattern']

        for instruction in disassembly:
            if re.match(pattern, instruction):
                vulnerabilities.append({
                    'type': vuln_name,
                    'description': vuln_info['description'],
                    'severity': vuln_info['severity'],
                    'instruction': instruction
                })

    return vulnerabilities

# Patch generation strategies
PATCH_STRATEGIES = {
    'nop_padding': {
        'description': 'Replace vulnerable code with NOPs',
        'risk': 'low',
        'effectiveness': 'medium'
    },
    'bounds_check': {
        'description': 'Add bounds checking before access',
        'risk': 'low',
        'effectiveness': 'high'
    },
    'return_early': {
        'description': 'Add early return to skip vulnerable code',
        'risk': 'medium',
        'effectiveness': 'high'
    },
    'exception_handler': {
        'description': 'Wrap in exception handler',
        'risk': 'medium',
        'effectiveness': 'medium'
    }
}

def generate_patches(vulnerabilities):
    """Generate patches for vulnerabilities."""
    patches = []

    for vuln in vulnerabilities:
        if vuln['severity'] == 'high':
            strategy = 'bounds_check'
        elif vuln['severity'] == 'medium':
            strategy = 'return_early'
        else:
            strategy = 'nop_padding'

        patch = {
            'vulnerability': vuln,
            'strategy': strategy,
            'description': PATCH_STRATEGIES[strategy]['description'],
            'risk': PATCH_STRATEGIES[strategy]['risk']
        }

        patches.append(patch)

    return patches

# Technology stack detection
TECHNOLOGY_SIGNATURES = {
    'frameworks': {
        'Django': ['django', 'csrf_token', 'django.core'],
        'Flask': ['flask', 'werkzeug', 'jinja2'],
        'React': ['react', 'react-dom', '__REACT_DEVTOOLS_GLOBAL_HOOK__'],
        'Vue': ['vue', '__VUE__', 'Vue.js'],
        'Angular': ['angular', 'ng-app', 'ng-controller']
    },
    'databases': {
        'PostgreSQL': ['psycopg2', 'pg_', 'postgres'],
        'MySQL': ['mysql', 'mysqli', 'PDO'],
        'MongoDB': ['mongodb', 'mongoose', 'mongo'],
        'Redis': ['redis', 'ioredis', 'redis-py']
    },
    'servers': {
        'Apache': ['Apache', 'mod_', 'httpd'],
        'Nginx': ['nginx', 'Nginx'],
        'IIS': ['IIS', 'ASP.NET'],
        'Node.js': ['Node.js', 'Express', 'npm']
    }
}

def detect_technologies(html_content, headers, js_code):
    """Detect technologies in web application."""
    detected = {
        'frameworks': [],
        'databases': [],
        'servers': []
    }

    content = html_content + js_code + str(headers)

    for category, signatures in TECHNOLOGY_SIGNATURES.items():
        for tech, patterns in signatures.items():
            for pattern in patterns:
                if pattern.lower() in content.lower():
                    detected[category].append(tech)

    return detected

# API endpoint patterns
API_PATTERNS = {
    'rest': r'/api/v\d+/[a-z_/]+',
    'graphql': r'/graphql',
    'websocket': r'wss?://',
    'rpc': r'/rpc',
    'soap': r'\.wsdl$'
}

def discover_api_endpoints(traffic_data):
    """Discover API endpoints from traffic."""
    endpoints = []

    for request in traffic_data:
        url = request['url']
        method = request['method']

        for api_type, pattern in API_PATTERNS.items():
            if re.match(pattern, url):
                endpoints.append({
                    'url': url,
                    'method': method,
                    'type': api_type,
                    'parameters': extract_parameters(request),
                    'authentication': detect_authentication(request)
                })

    return endpoints

# Hierarchical memory with importance-based promotion
class HierarchicalMemoryAgent:
    """Agent with hierarchical memory."""

    def __init__(self):
        self.recent = []  # Recent messages (window size: 3)
        self.important = []  # Important messages (size: 5)
        self.archived = []  # Archived messages (unlimited)

    def add_message(self, message, importance=0.5):
        """Add message with importance score."""
        msg = {
            'content': message,
            'importance': importance,
            'timestamp': time.time()
        }

        if importance > 0.8:
            # High importance: promote to important
            self.important.append(msg)
            if len(self.important) > 5:
                self.archived.append(self.important.pop(0))
        else:
            # Normal: add to recent
            self.recent.append(msg)
            if len(self.recent) > 3:
                self.archived.append(self.recent.pop(0))

    def get_context(self, max_messages=10):
        """Get context for LLM."""
        context = []
        context.extend(self.recent)
        context.extend(self.important)

        # Sort by timestamp (most recent first)
        context.sort(key=lambda x: x['timestamp'], reverse=True)

        return context[:max_messages]

# Retrieval-based memory with semantic search
class RetrievalMemoryAgent:
    """Agent with retrieval-based memory."""

    def __init__(self, db, embedding_gen):
        self.db = db
        self.embedding_gen = embedding_gen

    def store_memory(self, content, metadata=None):
        """Store memory with embedding."""
        embedding = self.embedding_gen.generate_embedding(content)

        self.db.execute_query(
            "INSERT INTO memory (content, embedding, metadata) VALUES (%s, %s, %s)",
            (content, embedding, json.dumps(metadata or {}))
        )

    def retrieve_memory(self, query, limit=5, threshold=0.7):
        """Retrieve relevant memories."""
        query_embedding = self.embedding_gen.generate_embedding(query)

        results = self.db.search_similar_instructions(
            embedding=query_embedding,
            limit=limit,
            threshold=threshold
        )

        return results

    def get_context(self, query):
        """Get context for LLM based on query."""
        memories = self.retrieve_memory(query)

        context = "\n".join([
            f"- {m['content']} (relevance: {m['similarity']:.2%})"
            for m in memories
        ])

        return context

# Knowledge base with semantic search
class KnowledgeBase:
    """Semantic knowledge base."""

    def __init__(self, db, embedding_gen):
        self.db = db
        self.embedding_gen = embedding_gen

    def add_knowledge(self, content, category, tags=None):
        """Add knowledge to base."""
        embedding = self.embedding_gen.generate_embedding(content)

        self.db.execute_query(
            "INSERT INTO knowledge_base (content, category, tags, embedding) "
            "VALUES (%s, %s, %s, %s)",
            (content, category, json.dumps(tags or []), embedding)
        )

    def search(self, query, category=None, limit=10):
        """Search knowledge base."""
        query_embedding = self.embedding_gen.generate_embedding(query)

        sql = """
            SELECT content, category, tags,
                   1 - (embedding <=> %s::vector) AS similarity
            FROM knowledge_base
            WHERE 1 - (embedding <=> %s::vector) >= 0.7
        """
        params = [query_embedding, query_embedding]

        if category:
            sql += " AND category = %s"
            params.append(category)

        sql += " ORDER BY embedding <=> %s::vector LIMIT %s"
        params.extend([query_embedding, limit])

        return self.db.execute_query(sql, tuple(params))

    def get_statistics(self):
        """Get knowledge base statistics."""
        stats = self.db.execute_query(
            "SELECT category, COUNT(*) as count FROM knowledge_base GROUP BY category"
        )

        return {s['category']: s['count'] for s in stats}

# Analyze multiple binaries efficiently
def batch_analyze_binaries(binary_paths, batch_size=5):
    """Analyze multiple binaries with batching."""
    from concurrent.futures import ThreadPoolExecutor

    oa = OrchestratingAgent()
    results = []

    with ThreadPoolExecutor(max_workers=batch_size) as executor:
        futures = [
            executor.submit(oa.run, path)
            for path in binary_paths
        ]

        for future in futures:
            try:
                result = future.result(timeout=300)
                results.append(result)
            except Exception as e:
                logger.error(f"Analysis failed: {e}")
                results.append({'error': str(e)})

    return results

# Usage
binaries = [
    '/binaries/app1.exe',
    '/binaries/app2.exe',
    '/binaries/app3.exe'
]

results = batch_analyze_binaries(binaries, batch_size=3)
for result in results:
    print(f"Binary: {result.get('binary_hash', 'ERROR')}")
    print(f"Success: {result.get('success', False)}")

# Generate embeddings for large datasets
def batch_generate_embeddings(texts, batch_size=32, cache=None):
    """Generate embeddings with caching."""
    from src.utils.embeddings_v2 import EmbeddingGenerator

    embedding_gen = EmbeddingGenerator(batch_size=batch_size)
    embeddings = []

    for i in range(0, len(texts), batch_size):
        batch = texts[i:i+batch_size]

        # Check cache
        if cache:
            batch_embeddings = []
            uncached = []
            uncached_indices = []

            for j, text in enumerate(batch):
                cached = cache.get(f"embedding:{hash(text)}")
                if cached:
                    batch_embeddings.append(cached)
                else:
                    uncached.append(text)
                    uncached_indices.append(j)

            # Generate uncached
            if uncached:
                new_embeddings = embedding_gen.batch_encode(uncached)
                for text, embedding in zip(uncached, new_embeddings):
                    cache.set(f"embedding:{hash(text)}", embedding)
                    batch_embeddings.insert(uncached_indices[len(batch_embeddings)], embedding)
        else:
            batch_embeddings = embedding_gen.batch_encode(batch)

        embeddings.extend(batch_embeddings)
        logger.info(f"Generated {len(embeddings)}/{len(texts)} embeddings")

    return embeddings

# Execute agents asynchronously
import asyncio

async def execute_agents_async(agents, task):
    """Execute agents concurrently."""
    tasks = [
        asyncio.create_task(agent.execute_async(task))
        for agent in agents
    ]

    results = await asyncio.gather(*tasks, return_exceptions=True)
    return results

# Usage
async def main():
    agents = [
        ReconnaissanceAgent(orchestrator),
        TrafficInterceptionAgent(orchestrator),
        JavaScriptAnalysisAgent(orchestrator)
    ]

    task = {'target_url': 'https://api.example.com'}
    results = await execute_agents_async(agents, task)

    for agent, result in zip(agents, results):
        print(f"{agent.name}: {result}")

asyncio.run(main())

# Stream results as they become available
def stream_analysis_results(binary_paths):
    """Stream analysis results."""
    oa = OrchestratingAgent()

    for binary_path in binary_paths:
        try:
            result = oa.run(binary_path)
            yield {
                'status': 'success',
                'binary_path': binary_path,
                'result': result
            }
        except Exception as e:
            yield {
                'status': 'error',
                'binary_path': binary_path,
                'error': str(e)
            }

# Usage
for result in stream_analysis_results(binary_paths):
    if result['status'] == 'success':
        print(f"✓ {result['binary_path']}")
    else:
        print(f"✗ {result['binary_path']}: {result['error']}")

# Distributed cache with Redis cluster
class DistributedCache:
    """Distributed cache using Redis cluster."""

    def __init__(self, nodes):
        from rediscluster import RedisCluster

        self.cluster = RedisCluster(
            startup_nodes=nodes,
            skip_full_coverage_check=True
        )

    def get(self, key):
        """Get from distributed cache."""
        value = self.cluster.get(key)
        return json.loads(value) if value else None

    def set(self, key, value, ttl=3600):
        """Set in distributed cache."""
        self.cluster.setex(
            key,
            ttl,
            json.dumps(value)
        )

    def delete(self, key):
        """Delete from distributed cache."""
        self.cluster.delete(key)

    def flush(self):
        """Flush all cache."""
        self.cluster.flushall()

# Usage
nodes = [
    {'host': 'redis-1', 'port': 6379},
    {'host': 'redis-2', 'port': 6379},
    {'host': 'redis-3', 'port': 6379}
]

cache = DistributedCache(nodes)
cache.set('key', {'data': 'value'})
result = cache.get('key')

# Pre-populate cache with frequently accessed data
def warm_cache(cache, db):
    """Warm cache with frequently accessed data."""

    # Get frequently analyzed binaries
    frequent_binaries = db.execute_query("""
        SELECT file_hash, analysis_result
        FROM analysis_results
        WHERE created_at > NOW() - INTERVAL '7 days'
        ORDER BY access_count DESC
        LIMIT 1000
    """)

    for binary in frequent_binaries:
        cache.set(
            f"analysis:{binary['file_hash']}",
            binary['analysis_result'],
            ttl=604800  # 7 days
        )

    logger.info(f"Warmed cache with {len(frequent_binaries)} entries")

# Define custom metrics
from prometheus_client import Counter, Histogram, Gauge

# Counters
binary_analysis_total = Counter(
    'binary_analysis_total',
    'Total binary analyses',
    ['status', 'architecture']
)

vulnerability_detected_total = Counter(
    'vulnerability_detected_total',
    'Total vulnerabilities detected',
    ['type', 'severity']
)

# Histograms
analysis_duration_seconds = Histogram(
    'analysis_duration_seconds',
    'Analysis duration',
    buckets=(1, 2, 5, 10, 30, 60, 120)
)

patch_size_bytes = Histogram(
    'patch_size_bytes',
    'Patch size in bytes',
    buckets=(10, 50, 100, 500, 1000, 5000)
)

# Gauges
active_analyses = Gauge(
    'active_analyses',
    'Number of active analyses'
)

cache_size_bytes = Gauge(
    'cache_size_bytes',
    'Cache size in bytes'
)

# Usage
@active_analyses.track_inprogress()
def analyze_binary(binary_path):
    """Analyze binary with metrics."""
    with analysis_duration_seconds.time():
        result = oa.run(binary_path)

    binary_analysis_total.labels(
        status='success' if result['success'] else 'failed',
        architecture=result['metadata']['architecture']
    ).inc()

    for vuln in result.get('vulnerabilities', []):
        vulnerability_detected_total.labels(
            type=vuln['type'],
            severity=vuln['severity']
        ).inc()

    return result

# prometheus-alerts.yml
groups:
  - name: raverse_alerts
    rules:
      # High error rate
      - alert: HighAnalysisErrorRate
        expr: rate(binary_analysis_total{status="failed"}[5m]) > 0.1
        for: 5m
        annotations:
          summary: "High binary analysis error rate"
          description: "Error rate is {{ $value | humanizePercentage }}"

      # Slow analysis
      - alert: SlowAnalysis
        expr: histogram_quantile(0.95, analysis_duration_seconds) > 30
        for: 10m
        annotations:
          summary: "Analysis taking too long"
          description: "p95 latency is {{ $value }}s"

      # Cache efficiency
      - alert: LowCacheHitRatio
        expr: cache_hit_ratio < 0.5
        for: 15m
        annotations:
          summary: "Low cache hit ratio"
          description: "Cache hit ratio is {{ $value | humanizePercentage }}"

      # Database connection pool
      - alert: DatabaseConnectionPoolExhausted
        expr: database_connection_pool_size >= 20
        for: 2m
        annotations:
          summary: "Database connection pool exhausted"
          description: "Active connections: {{ $value }}"

# Analyze and optimize queries
def analyze_query_performance(db, query):
    """Analyze query performance."""

    # Get query plan
    plan = db.execute_query(f"EXPLAIN ANALYZE {query}")

    # Extract metrics
    metrics = {
        'total_cost': None,
        'rows': None,
        'execution_time': None,
        'planning_time': None
    }

    for row in plan:
        if 'Total Cost' in row:
            metrics['total_cost'] = float(row.split(':')[1])
        elif 'Rows' in row:
            metrics['rows'] = int(row.split(':')[1])
        elif 'Execution Time' in row:
            metrics['execution_time'] = float(row.split(':')[1])
        elif 'Planning Time' in row:
            metrics['planning_time'] = float(row.split(':')[1])

    return metrics

# Usage
query = """
    SELECT * FROM code_embeddings
    WHERE 1 - (embedding <=> query_embedding::vector) >= 0.7
    ORDER BY embedding <=> query_embedding::vector
    LIMIT 10
"""

metrics = analyze_query_performance(db, query)
print(f"Total Cost: {metrics['total_cost']}")
print(f"Execution Time: {metrics['execution_time']}ms")

# Optimize indexes
def optimize_indexes(db):
    """Optimize database indexes."""

    # Analyze index usage
    index_stats = db.execute_query("""
        SELECT schemaname, tablename, indexname, idx_scan, idx_tup_read, idx_tup_fetch
        FROM pg_stat_user_indexes
        ORDER BY idx_scan DESC
    """)

    # Identify unused indexes
    unused_indexes = [
        idx for idx in index_stats
        if idx['idx_scan'] == 0
    ]

    # Identify inefficient indexes
    inefficient_indexes = [
        idx for idx in index_stats
        if idx['idx_tup_read'] > 0 and idx['idx_tup_fetch'] / idx['idx_tup_read'] < 0.1
    ]

    logger.info(f"Unused indexes: {len(unused_indexes)}")
    logger.info(f"Inefficient indexes: {len(inefficient_indexes)}")

    # Reindex
    for idx in inefficient_indexes:
        db.execute_query(f"REINDEX INDEX {idx['indexname']}")
        logger.info(f"Reindexed: {idx['indexname']}")

# Complete workflow from binary to patched binary
from src.agents.orchestrator import OrchestratingAgent
from src.utils.database import DatabaseManager
from src.utils.cache import CacheManager

# Initialize
oa = OrchestratingAgent()
db = DatabaseManager()
cache = CacheManager()

# 1. Analyze binary
binary_path = '/binaries/vulnerable_app.exe'
result = oa.run(binary_path)

# 2. Check results
if result['success']:
    print(f"✓ Analysis successful")
    print(f"  Binary Hash: {result['binary_hash']}")
    print(f"  Vulnerabilities: {len(result.get('vulnerabilities', []))}")
    print(f"  Patches Generated: {len(result.get('patches', []))}")

    # 3. Review patches
    for patch in result['patches']:
        print(f"\n  Patch: {patch['type']}")
        print(f"  Address: {hex(patch['address'])}")
        print(f"  Risk: {patch['risk']}")

    # 4. Apply patches
    patched_binary = result['patched_binary_path']
    print(f"\n✓ Patched binary: {patched_binary}")

    # 5. Verify patches
    verification = result['verification']
    print(f"\n  Verification:")
    print(f"  - Structure Valid: {verification['structure_valid']}")
    print(f"  - Patches Applied: {verification['patch_applied']}")
    print(f"  - Functionality OK: {verification['functionality_ok']}")
else:
    print(f"✗ Analysis failed: {result.get('error')}")

# Discover and document APIs
from src.agents.online_orchestrator import OnlineOrchestrationAgent

# Initialize
oa = OnlineOrchestrationAgent(
    api_key='sk-or-v1-...',
    model='meta-llama/llama-3.3-70b-instruct:free'
)

# Execute online analysis
result = oa.execute(
    target_url='https://api.example.com',
    scope={
        'target_url': 'https://api.example.com',
        'allowed_domains': ['api.example.com', '*.example.com'],
        'max_depth': 3,
        'max_urls': 1000
    },
    options={
        'recon': {'detect_technologies': True},
        'traffic': {'duration_seconds': 60},
        'api_discovery': {'detect_rest': True, 'detect_graphql': True},
        'security': {'check_vulnerabilities': True}
    }
)

# Process results
print("=== API Discovery Results ===\n")

# Technologies
print("Technologies Detected:")
for tech in result['recon']['technologies']:
    print(f"  - {tech}")

# API Endpoints
print("\nAPI Endpoints:")
for endpoint in result['api_reeng']['endpoints']:
    print(f"  {endpoint['method']} {endpoint['url']}")
    if endpoint['parameters']:
        for param in endpoint['parameters']:
            print(f"    - {param['name']}: {param['type']}")

# Vulnerabilities
print("\nVulnerabilities:")
for vuln in result['security']['vulnerabilities']:
    print(f"  [{vuln['severity']}] {vuln['type']}")
    print(f"    {vuln['description']}")

# Export OpenAPI spec
openapi_spec = result['api_reeng']['openapi_spec']
with open('api_spec.json', 'w') as f:
    json.dump(openapi_spec, f, indent=2)
print("\n✓ OpenAPI spec exported to api_spec.json")

# Use RAG for intelligent analysis
from src.agents.online_rag_orchestrator_agent import RAGOrchestratorAgent
from src.utils.semantic_search import SemanticSearchEngine

# Initialize
rag = RAGOrchestratorAgent(
    api_key='sk-or-v1-...',
    model='meta-llama/llama-3.3-70b-instruct:free'
)

# Execute RAG query
result = rag.execute({
    'query': 'What are the common binary patching techniques?',
    'context': 'Binary analysis and security patching'
})

# Process results
print("=== RAG Analysis Results ===\n")

print("Retrieved Knowledge:")
for knowledge in result['retrieved_knowledge']:
    print(f"  - {knowledge['content']}")
    print(f"    Relevance: {knowledge['similarity']:.2%}\n")

print("Generated Response:")
print(result['generated_response'])

print(f"\nConfidence: {result['confidence']:.2%}")

def detect_binary_format(binary_path):
    """Detect binary format."""
    with open(binary_path, 'rb') as f:
        magic = f.read(4)

    if magic == b'\x7fELF':
        return 'ELF'
    elif magic == b'MZ':
        return 'PE'
    elif magic == b'\xfe\xed\xfa':
        return 'Mach-O'
    elif magic == b'\x00\x61\x73\x6d':
        return 'WebAssembly'
    else:
        return 'Unknown'

-- Binaries table
CREATE TABLE raverse.binaries (
    id SERIAL PRIMARY KEY,
    file_name VARCHAR(255),
    file_path TEXT,
    file_hash VARCHAR(64) UNIQUE,
    file_size BIGINT,
    file_type VARCHAR(50),
    architecture VARCHAR(50),
    metadata JSONB,
    created_at TIMESTAMP DEFAULT NOW(),
    updated_at TIMESTAMP DEFAULT NOW()
);

-- Disassembly cache
CREATE TABLE raverse.disassembly_cache (
    id SERIAL PRIMARY KEY,
    binary_id INTEGER REFERENCES raverse.binaries(id),
    address BIGINT,
    instruction VARCHAR(255),
    opcode BYTEA,
    operands TEXT,
    disassembly_text TEXT,
    embedding vector(384),
    created_at TIMESTAMP DEFAULT NOW()
);

-- Code embeddings
CREATE TABLE raverse.code_embeddings (
    id SERIAL PRIMARY KEY,
    binary_hash VARCHAR(64),
    code_snippet TEXT,
    embedding vector(384),
    metadata JSONB,
    created_at TIMESTAMP DEFAULT NOW()
);

-- Analysis results
CREATE TABLE raverse.analysis_results (
    id SERIAL PRIMARY KEY,
    binary_id INTEGER REFERENCES raverse.binaries(id),
    result_data JSONB,
    vulnerabilities JSONB,
    patches JSONB,
    verification JSONB,
    created_at TIMESTAMP DEFAULT NOW()
);

-- Knowledge base
CREATE TABLE raverse.knowledge_base (
    id SERIAL PRIMARY KEY,
    content TEXT,
    category VARCHAR(100),
    tags TEXT[],
    embedding vector(384),
    metadata JSONB,
    created_at TIMESTAMP DEFAULT NOW()
);

-- Connection settings
max_connections = 200
superuser_reserved_connections = 10

-- Memory settings
shared_buffers = 4GB
effective_cache_size = 12GB
work_mem = 20MB
maintenance_work_mem = 1GB

-- WAL settings
wal_buffers = 16MB
checkpoint_completion_target = 0.9
wal_level = replica

-- Query planning
random_page_cost = 1.1
effective_io_concurrency = 200

-- HNSW index settings
hnsw.ef_construction = 128
hnsw.ef_search = 200

# Memory settings
maxmemory 2gb
maxmemory-policy allkeys-lru

# Persistence
save 900 1
save 300 10
save 60 10000

# Replication
repl-diskless-sync yes
repl-diskless-sync-delay 5

# Cluster settings (if using cluster)
cluster-enabled yes
cluster-node-timeout 15000

# Ingress rules
Allow 8000/tcp from 0.0.0.0/0  # Application
Allow 5432/tcp from 10.0.0.0/8  # PostgreSQL (internal only)
Allow 6379/tcp from 10.0.0.0/8  # Redis (internal only)
Allow 9090/tcp from 10.0.0.0/8  # Prometheus (internal only)
Allow 3000/tcp from 10.0.0.0/8  # Grafana (internal only)

# Egress rules
Allow 443/tcp to 0.0.0.0/0  # HTTPS (OpenRouter API)
Allow 53/udp to 0.0.0.0/0   # DNS

# Nginx configuration
upstream raverse_backend {
    least_conn;
    server raverse-1:8000 weight=1;
    server raverse-2:8000 weight=1;
    server raverse-3:8000 weight=1;
}

server {
    listen 80;
    server_name api.raverse.example.com;

    location / {
        proxy_pass http://raverse_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # Timeouts
        proxy_connect_timeout 60s;
        proxy_send_timeout 60s;
        proxy_read_timeout 60s;
    }
}

# DEBUG: Detailed execution flow
logger.debug(f"Processing instruction: {instruction}")

# INFO: Major milestones
logger.info(f"Starting analysis of {binary_path}")

# WARNING: Recoverable errors
logger.warning(f"Patch verification failed, using fallback")

# ERROR: Unrecoverable errors
logger.error(f"Analysis failed: {error}")

# CRITICAL: System-level failures
logger.critical(f"Database connection lost")

# JSON structured logging
import json

log_entry = {
    'timestamp': datetime.utcnow().isoformat(),
    'level': 'INFO',
    'logger': 'orchestrator',
    'message': 'Analysis complete',
    'binary_id': 123,
    'execution_time_ms': 5432,
    'success': True,
    'tags': ['offline', 'analysis']
}

logger.info(json.dumps(log_entry))

┌─────────────────────────────────────────────────────────┐
│                    API Gateway                          │
│              (Load Balancer + Auth)                     │
└────────────────────┬────────────────────────────────────┘
                     │
        ┌────────────┼────────────┐
        │            │            │
   ┌────▼────┐  ┌────▼────┐  ┌────▼───┐
   │ Offline │  │ Online  │  │  RAG   │
   │ Service │  │ Service │  │Service │
   └────┬────┘  └───┬─────┘  └───┬────┘
        │           │            │
        └───────────┼────────────┘
                    │
        ┌───────────┼───────────┐
        │           │           │
   ┌────▼─────┐  ┌──▼────┐  ┌───▼──────┐
   │PostgreSQL│  │Redis  │  │Prometheus│
   │+ pgvector│  │Cluster│  │+ Grafana │
   └──────────┘  └───────┘  └──────────┘

# Event-driven agent communication
class EventBus:
    """Central event bus for agent communication."""

    def __init__(self):
        self.subscribers = {}

    def subscribe(self, event_type, handler):
        """Subscribe to event."""
        if event_type not in self.subscribers:
            self.subscribers[event_type] = []
        self.subscribers[event_type].append(handler)

    def publish(self, event_type, data):
        """Publish event."""
        if event_type in self.subscribers:
            for handler in self.subscribers[event_type]:
                handler(data)

# Usage
event_bus = EventBus()

# Subscribe to events
event_bus.subscribe('analysis_complete', on_analysis_complete)
event_bus.subscribe('vulnerability_detected', on_vulnerability_detected)

# Publish events
event_bus.publish('analysis_complete', {
    'binary_id': 123,
    'success': True
})

# Separate read and write models
class CommandHandler:
    """Handle write operations."""

    def __init__(self, db):
        self.db = db

    def create_analysis(self, binary_path):
        """Create analysis (write)."""
        result = analyze_binary(binary_path)
        self.db.save_analysis(result)
        return result

class QueryHandler:
    """Handle read operations."""

    def __init__(self, db, cache):
        self.db = db
        self.cache = cache

    def get_analysis(self, binary_id):
        """Get analysis (read)."""
        # Try cache first
        cached = self.cache.get(f"analysis:{binary_id}")
        if cached:
            return cached

        # Query database
        result = self.db.get_analysis(binary_id)
        self.cache.set(f"analysis:{binary_id}", result)
        return result

# Prevent cascading failures
class CircuitBreaker:
    """Circuit breaker for fault tolerance."""

    def __init__(self, failure_threshold=5, timeout=60):
        self.failure_threshold = failure_threshold
        self.timeout = timeout
        self.failure_count = 0
        self.last_failure_time = None
        self.state = 'CLOSED'  # CLOSED, OPEN, HALF_OPEN

    def call(self, func, *args, **kwargs):
        """Call function with circuit breaker."""
        if self.state == 'OPEN':
            if time.time() - self.last_failure_time > self.timeout:
                self.state = 'HALF_OPEN'
            else:
                raise Exception("Circuit breaker is OPEN")

        try:
            result = func(*args, **kwargs)
            self.on_success()
            return result
        except Exception as e:
            self.on_failure()
            raise

    def on_success(self):
        """Handle successful call."""
        self.failure_count = 0
        self.state = 'CLOSED'

    def on_failure(self):
        """Handle failed call."""
        self.failure_count += 1
        self.last_failure_time = time.time()

        if self.failure_count >= self.failure_threshold:
            self.state = 'OPEN'

# Usage
breaker = CircuitBreaker(failure_threshold=5, timeout=60)

try:
    result = breaker.call(oa.call_openrouter, prompt)
except Exception as e:
    logger.error(f"API call failed: {e}")

# Retry with exponential backoff
def retry_with_backoff(func, max_retries=3, base_delay=1):
    """Retry function with exponential backoff."""
    for attempt in range(max_retries):
        try:
            return func()
        except Exception as e:
            if attempt == max_retries - 1:
                raise

            delay = base_delay * (2 ** attempt)
            logger.warning(f"Attempt {attempt + 1} failed, retrying in {delay}s: {e}")
            time.sleep(delay)

# Usage
result = retry_with_backoff(
    lambda: oa.call_openrouter(prompt),
    max_retries=3,
    base_delay=1
)

User Input
    ↓
┌─────────────────────────────────────┐
│   Input Validation & Sanitization   │
└────────────┬────────────────────────┘
             ↓
┌─────────────────────────────────────┐
│   Check Cache (L1 → L2 → L3)        │
└────────────┬────────────────────────┘
             ↓
        ┌────┴─────┐
        │          │
    Cache Hit   Cache Miss
        │          │
        │      ┌───▼──────────────────┐
        │      │  Execute Analysis    │
        │      │  (Agent Pipeline)    │
        │      └───┬──────────────────┘
        │          ↓
        │      ┌─────────────────────┐
        │      │  Store in Cache     │
        │      │  (L1 + L2 + L3)     │
        │      └───┬─────────────────┘
        │          │
        └──────┬───┘
               ↓
        ┌─────────────────────┐
        │  Format Response    │
        └────────┬────────────┘
                 ↓
           User Output

1. REQUEST RECEIVED
   ├─ Parse request
   ├─ Extract parameters
   └─ Validate input

2. AUTHENTICATION & AUTHORIZATION
   ├─ Verify API key
   ├─ Check permissions
   └─ Rate limiting

3. CACHE LOOKUP
   ├─ Check L1 (memory)
   ├─ Check L2 (Redis)
   └─ Check L3 (database)

4. ANALYSIS EXECUTION
   ├─ Initialize agents
   ├─ Execute pipeline
   ├─ Collect results
   └─ Generate report

5. RESULT STORAGE
   ├─ Store in database
   ├─ Cache result
   ├─ Update metrics
   └─ Log event

6. RESPONSE FORMATTING
   ├─ Format output
   ├─ Add metadata
   └─ Return to client

# Comprehensive error handling
class ErrorHandler:
    """Centralized error handling."""

    ERROR_CODES = {
        'INVALID_INPUT': 400,
        'UNAUTHORIZED': 401,
        'FORBIDDEN': 403,
        'NOT_FOUND': 404,
        'RATE_LIMITED': 429,
        'INTERNAL_ERROR': 500,
        'SERVICE_UNAVAILABLE': 503
    }

    @staticmethod
    def handle_error(error_type, message, details=None):
        """Handle error and return response."""
        status_code = ErrorHandler.ERROR_CODES.get(error_type, 500)

        response = {
            'error': error_type,
            'message': message,
            'status_code': status_code
        }

        if details:
            response['details'] = details

        logger.error(f"{error_type}: {message}", extra=details or {})

        return response, status_code

# Usage
try:
    result = oa.run(binary_path)
except FileNotFoundError as e:
    return ErrorHandler.handle_error(
        'NOT_FOUND',
        f"Binary not found: {binary_path}",
        {'path': binary_path}
    )
except ValueError as e:
    return ErrorHandler.handle_error(
        'INVALID_INPUT',
        str(e)
    )
except Exception as e:
    return ErrorHandler.handle_error(
        'INTERNAL_ERROR',
        'An unexpected error occurred',
        {'error': str(e)}
    )

# Agent state management
class AgentState:
    """Manage agent execution state."""

    def __init__(self):
        self.state = 'IDLE'
        self.current_task = None
        self.progress = 0
        self.start_time = None
        self.end_time = None

    def start_task(self, task):
        """Start task execution."""
        self.state = 'RUNNING'
        self.current_task = task
        self.progress = 0
        self.start_time = time.time()

    def update_progress(self, progress):
        """Update task progress."""
        self.progress = progress

    def complete_task(self):
        """Complete task execution."""
        self.state = 'IDLE'
        self.end_time = time.time()
        self.current_task = None

    def get_status(self):
        """Get current status."""
        return {
            'state': self.state,
            'task': self.current_task,
            'progress': self.progress,
            'elapsed_time': time.time() - self.start_time if self.start_time else 0
        }

# Dependency injection for loose coupling
class Container:
    """Dependency injection container."""

    def __init__(self):
        self.services = {}

    def register(self, name, factory):
        """Register service factory."""
        self.services[name] = factory

    def get(self, name):
        """Get service instance."""
        if name not in self.services:
            raise ValueError(f"Service not found: {name}")
        return self.services[name]()

# Setup
container = Container()
container.register('db', lambda: DatabaseManager())
container.register('cache', lambda: CacheManager())
container.register('embedding_gen', lambda: EmbeddingGenerator())

# Usage
db = container.get('db')
cache = container.get('cache')
embedding_gen = container.get('embedding_gen')

-- Use EXPLAIN ANALYZE to optimize queries
EXPLAIN ANALYZE
SELECT * FROM code_embeddings
WHERE 1 - (embedding <=> query_embedding::vector) >= 0.7
ORDER BY embedding <=> query_embedding::vector
LIMIT 10;

-- Create appropriate indexes
CREATE INDEX idx_embeddings_hnsw ON code_embeddings
USING hnsw (embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);

-- Use VACUUM and ANALYZE
VACUUM ANALYZE code_embeddings;

-- Monitor slow queries
SET log_min_duration_statement = 100;  -- Log queries > 100ms

# Optimize connection pool
db = DatabaseManager(
    pool_size=10,           # Minimum connections
    max_overflow=20,        # Maximum overflow
    pool_recycle=3600,      # Recycle after 1 hour
    pool_pre_ping=True      # Test before use
)

# Monitor pool
pool_status = db.get_pool_status()
print(f"Active connections: {pool_status['active']}")
print(f"Idle connections: {pool_status['idle']}")
print(f"Overflow connections: {pool_status['overflow']}")

# Optimize memory usage
import gc

# Disable automatic garbage collection during analysis
gc.disable()

try:
    result = oa.run(binary_path)
finally:
    # Force garbage collection
    gc.collect()
    gc.enable()

# Monitor memory
import psutil
process = psutil.Process()
memory_info = process.memory_info()
print(f"RSS: {memory_info.rss / 1024 / 1024:.2f} MB")
print(f"VMS: {memory_info.vms / 1024 / 1024:.2f} MB")

# Optimize batch processing
def process_in_batches(items, batch_size=100, processor=None):
    """Process items in batches."""
    results = []

    for i in range(0, len(items), batch_size):
        batch = items[i:i+batch_size]
        batch_results = processor(batch)
        results.extend(batch_results)

        # Log progress
        logger.info(f"Processed {len(results)}/{len(items)} items")

    return results

# Usage
embeddings = process_in_batches(
    texts,
    batch_size=32,
    processor=embedding_gen.batch_encode
)

# src/agents/custom_agent.py
from src.agents.online_base_agent import OnlineBaseAgent

class CustomAnalysisAgent(OnlineBaseAgent):
    """Custom agent for specialized analysis."""

    def __init__(self, orchestrator, api_key, model):
        super().__init__(
            name="CustomAnalysis",
            orchestrator=orchestrator,
            api_key=api_key,
            model=model
        )
        self.logger.info(f"Initialized {self.name}")

def _execute_impl(self, task):
    """Implement agent logic."""
    try:
        # Validate input
        if not self._validate_input(task):
            return {'error': 'Invalid input'}

        # Extract parameters
        data = task.get('data')

        # Process data
        result = self._process_data(data)

        # Return result
        return {
            'status': 'success',
            'result': result
        }
    except Exception as e:
        self.logger.exception(f"Error in {self.name}: {e}")
        return {'error': str(e)}

def _validate_input(self, task):
    """Validate input parameters."""
    required_fields = ['data']
    return all(field in task for field in required_fields)

def _process_data(self, data):
    """Process data."""
    # Implementation here
    return {'processed': data}

# In orchestrator
from src.agents.custom_agent import CustomAnalysisAgent

self.agents['CUSTOM'] = CustomAnalysisAgent(
    orchestrator=self,
    api_key=self.api_key,
    model=self.model
)

# src/memory/custom_memory.py
class CustomMemory:
    """Custom memory strategy."""

    def __init__(self, config=None):
        self.config = config or {}
        self.memory = []

    def add(self, item):
        """Add item to memory."""
        self.memory.append(item)

    def get(self):
        """Get memory context."""
        return self.memory

    def clear(self):
        """Clear memory."""
        self.memory = []

class AgentWithCustomMemory:
    """Agent using custom memory."""

    def __init__(self):
        self.memory = CustomMemory()

    def execute(self, task):
        """Execute with memory."""
        # Add to memory
        self.memory.add({'task': task, 'timestamp': time.time()})

        # Get context
        context = self.memory.get()

        # Process
        result = self._process(task, context)

        return result

# src/cache/custom_cache.py
class CustomCacheBackend:
    """Custom cache backend."""

    def __init__(self):
        self.store = {}

    def get(self, key):
        """Get from cache."""
        return self.store.get(key)

    def set(self, key, value, ttl=None):
        """Set in cache."""
        self.store[key] = {
            'value': value,
            'ttl': ttl,
            'created_at': time.time()
        }

    def delete(self, key):
        """Delete from cache."""
        if key in self.store:
            del self.store[key]

    def flush(self):
        """Flush cache."""
        self.store = {}

# Use custom cache
cache = CustomCacheBackend()

# Store result
cache.set('analysis:123', result, ttl=3600)

# Retrieve result
cached = cache.get('analysis:123')

# src/monitoring/custom_metrics.py
from prometheus_client import Counter, Histogram, Gauge

# Custom metrics
custom_analysis_total = Counter(
    'custom_analysis_total',
    'Total custom analyses',
    ['status']
)

custom_analysis_duration = Histogram(
    'custom_analysis_duration_seconds',
    'Custom analysis duration',
    buckets=(1, 5, 10, 30, 60)
)

custom_result_quality = Gauge(
    'custom_result_quality',
    'Quality score of custom analysis results'
)

# Track metrics
@custom_analysis_duration.time()
def custom_analysis(data):
    """Perform custom analysis."""
    result = analyze(data)

    custom_analysis_total.labels(
        status='success' if result['success'] else 'failed'
    ).inc()

    custom_result_quality.set(result.get('quality_score', 0))

    return result

POST /api/v1/analyze
├─ Request: { binary_path: string }
├─ Response: { success: bool, result: object }
└─ Status: 200 (success), 400 (invalid), 500 (error)

GET /api/v1/analysis/{id}
├─ Request: None
├─ Response: { analysis: object }
└─ Status: 200 (found), 404 (not found)

POST /api/v1/online-analysis
├─ Request: { target_url: string, scope: object, options: object }
├─ Response: { success: bool, result: object }
└─ Status: 200 (success), 400 (invalid), 500 (error)

GET /api/v1/health
├─ Request: None
├─ Response: { status: string, version: string }
└─ Status: 200 (healthy), 503 (unhealthy)

ws://localhost:8000/ws/analysis/{id}
├─ Message: { type: string, data: object }
├─ Types: progress, complete, error
└─ Closes: On analysis complete or error

{
  "success": true,
  "binary_id": 123,
  "binary_hash": "abc123...",
  "metadata": {
    "file_name": "app.exe",
    "file_size": 1024000,
    "architecture": "x64",
    "file_type": "PE"
  },
  "disassembly": {
    "functions": [...],
    "instructions": [...]
  },
  "vulnerabilities": [
    {
      "type": "buffer_overflow",
      "severity": "high",
      "address": "0x401000",
      "description": "..."
    }
  ],
  "patches": [
    {
      "address": "0x401000",
      "original": "...",
      "patched": "...",
      "strategy": "bounds_check"
    }
  ],
  "verification": {
    "structure_valid": true,
    "patch_applied": true,
    "functionality_ok": true
  },
  "execution_time_ms": 5432
}

{
  "text": "cmp eax, 0x0; je 0x401000",
  "embedding": [0.1, 0.2, ..., 0.384],
  "dimensions": 384,
  "model": "all-MiniLM-L6-v2",
  "generated_at": "2025-10-26T10:30:00Z"
}

agents:
  daa:
    enabled: true
    timeout: 30
    memory_preset: medium

  lima:
    enabled: true
    timeout: 30
    memory_preset: medium

  pea:
    enabled: true
    timeout: 30
    memory_preset: light

  va:
    enabled: true
    timeout: 30
    memory_preset: light

database:
  host: localhost
  port: 5432
  user: raverse
  password: ${DB_PASSWORD}
  database: raverse_db
  pool:
    size: 10
    max_overflow: 20
    recycle: 3600

cache:
  backend: redis
  host: localhost
  port: 6379
  db: 0
  password: ${REDIS_PASSWORD}
  ttl:
    analysis: 604800  # 7 days
    embedding: 604800  # 7 days
    llm_response: 86400  # 1 day

# Build image
docker build -t raverse:latest .

# Run container
docker run -d --name raverse -p 8000:8000 raverse:latest

# View logs
docker logs -f raverse

# Stop container
docker stop raverse

# Remove container
docker rm raverse

# Docker Compose
docker-compose up -d
docker-compose down
docker-compose logs -f
docker-compose ps

# Connect to PostgreSQL
psql -h localhost -U raverse -d raverse_db

# Create database
createdb -U raverse raverse_db

# Backup database
pg_dump -U raverse raverse_db > backup.sql

# Restore database
psql -U raverse raverse_db < backup.sql

# Check connections
SELECT count(*) FROM pg_stat_activity;

# Kill idle connections
SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE state = 'idle';

# Connect to Redis
redis-cli -h localhost -p 6379

# Check connection
PING

# Get key
GET key

# Set key
SET key value

# Delete key
DEL key

# Flush all
FLUSHALL

# Monitor commands
MONITOR

# Create virtual environment
python -m venv venv

# Activate virtual environment
source venv/bin/activate  # Linux/Mac
venv\Scripts\activate  # Windows

# Install dependencies
pip install -r requirements.txt

# Run tests
pytest tests/ -v

# Run with coverage
pytest tests/ --cov=src --cov-report=html

# Format code
black src/ tests/

# Type checking
mypy src/

# Linting
ruff check src/

# Create namespace
kubectl create namespace raverse

# Deploy
kubectl apply -f deployment.yaml -n raverse

# Check pods
kubectl get pods -n raverse

# View logs
kubectl logs -f deployment/raverse -n raverse

# Port forward
kubectl port-forward svc/raverse 8000:8000 -n raverse

# Delete deployment
kubectl delete deployment raverse -n raverse

# API Configuration
OPENROUTER_API_KEY=sk-or-v1-...
OPENROUTER_MODEL=meta-llama/llama-3.3-70b-instruct:free
OPENROUTER_TIMEOUT=30
OPENROUTER_MAX_RETRIES=3

# Database Configuration
DB_HOST=localhost
DB_PORT=5432
DB_USER=raverse
DB_PASSWORD=your_password
DB_NAME=raverse_db
DB_POOL_SIZE=10
DB_MAX_OVERFLOW=20
DB_POOL_RECYCLE=3600

# Redis Configuration
REDIS_HOST=localhost
REDIS_PORT=6379
REDIS_DB=0
REDIS_PASSWORD=
REDIS_CLUSTER_MODE=false
REDIS_CACHE_TTL=604800

# Logging Configuration
LOG_LEVEL=INFO
LOG_FILE=logs/raverse.log
LOG_FORMAT=json
LOG_MAX_SIZE=104857600
LOG_BACKUP_COUNT=10

# Feature Configuration
ENABLE_VECTOR_SEARCH=true
ENABLE_RAG=true
ENABLE_CACHING=true
ENABLE_MONITORING=true
ENABLE_PROFILING=false

# Performance Configuration
BATCH_SIZE=32
MAX_CONCURRENT_ANALYSES=5
EMBEDDING_CACHE_SIZE=10000
VECTOR_SEARCH_LIMIT=10
VECTOR_SEARCH_THRESHOLD=0.7

# Monitoring Configuration
PROMETHEUS_PORT=9090
GRAFANA_PORT=3000
GRAFANA_PASSWORD=admin

# Deployment Configuration
ENVIRONMENT=production
DEBUG=false
WORKERS=4
WORKER_CLASS=uvicorn.workers.UvicornWorker

RAVERSE/
├── src/
│   ├── agents/
│   │   ├── __init__.py
│   │   ├── orchestrator.py
│   │   ├── online_orchestrator.py
│   │   ├── disassembly_agent.py
│   │   ├── logic_identification.py
│   │   ├── patching_execution.py
│   │   ├── verification.py
│   │   ├── reconnaissance_agent.py
│   │   ├── traffic_interception_agent.py
│   │   ├── javascript_analysis_agent.py
│   │   ├── api_reverse_engineering_agent.py
│   │   ├── rag_orchestrator_agent.py
│   │   ├── knowledge_base_agent.py
│   │   └── ... (more agents)
│   ├── utils/
│   │   ├── __init__.py
│   │   ├── database.py
│   │   ├── cache.py
│   │   ├── embeddings_v2.py
│   │   ├── semantic_search.py
│   │   ├── binary_analyzer.py
│   │   ├── multi_level_cache.py
│   │   └── ... (more utilities)
│   ├── config/
│   │   ├── __init__.py
│   │   ├── settings.py
│   │   ├── agent_memory_config.py
│   │   ├── deepcrawler_config.py
│   │   └── ... (more configs)
│   ├── main.py
│   └── raverse_online_cli.py
├── tests/
│   ├── unit/
│   ├── integration/
│   ├── performance/
│   ├── conftest.py
│   └── ... (more tests)
├── docker/
│   ├── postgres/
│   ├── redis/
│   ├── prometheus/
│   ├── grafana/
│   └── Dockerfile
├── docs/
│   ├── ARCHITECTURE.md
│   ├── PRODUCTION_DEPLOYMENT_GUIDE.md
│   ├── QUICK_START_AI_FEATURES.md
│   ├── DEEPCRAWLER_USER_GUIDE.md
│   ├── MEMORY_INTEGRATION_MIGRATION_GUIDE.md
│   ├── A2A_PROTOCOL_DESIGN.md
│   └── archive/
├── examples/
│   ├── scope_example.json
│   ├── options_example.json
│   └── ... (more examples)
├── .github/
│   └── workflows/
│       ├── ci.yml
│       └── docker.yml
├── .env.example
├── .gitignore
├── requirements.txt
├── docker-compose.yml
├── docker-compose.prod.yml
├── Dockerfile
├── README.md
├── LICENSE
└── CONTRIBUTING.md

Problem: Analysis is slow
├─ Check binary size
│  ├─ If >100MB: Split into chunks
│  └─ If <100MB: Continue
├─ Check LLM model
│  ├─ If free model: Try paid model
│  └─ If paid model: Continue
├─ Check system resources
│  ├─ If CPU high: Reduce workers
│  ├─ If RAM high: Reduce batch size
│  └─ If disk high: Clean cache
└─ Check database
   ├─ Run EXPLAIN ANALYZE
   ├─ Check indexes
   └─ Optimize queries

Problem: High memory usage
├─ Check memory preset
│  ├─ If heavy: Switch to medium
│  └─ If medium: Switch to light
├─ Check batch size
│  ├─ If 32: Reduce to 16
│  └─ If 16: Reduce to 8
├─ Check cache size
│  ├─ If large: Reduce cache
│  └─ If small: Continue
└─ Check embeddings
   ├─ Clear embedding cache
   └─ Reduce embedding dimensions

Problem: Database connection errors
├─ Check PostgreSQL status
│  ├─ If down: Start PostgreSQL
│  └─ If up: Continue
├─ Check connection pool
│  ├─ If exhausted: Increase pool size
│  └─ If available: Continue
├─ Check credentials
│  ├─ If wrong: Update .env
│  └─ If correct: Continue
└─ Check network
   ├─ Test connectivity
   └─ Check firewall rules

Problem: Vector search returns no results
├─ Check embeddings exist
│  ├─ If missing: Generate embeddings
│  └─ If present: Continue
├─ Check index exists
│  ├─ If missing: Create index
│  └─ If present: Continue
├─ Check similarity threshold
│  ├─ If high: Lower threshold
│  └─ If low: Continue
└─ Check query embedding
   ├─ Verify embedding generated
   └─ Check embedding dimensions

# Identify functions in binary
def identify_functions(binary):
    """Identify functions in binary."""
    functions = []

    # Entry point
    functions.append({
        'address': binary.entry_point,
        'name': '_start',
        'type': 'entry'
    })

    # Exported functions
    for export in binary.exports:
        functions.append({
            'address': export.address,
            'name': export.name,
            'type': 'export'
        })

    # Imported functions
    for import_ in binary.imports:
        functions.append({
            'address': import_.address,
            'name': import_.name,
            'type': 'import'
        })

    # Discovered functions
    for address in discover_function_starts(binary):
        functions.append({
            'address': address,
            'name': f'sub_{address:x}',
            'type': 'discovered'
        })

    return functions

# Detect common vulnerabilities
def detect_vulnerabilities(binary):
    """Detect vulnerabilities in binary."""
    vulnerabilities = []

    # Buffer overflow patterns
    for func in binary.functions:
        if has_unbounded_copy(func):
            vulnerabilities.append({
                'type': 'buffer_overflow',
                'severity': 'high',
                'address': func.address,
                'description': 'Unbounded buffer copy detected'
            })

    # Format string vulnerabilities
    for func in binary.functions:
        if has_format_string(func):
            vulnerabilities.append({
                'type': 'format_string',
                'severity': 'high',
                'address': func.address,
                'description': 'Format string vulnerability detected'
            })

    # Use-after-free
    for func in binary.functions:
        if has_use_after_free(func):
            vulnerabilities.append({
                'type': 'use_after_free',
                'severity': 'critical',
                'address': func.address,
                'description': 'Use-after-free vulnerability detected'
            })

    return vulnerabilities

# Optimize memory usage
class MemoryOptimizer:
    """Optimize memory usage during analysis."""

    @staticmethod
    def enable_streaming(binary_path):
        """Enable streaming for large binaries."""
        with open(binary_path, 'rb') as f:
            while True:
                chunk = f.read(1024 * 1024)  # 1MB chunks
                if not chunk:
                    break
                yield chunk

    @staticmethod
    def enable_compression(data):
        """Enable compression for cached data."""
        import zlib
        return zlib.compress(data, level=6)

    @staticmethod
    def enable_lazy_loading(binary):
        """Enable lazy loading of sections."""
        class LazySection:
            def __init__(self, binary, section_name):
                self.binary = binary
                self.section_name = section_name
                self._data = None

            @property
            def data(self):
                if self._data is None:
                    self._data = self.binary.load_section(self.section_name)
                return self._data

        return LazySection(binary, 'text')

# Generate embeddings for code
def generate_embeddings(code_snippets):
    """Generate embeddings for code snippets."""
    from sentence_transformers import SentenceTransformer

    model = SentenceTransformer('all-MiniLM-L6-v2')
    embeddings = model.encode(code_snippets, show_progress_bar=True)

    return embeddings

# Store embeddings in pgvector
def store_embeddings(embeddings, code_snippets):
    """Store embeddings in PostgreSQL."""
    import psycopg2
    from pgvector.psycopg2 import register_vector

    conn = psycopg2.connect("dbname=raverse_db user=raverse")
    register_vector(conn)

    cur = conn.cursor()

    for embedding, snippet in zip(embeddings, code_snippets):
        cur.execute(
            "INSERT INTO code_embeddings (code_snippet, embedding) VALUES (%s, %s)",
            (snippet, embedding)
        )

    conn.commit()
    cur.close()
    conn.close()

# Optimize vector search
def optimize_vector_search(query, limit=10, threshold=0.7):
    """Optimize vector search with caching and indexing."""
    import psycopg2
    from pgvector.psycopg2 import register_vector

    conn = psycopg2.connect("dbname=raverse_db user=raverse")
    register_vector(conn)

    cur = conn.cursor()

    # Generate query embedding
    from sentence_transformers import SentenceTransformer
    model = SentenceTransformer('all-MiniLM-L6-v2')
    query_embedding = model.encode([query])[0]

    # Search with HNSW index
    cur.execute("""
        SELECT id, code_snippet, 1 - (embedding <=> %s) as similarity
        FROM code_embeddings
        WHERE 1 - (embedding <=> %s) >= %s
        ORDER BY embedding <=> %s
        LIMIT %s
    """, (query_embedding, query_embedding, threshold, query_embedding, limit))

    results = cur.fetchall()
    cur.close()
    conn.close()

    return results

# Agent-to-Agent communication
class A2AProtocol:
    """Agent-to-Agent communication protocol."""

    def __init__(self, redis_client):
        self.redis = redis_client

    def send_message(self, from_agent, to_agent, message):
        """Send message from one agent to another."""
        channel = f"agent:{to_agent}:messages"

        payload = {
            'from': from_agent,
            'to': to_agent,
            'message': message,
            'timestamp': time.time()
        }

        self.redis.publish(channel, json.dumps(payload))

    def subscribe_to_messages(self, agent_name):
        """Subscribe to messages for agent."""
        channel = f"agent:{agent_name}:messages"
        pubsub = self.redis.pubsub()
        pubsub.subscribe(channel)

        for message in pubsub.listen():
            if message['type'] == 'message':
                yield json.loads(message['data'])

    def audit_message(self, from_agent, to_agent, message):
        """Audit message in database."""
        # Store in PostgreSQL for audit trail
        pass

# Profile CPU usage
import cProfile
import pstats

def profile_analysis(binary_path):
    """Profile analysis execution."""
    profiler = cProfile.Profile()
    profiler.enable()

    # Run analysis
    result = oa.run(binary_path)

    profiler.disable()

    # Print stats
    stats = pstats.Stats(profiler)
    stats.sort_stats('cumulative')
    stats.print_stats(20)  # Top 20 functions

    return result

# Profile memory usage
from memory_profiler import profile

@profile
def analyze_binary(binary_path):
    """Analyze binary with memory profiling."""
    binary = load_binary(binary_path)
    disassembly = disassemble(binary)
    analysis = analyze(disassembly)
    return analysis

# Profile latency
import time

def profile_latency(func, *args, **kwargs):
    """Profile function latency."""
    start = time.perf_counter()
    result = func(*args, **kwargs)
    end = time.perf_counter()

    latency_ms = (end - start) * 1000
    print(f"Latency: {latency_ms:.2f}ms")

    return result

# Daily backup script
#!/bin/bash

BACKUP_DIR="/backups/raverse"
DATE=$(date +%Y%m%d_%H%M%S)

# PostgreSQL backup
pg_dump -U raverse raverse_db | gzip > $BACKUP_DIR/db_$DATE.sql.gz

# Redis backup
redis-cli BGSAVE
cp /var/lib/redis/dump.rdb $BACKUP_DIR/redis_$DATE.rdb

# Upload to S3
aws s3 cp $BACKUP_DIR s3://raverse-backups/ --recursive

# Cleanup old backups (keep 30 days)
find $BACKUP_DIR -mtime +30 -delete

# Restore from backup
#!/bin/bash

BACKUP_FILE=$1

# Restore PostgreSQL
gunzip -c $BACKUP_FILE | psql -U raverse raverse_db

# Restore Redis
redis-cli SHUTDOWN
cp $BACKUP_FILE /var/lib/redis/dump.rdb
redis-server

# Verify
psql -U raverse raverse_db -c "SELECT COUNT(*) FROM binaries;"
redis-cli PING

# Comprehensive audit logging
class AuditLogger:
    """Log all actions for compliance."""

    def __init__(self, db):
        self.db = db

    def log_action(self, user, action, resource, result):
        """Log action for audit trail."""
        audit_entry = {
            'user': user,
            'action': action,
            'resource': resource,
            'result': result,
            'timestamp': datetime.utcnow(),
            'ip_address': get_client_ip(),
            'user_agent': get_user_agent()
        }

        self.db.insert('audit_log', audit_entry)

    def get_audit_trail(self, resource_id, days=90):
        """Get audit trail for resource."""
        cutoff = datetime.utcnow() - timedelta(days=days)

        return self.db.query(
            "SELECT * FROM audit_log WHERE resource = %s AND timestamp > %s",
            (resource_id, cutoff)
        )

# Generate compliance reports
def generate_compliance_report(start_date, end_date):
    """Generate compliance report."""
    report = {
        'period': f"{start_date} to {end_date}",
        'total_analyses': count_analyses(start_date, end_date),
        'total_vulnerabilities': count_vulnerabilities(start_date, end_date),
        'total_patches': count_patches(start_date, end_date),
        'patch_success_rate': calculate_patch_success_rate(start_date, end_date),
        'audit_entries': count_audit_entries(start_date, end_date),
        'security_incidents': count_security_incidents(start_date, end_date),
        'compliance_status': 'COMPLIANT'
    }

    return report

# Complete end-to-end analysis workflow
from src.agents.orchestrator import OfflineOrchestrator
from src.utils.database import DatabaseManager
from src.utils.cache import CacheManager
from src.config.agent_memory_config import MEMORY_PRESETS

# Initialize components
db = DatabaseManager()
cache = CacheManager()
orchestrator = OfflineOrchestrator(
    api_key="sk-or-v1-...",
    model="meta-llama/llama-3.3-70b-instruct:free",
    memory_preset=MEMORY_PRESETS['medium']
)

# Run analysis
binary_path = "/path/to/binary"
result = orchestrator.run(binary_path)

# Process results
print(f"Analysis Status: {result['status']}")
print(f"Vulnerabilities Found: {len(result['vulnerabilities'])}")
print(f"Patches Generated: {len(result['patches'])}")
print(f"Verification: {result['verification']}")

# Store in database
db.save_analysis(result)

# Cache result
cache.set(f"analysis:{result['binary_id']}", result, ttl=604800)

# Return to user
return {
    'success': True,
    'analysis_id': result['binary_id'],
    'vulnerabilities': result['vulnerabilities'],
    'patches': result['patches']
}

# Implement custom vulnerability detection agent
from src.agents.online_base_agent import OnlineBaseAgent

class CustomVulnerabilityDetector(OnlineBaseAgent):
    """Custom agent for detecting specific vulnerabilities."""

    def __init__(self, orchestrator, api_key, model):
        super().__init__(
            name="CustomVulnDetector",
            orchestrator=orchestrator,
            api_key=api_key,
            model=model
        )

    def _execute_impl(self, task):
        """Execute custom vulnerability detection."""
        try:
            # Extract binary data
            binary_data = task.get('binary_data')
            analysis_type = task.get('type', 'all')

            # Prepare prompt
            prompt = self._prepare_prompt(binary_data, analysis_type)

            # Call LLM
            response = self.orchestrator.call_openrouter(prompt)

            # Parse response
            vulnerabilities = self._parse_response(response)

            # Validate findings
            validated = self._validate_findings(vulnerabilities)

            return {
                'status': 'success',
                'vulnerabilities': validated,
                'confidence': self._calculate_confidence(validated)
            }
        except Exception as e:
            self.logger.exception(f"Error in {self.name}: {e}")
            return {'status': 'error', 'error': str(e)}

    def _prepare_prompt(self, binary_data, analysis_type):
        """Prepare analysis prompt."""
        return f"""
        Analyze the following binary data for {analysis_type} vulnerabilities:

        {binary_data}

        Provide:
        1. List of vulnerabilities found
        2. Severity level for each
        3. Recommended patches
        4. Confidence score
        """

    def _parse_response(self, response):
        """Parse LLM response."""
        # Parse response and extract vulnerabilities
        return []

    def _validate_findings(self, vulnerabilities):
        """Validate findings."""
        # Validate each finding
        return vulnerabilities

    def _calculate_confidence(self, vulnerabilities):
        """Calculate overall confidence."""
        if not vulnerabilities:
            return 0.0
        return sum(v.get('confidence', 0) for v in vulnerabilities) / len(vulnerabilities)

# Integrate with external threat intelligence
class ThreatIntelligenceIntegration:
    """Integrate with external threat intelligence feeds."""

    def __init__(self, api_key):
        self.api_key = api_key

    def check_vulnerability_database(self, cve_id):
        """Check external vulnerability database."""
        import requests

        response = requests.get(
            f"https://services.nvd.nist.gov/rest/json/cves/1.0/{cve_id}",
            headers={'Accept': 'application/json'}
        )

        return response.json()

    def check_malware_database(self, file_hash):
        """Check external malware database."""
        import requests

        response = requests.get(
            f"https://www.virustotal.com/api/v3/files/{file_hash}",
            headers={'x-apikey': self.api_key}
        )

        return response.json()

    def get_exploit_information(self, cve_id):
        """Get exploit information."""
        import requests

        response = requests.get(
            f"https://exploit-db.com/api/search?cve={cve_id}",
            headers={'Authorization': f'Bearer {self.api_key}'}
        )

        return response.json()

# Process multiple binaries in batch
def batch_analyze_binaries(binary_paths, batch_size=5):
    """Analyze multiple binaries in batches."""
    from concurrent.futures import ThreadPoolExecutor, as_completed

    results = []

    with ThreadPoolExecutor(max_workers=batch_size) as executor:
        # Submit all tasks
        futures = {
            executor.submit(analyze_single_binary, path): path
            for path in binary_paths
        }

        # Process completed tasks
        for future in as_completed(futures):
            path = futures[future]
            try:
                result = future.result()
                results.append(result)
                print(f"✓ Completed: {path}")
            except Exception as e:
                print(f"✗ Failed: {path} - {e}")
                results.append({'path': path, 'error': str(e)})

    return results

def analyze_single_binary(binary_path):
    """Analyze single binary."""
    orchestrator = OfflineOrchestrator(...)
    return orchestrator.run(binary_path)

# Setup monitoring and alerting
from prometheus_client import Counter, Histogram, Gauge, start_http_server

# Define metrics
analysis_total = Counter(
    'raverse_analysis_total',
    'Total analyses performed',
    ['status']
)

analysis_duration = Histogram(
    'raverse_analysis_duration_seconds',
    'Analysis duration in seconds',
    buckets=(1, 5, 10, 30, 60, 120)
)

vulnerability_count = Gauge(
    'raverse_vulnerabilities_total',
    'Total vulnerabilities detected'
)

patch_success_rate = Gauge(
    'raverse_patch_success_rate',
    'Patch success rate'
)

# Start Prometheus metrics server
start_http_server(8000)

# Use metrics in code
@analysis_duration.time()
def run_analysis(binary_path):
    """Run analysis with metrics."""
    try:
        result = orchestrator.run(binary_path)
        analysis_total.labels(status='success').inc()
        vulnerability_count.set(len(result['vulnerabilities']))
        return result
    except Exception as e:
        analysis_total.labels(status='error').inc()
        raise

# FastAPI endpoint for analysis
from fastapi import FastAPI, File, UploadFile, HTTPException
from fastapi.responses import JSONResponse

app = FastAPI()

@app.post("/api/v1/analyze")
async def analyze_binary(file: UploadFile = File(...)):
    """Analyze uploaded binary."""
    try:
        # Save uploaded file
        contents = await file.read()
        binary_path = f"/tmp/{file.filename}"

        with open(binary_path, 'wb') as f:
            f.write(contents)

        # Run analysis
        result = orchestrator.run(binary_path)

        # Return result
        return JSONResponse({
            'success': True,
            'analysis_id': result['binary_id'],
            'vulnerabilities': result['vulnerabilities'],
            'patches': result['patches']
        })

    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/analysis/{analysis_id}")
async def get_analysis(analysis_id: int):
    """Get analysis result."""
    try:
        # Check cache first
        cached = cache.get(f"analysis:{analysis_id}")
        if cached:
            return JSONResponse(cached)

        # Query database
        result = db.get_analysis(analysis_id)

        if not result:
            raise HTTPException(status_code=404, detail="Analysis not found")

        return JSONResponse(result)

    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/health")
async def health_check():
    """Health check endpoint."""
    return JSONResponse({
        'status': 'healthy',
        'version': '2.0.0',
        'timestamp': datetime.utcnow().isoformat()
    })

# Comprehensive testing example
import pytest
from unittest.mock import Mock, patch

class TestOfflineOrchestrator:
    """Test offline orchestrator."""

    @pytest.fixture
    def orchestrator(self):
        """Create orchestrator instance."""
        return OfflineOrchestrator(
            api_key="test-key",
            model="test-model",
            memory_preset=MEMORY_PRESETS['light']
        )

    def test_analyze_valid_binary(self, orchestrator):
        """Test analyzing valid binary."""
        result = orchestrator.run("tests/fixtures/test_binary")

        assert result['status'] == 'success'
        assert 'vulnerabilities' in result
        assert 'patches' in result

    def test_analyze_invalid_binary(self, orchestrator):
        """Test analyzing invalid binary."""
        with pytest.raises(FileNotFoundError):
            orchestrator.run("nonexistent/binary")

    def test_cache_hit(self, orchestrator):
        """Test cache hit."""
        binary_path = "tests/fixtures/test_binary"

        # First run
        result1 = orchestrator.run(binary_path)

        # Second run (should hit cache)
        result2 = orchestrator.run(binary_path)

        assert result1 == result2

    @patch('src.utils.openrouter.call_openrouter')
    def test_llm_integration(self, mock_llm, orchestrator):
        """Test LLM integration."""
        mock_llm.return_value = "Test response"

        result = orchestrator.run("tests/fixtures/test_binary")

        assert mock_llm.called
        assert result['status'] == 'success'