commit d708c8b58c05479fd1451719a52ac3945084bae4 Author: Till Tomczak Date: Tue Jul 28 11:04:37 2026 +0200 Initial commit: Browser Pilot - Agentic Browser Control Extension - Side Panel with Chat UI, Quick Actions, Dev Tools - 22 browser automation tools (DOM, JS, screenshots, navigation) - MCP server (zero-dep, Node 18+) for external AI clients - Supports OpenRouter, Ollama, Anthropic, OpenAI, custom endpoints - Chrome/Brave MV3 extension with content script console capture diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3931b2e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.env +*.log +dist/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..31fcedf --- /dev/null +++ b/README.md @@ -0,0 +1,207 @@ +# πŸš€ Browser Pilot + +**Agentic Browser Control – wie im Screenshot: Side Panel Chat + MCP Server + eigenstΓ€ndige Browser-KI.** + +Eine Chrome/Brave Extension die drei Dinge gleichzeitig kann: + +1. **πŸ—¨οΈ In-Browser Agent (Side Panel)** – Chat-UI direkt im Browser, die KI steuert die Seite +2. **πŸ”Œ MCP Server** – externe KI-Clients (Claude Desktop, Cursor, du als Agent) kΓΆnnen den Browser fernsteuern +3. **πŸ› οΈ Dev Tools** – JS Console, CSS Selector Tester, direkte Tool-AusfΓΌhrung + +![Architecture](https://img.shields.io/badge/MV3-Chrome%20Extension-blue) ![Zero Deps](https://img.shields.io/badge/Server-Zero%20Dependencies-green) + +--- + +## πŸ—οΈ Architektur + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ BROWSER (Chrome/Brave) β”‚ +β”‚ β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Side Panel (UI) β”‚ β”‚ Background Service Worker β”‚ β”‚ +β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ β”‚ β€’ Chat mit LLM │◄──►│ β€’ 22 Browser-Tools β”‚ β”‚ +β”‚ β”‚ β€’ Quick Actions β”‚ β”‚ β€’ WebSocket β†’ MCP Server β”‚ β”‚ +β”‚ β”‚ β€’ JS Console β”‚ β”‚ β€’ Message Router β”‚ β”‚ +β”‚ β”‚ β€’ Selector Test β”‚ β”‚ β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ +β”‚ β”‚ Content Script β”‚β—„β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ (jede Seite) β”‚ β”‚ +β”‚ β”‚ β€’ Console Capture β”‚ β”‚ +β”‚ β”‚ β€’ Error Tracking β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β–² β–² + β”‚ LLM API β”‚ WebSocket + β”‚ (OpenRouter/Ollama/etc) β”‚ + β–Ό β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ LLM Provider β”‚ β”‚ MCP Server (Node.js) β”‚ +β”‚ (dein Endpoint) β”‚ β”‚ Port 9224 β”‚ +β”‚ β”‚ β”‚ β”‚ +β”‚ β€’ OpenRouter β”‚ β”‚ β€’ HTTP MCP Protocol β”‚ +β”‚ β€’ Ollama (lokal) β”‚ β”‚ β€’ WS Bridge β”‚ +β”‚ β€’ Anthropic β”‚ β”‚ β€’ Zero Dependencies β”‚ +β”‚ β€’ OpenAI β”‚ β”‚ β”‚ +β”‚ β€’ Custom β”‚ β”‚ Clients: β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β€’ Claude Desktop β”‚ + β”‚ β€’ Cursor β”‚ + β”‚ β€’ Dieser Agent (du) β”‚ + β”‚ β€’ Beliebiger MCP-Clientβ”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +--- + +## ⚑ Quick Start + +### 1. Extension installieren + +```bash +# In Chrome/Brave: +# 1. chrome://extensions/ ΓΆffnen +# 2. Developer Mode aktivieren +# 3. "Load unpacked" β†’ browser-pilot/ Ordner wΓ€hlen +``` + +### 2. Side Panel nutzen (eigenstΓ€ndig im Browser) + +1. Extension-Icon klicken β†’ Side Panel ΓΆffnet sich +2. βš™οΈ Settings β†’ API Key + Endpoint konfigurieren +3. Lostippen: "Was steht auf dieser Seite?" oder "Klicke auf den Login-Button" + +### 3. MCP Server starten (fΓΌr externe KI-Anbindung) + +```bash +cd browser-pilot/server +node index.js +``` + +### 4. In Claude Desktop / Cursor / etc. einbinden + +```json +{ + "mcpServers": { + "browser-pilot": { + "type": "streamableHttp", + "url": "http://127.0.0.1:9224/mcp" + } + } +} +``` + +--- + +## πŸ—¨οΈ Side Panel Features + +| Tab | Funktion | +|-----|----------| +| **Fragen** | Chat mit KI – stellt Fragen ΓΌber die Seite, lΓ€sst Aufgaben ausfΓΌhren | +| **Handeln** | Quick Actions: Screenshot, Seitentext, Links, Formulare, Zusammenfassung | +| **Dev** | JS Console (Ctrl+Enter), CSS Selector Tester mit Highlighting | + +### UnterstΓΌtzte LLM Provider + +| Provider | Base URL | Modelle | +|----------|----------|---------| +| OpenRouter | `https://openrouter.ai/api/v1` | Alle (Claude, GPT, Gemini, ...) | +| Ollama | `http://localhost:11434/v1` | qwen2.5, llama3, mistral, ... | +| Anthropic | `https://api.anthropic.com/v1` | Claude Sonnet/Haiku/Opus | +| OpenAI | `https://api.openai.com/v1` | GPT-4o, o1, ... | +| Custom | Beliebig | Jeder OpenAI-kompatible Endpoint | + +--- + +## πŸ”§ 22 Browser-Tools + +| Tool | Beschreibung | +|------|-------------| +| `browser_navigate` | Zu URL navigieren | +| `browser_go_back/forward` | History-Navigation | +| `browser_reload` | Seite neu laden | +| `browser_get_tabs` | Alle Tabs auflisten | +| `browser_switch_tab` | Tab wechseln | +| `browser_close_tab` | Tab schließen | +| `browser_screenshot` | Screenshot (base64 PNG) | +| `browser_get_page_info` | URL, Titel, Viewport, Scroll | +| `browser_get_text` | Textinhalt der Seite/Element | +| `browser_get_html` | HTML der Seite/Element | +| `browser_query_selector` | Elemente per CSS-Selektor | +| `browser_get_interactive_elements` | Alle klickbaren Elemente | +| `browser_click` | Element anklicken | +| `browser_type` | Text in Input eingeben | +| `browser_select` | Option in Select wΓ€hlen | +| `browser_scroll` | Seite/Element scrollen | +| `browser_execute_js` | Beliebiges JavaScript ausfΓΌhren | +| `browser_press_key` | Taste drΓΌcken (mit Modifiern) | +| `browser_wait` | Auf Element/Timeout warten | +| `browser_get_console_logs` | Console-Logs auslesen | +| `browser_highlight` | Element visuell hervorheben | + +--- + +## πŸ”’ Compliance & Sicherheit + +- **Eigene Endpoints** – du bestimmst welches LLM, welcher Server +- **Kein Cloud-Zwang** – funktioniert komplett offline mit Ollama +- **Kein Tracking** – null Telemetrie, null externe Calls (außer dein LLM) +- **Lokaler MCP Server** – bindet nur auf 127.0.0.1 +- **Optional Auth Token** – `AUTH_TOKEN=xxx` Environment Variable +- **Zero Dependencies** – Server braucht kein `npm install` + +--- + +## πŸ“ Projektstruktur + +``` +browser-pilot/ +β”œβ”€β”€ manifest.json # Chrome MV3 Manifest +β”œβ”€β”€ background.js # Service Worker (Tools + WS Bridge) +β”œβ”€β”€ content.js # Content Script (Console Capture) +β”œβ”€β”€ sidepanel.html # Side Panel UI +β”œβ”€β”€ sidepanel.css # Styles (Dark Theme) +β”œβ”€β”€ sidepanel.js # Agent Loop + Chat Logic +β”œβ”€β”€ icons/ # Extension Icons +└── server/ + β”œβ”€β”€ index.js # MCP Server (zero-dep, Node 18+) + └── package.json +``` + +--- + +## πŸ’‘ Beispiele + +### Im Side Panel: +``` +"Fasse diese Seite zusammen" +"FΓΌlle das Kontaktformular mit Testdaten aus" +"Finde alle Links die auf externe Seiten zeigen" +"Klicke auf 'Weiter' und warte bis die nΓ€chste Seite geladen ist" +"Sperre die Beobachtungsliste" (wie im Screenshot) +``` + +### Via MCP (Claude Desktop / dieser Agent): +``` +Tool: browser_navigate β†’ {"url": "https://servicenow.mercedes-benz.com"} +Tool: browser_get_interactive_elements β†’ {} +Tool: browser_click β†’ {"selector": "#resolve-btn"} +Tool: browser_screenshot β†’ {} +``` + +--- + +## πŸš€ Entwicklung + +```bash +# Server im Watch-Mode: +cd server && node --watch index.js + +# Extension: Dateien editieren β†’ chrome://extensions/ β†’ Reload +``` + +--- + +MIT License. Keine AbhΓ€ngigkeiten. Dein Browser, dein LLM, deine Regeln. πŸ”₯ diff --git a/background.js b/background.js new file mode 100644 index 0000000..e031c78 --- /dev/null +++ b/background.js @@ -0,0 +1,428 @@ +/** + * Browser Pilot – Background Service Worker + * + * Three roles: + * 1. Tool execution engine (DOM, screenshots, JS, navigation) + * 2. WebSocket bridge for external MCP server + * 3. Message router between side panel, content scripts, and external clients + */ + +// ============================================================ +// Tool Registry +// ============================================================ +const TOOLS = { + browser_navigate: { + description: 'Navigate to a URL', + params: { url: 'string', newTab: 'boolean?' }, + handler: async ({ url, newTab }) => { + if (newTab) { + const tab = await chrome.tabs.create({ url }); + return { tabId: tab.id, url }; + } + const tab = await getActiveTab(); + await chrome.tabs.update(tab.id, { url }); + return { tabId: tab.id, url }; + } + }, + browser_go_back: { + description: 'Go back in history', + params: {}, + handler: async () => { const t = await getActiveTab(); await chrome.tabs.goBack(t.id); return { ok: true }; } + }, + browser_go_forward: { + description: 'Go forward in history', + params: {}, + handler: async () => { const t = await getActiveTab(); await chrome.tabs.goForward(t.id); return { ok: true }; } + }, + browser_reload: { + description: 'Reload current page', + params: {}, + handler: async () => { const t = await getActiveTab(); await chrome.tabs.reload(t.id); return { ok: true }; } + }, + browser_get_tabs: { + description: 'List all open tabs', + params: {}, + handler: async () => { + const tabs = await chrome.tabs.query({}); + return tabs.map(t => ({ id: t.id, title: t.title, url: t.url, active: t.active })); + } + }, + browser_switch_tab: { + description: 'Switch to tab by ID', + params: { tabId: 'number' }, + handler: async ({ tabId }) => { + await chrome.tabs.update(tabId, { active: true }); + const t = await chrome.tabs.get(tabId); + await chrome.windows.update(t.windowId, { focused: true }); + return { ok: true, tabId }; + } + }, + browser_close_tab: { + description: 'Close a tab', + params: { tabId: 'number' }, + handler: async ({ tabId }) => { await chrome.tabs.remove(tabId); return { ok: true }; } + }, + browser_screenshot: { + description: 'Screenshot visible viewport (base64 PNG)', + params: {}, + handler: async () => { + const tab = await getActiveTab(); + const dataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, { format: 'png' }); + return { image: dataUrl }; + } + }, + browser_get_page_info: { + description: 'Get page URL, title, viewport, scroll position', + params: {}, + handler: async () => { + const tab = await getActiveTab(); + const info = await execInTab(tab.id, () => ({ + title: document.title, + url: location.href, + viewport: { w: window.innerWidth, h: window.innerHeight }, + scroll: { x: window.scrollX, y: window.scrollY }, + docHeight: document.documentElement.scrollHeight + })); + return { ...info, tabId: tab.id }; + } + }, + browser_get_text: { + description: 'Get text content of page or element', + params: { selector: 'string?' }, + handler: async ({ selector }) => { + const tab = await getActiveTab(); + return await execInTab(tab.id, (sel) => { + const el = sel ? document.querySelector(sel) : document.body; + if (!el) return { error: 'Element not found: ' + sel }; + return el.innerText; + }, [selector || null]); + } + }, + browser_get_html: { + description: 'Get HTML of page or element', + params: { selector: 'string?', outer: 'boolean?' }, + handler: async ({ selector, outer = true }) => { + const tab = await getActiveTab(); + return await execInTab(tab.id, (sel, isOuter) => { + const el = sel ? document.querySelector(sel) : document.documentElement; + if (!el) return { error: 'Element not found: ' + sel }; + return isOuter ? el.outerHTML : el.innerHTML; + }, [selector || null, outer]); + } + }, + browser_query_selector: { + description: 'Query elements – returns tag, text, attributes, bounding box', + params: { selector: 'string', limit: 'number?' }, + handler: async ({ selector, limit = 20 }) => { + const tab = await getActiveTab(); + return await execInTab(tab.id, (sel, lim) => { + return Array.from(document.querySelectorAll(sel)).slice(0, lim).map(el => { + const r = el.getBoundingClientRect(); + const attrs = {}; + for (const a of el.attributes) attrs[a.name] = a.value; + return { tag: el.tagName.toLowerCase(), text: (el.innerText||'').slice(0,200), attrs, box: {x:r.x,y:r.y,w:r.width,h:r.height} }; + }); + }, [selector, limit]); + } + }, + browser_get_interactive_elements: { + description: 'Find all interactive elements (buttons, links, inputs) with selectors', + params: { viewportOnly: 'boolean?' }, + handler: async ({ viewportOnly = true }) => { + const tab = await getActiveTab(); + return await execInTab(tab.id, (vpOnly) => { + const sels = 'a[href], button, input, select, textarea, [role="button"], [onclick], [tabindex]:not([tabindex="-1"])'; + const els = Array.from(document.querySelectorAll(sels)); + const results = []; + for (const el of els) { + const r = el.getBoundingClientRect(); + if (r.width === 0 && r.height === 0) continue; + if (vpOnly && (r.bottom < 0 || r.top > window.innerHeight || r.right < 0 || r.left > window.innerWidth)) continue; + const attrs = {}; + for (const a of ['id','class','name','type','href','value','placeholder','aria-label','role','title']) + if (el.getAttribute(a)) attrs[a] = el.getAttribute(a); + let selector = ''; + if (el.id) selector = '#' + el.id; + else if (el.name) selector = el.tagName.toLowerCase() + '[name="' + el.name + '"]'; + else { + const parts = []; let cur = el; + while (cur && cur !== document.body && parts.length < 4) { + let s = cur.tagName.toLowerCase(); + if (cur.id) { parts.unshift('#' + cur.id); break; } + const parent = cur.parentElement; + if (parent) { + const sibs = Array.from(parent.children).filter(c => c.tagName === cur.tagName); + if (sibs.length > 1) s += ':nth-of-type(' + (sibs.indexOf(cur)+1) + ')'; + } + parts.unshift(s); cur = cur.parentElement; + } + selector = parts.join(' > '); + } + results.push({ tag: el.tagName.toLowerCase(), text: (el.innerText||el.value||'').slice(0,80), attrs, selector, box:{x:Math.round(r.x),y:Math.round(r.y),w:Math.round(r.width),h:Math.round(r.height)} }); + if (results.length >= 50) break; + } + return results; + }, [viewportOnly]); + } + }, + browser_click: { + description: 'Click element by CSS selector', + params: { selector: 'string' }, + handler: async ({ selector }) => { + const tab = await getActiveTab(); + return await execInTab(tab.id, (sel) => { + const el = document.querySelector(sel); + if (!el) return { error: 'Not found: ' + sel }; + el.scrollIntoView({ block: 'center', behavior: 'instant' }); + el.click(); + return { clicked: sel, tag: el.tagName.toLowerCase(), text: (el.innerText||'').slice(0,80) }; + }, [selector]); + } + }, + browser_type: { + description: 'Type text into input/textarea', + params: { selector: 'string', text: 'string', clear: 'boolean?' }, + handler: async ({ selector, text, clear = true }) => { + const tab = await getActiveTab(); + return await execInTab(tab.id, (sel, txt, clr) => { + const el = document.querySelector(sel); + if (!el) return { error: 'Not found: ' + sel }; + el.focus(); + if (clr) { el.value = ''; el.dispatchEvent(new Event('input', {bubbles:true})); } + // Simulate character-by-character for React/Angular + for (const ch of txt) { + el.value += ch; + el.dispatchEvent(new Event('input', {bubbles:true})); + } + el.dispatchEvent(new Event('change', {bubbles:true})); + return { typed: txt, selector: sel }; + }, [selector, text, clear]); + } + }, + browser_select: { + description: 'Select option in ', inputSchema: { type: 'object', properties: { selector: { type: 'string' }, value: { type: 'string' } }, required: ['selector', 'value'] } }, + { name: 'browser_scroll', description: 'Scroll page or element', inputSchema: { type: 'object', properties: { direction: { type: 'string', enum: ['up','down','left','right'] }, amount: { type: 'number' }, selector: { type: 'string' } }, required: ['direction'] } }, + { name: 'browser_execute_js', description: 'Execute JavaScript in page context', inputSchema: { type: 'object', properties: { code: { type: 'string' } }, required: ['code'] } }, + { name: 'browser_press_key', description: 'Press keyboard key', inputSchema: { type: 'object', properties: { key: { type: 'string' }, modifiers: { type: 'array', items: { type: 'string' } } }, required: ['key'] } }, + { name: 'browser_wait', description: 'Wait for element or timeout', inputSchema: { type: 'object', properties: { selector: { type: 'string' }, timeout: { type: 'number' } } } }, + { name: 'browser_get_console_logs', description: 'Get console logs', inputSchema: { type: 'object', properties: { clear: { type: 'boolean' } } } }, + { name: 'browser_highlight', description: 'Highlight element visually', inputSchema: { type: 'object', properties: { selector: { type: 'string' }, color: { type: 'string' } }, required: ['selector'] } }, +]; + +// ============================================================ +// MCP JSON-RPC Handler +// ============================================================ +async function handleMcp(body) { + const { method, params, id } = body; + + if (method === 'initialize') { + return { jsonrpc: '2.0', id, result: { protocolVersion: '2024-11-05', capabilities: { tools: {} }, serverInfo: { name: 'browser-pilot', version: '1.0.0' } } }; + } + if (method === 'notifications/initialized') return null; + if (method === 'tools/list') { + return { jsonrpc: '2.0', id, result: { tools: TOOLS } }; + } + if (method === 'tools/call') { + const { name, arguments: args } = params; + try { + const resp = await callExtension(name, args || {}); + const content = []; + if (resp.success) { + if (resp.result?.image) { + const b64 = resp.result.image.replace(/^data:image\/png;base64,/, ''); + content.push({ type: 'image', data: b64, mimeType: 'image/png' }); + } else { + content.push({ type: 'text', text: JSON.stringify(resp.result, null, 2) }); + } + } else { + content.push({ type: 'text', text: 'Error: ' + resp.error }); + } + return { jsonrpc: '2.0', id, result: { content, isError: !resp.success } }; + } catch (e) { + return { jsonrpc: '2.0', id, result: { content: [{ type: 'text', text: 'Error: ' + e.message }], isError: true } }; + } + } + return { jsonrpc: '2.0', id, error: { code: -32601, message: 'Method not found: ' + method } }; +} + +// ============================================================ +// HTTP Server +// ============================================================ +const server = http.createServer(async (req, res) => { + const cors = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization, Mcp-Session-Id', + 'Access-Control-Expose-Headers': 'Mcp-Session-Id' + }; + + if (req.method === 'OPTIONS') { res.writeHead(204, cors); res.end(); return; } + + // Auth + if (AUTH_TOKEN && req.headers.authorization !== `Bearer ${AUTH_TOKEN}`) { + res.writeHead(401, { ...cors, 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Unauthorized' })); + return; + } + + const url = new URL(req.url, `http://localhost:${PORT}`); + + // Health + if (url.pathname === '/health') { + res.writeHead(200, { ...cors, 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'ok', extensionConnected: !!extensionWs?.alive, tools: TOOLS.length })); + return; + } + + // Tools list (convenience) + if (url.pathname === '/tools' && req.method === 'GET') { + res.writeHead(200, { ...cors, 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ tools: TOOLS })); + return; + } + + // Direct execute (convenience) + if (url.pathname === '/execute' && req.method === 'POST') { + let body = ''; for await (const c of req) body += c; + try { + const { tool, params } = JSON.parse(body); + const result = await callExtension(tool, params || {}); + res.writeHead(200, { ...cors, 'Content-Type': 'application/json' }); + res.end(JSON.stringify(result)); + } catch (e) { + res.writeHead(500, { ...cors, 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: false, error: e.message })); + } + return; + } + + // MCP endpoint + if (url.pathname === '/mcp') { + if (req.method === 'GET') { + // SSE + res.writeHead(200, { ...cors, 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' }); + res.write('event: open\ndata: {"status":"connected"}\n\n'); + const ka = setInterval(() => res.write(': ping\n\n'), 15000); + req.on('close', () => clearInterval(ka)); + return; + } + if (req.method === 'POST') { + let body = ''; for await (const c of req) body += c; + try { + const parsed = JSON.parse(body); + const result = await handleMcp(parsed); + if (result) { + res.writeHead(200, { ...cors, 'Content-Type': 'application/json' }); + res.end(JSON.stringify(result)); + } else { + res.writeHead(204, cors); res.end(); + } + } catch (e) { + res.writeHead(400, { ...cors, 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ jsonrpc: '2.0', error: { code: -32700, message: e.message } })); + } + return; + } + } + + res.writeHead(404, cors); res.end('Not found'); +}); + +// WebSocket upgrade +server.on('upgrade', (req, socket, head) => { + const key = req.headers['sec-websocket-key']; + if (!key) { socket.destroy(); return; } + + const accept = crypto.createHash('sha1').update(key + WS_GUID).digest('base64'); + socket.write([ + 'HTTP/1.1 101 Switching Protocols', + 'Upgrade: websocket', + 'Connection: Upgrade', + `Sec-WebSocket-Accept: ${accept}`, + '', '' + ].join('\r\n')); + + const ws = new WsConnection(socket); + extensionWs = ws; + console.log('βœ“ Extension connected'); + + ws.onMessage = (data) => { + try { + const msg = JSON.parse(data); + if (msg.id && pendingCalls.has(msg.id)) { + pendingCalls.get(msg.id).resolve(msg); + pendingCalls.delete(msg.id); + } + } catch {} + }; + + ws.onClose = () => { + console.log('βœ— Extension disconnected'); + if (extensionWs === ws) extensionWs = null; + for (const [id, { reject }] of pendingCalls) reject(new Error('Extension disconnected')); + pendingCalls.clear(); + }; +}); + +server.listen(PORT, '127.0.0.1', () => { + console.log(` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ πŸš€ Browser Pilot – MCP Server β”‚ +β”‚ β”‚ +β”‚ MCP: http://127.0.0.1:${PORT}/mcp β”‚ +β”‚ Health: http://127.0.0.1:${PORT}/health β”‚ +β”‚ WS: ws://127.0.0.1:${PORT} (extension bridge) β”‚ +β”‚ β”‚ +β”‚ Waiting for extension... β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + `); +}); diff --git a/server/package.json b/server/package.json new file mode 100644 index 0000000..ade604f --- /dev/null +++ b/server/package.json @@ -0,0 +1,13 @@ +{ + "name": "browser-pilot-server", + "version": "1.0.0", + "description": "MCP server + WebSocket bridge for Browser Pilot extension", + "type": "module", + "main": "index.js", + "scripts": { + "start": "node index.js", + "dev": "node --watch index.js" + }, + "dependencies": {}, + "license": "MIT" +} diff --git a/sidepanel.css b/sidepanel.css new file mode 100644 index 0000000..6d8b2b4 --- /dev/null +++ b/sidepanel.css @@ -0,0 +1,415 @@ +/* Browser Pilot – Side Panel Styles */ +:root { + --bg: #0d1117; + --bg2: #161b22; + --bg3: #21262d; + --border: #30363d; + --text: #e6edf3; + --text2: #8b949e; + --accent: #ff6b35; + --accent2: #ff8c5a; + --green: #3fb950; + --red: #f85149; + --blue: #58a6ff; + --radius: 8px; + --font: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; + --mono: 'JetBrains Mono', 'Fira Code', 'Cascadia Code', monospace; +} + +* { margin: 0; padding: 0; box-sizing: border-box; } + +body { + font-family: var(--font); + background: var(--bg); + color: var(--text); + height: 100vh; + display: flex; + flex-direction: column; + overflow: hidden; + font-size: 13px; +} + +/* Header */ +.header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 12px; + background: var(--bg2); + border-bottom: 1px solid var(--border); + flex-shrink: 0; +} +.header-left { display: flex; align-items: center; gap: 8px; } +.logo { font-size: 18px; } +.title { font-weight: 600; font-size: 14px; } +.header-right { display: flex; align-items: center; gap: 8px; } +.provider-select { + background: var(--bg3); + border: 1px solid var(--border); + color: var(--text); + border-radius: 4px; + padding: 3px 6px; + font-size: 11px; +} +.status-dot { + width: 8px; height: 8px; + border-radius: 50%; + background: var(--red); + transition: background 0.3s; +} +.status-dot.connected { background: var(--green); } +.icon-btn { + background: none; + border: none; + cursor: pointer; + font-size: 16px; + padding: 2px; +} + +/* Tabs */ +.tabs { + display: flex; + background: var(--bg2); + border-bottom: 1px solid var(--border); + flex-shrink: 0; +} +.tab { + flex: 1; + padding: 8px; + background: none; + border: none; + color: var(--text2); + font-size: 12px; + font-weight: 500; + cursor: pointer; + border-bottom: 2px solid transparent; + transition: all 0.2s; +} +.tab:hover { color: var(--text); background: var(--bg3); } +.tab.active { color: var(--accent); border-bottom-color: var(--accent); } + +/* Tab Content */ +.tab-content { + display: none; + flex: 1; + overflow: hidden; + flex-direction: column; +} +.tab-content.active { display: flex; } + +/* Messages */ +.messages { + flex: 1; + overflow-y: auto; + padding: 12px; + display: flex; + flex-direction: column; + gap: 10px; +} +.message { + max-width: 95%; + padding: 10px 12px; + border-radius: var(--radius); + font-size: 13px; + line-height: 1.5; + word-wrap: break-word; +} +.message.assistant { + background: var(--bg2); + border: 1px solid var(--border); + align-self: flex-start; +} +.message.user { + background: var(--accent); + color: white; + align-self: flex-end; + border-radius: var(--radius) var(--radius) 4px var(--radius); +} +.message.tool { + background: var(--bg3); + border: 1px solid var(--border); + font-family: var(--mono); + font-size: 11px; + color: var(--text2); + align-self: flex-start; + max-width: 100%; +} +.message.error { + background: #2d1b1b; + border: 1px solid var(--red); + color: var(--red); +} +.message-content { white-space: pre-wrap; } +.message-content strong { color: var(--accent); } +.message-content code { + background: var(--bg3); + padding: 1px 4px; + border-radius: 3px; + font-family: var(--mono); + font-size: 11px; +} +.tool-label { + font-size: 10px; + color: var(--accent); + font-weight: 600; + margin-bottom: 4px; + text-transform: uppercase; +} + +/* Input Area */ +.input-area { + padding: 10px 12px; + border-top: 1px solid var(--border); + background: var(--bg2); + flex-shrink: 0; +} +.input-row { + display: flex; + gap: 8px; + align-items: flex-end; +} +.input-row textarea { + flex: 1; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text); + padding: 8px 12px; + font-family: var(--font); + font-size: 13px; + resize: none; + max-height: 120px; + line-height: 1.4; +} +.input-row textarea:focus { outline: none; border-color: var(--accent); } +.send-btn { + width: 36px; height: 36px; + border-radius: 50%; + background: var(--accent); + border: none; + color: white; + font-size: 14px; + cursor: pointer; + flex-shrink: 0; + transition: background 0.2s; +} +.send-btn:hover { background: var(--accent2); } +.send-btn:disabled { opacity: 0.5; cursor: not-allowed; } +.input-options { + display: flex; + align-items: center; + justify-content: space-between; + margin-top: 6px; +} +.checkbox-label { + font-size: 11px; + color: var(--text2); + display: flex; + align-items: center; + gap: 4px; + cursor: pointer; +} +.stop-btn { + background: var(--red); + color: white; + border: none; + border-radius: 4px; + padding: 4px 10px; + font-size: 11px; + cursor: pointer; +} +.hidden { display: none !important; } + +/* Actions Tab */ +.actions-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; + padding: 12px; +} +.action-btn { + background: var(--bg2); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text); + padding: 12px 10px; + font-size: 12px; + cursor: pointer; + transition: all 0.2s; + text-align: left; +} +.action-btn:hover { border-color: var(--accent); background: var(--bg3); } +.action-result { + flex: 1; + overflow-y: auto; + padding: 12px; + font-family: var(--mono); + font-size: 11px; + white-space: pre-wrap; + color: var(--text2); +} + +/* Dev Tab */ +.dev-section { + padding: 12px; + border-bottom: 1px solid var(--border); +} +.dev-section h3 { + font-size: 12px; + color: var(--accent); + margin-bottom: 8px; + text-transform: uppercase; + letter-spacing: 0.5px; +} +.code-input, .code-input-line { + width: 100%; + background: var(--bg); + border: 1px solid var(--border); + border-radius: 4px; + color: var(--text); + font-family: var(--mono); + font-size: 12px; + padding: 8px; + resize: vertical; +} +.code-input-line { resize: none; height: 32px; } +.code-input:focus, .code-input-line:focus { outline: none; border-color: var(--accent); } +.exec-btn { + margin-top: 6px; + background: var(--bg3); + border: 1px solid var(--border); + color: var(--text); + border-radius: 4px; + padding: 5px 12px; + font-size: 11px; + cursor: pointer; +} +.exec-btn:hover { border-color: var(--accent); } +.code-output { + margin-top: 8px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: 4px; + padding: 8px; + font-family: var(--mono); + font-size: 11px; + color: var(--green); + max-height: 200px; + overflow-y: auto; + white-space: pre-wrap; + word-break: break-all; +} +.code-output.small { font-size: 10px; color: var(--text2); } +.mcp-status { + display: flex; + align-items: center; + gap: 8px; + font-size: 12px; + margin-bottom: 8px; +} +.mcp-dot { + width: 8px; height: 8px; + border-radius: 50%; + background: var(--green); +} +.mcp-dot.offline { background: var(--red); } + +/* Modal */ +.modal { + position: fixed; + inset: 0; + background: rgba(0,0,0,0.7); + display: flex; + align-items: center; + justify-content: center; + z-index: 1000; +} +.modal-content { + background: var(--bg2); + border: 1px solid var(--border); + border-radius: 12px; + width: 90%; + max-width: 400px; + max-height: 90vh; + overflow-y: auto; + padding: 20px; +} +.modal-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 16px; +} +.modal-header h2 { font-size: 16px; } +.close-btn { + background: none; + border: none; + color: var(--text2); + font-size: 18px; + cursor: pointer; +} +.form-group { margin-bottom: 12px; } +.form-group label { + display: block; + font-size: 11px; + color: var(--text2); + margin-bottom: 4px; + font-weight: 500; +} +.form-group input, .form-group textarea { + width: 100%; + background: var(--bg); + border: 1px solid var(--border); + border-radius: 4px; + color: var(--text); + padding: 8px; + font-size: 12px; + font-family: var(--mono); +} +.form-group input:focus, .form-group textarea:focus { outline: none; border-color: var(--accent); } +.save-btn { + width: 100%; + background: var(--accent); + color: white; + border: none; + border-radius: var(--radius); + padding: 10px; + font-size: 13px; + font-weight: 600; + cursor: pointer; + margin-top: 8px; +} +.save-btn:hover { background: var(--accent2); } + +/* Scrollbar */ +::-webkit-scrollbar { width: 6px; } +::-webkit-scrollbar-track { background: var(--bg); } +::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } +::-webkit-scrollbar-thumb:hover { background: var(--text2); } + +/* Loading animation */ +.typing-indicator { + display: flex; + gap: 4px; + padding: 4px 0; +} +.typing-indicator span { + width: 6px; height: 6px; + background: var(--text2); + border-radius: 50%; + animation: bounce 1.4s infinite; +} +.typing-indicator span:nth-child(2) { animation-delay: 0.2s; } +.typing-indicator span:nth-child(3) { animation-delay: 0.4s; } +@keyframes bounce { + 0%, 60%, 100% { transform: translateY(0); } + 30% { transform: translateY(-4px); } +} + +/* Screenshot in chat */ +.message img { + max-width: 100%; + border-radius: 4px; + margin-top: 6px; + border: 1px solid var(--border); +} diff --git a/sidepanel.html b/sidepanel.html new file mode 100644 index 0000000..52830ab --- /dev/null +++ b/sidepanel.html @@ -0,0 +1,138 @@ + + + + + + Browser Pilot + + + + +
+
+ + Browser Pilot +
+
+ +
+ +
+
+ + + + + +
+
+
+
+ Hallo! Ich bin Browser Pilot. Ich kann diese Seite lesen, Fragen beantworten, auf Buttons klicken, Formulare ausfΓΌllen und mehrstufige Aufgaben automatisieren. +
+
+
+
+
+ + +
+
+ + +
+
+
+ + +
+
+ + + + + + + + +
+
+
+ + +
+
+

JavaScript Console

+ + +

+    
+
+

CSS Selector Test

+ + +

+    
+
+

MCP Server Status

+
+ MCP Server nicht verbunden +
+
{
+  "mcpServers": {
+    "browser-pilot": {
+      "url": "http://127.0.0.1:9224/mcp"
+    }
+  }
+}
+
+
+ + + + + + + diff --git a/sidepanel.js b/sidepanel.js new file mode 100644 index 0000000..97dc4ca --- /dev/null +++ b/sidepanel.js @@ -0,0 +1,632 @@ +/** + * Browser Pilot – Side Panel Logic + * + * This is the in-browser agent. It: + * 1. Provides a chat UI for the user + * 2. Connects to a configurable LLM endpoint (OpenRouter, Anthropic, Ollama, custom) + * 3. Runs an agentic tool-use loop: LLM decides which browser tools to call + * 4. Shows results in real-time + */ + +// ============================================================ +// State +// ============================================================ +let config = { + baseUrl: 'https://openrouter.ai/api/v1', + model: 'anthropic/claude-sonnet-4-20250514', + apiKey: '', + systemPrompt: '', + maxIterations: 15, + provider: 'openrouter' +}; + +let isRunning = false; +let abortController = null; +let conversationHistory = []; + +// ============================================================ +// DOM refs +// ============================================================ +const $ = (sel) => document.querySelector(sel); +const chatMessages = $('#chatMessages'); +const chatInput = $('#chatInput'); +const sendBtn = $('#sendBtn'); +const stopBtn = $('#stopBtn'); +const settingsBtn = $('#settingsBtn'); +const settingsModal = $('#settingsModal'); +const closeSettings = $('#closeSettings'); +const saveSettingsBtn = $('#saveSettings'); +const statusDot = $('#statusDot'); +const providerSelect = $('#providerSelect'); +const actionResult = $('#actionResult'); + +// ============================================================ +// Init +// ============================================================ +document.addEventListener('DOMContentLoaded', async () => { + await loadConfig(); + setupTabs(); + setupChat(); + setupActions(); + setupDev(); + setupSettings(); + checkConnection(); +}); + +async function loadConfig() { + const stored = await chrome.storage.local.get(['bp_config']); + if (stored.bp_config) { + config = { ...config, ...stored.bp_config }; + } + providerSelect.value = config.provider || 'custom'; + applyProviderDefaults(config.provider); +} + +async function saveConfig() { + await chrome.storage.local.set({ bp_config: config }); +} + +// ============================================================ +// Tabs +// ============================================================ +function setupTabs() { + document.querySelectorAll('.tab').forEach(tab => { + tab.addEventListener('click', () => { + document.querySelectorAll('.tab').forEach(t => t.classList.remove('active')); + document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active')); + tab.classList.add('active'); + $(`#tab-${tab.dataset.tab}`).classList.add('active'); + }); + }); +} + +// ============================================================ +// Chat +// ============================================================ +function setupChat() { + sendBtn.addEventListener('click', sendMessage); + chatInput.addEventListener('keydown', (e) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + sendMessage(); + } + }); + // Auto-resize textarea + chatInput.addEventListener('input', () => { + chatInput.style.height = 'auto'; + chatInput.style.height = Math.min(chatInput.scrollHeight, 120) + 'px'; + }); + stopBtn.addEventListener('click', stopAgent); +} + +async function sendMessage() { + const text = chatInput.value.trim(); + if (!text || isRunning) return; + + chatInput.value = ''; + chatInput.style.height = 'auto'; + addMessage('user', text); + + conversationHistory.push({ role: 'user', content: text }); + await runAgentLoop(); +} + +function stopAgent() { + if (abortController) abortController.abort(); + isRunning = false; + stopBtn.classList.add('hidden'); + sendBtn.disabled = false; +} + +// ============================================================ +// Agent Loop – the core +// ============================================================ +async function runAgentLoop() { + isRunning = true; + sendBtn.disabled = true; + stopBtn.classList.remove('hidden'); + abortController = new AbortController(); + + const tools = await getToolsForLLM(); + const systemPrompt = config.systemPrompt || getDefaultSystemPrompt(); + + let iterations = 0; + + try { + while (iterations < config.maxIterations && isRunning) { + iterations++; + + // Show thinking indicator + const thinkingEl = addTypingIndicator(); + + // Call LLM + const messages = [ + { role: 'system', content: systemPrompt }, + ...conversationHistory + ]; + + let response; + try { + response = await callLLM(messages, tools); + } catch (err) { + removeElement(thinkingEl); + if (err.name === 'AbortError') break; + addMessage('error', `LLM Error: ${err.message}`); + break; + } + + removeElement(thinkingEl); + + const assistantMsg = response.choices?.[0]?.message; + if (!assistantMsg) { + addMessage('error', 'No response from LLM'); + break; + } + + // Add to history + conversationHistory.push(assistantMsg); + + // If there's text content, show it + if (assistantMsg.content) { + addMessage('assistant', assistantMsg.content); + } + + // If no tool calls, we're done + if (!assistantMsg.tool_calls || assistantMsg.tool_calls.length === 0) { + break; + } + + // Execute tool calls + for (const toolCall of assistantMsg.tool_calls) { + if (!isRunning) break; + + const funcName = toolCall.function.name; + let args = {}; + try { args = JSON.parse(toolCall.function.arguments || '{}'); } catch {} + + // Show tool call in chat + addMessage('tool', `πŸ”§ ${funcName}\n${JSON.stringify(args, null, 2)}`); + + // Execute via background + let result; + try { + const resp = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: funcName, params: args }); + if (resp.success) { + result = JSON.stringify(resp.result, null, 2); + // Show screenshot inline + if (resp.result?.image) { + addScreenshot(resp.result.image); + result = '(screenshot captured)'; + } else { + // Truncate long results in display + const display = result.length > 500 ? result.slice(0, 500) + '\n...' : result; + addMessage('tool', `βœ“ Result:\n${display}`); + } + } else { + result = `Error: ${resp.error}`; + addMessage('error', result); + } + } catch (err) { + result = `Error: ${err.message}`; + addMessage('error', result); + } + + // Add tool result to conversation + conversationHistory.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: result.slice(0, 8000) // Limit context size + }); + } + } + + if (iterations >= config.maxIterations) { + addMessage('assistant', '⚠️ Maximale Iterationen erreicht. Stoppe.'); + } + } finally { + isRunning = false; + sendBtn.disabled = false; + stopBtn.classList.add('hidden'); + } +} + +// ============================================================ +// LLM API Call +// ============================================================ +async function callLLM(messages, tools) { + const payload = { + model: config.model, + messages, + tools, + tool_choice: 'auto', + temperature: 0.1, + max_tokens: 4096 + }; + + // Provider-specific headers + const headers = { + 'Content-Type': 'application/json' + }; + + if (config.apiKey) { + if (config.provider === 'anthropic') { + headers['x-api-key'] = config.apiKey; + headers['anthropic-version'] = '2023-06-01'; + } else { + headers['Authorization'] = `Bearer ${config.apiKey}`; + } + } + + // Anthropic uses a different format + let url = `${config.baseUrl}/chat/completions`; + if (config.provider === 'anthropic') { + url = `${config.baseUrl}/messages`; + // Convert to Anthropic format + const anthropicPayload = convertToAnthropicFormat(messages, tools, config.model); + const resp = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify(anthropicPayload), + signal: abortController.signal + }); + if (!resp.ok) { + const err = await resp.text(); + throw new Error(`${resp.status}: ${err}`); + } + const data = await resp.json(); + return convertFromAnthropicFormat(data); + } + + const resp = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify(payload), + signal: abortController.signal + }); + + if (!resp.ok) { + const err = await resp.text(); + throw new Error(`${resp.status}: ${err}`); + } + + return await resp.json(); +} + +function convertToAnthropicFormat(messages, tools, model) { + const system = messages.find(m => m.role === 'system')?.content || ''; + const msgs = messages.filter(m => m.role !== 'system').map(m => { + if (m.role === 'tool') { + return { role: 'user', content: [{ type: 'tool_result', tool_use_id: m.tool_call_id, content: m.content }] }; + } + if (m.tool_calls) { + return { + role: 'assistant', + content: [ + ...(m.content ? [{ type: 'text', text: m.content }] : []), + ...m.tool_calls.map(tc => ({ type: 'tool_use', id: tc.id, name: tc.function.name, input: JSON.parse(tc.function.arguments || '{}') })) + ] + }; + } + return m; + }); + + const anthropicTools = tools.map(t => ({ + name: t.function.name, + description: t.function.description, + input_schema: t.function.parameters + })); + + return { model, system, messages: msgs, tools: anthropicTools, max_tokens: 4096 }; +} + +function convertFromAnthropicFormat(data) { + const content = data.content || []; + const textParts = content.filter(c => c.type === 'text').map(c => c.text).join(''); + const toolUses = content.filter(c => c.type === 'tool_use'); + + const message = { + role: 'assistant', + content: textParts || null, + tool_calls: toolUses.length > 0 ? toolUses.map(tu => ({ + id: tu.id, + type: 'function', + function: { name: tu.name, arguments: JSON.stringify(tu.input) } + })) : undefined + }; + + return { choices: [{ message, finish_reason: data.stop_reason === 'end_turn' ? 'stop' : 'tool_calls' }] }; +} + +// ============================================================ +// Tools schema for LLM +// ============================================================ +async function getToolsForLLM() { + const resp = await chrome.runtime.sendMessage({ type: 'GET_TOOLS' }); + return resp.tools.map(t => ({ + type: 'function', + function: { + name: t.name, + description: t.description, + parameters: buildJsonSchema(t.params) + } + })); +} + +function buildJsonSchema(params) { + const properties = {}; + const required = []; + for (const [key, type] of Object.entries(params)) { + const isOptional = type.endsWith('?'); + const baseType = type.replace('?', ''); + properties[key] = { type: baseType === 'array' ? 'array' : baseType }; + if (!isOptional) required.push(key); + } + return { type: 'object', properties, required }; +} + +// ============================================================ +// System Prompt +// ============================================================ +function getDefaultSystemPrompt() { + return `Du bist Browser Pilot, ein KI-Agent der einen Webbrowser steuert. + +Du hast Zugriff auf Browser-Automatisierungs-Tools. Nutze sie Schritt fΓΌr Schritt um die Aufgabe des Nutzers zu erledigen. + +Regeln: +- Starte mit browser_get_page_info oder browser_get_interactive_elements um den aktuellen Zustand zu verstehen +- Nutze browser_screenshot wenn du die Seite visuell sehen musst +- Nutze browser_execute_js fΓΌr komplexe DOM-Operationen +- Sei methodisch: navigieren β†’ beobachten β†’ handeln β†’ verifizieren +- Wenn du fertig bist, fasse zusammen was du getan hast +- Antworte auf Deutsch wenn der Nutzer Deutsch spricht +- Bei Fragen ΓΌber den Seiteninhalt: lies zuerst die Seite mit browser_get_text + +VerfΓΌgbare Tools: +- browser_navigate: Zu URL navigieren +- browser_get_page_info: Seiten-URL, Titel, Viewport +- browser_get_text: Text der Seite oder eines Elements +- browser_get_html: HTML der Seite oder eines Elements +- browser_query_selector: Elemente abfragen mit CSS-Selektor +- browser_get_interactive_elements: Alle klickbaren Elemente finden +- browser_click: Element anklicken +- browser_type: Text in Input eingeben +- browser_select: Option in Select wΓ€hlen +- browser_scroll: Seite scrollen +- browser_execute_js: JavaScript ausfΓΌhren +- browser_screenshot: Screenshot machen +- browser_press_key: Taste drΓΌcken +- browser_wait: Auf Element oder Timeout warten +- browser_highlight: Element visuell hervorheben +- browser_get_console_logs: Console-Logs lesen`; +} + +// ============================================================ +// UI Helpers +// ============================================================ +function addMessage(type, content) { + const div = document.createElement('div'); + div.className = `message ${type}`; + + if (type === 'tool') { + div.innerHTML = `
Tool
${escapeHtml(content)}
`; + } else { + div.innerHTML = `
${formatContent(content)}
`; + } + + chatMessages.appendChild(div); + chatMessages.scrollTop = chatMessages.scrollHeight; + return div; +} + +function addScreenshot(dataUrl) { + const div = document.createElement('div'); + div.className = 'message assistant'; + div.innerHTML = `
Screenshot
Screenshot`; + chatMessages.appendChild(div); + chatMessages.scrollTop = chatMessages.scrollHeight; +} + +function addTypingIndicator() { + const div = document.createElement('div'); + div.className = 'message assistant'; + div.innerHTML = '
'; + chatMessages.appendChild(div); + chatMessages.scrollTop = chatMessages.scrollHeight; + return div; +} + +function removeElement(el) { + if (el && el.parentNode) el.parentNode.removeChild(el); +} + +function escapeHtml(text) { + return text.replace(/&/g, '&').replace(//g, '>'); +} + +function formatContent(text) { + // Basic markdown-like formatting + let html = escapeHtml(text); + html = html.replace(/\*\*(.*?)\*\*/g, '$1'); + html = html.replace(/`(.*?)`/g, '$1'); + return html; +} + +// ============================================================ +// Quick Actions (Act Tab) +// ============================================================ +function setupActions() { + document.querySelectorAll('.action-btn').forEach(btn => { + btn.addEventListener('click', () => executeAction(btn.dataset.action)); + }); +} + +async function executeAction(action) { + actionResult.textContent = 'Executing...'; + + try { + let result; + switch (action) { + case 'screenshot': + result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_screenshot', params: {} }); + if (result.success && result.result.image) { + actionResult.innerHTML = ``; + return; + } + break; + case 'get-text': + result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_text', params: {} }); + break; + case 'get-links': + result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector: 'a[href]', limit: 50 } }); + break; + case 'get-forms': + result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector: 'form, input, select, textarea', limit: 30 } }); + break; + case 'get-interactive': + result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_interactive_elements', params: {} }); + break; + case 'console-logs': + result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_console_logs', params: {} }); + break; + case 'page-info': + result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_page_info', params: {} }); + break; + case 'summarize': + // Get text and send to LLM for summary + const textResult = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_text', params: {} }); + if (textResult.success) { + // Switch to chat tab and ask for summary + document.querySelector('[data-tab="chat"]').click(); + chatInput.value = 'Fasse den Inhalt dieser Seite zusammen.'; + sendMessage(); + return; + } + break; + } + + if (result?.success) { + actionResult.textContent = JSON.stringify(result.result, null, 2); + } else { + actionResult.textContent = 'Error: ' + (result?.error || 'Unknown'); + } + } catch (err) { + actionResult.textContent = 'Error: ' + err.message; + } +} + +// ============================================================ +// Dev Tab +// ============================================================ +function setupDev() { + $('#execJsBtn').addEventListener('click', async () => { + const code = $('#jsInput').value; + try { + const result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_execute_js', params: { code } }); + $('#jsOutput').textContent = result.success + ? JSON.stringify(result.result, null, 2) + : 'Error: ' + result.error; + $('#jsOutput').style.color = result.success ? 'var(--green)' : 'var(--red)'; + } catch (err) { + $('#jsOutput').textContent = 'Error: ' + err.message; + $('#jsOutput').style.color = 'var(--red)'; + } + }); + + $('#testSelectorBtn').addEventListener('click', async () => { + const selector = $('#selectorInput').value; + try { + const result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector, limit: 10 } }); + if (result.success) { + $('#selectorOutput').textContent = JSON.stringify(result.result, null, 2); + $('#selectorOutput').style.color = 'var(--green)'; + // Highlight first match + if (result.result?.length > 0) { + await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_highlight', params: { selector } }); + } + } else { + $('#selectorOutput').textContent = 'Error: ' + result.error; + $('#selectorOutput').style.color = 'var(--red)'; + } + } catch (err) { + $('#selectorOutput').textContent = 'Error: ' + err.message; + } + }); + + // JS input: Ctrl+Enter to execute + $('#jsInput').addEventListener('keydown', (e) => { + if (e.key === 'Enter' && e.ctrlKey) { + e.preventDefault(); + $('#execJsBtn').click(); + } + }); +} + +// ============================================================ +// Settings +// ============================================================ +function setupSettings() { + settingsBtn.addEventListener('click', () => { + $('#cfgBaseUrl').value = config.baseUrl; + $('#cfgModel').value = config.model; + $('#cfgApiKey').value = config.apiKey; + $('#cfgSystemPrompt').value = config.systemPrompt; + $('#cfgMaxIter').value = config.maxIterations; + settingsModal.classList.remove('hidden'); + }); + + closeSettings.addEventListener('click', () => settingsModal.classList.add('hidden')); + + saveSettingsBtn.addEventListener('click', async () => { + config.baseUrl = $('#cfgBaseUrl').value.trim(); + config.model = $('#cfgModel').value.trim(); + config.apiKey = $('#cfgApiKey').value.trim(); + config.systemPrompt = $('#cfgSystemPrompt').value.trim(); + config.maxIterations = parseInt($('#cfgMaxIter').value) || 15; + config.provider = providerSelect.value; + await saveConfig(); + settingsModal.classList.add('hidden'); + checkConnection(); + }); + + providerSelect.addEventListener('change', (e) => { + config.provider = e.target.value; + applyProviderDefaults(e.target.value); + }); +} + +function applyProviderDefaults(provider) { + switch (provider) { + case 'openrouter': + config.baseUrl = 'https://openrouter.ai/api/v1'; + config.model = 'anthropic/claude-sonnet-4-20250514'; + break; + case 'ollama': + config.baseUrl = 'http://localhost:11434/v1'; + config.model = 'qwen2.5:14b'; + break; + case 'anthropic': + config.baseUrl = 'https://api.anthropic.com/v1'; + config.model = 'claude-sonnet-4-20250514'; + break; + case 'openai': + config.baseUrl = 'https://api.openai.com/v1'; + config.model = 'gpt-4o'; + break; + } +} + +// ============================================================ +// Connection check +// ============================================================ +async function checkConnection() { + try { + const resp = await chrome.runtime.sendMessage({ type: 'PING' }); + if (resp?.pong) { + statusDot.classList.add('connected'); + statusDot.title = `Connected – ${resp.tools} tools`; + } + } catch { + statusDot.classList.remove('connected'); + statusDot.title = 'Disconnected'; + } +}