/** * Browser Pilot – Side Panel Logic * * This is the in-browser agent. It: * 1. Provides a chat UI for the user * 2. Connects to a configurable LLM endpoint (OpenRouter, Anthropic, Ollama, custom) * 3. Runs an agentic tool-use loop: LLM decides which browser tools to call * 4. Shows results in real-time */ // ============================================================ // State // ============================================================ let config = { baseUrl: 'https://openrouter.ai/api/v1', model: 'anthropic/claude-sonnet-4-20250514', apiKey: '', systemPrompt: '', maxIterations: 15, provider: 'openrouter' }; let isRunning = false; let abortController = null; let conversationHistory = []; // ============================================================ // DOM refs // ============================================================ const $ = (sel) => document.querySelector(sel); const chatMessages = $('#chatMessages'); const chatInput = $('#chatInput'); const sendBtn = $('#sendBtn'); const stopBtn = $('#stopBtn'); const settingsBtn = $('#settingsBtn'); const settingsModal = $('#settingsModal'); const closeSettings = $('#closeSettings'); const saveSettingsBtn = $('#saveSettings'); const statusDot = $('#statusDot'); const providerSelect = $('#providerSelect'); const actionResult = $('#actionResult'); // ============================================================ // Init // ============================================================ document.addEventListener('DOMContentLoaded', async () => { await loadConfig(); setupTabs(); setupChat(); setupActions(); setupDev(); setupSettings(); checkConnection(); }); async function loadConfig() { const stored = await chrome.storage.local.get(['bp_config']); if (stored.bp_config) { config = { ...config, ...stored.bp_config }; } providerSelect.value = config.provider || 'custom'; applyProviderDefaults(config.provider); } async function saveConfig() { await chrome.storage.local.set({ bp_config: config }); } // ============================================================ // Tabs // ============================================================ function setupTabs() { document.querySelectorAll('.tab').forEach(tab => { tab.addEventListener('click', () => { document.querySelectorAll('.tab').forEach(t => t.classList.remove('active')); document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active')); tab.classList.add('active'); $(`#tab-${tab.dataset.tab}`).classList.add('active'); }); }); } // ============================================================ // Chat // ============================================================ function setupChat() { sendBtn.addEventListener('click', sendMessage); chatInput.addEventListener('keydown', (e) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); sendMessage(); } }); // Auto-resize textarea chatInput.addEventListener('input', () => { chatInput.style.height = 'auto'; chatInput.style.height = Math.min(chatInput.scrollHeight, 120) + 'px'; }); stopBtn.addEventListener('click', stopAgent); } async function sendMessage() { const text = chatInput.value.trim(); if (!text || isRunning) return; chatInput.value = ''; chatInput.style.height = 'auto'; addMessage('user', text); conversationHistory.push({ role: 'user', content: text }); await runAgentLoop(); } function stopAgent() { if (abortController) abortController.abort(); isRunning = false; stopBtn.classList.add('hidden'); sendBtn.disabled = false; } // ============================================================ // Agent Loop – the core // ============================================================ async function runAgentLoop() { isRunning = true; sendBtn.disabled = true; stopBtn.classList.remove('hidden'); abortController = new AbortController(); const tools = await getToolsForLLM(); const systemPrompt = config.systemPrompt || getDefaultSystemPrompt(); let iterations = 0; try { while (iterations < config.maxIterations && isRunning) { iterations++; // Show thinking indicator const thinkingEl = addTypingIndicator(); // Call LLM const messages = [ { role: 'system', content: systemPrompt }, ...conversationHistory ]; let response; try { response = await callLLM(messages, tools); } catch (err) { removeElement(thinkingEl); if (err.name === 'AbortError') break; addMessage('error', `LLM Error: ${err.message}`); break; } removeElement(thinkingEl); const assistantMsg = response.choices?.[0]?.message; if (!assistantMsg) { addMessage('error', 'No response from LLM'); break; } // Add to history conversationHistory.push(assistantMsg); // If there's text content, show it if (assistantMsg.content) { addMessage('assistant', assistantMsg.content); } // If no tool calls, we're done if (!assistantMsg.tool_calls || assistantMsg.tool_calls.length === 0) { break; } // Execute tool calls for (const toolCall of assistantMsg.tool_calls) { if (!isRunning) break; const funcName = toolCall.function.name; let args = {}; try { args = JSON.parse(toolCall.function.arguments || '{}'); } catch {} // Show tool call in chat addMessage('tool', `🔧 ${funcName}\n${JSON.stringify(args, null, 2)}`); // Execute via background let result; try { const resp = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: funcName, params: args }); if (resp.success) { result = JSON.stringify(resp.result, null, 2); // Show screenshot inline if (resp.result?.image) { addScreenshot(resp.result.image); result = '(screenshot captured)'; } else { // Truncate long results in display const display = result.length > 500 ? result.slice(0, 500) + '\n...' : result; addMessage('tool', `✓ Result:\n${display}`); } } else { result = `Error: ${resp.error}`; addMessage('error', result); } } catch (err) { result = `Error: ${err.message}`; addMessage('error', result); } // Add tool result to conversation conversationHistory.push({ role: 'tool', tool_call_id: toolCall.id, content: result.slice(0, 8000) // Limit context size }); } } if (iterations >= config.maxIterations) { addMessage('assistant', '⚠️ Maximale Iterationen erreicht. Stoppe.'); } } finally { isRunning = false; sendBtn.disabled = false; stopBtn.classList.add('hidden'); } } // ============================================================ // LLM API Call // ============================================================ async function callLLM(messages, tools) { const payload = { model: config.model, messages, tools, tool_choice: 'auto', temperature: 0.1, max_tokens: 4096 }; // Provider-specific headers const headers = { 'Content-Type': 'application/json' }; if (config.apiKey) { if (config.provider === 'anthropic') { headers['x-api-key'] = config.apiKey; headers['anthropic-version'] = '2023-06-01'; } else { headers['Authorization'] = `Bearer ${config.apiKey}`; } } // Anthropic uses a different format let url = `${config.baseUrl}/chat/completions`; if (config.provider === 'anthropic') { url = `${config.baseUrl}/messages`; // Convert to Anthropic format const anthropicPayload = convertToAnthropicFormat(messages, tools, config.model); const resp = await fetch(url, { method: 'POST', headers, body: JSON.stringify(anthropicPayload), signal: abortController.signal }); if (!resp.ok) { const err = await resp.text(); throw new Error(`${resp.status}: ${err}`); } const data = await resp.json(); return convertFromAnthropicFormat(data); } const resp = await fetch(url, { method: 'POST', headers, body: JSON.stringify(payload), signal: abortController.signal }); if (!resp.ok) { const err = await resp.text(); throw new Error(`${resp.status}: ${err}`); } return await resp.json(); } function convertToAnthropicFormat(messages, tools, model) { const system = messages.find(m => m.role === 'system')?.content || ''; const msgs = messages.filter(m => m.role !== 'system').map(m => { if (m.role === 'tool') { return { role: 'user', content: [{ type: 'tool_result', tool_use_id: m.tool_call_id, content: m.content }] }; } if (m.tool_calls) { return { role: 'assistant', content: [ ...(m.content ? [{ type: 'text', text: m.content }] : []), ...m.tool_calls.map(tc => ({ type: 'tool_use', id: tc.id, name: tc.function.name, input: JSON.parse(tc.function.arguments || '{}') })) ] }; } return m; }); const anthropicTools = tools.map(t => ({ name: t.function.name, description: t.function.description, input_schema: t.function.parameters })); return { model, system, messages: msgs, tools: anthropicTools, max_tokens: 4096 }; } function convertFromAnthropicFormat(data) { const content = data.content || []; const textParts = content.filter(c => c.type === 'text').map(c => c.text).join(''); const toolUses = content.filter(c => c.type === 'tool_use'); const message = { role: 'assistant', content: textParts || null, tool_calls: toolUses.length > 0 ? toolUses.map(tu => ({ id: tu.id, type: 'function', function: { name: tu.name, arguments: JSON.stringify(tu.input) } })) : undefined }; return { choices: [{ message, finish_reason: data.stop_reason === 'end_turn' ? 'stop' : 'tool_calls' }] }; } // ============================================================ // Tools schema for LLM // ============================================================ async function getToolsForLLM() { const resp = await chrome.runtime.sendMessage({ type: 'GET_TOOLS' }); return resp.tools.map(t => ({ type: 'function', function: { name: t.name, description: t.description, parameters: buildJsonSchema(t.params) } })); } function buildJsonSchema(params) { const properties = {}; const required = []; for (const [key, type] of Object.entries(params)) { const isOptional = type.endsWith('?'); const baseType = type.replace('?', ''); properties[key] = { type: baseType === 'array' ? 'array' : baseType }; if (!isOptional) required.push(key); } return { type: 'object', properties, required }; } // ============================================================ // System Prompt // ============================================================ function getDefaultSystemPrompt() { return `Du bist Browser Pilot, ein KI-Agent der einen Webbrowser steuert. Du hast Zugriff auf Browser-Automatisierungs-Tools. Nutze sie Schritt für Schritt um die Aufgabe des Nutzers zu erledigen. Regeln: - Starte mit browser_get_page_info oder browser_get_interactive_elements um den aktuellen Zustand zu verstehen - Nutze browser_screenshot wenn du die Seite visuell sehen musst - Nutze browser_execute_js für komplexe DOM-Operationen - Sei methodisch: navigieren → beobachten → handeln → verifizieren - Wenn du fertig bist, fasse zusammen was du getan hast - Antworte auf Deutsch wenn der Nutzer Deutsch spricht - Bei Fragen über den Seiteninhalt: lies zuerst die Seite mit browser_get_text Verfügbare Tools: - browser_navigate: Zu URL navigieren - browser_get_page_info: Seiten-URL, Titel, Viewport - browser_get_text: Text der Seite oder eines Elements - browser_get_html: HTML der Seite oder eines Elements - browser_query_selector: Elemente abfragen mit CSS-Selektor - browser_get_interactive_elements: Alle klickbaren Elemente finden - browser_click: Element anklicken - browser_type: Text in Input eingeben - browser_select: Option in Select wählen - browser_scroll: Seite scrollen - browser_execute_js: JavaScript ausführen - browser_screenshot: Screenshot machen - browser_press_key: Taste drücken - browser_wait: Auf Element oder Timeout warten - browser_highlight: Element visuell hervorheben - browser_get_console_logs: Console-Logs lesen`; } // ============================================================ // UI Helpers // ============================================================ function addMessage(type, content) { const div = document.createElement('div'); div.className = `message ${type}`; if (type === 'tool') { div.innerHTML = `
Tool
${escapeHtml(content)}
`; } else { div.innerHTML = `
${formatContent(content)}
`; } chatMessages.appendChild(div); chatMessages.scrollTop = chatMessages.scrollHeight; return div; } function addScreenshot(dataUrl) { const div = document.createElement('div'); div.className = 'message assistant'; div.innerHTML = `
Screenshot
Screenshot`; chatMessages.appendChild(div); chatMessages.scrollTop = chatMessages.scrollHeight; } function addTypingIndicator() { const div = document.createElement('div'); div.className = 'message assistant'; div.innerHTML = '
'; chatMessages.appendChild(div); chatMessages.scrollTop = chatMessages.scrollHeight; return div; } function removeElement(el) { if (el && el.parentNode) el.parentNode.removeChild(el); } function escapeHtml(text) { return text.replace(/&/g, '&').replace(//g, '>'); } function formatContent(text) { // Basic markdown-like formatting let html = escapeHtml(text); html = html.replace(/\*\*(.*?)\*\*/g, '$1'); html = html.replace(/`(.*?)`/g, '$1'); return html; } // ============================================================ // Quick Actions (Act Tab) // ============================================================ function setupActions() { document.querySelectorAll('.action-btn').forEach(btn => { btn.addEventListener('click', () => executeAction(btn.dataset.action)); }); } async function executeAction(action) { actionResult.textContent = 'Executing...'; try { let result; switch (action) { case 'screenshot': result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_screenshot', params: {} }); if (result.success && result.result.image) { actionResult.innerHTML = ``; return; } break; case 'get-text': result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_text', params: {} }); break; case 'get-links': result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector: 'a[href]', limit: 50 } }); break; case 'get-forms': result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector: 'form, input, select, textarea', limit: 30 } }); break; case 'get-interactive': result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_interactive_elements', params: {} }); break; case 'console-logs': result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_console_logs', params: {} }); break; case 'page-info': result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_page_info', params: {} }); break; case 'summarize': // Get text and send to LLM for summary const textResult = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_text', params: {} }); if (textResult.success) { // Switch to chat tab and ask for summary document.querySelector('[data-tab="chat"]').click(); chatInput.value = 'Fasse den Inhalt dieser Seite zusammen.'; sendMessage(); return; } break; } if (result?.success) { actionResult.textContent = JSON.stringify(result.result, null, 2); } else { actionResult.textContent = 'Error: ' + (result?.error || 'Unknown'); } } catch (err) { actionResult.textContent = 'Error: ' + err.message; } } // ============================================================ // Dev Tab // ============================================================ function setupDev() { $('#execJsBtn').addEventListener('click', async () => { const code = $('#jsInput').value; try { const result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_execute_js', params: { code } }); $('#jsOutput').textContent = result.success ? JSON.stringify(result.result, null, 2) : 'Error: ' + result.error; $('#jsOutput').style.color = result.success ? 'var(--green)' : 'var(--red)'; } catch (err) { $('#jsOutput').textContent = 'Error: ' + err.message; $('#jsOutput').style.color = 'var(--red)'; } }); $('#testSelectorBtn').addEventListener('click', async () => { const selector = $('#selectorInput').value; try { const result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector, limit: 10 } }); if (result.success) { $('#selectorOutput').textContent = JSON.stringify(result.result, null, 2); $('#selectorOutput').style.color = 'var(--green)'; // Highlight first match if (result.result?.length > 0) { await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_highlight', params: { selector } }); } } else { $('#selectorOutput').textContent = 'Error: ' + result.error; $('#selectorOutput').style.color = 'var(--red)'; } } catch (err) { $('#selectorOutput').textContent = 'Error: ' + err.message; } }); // JS input: Ctrl+Enter to execute $('#jsInput').addEventListener('keydown', (e) => { if (e.key === 'Enter' && e.ctrlKey) { e.preventDefault(); $('#execJsBtn').click(); } }); } // ============================================================ // Settings // ============================================================ function setupSettings() { settingsBtn.addEventListener('click', () => { $('#cfgBaseUrl').value = config.baseUrl; $('#cfgModel').value = config.model; $('#cfgApiKey').value = config.apiKey; $('#cfgSystemPrompt').value = config.systemPrompt; $('#cfgMaxIter').value = config.maxIterations; settingsModal.classList.remove('hidden'); }); closeSettings.addEventListener('click', () => settingsModal.classList.add('hidden')); saveSettingsBtn.addEventListener('click', async () => { config.baseUrl = $('#cfgBaseUrl').value.trim(); config.model = $('#cfgModel').value.trim(); config.apiKey = $('#cfgApiKey').value.trim(); config.systemPrompt = $('#cfgSystemPrompt').value.trim(); config.maxIterations = parseInt($('#cfgMaxIter').value) || 15; config.provider = providerSelect.value; await saveConfig(); settingsModal.classList.add('hidden'); checkConnection(); }); providerSelect.addEventListener('change', (e) => { config.provider = e.target.value; applyProviderDefaults(e.target.value); }); } function applyProviderDefaults(provider) { switch (provider) { case 'openrouter': config.baseUrl = 'https://openrouter.ai/api/v1'; config.model = 'anthropic/claude-sonnet-4-20250514'; break; case 'ollama': config.baseUrl = 'http://localhost:11434/v1'; config.model = 'qwen2.5:14b'; break; case 'anthropic': config.baseUrl = 'https://api.anthropic.com/v1'; config.model = 'claude-sonnet-4-20250514'; break; case 'openai': config.baseUrl = 'https://api.openai.com/v1'; config.model = 'gpt-4o'; break; } } // ============================================================ // Connection check // ============================================================ async function checkConnection() { try { const resp = await chrome.runtime.sendMessage({ type: 'PING' }); if (resp?.pong) { statusDot.classList.add('connected'); statusDot.title = `Connected – ${resp.tools} tools`; } } catch { statusDot.classList.remove('connected'); statusDot.title = 'Disconnected'; } }