Initial commit: Browser Pilot - Agentic Browser Control Extension

- Side Panel with Chat UI, Quick Actions, Dev Tools - 22 browser automation tools (DOM, JS, screenshots, navigation) - MCP server (zero-dep, Node 18+) for external AI clients - Supports OpenRouter, Ollama, Anthropic, OpenAI, custom endpoints - Chrome/Brave MV3 extension with content script console capture
2026-07-28 11:04:37 +02:00
commit d708c8b58c
13 changed files with 2258 additions and 0 deletions
@@ -0,0 +1,632 @@
+/**
+ * Browser Pilot – Side Panel Logic
+ * 
+ * This is the in-browser agent. It:
+ * 1. Provides a chat UI for the user
+ * 2. Connects to a configurable LLM endpoint (OpenRouter, Anthropic, Ollama, custom)
+ * 3. Runs an agentic tool-use loop: LLM decides which browser tools to call
+ * 4. Shows results in real-time
+ */
+
+// ============================================================
+// State
+// ============================================================
+let config = {
+  baseUrl: 'https://openrouter.ai/api/v1',
+  model: 'anthropic/claude-sonnet-4-20250514',
+  apiKey: '',
+  systemPrompt: '',
+  maxIterations: 15,
+  provider: 'openrouter'
+};
+
+let isRunning = false;
+let abortController = null;
+let conversationHistory = [];
+
+// ============================================================
+// DOM refs
+// ============================================================
+const $ = (sel) => document.querySelector(sel);
+const chatMessages = $('#chatMessages');
+const chatInput = $('#chatInput');
+const sendBtn = $('#sendBtn');
+const stopBtn = $('#stopBtn');
+const settingsBtn = $('#settingsBtn');
+const settingsModal = $('#settingsModal');
+const closeSettings = $('#closeSettings');
+const saveSettingsBtn = $('#saveSettings');
+const statusDot = $('#statusDot');
+const providerSelect = $('#providerSelect');
+const actionResult = $('#actionResult');
+
+// ============================================================
+// Init
+// ============================================================
+document.addEventListener('DOMContentLoaded', async () => {
+  await loadConfig();
+  setupTabs();
+  setupChat();
+  setupActions();
+  setupDev();
+  setupSettings();
+  checkConnection();
+});
+
+async function loadConfig() {
+  const stored = await chrome.storage.local.get(['bp_config']);
+  if (stored.bp_config) {
+    config = { ...config, ...stored.bp_config };
+  }
+  providerSelect.value = config.provider || 'custom';
+  applyProviderDefaults(config.provider);
+}
+
+async function saveConfig() {
+  await chrome.storage.local.set({ bp_config: config });
+}
+
+// ============================================================
+// Tabs
+// ============================================================
+function setupTabs() {
+  document.querySelectorAll('.tab').forEach(tab => {
+    tab.addEventListener('click', () => {
+      document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
+      document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active'));
+      tab.classList.add('active');
+      $(`#tab-${tab.dataset.tab}`).classList.add('active');
+    });
+  });
+}
+
+// ============================================================
+// Chat
+// ============================================================
+function setupChat() {
+  sendBtn.addEventListener('click', sendMessage);
+  chatInput.addEventListener('keydown', (e) => {
+    if (e.key === 'Enter' && !e.shiftKey) {
+      e.preventDefault();
+      sendMessage();
+    }
+  });
+  // Auto-resize textarea
+  chatInput.addEventListener('input', () => {
+    chatInput.style.height = 'auto';
+    chatInput.style.height = Math.min(chatInput.scrollHeight, 120) + 'px';
+  });
+  stopBtn.addEventListener('click', stopAgent);
+}
+
+async function sendMessage() {
+  const text = chatInput.value.trim();
+  if (!text || isRunning) return;
+
+  chatInput.value = '';
+  chatInput.style.height = 'auto';
+  addMessage('user', text);
+
+  conversationHistory.push({ role: 'user', content: text });
+  await runAgentLoop();
+}
+
+function stopAgent() {
+  if (abortController) abortController.abort();
+  isRunning = false;
+  stopBtn.classList.add('hidden');
+  sendBtn.disabled = false;
+}
+
+// ============================================================
+// Agent Loop – the core
+// ============================================================
+async function runAgentLoop() {
+  isRunning = true;
+  sendBtn.disabled = true;
+  stopBtn.classList.remove('hidden');
+  abortController = new AbortController();
+
+  const tools = await getToolsForLLM();
+  const systemPrompt = config.systemPrompt || getDefaultSystemPrompt();
+
+  let iterations = 0;
+
+  try {
+    while (iterations < config.maxIterations && isRunning) {
+      iterations++;
+
+      // Show thinking indicator
+      const thinkingEl = addTypingIndicator();
+
+      // Call LLM
+      const messages = [
+        { role: 'system', content: systemPrompt },
+        ...conversationHistory
+      ];
+
+      let response;
+      try {
+        response = await callLLM(messages, tools);
+      } catch (err) {
+        removeElement(thinkingEl);
+        if (err.name === 'AbortError') break;
+        addMessage('error', `LLM Error: ${err.message}`);
+        break;
+      }
+
+      removeElement(thinkingEl);
+
+      const assistantMsg = response.choices?.[0]?.message;
+      if (!assistantMsg) {
+        addMessage('error', 'No response from LLM');
+        break;
+      }
+
+      // Add to history
+      conversationHistory.push(assistantMsg);
+
+      // If there's text content, show it
+      if (assistantMsg.content) {
+        addMessage('assistant', assistantMsg.content);
+      }
+
+      // If no tool calls, we're done
+      if (!assistantMsg.tool_calls || assistantMsg.tool_calls.length === 0) {
+        break;
+      }
+
+      // Execute tool calls
+      for (const toolCall of assistantMsg.tool_calls) {
+        if (!isRunning) break;
+
+        const funcName = toolCall.function.name;
+        let args = {};
+        try { args = JSON.parse(toolCall.function.arguments || '{}'); } catch {}
+
+        // Show tool call in chat
+        addMessage('tool', `🔧 ${funcName}\n${JSON.stringify(args, null, 2)}`);
+
+        // Execute via background
+        let result;
+        try {
+          const resp = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: funcName, params: args });
+          if (resp.success) {
+            result = JSON.stringify(resp.result, null, 2);
+            // Show screenshot inline
+            if (resp.result?.image) {
+              addScreenshot(resp.result.image);
+              result = '(screenshot captured)';
+            } else {
+              // Truncate long results in display
+              const display = result.length > 500 ? result.slice(0, 500) + '\n...' : result;
+              addMessage('tool', `✓ Result:\n${display}`);
+            }
+          } else {
+            result = `Error: ${resp.error}`;
+            addMessage('error', result);
+          }
+        } catch (err) {
+          result = `Error: ${err.message}`;
+          addMessage('error', result);
+        }
+
+        // Add tool result to conversation
+        conversationHistory.push({
+          role: 'tool',
+          tool_call_id: toolCall.id,
+          content: result.slice(0, 8000) // Limit context size
+        });
+      }
+    }
+
+    if (iterations >= config.maxIterations) {
+      addMessage('assistant', '⚠️ Maximale Iterationen erreicht. Stoppe.');
+    }
+  } finally {
+    isRunning = false;
+    sendBtn.disabled = false;
+    stopBtn.classList.add('hidden');
+  }
+}
+
+// ============================================================
+// LLM API Call
+// ============================================================
+async function callLLM(messages, tools) {
+  const payload = {
+    model: config.model,
+    messages,
+    tools,
+    tool_choice: 'auto',
+    temperature: 0.1,
+    max_tokens: 4096
+  };
+
+  // Provider-specific headers
+  const headers = {
+    'Content-Type': 'application/json'
+  };
+
+  if (config.apiKey) {
+    if (config.provider === 'anthropic') {
+      headers['x-api-key'] = config.apiKey;
+      headers['anthropic-version'] = '2023-06-01';
+    } else {
+      headers['Authorization'] = `Bearer ${config.apiKey}`;
+    }
+  }
+
+  // Anthropic uses a different format
+  let url = `${config.baseUrl}/chat/completions`;
+  if (config.provider === 'anthropic') {
+    url = `${config.baseUrl}/messages`;
+    // Convert to Anthropic format
+    const anthropicPayload = convertToAnthropicFormat(messages, tools, config.model);
+    const resp = await fetch(url, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(anthropicPayload),
+      signal: abortController.signal
+    });
+    if (!resp.ok) {
+      const err = await resp.text();
+      throw new Error(`${resp.status}: ${err}`);
+    }
+    const data = await resp.json();
+    return convertFromAnthropicFormat(data);
+  }
+
+  const resp = await fetch(url, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(payload),
+    signal: abortController.signal
+  });
+
+  if (!resp.ok) {
+    const err = await resp.text();
+    throw new Error(`${resp.status}: ${err}`);
+  }
+
+  return await resp.json();
+}
+
+function convertToAnthropicFormat(messages, tools, model) {
+  const system = messages.find(m => m.role === 'system')?.content || '';
+  const msgs = messages.filter(m => m.role !== 'system').map(m => {
+    if (m.role === 'tool') {
+      return { role: 'user', content: [{ type: 'tool_result', tool_use_id: m.tool_call_id, content: m.content }] };
+    }
+    if (m.tool_calls) {
+      return {
+        role: 'assistant',
+        content: [
+          ...(m.content ? [{ type: 'text', text: m.content }] : []),
+          ...m.tool_calls.map(tc => ({ type: 'tool_use', id: tc.id, name: tc.function.name, input: JSON.parse(tc.function.arguments || '{}') }))
+        ]
+      };
+    }
+    return m;
+  });
+
+  const anthropicTools = tools.map(t => ({
+    name: t.function.name,
+    description: t.function.description,
+    input_schema: t.function.parameters
+  }));
+
+  return { model, system, messages: msgs, tools: anthropicTools, max_tokens: 4096 };
+}
+
+function convertFromAnthropicFormat(data) {
+  const content = data.content || [];
+  const textParts = content.filter(c => c.type === 'text').map(c => c.text).join('');
+  const toolUses = content.filter(c => c.type === 'tool_use');
+
+  const message = {
+    role: 'assistant',
+    content: textParts || null,
+    tool_calls: toolUses.length > 0 ? toolUses.map(tu => ({
+      id: tu.id,
+      type: 'function',
+      function: { name: tu.name, arguments: JSON.stringify(tu.input) }
+    })) : undefined
+  };
+
+  return { choices: [{ message, finish_reason: data.stop_reason === 'end_turn' ? 'stop' : 'tool_calls' }] };
+}
+
+// ============================================================
+// Tools schema for LLM
+// ============================================================
+async function getToolsForLLM() {
+  const resp = await chrome.runtime.sendMessage({ type: 'GET_TOOLS' });
+  return resp.tools.map(t => ({
+    type: 'function',
+    function: {
+      name: t.name,
+      description: t.description,
+      parameters: buildJsonSchema(t.params)
+    }
+  }));
+}
+
+function buildJsonSchema(params) {
+  const properties = {};
+  const required = [];
+  for (const [key, type] of Object.entries(params)) {
+    const isOptional = type.endsWith('?');
+    const baseType = type.replace('?', '');
+    properties[key] = { type: baseType === 'array' ? 'array' : baseType };
+    if (!isOptional) required.push(key);
+  }
+  return { type: 'object', properties, required };
+}
+
+// ============================================================
+// System Prompt
+// ============================================================
+function getDefaultSystemPrompt() {
+  return `Du bist Browser Pilot, ein KI-Agent der einen Webbrowser steuert.
+
+Du hast Zugriff auf Browser-Automatisierungs-Tools. Nutze sie Schritt für Schritt um die Aufgabe des Nutzers zu erledigen.
+
+Regeln:
+- Starte mit browser_get_page_info oder browser_get_interactive_elements um den aktuellen Zustand zu verstehen
+- Nutze browser_screenshot wenn du die Seite visuell sehen musst
+- Nutze browser_execute_js für komplexe DOM-Operationen
+- Sei methodisch: navigieren → beobachten → handeln → verifizieren
+- Wenn du fertig bist, fasse zusammen was du getan hast
+- Antworte auf Deutsch wenn der Nutzer Deutsch spricht
+- Bei Fragen über den Seiteninhalt: lies zuerst die Seite mit browser_get_text
+
+Verfügbare Tools:
+- browser_navigate: Zu URL navigieren
+- browser_get_page_info: Seiten-URL, Titel, Viewport
+- browser_get_text: Text der Seite oder eines Elements
+- browser_get_html: HTML der Seite oder eines Elements
+- browser_query_selector: Elemente abfragen mit CSS-Selektor
+- browser_get_interactive_elements: Alle klickbaren Elemente finden
+- browser_click: Element anklicken
+- browser_type: Text in Input eingeben
+- browser_select: Option in Select wählen
+- browser_scroll: Seite scrollen
+- browser_execute_js: JavaScript ausführen
+- browser_screenshot: Screenshot machen
+- browser_press_key: Taste drücken
+- browser_wait: Auf Element oder Timeout warten
+- browser_highlight: Element visuell hervorheben
+- browser_get_console_logs: Console-Logs lesen`;
+}
+
+// ============================================================
+// UI Helpers
+// ============================================================
+function addMessage(type, content) {
+  const div = document.createElement('div');
+  div.className = `message ${type}`;
+  
+  if (type === 'tool') {
+    div.innerHTML = `<div class="tool-label">Tool</div><div class="message-content">${escapeHtml(content)}</div>`;
+  } else {
+    div.innerHTML = `<div class="message-content">${formatContent(content)}</div>`;
+  }
+  
+  chatMessages.appendChild(div);
+  chatMessages.scrollTop = chatMessages.scrollHeight;
+  return div;
+}
+
+function addScreenshot(dataUrl) {
+  const div = document.createElement('div');
+  div.className = 'message assistant';
+  div.innerHTML = `<div class="tool-label">Screenshot</div><img src="${dataUrl}" alt="Screenshot">`;
+  chatMessages.appendChild(div);
+  chatMessages.scrollTop = chatMessages.scrollHeight;
+}
+
+function addTypingIndicator() {
+  const div = document.createElement('div');
+  div.className = 'message assistant';
+  div.innerHTML = '<div class="typing-indicator"><span></span><span></span><span></span></div>';
+  chatMessages.appendChild(div);
+  chatMessages.scrollTop = chatMessages.scrollHeight;
+  return div;
+}
+
+function removeElement(el) {
+  if (el && el.parentNode) el.parentNode.removeChild(el);
+}
+
+function escapeHtml(text) {
+  return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+function formatContent(text) {
+  // Basic markdown-like formatting
+  let html = escapeHtml(text);
+  html = html.replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>');
+  html = html.replace(/`(.*?)`/g, '<code>$1</code>');
+  return html;
+}
+
+// ============================================================
+// Quick Actions (Act Tab)
+// ============================================================
+function setupActions() {
+  document.querySelectorAll('.action-btn').forEach(btn => {
+    btn.addEventListener('click', () => executeAction(btn.dataset.action));
+  });
+}
+
+async function executeAction(action) {
+  actionResult.textContent = 'Executing...';
+  
+  try {
+    let result;
+    switch (action) {
+      case 'screenshot':
+        result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_screenshot', params: {} });
+        if (result.success && result.result.image) {
+          actionResult.innerHTML = `<img src="${result.result.image}" style="max-width:100%;border-radius:4px;">`;
+          return;
+        }
+        break;
+      case 'get-text':
+        result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_text', params: {} });
+        break;
+      case 'get-links':
+        result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector: 'a[href]', limit: 50 } });
+        break;
+      case 'get-forms':
+        result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector: 'form, input, select, textarea', limit: 30 } });
+        break;
+      case 'get-interactive':
+        result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_interactive_elements', params: {} });
+        break;
+      case 'console-logs':
+        result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_console_logs', params: {} });
+        break;
+      case 'page-info':
+        result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_page_info', params: {} });
+        break;
+      case 'summarize':
+        // Get text and send to LLM for summary
+        const textResult = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_text', params: {} });
+        if (textResult.success) {
+          // Switch to chat tab and ask for summary
+          document.querySelector('[data-tab="chat"]').click();
+          chatInput.value = 'Fasse den Inhalt dieser Seite zusammen.';
+          sendMessage();
+          return;
+        }
+        break;
+    }
+    
+    if (result?.success) {
+      actionResult.textContent = JSON.stringify(result.result, null, 2);
+    } else {
+      actionResult.textContent = 'Error: ' + (result?.error || 'Unknown');
+    }
+  } catch (err) {
+    actionResult.textContent = 'Error: ' + err.message;
+  }
+}
+
+// ============================================================
+// Dev Tab
+// ============================================================
+function setupDev() {
+  $('#execJsBtn').addEventListener('click', async () => {
+    const code = $('#jsInput').value;
+    try {
+      const result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_execute_js', params: { code } });
+      $('#jsOutput').textContent = result.success
+        ? JSON.stringify(result.result, null, 2)
+        : 'Error: ' + result.error;
+      $('#jsOutput').style.color = result.success ? 'var(--green)' : 'var(--red)';
+    } catch (err) {
+      $('#jsOutput').textContent = 'Error: ' + err.message;
+      $('#jsOutput').style.color = 'var(--red)';
+    }
+  });
+
+  $('#testSelectorBtn').addEventListener('click', async () => {
+    const selector = $('#selectorInput').value;
+    try {
+      const result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector, limit: 10 } });
+      if (result.success) {
+        $('#selectorOutput').textContent = JSON.stringify(result.result, null, 2);
+        $('#selectorOutput').style.color = 'var(--green)';
+        // Highlight first match
+        if (result.result?.length > 0) {
+          await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_highlight', params: { selector } });
+        }
+      } else {
+        $('#selectorOutput').textContent = 'Error: ' + result.error;
+        $('#selectorOutput').style.color = 'var(--red)';
+      }
+    } catch (err) {
+      $('#selectorOutput').textContent = 'Error: ' + err.message;
+    }
+  });
+
+  // JS input: Ctrl+Enter to execute
+  $('#jsInput').addEventListener('keydown', (e) => {
+    if (e.key === 'Enter' && e.ctrlKey) {
+      e.preventDefault();
+      $('#execJsBtn').click();
+    }
+  });
+}
+
+// ============================================================
+// Settings
+// ============================================================
+function setupSettings() {
+  settingsBtn.addEventListener('click', () => {
+    $('#cfgBaseUrl').value = config.baseUrl;
+    $('#cfgModel').value = config.model;
+    $('#cfgApiKey').value = config.apiKey;
+    $('#cfgSystemPrompt').value = config.systemPrompt;
+    $('#cfgMaxIter').value = config.maxIterations;
+    settingsModal.classList.remove('hidden');
+  });
+
+  closeSettings.addEventListener('click', () => settingsModal.classList.add('hidden'));
+  
+  saveSettingsBtn.addEventListener('click', async () => {
+    config.baseUrl = $('#cfgBaseUrl').value.trim();
+    config.model = $('#cfgModel').value.trim();
+    config.apiKey = $('#cfgApiKey').value.trim();
+    config.systemPrompt = $('#cfgSystemPrompt').value.trim();
+    config.maxIterations = parseInt($('#cfgMaxIter').value) || 15;
+    config.provider = providerSelect.value;
+    await saveConfig();
+    settingsModal.classList.add('hidden');
+    checkConnection();
+  });
+
+  providerSelect.addEventListener('change', (e) => {
+    config.provider = e.target.value;
+    applyProviderDefaults(e.target.value);
+  });
+}
+
+function applyProviderDefaults(provider) {
+  switch (provider) {
+    case 'openrouter':
+      config.baseUrl = 'https://openrouter.ai/api/v1';
+      config.model = 'anthropic/claude-sonnet-4-20250514';
+      break;
+    case 'ollama':
+      config.baseUrl = 'http://localhost:11434/v1';
+      config.model = 'qwen2.5:14b';
+      break;
+    case 'anthropic':
+      config.baseUrl = 'https://api.anthropic.com/v1';
+      config.model = 'claude-sonnet-4-20250514';
+      break;
+    case 'openai':
+      config.baseUrl = 'https://api.openai.com/v1';
+      config.model = 'gpt-4o';
+      break;
+  }
+}
+
+// ============================================================
+// Connection check
+// ============================================================
+async function checkConnection() {
+  try {
+    const resp = await chrome.runtime.sendMessage({ type: 'PING' });
+    if (resp?.pong) {
+      statusDot.classList.add('connected');
+      statusDot.title = `Connected – ${resp.tools} tools`;
+    }
+  } catch {
+    statusDot.classList.remove('connected');
+    statusDot.title = 'Disconnected';
+  }
+}