Files
core d708c8b58c Initial commit: Browser Pilot - Agentic Browser Control Extension
- Side Panel with Chat UI, Quick Actions, Dev Tools
- 22 browser automation tools (DOM, JS, screenshots, navigation)
- MCP server (zero-dep, Node 18+) for external AI clients
- Supports OpenRouter, Ollama, Anthropic, OpenAI, custom endpoints
- Chrome/Brave MV3 extension with content script console capture
2026-07-28 11:04:37 +02:00

633 lines
21 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Browser Pilot Side Panel Logic
*
* This is the in-browser agent. It:
* 1. Provides a chat UI for the user
* 2. Connects to a configurable LLM endpoint (OpenRouter, Anthropic, Ollama, custom)
* 3. Runs an agentic tool-use loop: LLM decides which browser tools to call
* 4. Shows results in real-time
*/
// ============================================================
// State
// ============================================================
let config = {
baseUrl: 'https://openrouter.ai/api/v1',
model: 'anthropic/claude-sonnet-4-20250514',
apiKey: '',
systemPrompt: '',
maxIterations: 15,
provider: 'openrouter'
};
let isRunning = false;
let abortController = null;
let conversationHistory = [];
// ============================================================
// DOM refs
// ============================================================
const $ = (sel) => document.querySelector(sel);
const chatMessages = $('#chatMessages');
const chatInput = $('#chatInput');
const sendBtn = $('#sendBtn');
const stopBtn = $('#stopBtn');
const settingsBtn = $('#settingsBtn');
const settingsModal = $('#settingsModal');
const closeSettings = $('#closeSettings');
const saveSettingsBtn = $('#saveSettings');
const statusDot = $('#statusDot');
const providerSelect = $('#providerSelect');
const actionResult = $('#actionResult');
// ============================================================
// Init
// ============================================================
document.addEventListener('DOMContentLoaded', async () => {
await loadConfig();
setupTabs();
setupChat();
setupActions();
setupDev();
setupSettings();
checkConnection();
});
async function loadConfig() {
const stored = await chrome.storage.local.get(['bp_config']);
if (stored.bp_config) {
config = { ...config, ...stored.bp_config };
}
providerSelect.value = config.provider || 'custom';
applyProviderDefaults(config.provider);
}
async function saveConfig() {
await chrome.storage.local.set({ bp_config: config });
}
// ============================================================
// Tabs
// ============================================================
function setupTabs() {
document.querySelectorAll('.tab').forEach(tab => {
tab.addEventListener('click', () => {
document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active'));
tab.classList.add('active');
$(`#tab-${tab.dataset.tab}`).classList.add('active');
});
});
}
// ============================================================
// Chat
// ============================================================
function setupChat() {
sendBtn.addEventListener('click', sendMessage);
chatInput.addEventListener('keydown', (e) => {
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault();
sendMessage();
}
});
// Auto-resize textarea
chatInput.addEventListener('input', () => {
chatInput.style.height = 'auto';
chatInput.style.height = Math.min(chatInput.scrollHeight, 120) + 'px';
});
stopBtn.addEventListener('click', stopAgent);
}
async function sendMessage() {
const text = chatInput.value.trim();
if (!text || isRunning) return;
chatInput.value = '';
chatInput.style.height = 'auto';
addMessage('user', text);
conversationHistory.push({ role: 'user', content: text });
await runAgentLoop();
}
function stopAgent() {
if (abortController) abortController.abort();
isRunning = false;
stopBtn.classList.add('hidden');
sendBtn.disabled = false;
}
// ============================================================
// Agent Loop the core
// ============================================================
async function runAgentLoop() {
isRunning = true;
sendBtn.disabled = true;
stopBtn.classList.remove('hidden');
abortController = new AbortController();
const tools = await getToolsForLLM();
const systemPrompt = config.systemPrompt || getDefaultSystemPrompt();
let iterations = 0;
try {
while (iterations < config.maxIterations && isRunning) {
iterations++;
// Show thinking indicator
const thinkingEl = addTypingIndicator();
// Call LLM
const messages = [
{ role: 'system', content: systemPrompt },
...conversationHistory
];
let response;
try {
response = await callLLM(messages, tools);
} catch (err) {
removeElement(thinkingEl);
if (err.name === 'AbortError') break;
addMessage('error', `LLM Error: ${err.message}`);
break;
}
removeElement(thinkingEl);
const assistantMsg = response.choices?.[0]?.message;
if (!assistantMsg) {
addMessage('error', 'No response from LLM');
break;
}
// Add to history
conversationHistory.push(assistantMsg);
// If there's text content, show it
if (assistantMsg.content) {
addMessage('assistant', assistantMsg.content);
}
// If no tool calls, we're done
if (!assistantMsg.tool_calls || assistantMsg.tool_calls.length === 0) {
break;
}
// Execute tool calls
for (const toolCall of assistantMsg.tool_calls) {
if (!isRunning) break;
const funcName = toolCall.function.name;
let args = {};
try { args = JSON.parse(toolCall.function.arguments || '{}'); } catch {}
// Show tool call in chat
addMessage('tool', `🔧 ${funcName}\n${JSON.stringify(args, null, 2)}`);
// Execute via background
let result;
try {
const resp = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: funcName, params: args });
if (resp.success) {
result = JSON.stringify(resp.result, null, 2);
// Show screenshot inline
if (resp.result?.image) {
addScreenshot(resp.result.image);
result = '(screenshot captured)';
} else {
// Truncate long results in display
const display = result.length > 500 ? result.slice(0, 500) + '\n...' : result;
addMessage('tool', `✓ Result:\n${display}`);
}
} else {
result = `Error: ${resp.error}`;
addMessage('error', result);
}
} catch (err) {
result = `Error: ${err.message}`;
addMessage('error', result);
}
// Add tool result to conversation
conversationHistory.push({
role: 'tool',
tool_call_id: toolCall.id,
content: result.slice(0, 8000) // Limit context size
});
}
}
if (iterations >= config.maxIterations) {
addMessage('assistant', '⚠️ Maximale Iterationen erreicht. Stoppe.');
}
} finally {
isRunning = false;
sendBtn.disabled = false;
stopBtn.classList.add('hidden');
}
}
// ============================================================
// LLM API Call
// ============================================================
async function callLLM(messages, tools) {
const payload = {
model: config.model,
messages,
tools,
tool_choice: 'auto',
temperature: 0.1,
max_tokens: 4096
};
// Provider-specific headers
const headers = {
'Content-Type': 'application/json'
};
if (config.apiKey) {
if (config.provider === 'anthropic') {
headers['x-api-key'] = config.apiKey;
headers['anthropic-version'] = '2023-06-01';
} else {
headers['Authorization'] = `Bearer ${config.apiKey}`;
}
}
// Anthropic uses a different format
let url = `${config.baseUrl}/chat/completions`;
if (config.provider === 'anthropic') {
url = `${config.baseUrl}/messages`;
// Convert to Anthropic format
const anthropicPayload = convertToAnthropicFormat(messages, tools, config.model);
const resp = await fetch(url, {
method: 'POST',
headers,
body: JSON.stringify(anthropicPayload),
signal: abortController.signal
});
if (!resp.ok) {
const err = await resp.text();
throw new Error(`${resp.status}: ${err}`);
}
const data = await resp.json();
return convertFromAnthropicFormat(data);
}
const resp = await fetch(url, {
method: 'POST',
headers,
body: JSON.stringify(payload),
signal: abortController.signal
});
if (!resp.ok) {
const err = await resp.text();
throw new Error(`${resp.status}: ${err}`);
}
return await resp.json();
}
function convertToAnthropicFormat(messages, tools, model) {
const system = messages.find(m => m.role === 'system')?.content || '';
const msgs = messages.filter(m => m.role !== 'system').map(m => {
if (m.role === 'tool') {
return { role: 'user', content: [{ type: 'tool_result', tool_use_id: m.tool_call_id, content: m.content }] };
}
if (m.tool_calls) {
return {
role: 'assistant',
content: [
...(m.content ? [{ type: 'text', text: m.content }] : []),
...m.tool_calls.map(tc => ({ type: 'tool_use', id: tc.id, name: tc.function.name, input: JSON.parse(tc.function.arguments || '{}') }))
]
};
}
return m;
});
const anthropicTools = tools.map(t => ({
name: t.function.name,
description: t.function.description,
input_schema: t.function.parameters
}));
return { model, system, messages: msgs, tools: anthropicTools, max_tokens: 4096 };
}
function convertFromAnthropicFormat(data) {
const content = data.content || [];
const textParts = content.filter(c => c.type === 'text').map(c => c.text).join('');
const toolUses = content.filter(c => c.type === 'tool_use');
const message = {
role: 'assistant',
content: textParts || null,
tool_calls: toolUses.length > 0 ? toolUses.map(tu => ({
id: tu.id,
type: 'function',
function: { name: tu.name, arguments: JSON.stringify(tu.input) }
})) : undefined
};
return { choices: [{ message, finish_reason: data.stop_reason === 'end_turn' ? 'stop' : 'tool_calls' }] };
}
// ============================================================
// Tools schema for LLM
// ============================================================
async function getToolsForLLM() {
const resp = await chrome.runtime.sendMessage({ type: 'GET_TOOLS' });
return resp.tools.map(t => ({
type: 'function',
function: {
name: t.name,
description: t.description,
parameters: buildJsonSchema(t.params)
}
}));
}
function buildJsonSchema(params) {
const properties = {};
const required = [];
for (const [key, type] of Object.entries(params)) {
const isOptional = type.endsWith('?');
const baseType = type.replace('?', '');
properties[key] = { type: baseType === 'array' ? 'array' : baseType };
if (!isOptional) required.push(key);
}
return { type: 'object', properties, required };
}
// ============================================================
// System Prompt
// ============================================================
function getDefaultSystemPrompt() {
return `Du bist Browser Pilot, ein KI-Agent der einen Webbrowser steuert.
Du hast Zugriff auf Browser-Automatisierungs-Tools. Nutze sie Schritt für Schritt um die Aufgabe des Nutzers zu erledigen.
Regeln:
- Starte mit browser_get_page_info oder browser_get_interactive_elements um den aktuellen Zustand zu verstehen
- Nutze browser_screenshot wenn du die Seite visuell sehen musst
- Nutze browser_execute_js für komplexe DOM-Operationen
- Sei methodisch: navigieren → beobachten → handeln → verifizieren
- Wenn du fertig bist, fasse zusammen was du getan hast
- Antworte auf Deutsch wenn der Nutzer Deutsch spricht
- Bei Fragen über den Seiteninhalt: lies zuerst die Seite mit browser_get_text
Verfügbare Tools:
- browser_navigate: Zu URL navigieren
- browser_get_page_info: Seiten-URL, Titel, Viewport
- browser_get_text: Text der Seite oder eines Elements
- browser_get_html: HTML der Seite oder eines Elements
- browser_query_selector: Elemente abfragen mit CSS-Selektor
- browser_get_interactive_elements: Alle klickbaren Elemente finden
- browser_click: Element anklicken
- browser_type: Text in Input eingeben
- browser_select: Option in Select wählen
- browser_scroll: Seite scrollen
- browser_execute_js: JavaScript ausführen
- browser_screenshot: Screenshot machen
- browser_press_key: Taste drücken
- browser_wait: Auf Element oder Timeout warten
- browser_highlight: Element visuell hervorheben
- browser_get_console_logs: Console-Logs lesen`;
}
// ============================================================
// UI Helpers
// ============================================================
function addMessage(type, content) {
const div = document.createElement('div');
div.className = `message ${type}`;
if (type === 'tool') {
div.innerHTML = `<div class="tool-label">Tool</div><div class="message-content">${escapeHtml(content)}</div>`;
} else {
div.innerHTML = `<div class="message-content">${formatContent(content)}</div>`;
}
chatMessages.appendChild(div);
chatMessages.scrollTop = chatMessages.scrollHeight;
return div;
}
function addScreenshot(dataUrl) {
const div = document.createElement('div');
div.className = 'message assistant';
div.innerHTML = `<div class="tool-label">Screenshot</div><img src="${dataUrl}" alt="Screenshot">`;
chatMessages.appendChild(div);
chatMessages.scrollTop = chatMessages.scrollHeight;
}
function addTypingIndicator() {
const div = document.createElement('div');
div.className = 'message assistant';
div.innerHTML = '<div class="typing-indicator"><span></span><span></span><span></span></div>';
chatMessages.appendChild(div);
chatMessages.scrollTop = chatMessages.scrollHeight;
return div;
}
function removeElement(el) {
if (el && el.parentNode) el.parentNode.removeChild(el);
}
function escapeHtml(text) {
return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
}
function formatContent(text) {
// Basic markdown-like formatting
let html = escapeHtml(text);
html = html.replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>');
html = html.replace(/`(.*?)`/g, '<code>$1</code>');
return html;
}
// ============================================================
// Quick Actions (Act Tab)
// ============================================================
function setupActions() {
document.querySelectorAll('.action-btn').forEach(btn => {
btn.addEventListener('click', () => executeAction(btn.dataset.action));
});
}
async function executeAction(action) {
actionResult.textContent = 'Executing...';
try {
let result;
switch (action) {
case 'screenshot':
result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_screenshot', params: {} });
if (result.success && result.result.image) {
actionResult.innerHTML = `<img src="${result.result.image}" style="max-width:100%;border-radius:4px;">`;
return;
}
break;
case 'get-text':
result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_text', params: {} });
break;
case 'get-links':
result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector: 'a[href]', limit: 50 } });
break;
case 'get-forms':
result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector: 'form, input, select, textarea', limit: 30 } });
break;
case 'get-interactive':
result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_interactive_elements', params: {} });
break;
case 'console-logs':
result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_console_logs', params: {} });
break;
case 'page-info':
result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_page_info', params: {} });
break;
case 'summarize':
// Get text and send to LLM for summary
const textResult = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_get_text', params: {} });
if (textResult.success) {
// Switch to chat tab and ask for summary
document.querySelector('[data-tab="chat"]').click();
chatInput.value = 'Fasse den Inhalt dieser Seite zusammen.';
sendMessage();
return;
}
break;
}
if (result?.success) {
actionResult.textContent = JSON.stringify(result.result, null, 2);
} else {
actionResult.textContent = 'Error: ' + (result?.error || 'Unknown');
}
} catch (err) {
actionResult.textContent = 'Error: ' + err.message;
}
}
// ============================================================
// Dev Tab
// ============================================================
function setupDev() {
$('#execJsBtn').addEventListener('click', async () => {
const code = $('#jsInput').value;
try {
const result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_execute_js', params: { code } });
$('#jsOutput').textContent = result.success
? JSON.stringify(result.result, null, 2)
: 'Error: ' + result.error;
$('#jsOutput').style.color = result.success ? 'var(--green)' : 'var(--red)';
} catch (err) {
$('#jsOutput').textContent = 'Error: ' + err.message;
$('#jsOutput').style.color = 'var(--red)';
}
});
$('#testSelectorBtn').addEventListener('click', async () => {
const selector = $('#selectorInput').value;
try {
const result = await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_query_selector', params: { selector, limit: 10 } });
if (result.success) {
$('#selectorOutput').textContent = JSON.stringify(result.result, null, 2);
$('#selectorOutput').style.color = 'var(--green)';
// Highlight first match
if (result.result?.length > 0) {
await chrome.runtime.sendMessage({ type: 'TOOL_CALL', tool: 'browser_highlight', params: { selector } });
}
} else {
$('#selectorOutput').textContent = 'Error: ' + result.error;
$('#selectorOutput').style.color = 'var(--red)';
}
} catch (err) {
$('#selectorOutput').textContent = 'Error: ' + err.message;
}
});
// JS input: Ctrl+Enter to execute
$('#jsInput').addEventListener('keydown', (e) => {
if (e.key === 'Enter' && e.ctrlKey) {
e.preventDefault();
$('#execJsBtn').click();
}
});
}
// ============================================================
// Settings
// ============================================================
function setupSettings() {
settingsBtn.addEventListener('click', () => {
$('#cfgBaseUrl').value = config.baseUrl;
$('#cfgModel').value = config.model;
$('#cfgApiKey').value = config.apiKey;
$('#cfgSystemPrompt').value = config.systemPrompt;
$('#cfgMaxIter').value = config.maxIterations;
settingsModal.classList.remove('hidden');
});
closeSettings.addEventListener('click', () => settingsModal.classList.add('hidden'));
saveSettingsBtn.addEventListener('click', async () => {
config.baseUrl = $('#cfgBaseUrl').value.trim();
config.model = $('#cfgModel').value.trim();
config.apiKey = $('#cfgApiKey').value.trim();
config.systemPrompt = $('#cfgSystemPrompt').value.trim();
config.maxIterations = parseInt($('#cfgMaxIter').value) || 15;
config.provider = providerSelect.value;
await saveConfig();
settingsModal.classList.add('hidden');
checkConnection();
});
providerSelect.addEventListener('change', (e) => {
config.provider = e.target.value;
applyProviderDefaults(e.target.value);
});
}
function applyProviderDefaults(provider) {
switch (provider) {
case 'openrouter':
config.baseUrl = 'https://openrouter.ai/api/v1';
config.model = 'anthropic/claude-sonnet-4-20250514';
break;
case 'ollama':
config.baseUrl = 'http://localhost:11434/v1';
config.model = 'qwen2.5:14b';
break;
case 'anthropic':
config.baseUrl = 'https://api.anthropic.com/v1';
config.model = 'claude-sonnet-4-20250514';
break;
case 'openai':
config.baseUrl = 'https://api.openai.com/v1';
config.model = 'gpt-4o';
break;
}
}
// ============================================================
// Connection check
// ============================================================
async function checkConnection() {
try {
const resp = await chrome.runtime.sendMessage({ type: 'PING' });
if (resp?.pong) {
statusDot.classList.add('connected');
statusDot.title = `Connected ${resp.tools} tools`;
}
} catch {
statusDot.classList.remove('connected');
statusDot.title = 'Disconnected';
}
}