core
d708c8b58c
- Side Panel with Chat UI, Quick Actions, Dev Tools - 22 browser automation tools (DOM, JS, screenshots, navigation) - MCP server (zero-dep, Node 18+) for external AI clients - Supports OpenRouter, Ollama, Anthropic, OpenAI, custom endpoints - Chrome/Brave MV3 extension with content script console capture
429 lines
17 KiB
JavaScript
429 lines
17 KiB
JavaScript
/**
|
||
* Browser Pilot – Background Service Worker
|
||
*
|
||
* Three roles:
|
||
* 1. Tool execution engine (DOM, screenshots, JS, navigation)
|
||
* 2. WebSocket bridge for external MCP server
|
||
* 3. Message router between side panel, content scripts, and external clients
|
||
*/
|
||
|
||
// ============================================================
|
||
// Tool Registry
|
||
// ============================================================
|
||
const TOOLS = {
|
||
browser_navigate: {
|
||
description: 'Navigate to a URL',
|
||
params: { url: 'string', newTab: 'boolean?' },
|
||
handler: async ({ url, newTab }) => {
|
||
if (newTab) {
|
||
const tab = await chrome.tabs.create({ url });
|
||
return { tabId: tab.id, url };
|
||
}
|
||
const tab = await getActiveTab();
|
||
await chrome.tabs.update(tab.id, { url });
|
||
return { tabId: tab.id, url };
|
||
}
|
||
},
|
||
browser_go_back: {
|
||
description: 'Go back in history',
|
||
params: {},
|
||
handler: async () => { const t = await getActiveTab(); await chrome.tabs.goBack(t.id); return { ok: true }; }
|
||
},
|
||
browser_go_forward: {
|
||
description: 'Go forward in history',
|
||
params: {},
|
||
handler: async () => { const t = await getActiveTab(); await chrome.tabs.goForward(t.id); return { ok: true }; }
|
||
},
|
||
browser_reload: {
|
||
description: 'Reload current page',
|
||
params: {},
|
||
handler: async () => { const t = await getActiveTab(); await chrome.tabs.reload(t.id); return { ok: true }; }
|
||
},
|
||
browser_get_tabs: {
|
||
description: 'List all open tabs',
|
||
params: {},
|
||
handler: async () => {
|
||
const tabs = await chrome.tabs.query({});
|
||
return tabs.map(t => ({ id: t.id, title: t.title, url: t.url, active: t.active }));
|
||
}
|
||
},
|
||
browser_switch_tab: {
|
||
description: 'Switch to tab by ID',
|
||
params: { tabId: 'number' },
|
||
handler: async ({ tabId }) => {
|
||
await chrome.tabs.update(tabId, { active: true });
|
||
const t = await chrome.tabs.get(tabId);
|
||
await chrome.windows.update(t.windowId, { focused: true });
|
||
return { ok: true, tabId };
|
||
}
|
||
},
|
||
browser_close_tab: {
|
||
description: 'Close a tab',
|
||
params: { tabId: 'number' },
|
||
handler: async ({ tabId }) => { await chrome.tabs.remove(tabId); return { ok: true }; }
|
||
},
|
||
browser_screenshot: {
|
||
description: 'Screenshot visible viewport (base64 PNG)',
|
||
params: {},
|
||
handler: async () => {
|
||
const tab = await getActiveTab();
|
||
const dataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, { format: 'png' });
|
||
return { image: dataUrl };
|
||
}
|
||
},
|
||
browser_get_page_info: {
|
||
description: 'Get page URL, title, viewport, scroll position',
|
||
params: {},
|
||
handler: async () => {
|
||
const tab = await getActiveTab();
|
||
const info = await execInTab(tab.id, () => ({
|
||
title: document.title,
|
||
url: location.href,
|
||
viewport: { w: window.innerWidth, h: window.innerHeight },
|
||
scroll: { x: window.scrollX, y: window.scrollY },
|
||
docHeight: document.documentElement.scrollHeight
|
||
}));
|
||
return { ...info, tabId: tab.id };
|
||
}
|
||
},
|
||
browser_get_text: {
|
||
description: 'Get text content of page or element',
|
||
params: { selector: 'string?' },
|
||
handler: async ({ selector }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (sel) => {
|
||
const el = sel ? document.querySelector(sel) : document.body;
|
||
if (!el) return { error: 'Element not found: ' + sel };
|
||
return el.innerText;
|
||
}, [selector || null]);
|
||
}
|
||
},
|
||
browser_get_html: {
|
||
description: 'Get HTML of page or element',
|
||
params: { selector: 'string?', outer: 'boolean?' },
|
||
handler: async ({ selector, outer = true }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (sel, isOuter) => {
|
||
const el = sel ? document.querySelector(sel) : document.documentElement;
|
||
if (!el) return { error: 'Element not found: ' + sel };
|
||
return isOuter ? el.outerHTML : el.innerHTML;
|
||
}, [selector || null, outer]);
|
||
}
|
||
},
|
||
browser_query_selector: {
|
||
description: 'Query elements – returns tag, text, attributes, bounding box',
|
||
params: { selector: 'string', limit: 'number?' },
|
||
handler: async ({ selector, limit = 20 }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (sel, lim) => {
|
||
return Array.from(document.querySelectorAll(sel)).slice(0, lim).map(el => {
|
||
const r = el.getBoundingClientRect();
|
||
const attrs = {};
|
||
for (const a of el.attributes) attrs[a.name] = a.value;
|
||
return { tag: el.tagName.toLowerCase(), text: (el.innerText||'').slice(0,200), attrs, box: {x:r.x,y:r.y,w:r.width,h:r.height} };
|
||
});
|
||
}, [selector, limit]);
|
||
}
|
||
},
|
||
browser_get_interactive_elements: {
|
||
description: 'Find all interactive elements (buttons, links, inputs) with selectors',
|
||
params: { viewportOnly: 'boolean?' },
|
||
handler: async ({ viewportOnly = true }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (vpOnly) => {
|
||
const sels = 'a[href], button, input, select, textarea, [role="button"], [onclick], [tabindex]:not([tabindex="-1"])';
|
||
const els = Array.from(document.querySelectorAll(sels));
|
||
const results = [];
|
||
for (const el of els) {
|
||
const r = el.getBoundingClientRect();
|
||
if (r.width === 0 && r.height === 0) continue;
|
||
if (vpOnly && (r.bottom < 0 || r.top > window.innerHeight || r.right < 0 || r.left > window.innerWidth)) continue;
|
||
const attrs = {};
|
||
for (const a of ['id','class','name','type','href','value','placeholder','aria-label','role','title'])
|
||
if (el.getAttribute(a)) attrs[a] = el.getAttribute(a);
|
||
let selector = '';
|
||
if (el.id) selector = '#' + el.id;
|
||
else if (el.name) selector = el.tagName.toLowerCase() + '[name="' + el.name + '"]';
|
||
else {
|
||
const parts = []; let cur = el;
|
||
while (cur && cur !== document.body && parts.length < 4) {
|
||
let s = cur.tagName.toLowerCase();
|
||
if (cur.id) { parts.unshift('#' + cur.id); break; }
|
||
const parent = cur.parentElement;
|
||
if (parent) {
|
||
const sibs = Array.from(parent.children).filter(c => c.tagName === cur.tagName);
|
||
if (sibs.length > 1) s += ':nth-of-type(' + (sibs.indexOf(cur)+1) + ')';
|
||
}
|
||
parts.unshift(s); cur = cur.parentElement;
|
||
}
|
||
selector = parts.join(' > ');
|
||
}
|
||
results.push({ tag: el.tagName.toLowerCase(), text: (el.innerText||el.value||'').slice(0,80), attrs, selector, box:{x:Math.round(r.x),y:Math.round(r.y),w:Math.round(r.width),h:Math.round(r.height)} });
|
||
if (results.length >= 50) break;
|
||
}
|
||
return results;
|
||
}, [viewportOnly]);
|
||
}
|
||
},
|
||
browser_click: {
|
||
description: 'Click element by CSS selector',
|
||
params: { selector: 'string' },
|
||
handler: async ({ selector }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (sel) => {
|
||
const el = document.querySelector(sel);
|
||
if (!el) return { error: 'Not found: ' + sel };
|
||
el.scrollIntoView({ block: 'center', behavior: 'instant' });
|
||
el.click();
|
||
return { clicked: sel, tag: el.tagName.toLowerCase(), text: (el.innerText||'').slice(0,80) };
|
||
}, [selector]);
|
||
}
|
||
},
|
||
browser_type: {
|
||
description: 'Type text into input/textarea',
|
||
params: { selector: 'string', text: 'string', clear: 'boolean?' },
|
||
handler: async ({ selector, text, clear = true }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (sel, txt, clr) => {
|
||
const el = document.querySelector(sel);
|
||
if (!el) return { error: 'Not found: ' + sel };
|
||
el.focus();
|
||
if (clr) { el.value = ''; el.dispatchEvent(new Event('input', {bubbles:true})); }
|
||
// Simulate character-by-character for React/Angular
|
||
for (const ch of txt) {
|
||
el.value += ch;
|
||
el.dispatchEvent(new Event('input', {bubbles:true}));
|
||
}
|
||
el.dispatchEvent(new Event('change', {bubbles:true}));
|
||
return { typed: txt, selector: sel };
|
||
}, [selector, text, clear]);
|
||
}
|
||
},
|
||
browser_select: {
|
||
description: 'Select option in <select>',
|
||
params: { selector: 'string', value: 'string' },
|
||
handler: async ({ selector, value }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (sel, val) => {
|
||
const el = document.querySelector(sel);
|
||
if (!el) return { error: 'Not found: ' + sel };
|
||
let opt = Array.from(el.options).find(o => o.value === val || o.text === val);
|
||
if (!opt) return { error: 'Option not found: ' + val };
|
||
el.value = opt.value;
|
||
el.dispatchEvent(new Event('change', {bubbles:true}));
|
||
return { selected: opt.text, value: opt.value };
|
||
}, [selector, value]);
|
||
}
|
||
},
|
||
browser_scroll: {
|
||
description: 'Scroll page or element',
|
||
params: { direction: 'string', amount: 'number?', selector: 'string?' },
|
||
handler: async ({ direction, amount = 500, selector }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (dir, amt, sel) => {
|
||
const target = sel ? document.querySelector(sel) : window;
|
||
const opts = { behavior: 'smooth' };
|
||
if (dir === 'down') (target === window ? window : target).scrollBy({top: amt, ...opts});
|
||
else if (dir === 'up') (target === window ? window : target).scrollBy({top: -amt, ...opts});
|
||
else if (dir === 'right') (target === window ? window : target).scrollBy({left: amt, ...opts});
|
||
else if (dir === 'left') (target === window ? window : target).scrollBy({left: -amt, ...opts});
|
||
return { scrolled: dir, amount: amt };
|
||
}, [direction, amount, selector || null]);
|
||
}
|
||
},
|
||
browser_execute_js: {
|
||
description: 'Execute JavaScript in page context, return result',
|
||
params: { code: 'string' },
|
||
handler: async ({ code }) => {
|
||
const tab = await getActiveTab();
|
||
const results = await chrome.scripting.executeScript({
|
||
target: { tabId: tab.id },
|
||
func: (js) => { try { const r = eval(js); return { value: JSON.parse(JSON.stringify(r ?? null)) }; } catch(e) { return { error: e.message }; } },
|
||
args: [code]
|
||
});
|
||
if (results?.[0]?.result?.error) throw new Error(results[0].result.error);
|
||
return results?.[0]?.result?.value;
|
||
}
|
||
},
|
||
browser_press_key: {
|
||
description: 'Press keyboard key (Enter, Escape, Tab, etc.)',
|
||
params: { key: 'string', modifiers: 'array?' },
|
||
handler: async ({ key, modifiers = [] }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (k, mods) => {
|
||
const opts = { key: k, bubbles: true, cancelable: true, ctrlKey: mods.includes('ctrl'), altKey: mods.includes('alt'), shiftKey: mods.includes('shift'), metaKey: mods.includes('meta') };
|
||
const t = document.activeElement || document.body;
|
||
t.dispatchEvent(new KeyboardEvent('keydown', opts));
|
||
t.dispatchEvent(new KeyboardEvent('keyup', opts));
|
||
return { pressed: k, modifiers: mods };
|
||
}, [key, modifiers]);
|
||
}
|
||
},
|
||
browser_wait: {
|
||
description: 'Wait for element or timeout',
|
||
params: { selector: 'string?', timeout: 'number?' },
|
||
handler: async ({ selector, timeout = 5000 }) => {
|
||
if (!selector) { await new Promise(r => setTimeout(r, timeout)); return { waited: timeout }; }
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (sel, ms) => {
|
||
return new Promise(resolve => {
|
||
const start = Date.now();
|
||
(function check() {
|
||
if (document.querySelector(sel)) return resolve({ found: true, elapsed: Date.now()-start });
|
||
if (Date.now()-start > ms) return resolve({ found: false, timeout: true });
|
||
requestAnimationFrame(check);
|
||
})();
|
||
});
|
||
}, [selector, timeout]);
|
||
}
|
||
},
|
||
browser_get_console_logs: {
|
||
description: 'Get captured console logs',
|
||
params: { clear: 'boolean?' },
|
||
handler: async ({ clear = false }) => {
|
||
const logs = [...consoleLogs];
|
||
if (clear) consoleLogs.length = 0;
|
||
return logs;
|
||
}
|
||
},
|
||
browser_highlight: {
|
||
description: 'Highlight an element visually (for debugging)',
|
||
params: { selector: 'string', color: 'string?' },
|
||
handler: async ({ selector, color = '#ff6b35' }) => {
|
||
const tab = await getActiveTab();
|
||
return await execInTab(tab.id, (sel, col) => {
|
||
const el = document.querySelector(sel);
|
||
if (!el) return { error: 'Not found: ' + sel };
|
||
el.style.outline = '3px solid ' + col;
|
||
el.style.outlineOffset = '2px';
|
||
setTimeout(() => { el.style.outline = ''; el.style.outlineOffset = ''; }, 3000);
|
||
return { highlighted: sel };
|
||
}, [selector, color]);
|
||
}
|
||
}
|
||
};
|
||
|
||
// ============================================================
|
||
// State
|
||
// ============================================================
|
||
let consoleLogs = [];
|
||
|
||
// ============================================================
|
||
// Helpers
|
||
// ============================================================
|
||
async function getActiveTab() {
|
||
const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
|
||
if (!tab) throw new Error('No active tab');
|
||
return tab;
|
||
}
|
||
|
||
async function execInTab(tabId, func, args = []) {
|
||
const results = await chrome.scripting.executeScript({ target: { tabId }, func, args });
|
||
if (results?.[0]?.error) throw new Error(results[0].error.message || 'Script error');
|
||
return results?.[0]?.result;
|
||
}
|
||
|
||
async function handleToolCall(name, params = {}) {
|
||
const tool = TOOLS[name];
|
||
if (!tool) throw new Error('Unknown tool: ' + name);
|
||
return await tool.handler(params);
|
||
}
|
||
|
||
// ============================================================
|
||
// Message handling (from side panel, content scripts, external)
|
||
// ============================================================
|
||
chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
|
||
if (msg.type === 'TOOL_CALL') {
|
||
handleToolCall(msg.tool, msg.params || {})
|
||
.then(result => sendResponse({ success: true, result }))
|
||
.catch(err => sendResponse({ success: false, error: err.message }));
|
||
return true;
|
||
}
|
||
if (msg.type === 'GET_TOOLS') {
|
||
const tools = Object.entries(TOOLS).map(([name, t]) => ({ name, description: t.description, params: t.params }));
|
||
sendResponse({ tools });
|
||
return true;
|
||
}
|
||
if (msg.type === 'CONSOLE_LOG') {
|
||
consoleLogs.push({ level: msg.level, text: msg.text, ts: Date.now(), url: sender.tab?.url });
|
||
if (consoleLogs.length > 200) consoleLogs.shift();
|
||
}
|
||
if (msg.type === 'PING') {
|
||
sendResponse({ pong: true, version: '1.0.0', tools: Object.keys(TOOLS).length });
|
||
return true;
|
||
}
|
||
});
|
||
|
||
// External messages (from MCP server page or other extensions)
|
||
chrome.runtime.onMessageExternal.addListener((msg, sender, sendResponse) => {
|
||
if (msg.type === 'TOOL_CALL') {
|
||
handleToolCall(msg.tool, msg.params || {})
|
||
.then(result => sendResponse({ success: true, result }))
|
||
.catch(err => sendResponse({ success: false, error: err.message }));
|
||
return true;
|
||
}
|
||
if (msg.type === 'GET_TOOLS') {
|
||
const tools = Object.entries(TOOLS).map(([name, t]) => ({ name, description: t.description, params: t.params }));
|
||
sendResponse({ tools });
|
||
return true;
|
||
}
|
||
});
|
||
|
||
// ============================================================
|
||
// Side Panel activation on icon click
|
||
// ============================================================
|
||
chrome.action.onClicked.addListener(async (tab) => {
|
||
await chrome.sidePanel.open({ windowId: tab.windowId });
|
||
});
|
||
|
||
// Ensure side panel is available
|
||
chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true }).catch(() => {});
|
||
|
||
// ============================================================
|
||
// Keep alive
|
||
// ============================================================
|
||
chrome.alarms.create('keepAlive', { periodInMinutes: 0.4 });
|
||
chrome.alarms.onAlarm.addListener(() => {});
|
||
|
||
// ============================================================
|
||
// WebSocket bridge to MCP server (connects when server is running)
|
||
// ============================================================
|
||
let ws = null;
|
||
let wsRetry = null;
|
||
|
||
function connectWs() {
|
||
if (ws && ws.readyState === WebSocket.OPEN) return;
|
||
try {
|
||
ws = new WebSocket('ws://127.0.0.1:9224');
|
||
ws.onopen = () => {
|
||
console.log('[BrowserPilot] Connected to MCP bridge');
|
||
if (wsRetry) { clearInterval(wsRetry); wsRetry = null; }
|
||
ws.send(JSON.stringify({ type: 'hello', version: '1.0.0', tools: Object.keys(TOOLS).length }));
|
||
};
|
||
ws.onmessage = async (e) => {
|
||
try {
|
||
const msg = JSON.parse(e.data);
|
||
if (msg.type === 'TOOL_CALL' && msg.id) {
|
||
try {
|
||
const result = await handleToolCall(msg.tool, msg.params || {});
|
||
ws.send(JSON.stringify({ id: msg.id, success: true, result }));
|
||
} catch (err) {
|
||
ws.send(JSON.stringify({ id: msg.id, success: false, error: err.message }));
|
||
}
|
||
}
|
||
} catch (err) { console.error('[BrowserPilot] WS msg error:', err); }
|
||
};
|
||
ws.onclose = () => { ws = null; scheduleWsRetry(); };
|
||
ws.onerror = () => { ws = null; scheduleWsRetry(); };
|
||
} catch (e) { scheduleWsRetry(); }
|
||
}
|
||
|
||
function scheduleWsRetry() {
|
||
if (wsRetry) return;
|
||
wsRetry = setInterval(connectWs, 5000);
|
||
}
|
||
|
||
connectWs();
|
||
|
||
console.log('[BrowserPilot] Service worker loaded –', Object.keys(TOOLS).length, 'tools');
|