Files
nexus-browser-pilot-extension/src/background/index.ts
T
coreandClaude Opus 5 f943261c3c feat: Gemini bridge, browser extension hardening, shared memory
Nexus gateway (measured 2026-08-07, not assumed):
- Gemini speaks Google-GenAI (/v1beta/models/{id}:streamGenerateContent, header
  api-key), NOT the Azure-OpenAI path — that returned 404 "no Route matched" and
  was the cause of the reported failures. New agent/gemini_bridge.py translates
  Bedrock Converse <-> Gemini in both directions.
- Only gemini-3.6-flash is subscribed; 2.5-flash/2.5-pro/3.1-flash-lite give 403,
  every other name 404. Catalog corrected.
- Four Gemini rules, each previously an HTTP 400, now covered by tests:
  thought signatures are mandatory, they belong to the TURN (not the individual
  call), functionResponse turns must be homogeneous, arrays need `items`.
- Prompt caching is NOT available: cachePoint is accepted and ignored.

System prompt:
- Was an f-string; a code sample containing braces broke build_system_prompt at
  request time (CLI and web both 500, import stayed green). Now a plain template
  with __TOKEN__ placeholders. Regression guards in tests/test_system_prompt.py.

CLI:
- `agent resume` now prints the stored transcript. The history was always loaded
  into the model context, only the terminal stayed empty.

Memory (new, all three surfaces):
- agent/memory.py stores notes about the user in one local file, written
  atomically; memory.cnull.net remains an optional mirror that can never fail a
  write. Tools memory_save/search/forget, injected into the prompt with a budget.
  HTTP surface /api/memory for the extension.

Browser extension (agent/extension, first commit of the source):
- driveMode 'direct' talks to Nexus without the Python broker: Claude via
  Bedrock converse, GPT via Azure-OpenAI, Gemini via Google-GenAI.
- browser_type no longer guesses the focused element — that wrote whole mails
  into Outlook's subject line. Read-back now reports where the text actually
  landed, so a mis-target is visible instead of silent.
- aria-labelledby is resolved across all ids (it is a list); contenteditable is
  interactive and marked editable. Without this, subject and message body look
  identical to the model.
- Hard block against sending mail, independent of riskMode.
- Runs survive the panel: events are buffered and replayed by sequence number.
- Image input (paste, file, drag&drop), on-page glow/spotlight, memory tools.

Cost: fixed tokens per round 11434 -> 6540 (-43%) by trimming tool schemas,
dropping gateway docs from the browser prompt and sending site knowledge only
where it applies.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-07 17:31:26 +02:00

1760 lines
72 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Nexus Browser Pilot Background Service Worker
*
* Der Service Worker ist die einzige Stelle, an der Browser-Tools wirklich laufen.
* Alle drei Antriebe münden in denselben Engpass runTool():
*
* Broker (Python, WebSocket) ─┐
* MCP-Bridge (WebSocket) ─┼─→ runTool() → Risiko-Gate → dispatchTool()
* Eingebauter Loop (agent.ts)─┘
*
* Dadurch gilt der Risikomodus für jeden Antrieb, nicht nur für den Chat im Panel.
* Der Agent-Loop lebt bewusst hier und nicht im Side Panel: ein geschlossenes Panel
* darf einen laufenden Auftrag nicht abbrechen.
*/
import type {
BrokerMessage,
ExtHello,
ExtToolResult,
ExtUserMessage,
ExtAbort,
ExtApprovalResponse,
PanelMessage,
RiskVerdict,
InteractionMethod,
} from '../shared/protocol';
import {
loadConfig,
patchConfig,
EXTENSION_VERSION,
RECONNECT_INTERVAL_MS,
RECONNECT_MAX_ATTEMPTS,
type ExtensionConfig,
} from '../shared/config';
import { BROWSER_TOOL_SCHEMAS } from '../shared/tool-schemas.generated';
import * as cdp from './cdp';
import { runComputer } from './computer';
import { gate, setScaleProvider } from './risk';
import { localSave as memoryLocalSave, localSearch as memoryLocalSearch } from './memory';
import { runAgentTurn, abortAgent, isAgentRunning, resetConversation, queueMessage } from './agent';
import type { LlmToolDef } from './llm';
// ─── State ────────────────────────────────────────────────────────────────────
let ws: WebSocket | null = null;
let reconnectAttempts = 0;
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
let sessionSecret = '';
let panelPort: chrome.runtime.Port | null = null;
/**
* Target-Tab: Der Agent arbeitet auf diesem Tab, unabhängig davon, welcher Tab
* gerade im Vordergrund ist. Wird beim ersten Tool-Aufruf auf den aktiven Tab
* gesetzt und bleibt dort, bis der Nutzer explizit wechselt (browser_tabs_select)
* oder eine neue Konversation startet.
*/
let targetTabId: number | null = null;
/**
* Pause zwischen Tool-Aufrufen in ms. Macht den Agent weniger hektisch und gibt
* Seiten Zeit zum Rendern. Konfigurierbar über die Config.
*/
const TOOL_PACE_MS = 600;
/** Offene Risiko-Rückfragen: requestId → Auflöser. */
const pendingRisk = new Map<string, (approved: boolean) => void>();
/**
* Wartezeit für eine Risiko-Rückfrage. Läuft sie ab, wird AUSGEFÜHRT, nicht abgebrochen.
* Der Nutzer hat Autonomie verlangt; Schweigen darf keinen stillen Abbruch bedeuten.
* Nur eine ausdrückliche Ablehnung stoppt.
*/
const RISK_TIMEOUT_MS = 120_000;
/** Konsolenpuffer, gefüllt aus dem Content-Script. */
const consoleLogs: Array<{ level: string; text: string; ts: number; url?: string }> = [];
const MAX_CONSOLE_LOGS = 300;
/** Letzter Screenshot-Maßstab je Tab — für die Koordinaten-Rückrechnung. */
const lastScale = new Map<number, number>();
// Die Risiko-Engine muss Modell-Koordinaten in CSS-Pixel zurückrechnen können,
// bevor sie per elementFromPoint nachsieht, worauf da eigentlich geklickt wird.
setScaleProvider((tabId) => lastScale.get(tabId) ?? 1);
// ─── Session Secret ───────────────────────────────────────────────────────────
async function getOrCreateSecret(): Promise<string> {
const stored = await chrome.storage.session.get(['nexus_secret']);
if (stored.nexus_secret) return stored.nexus_secret;
const secret = crypto.randomUUID();
await chrome.storage.session.set({ nexus_secret: secret });
return secret;
}
// ─── WebSocket zum Antrieb (Broker oder MCP-Bridge) ───────────────────────────
async function connect(): Promise<void> {
if (ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) return;
const config = await loadConfig();
sessionSecret = await getOrCreateSecret();
try {
ws = new WebSocket(config.brokerUrl);
} catch (e) {
console.error('[Nexus] WS connect error:', e);
scheduleReconnect();
return;
}
ws.onopen = () => {
reconnectAttempts = 0;
const hello: ExtHello = { type: 'hello', version: EXTENSION_VERSION, secret: sessionSecret };
ws!.send(JSON.stringify(hello));
sendToPanel({ type: 'connection_status', connected: true });
};
ws.onmessage = (event) => {
try {
handleBrokerMessage(JSON.parse(event.data as string) as BrokerMessage);
} catch (e) {
console.error('[Nexus] Failed to parse broker message:', e);
}
};
ws.onclose = () => {
ws = null;
sendToPanel({ type: 'connection_status', connected: false });
scheduleReconnect();
};
ws.onerror = () => { /* onclose folgt */ };
}
function scheduleReconnect(): void {
if (reconnectTimer) return;
if (reconnectAttempts >= RECONNECT_MAX_ATTEMPTS) {
sendToPanel({ type: 'connection_status', connected: false, maxRetries: true });
return;
}
reconnectAttempts++;
const delay = Math.min(RECONNECT_INTERVAL_MS * reconnectAttempts, 30_000);
reconnectTimer = setTimeout(() => { reconnectTimer = null; connect(); }, delay);
}
function sendToBroker(msg: object): void {
if (ws && ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(msg));
}
async function handleBrokerMessage(msg: BrokerMessage): Promise<void> {
switch (msg.type) {
case 'welcome':
case 'text_delta':
case 'round':
case 'done':
case 'error':
case 'approval_request':
case 'tool_result_event':
case 'mcp_status':
sendToPanel(msg);
break;
case 'tool_call':
await executeForBroker(msg.callId, msg.name, msg.input);
break;
case 'ping':
sendToBroker({ type: 'pong', ts: msg.ts });
break;
}
}
// ─── Tool-Ausführung ──────────────────────────────────────────────────────────
async function executeForBroker(callId: string, name: string, input: Record<string, unknown>): Promise<void> {
const start = performance.now();
let method: InteractionMethod = 'synthetic';
try {
sendToPanel({ type: 'tool_executing', callId, name, input });
const result = await runTool(name, input);
method = (result?._method as InteractionMethod) || 'synthetic';
if (result && typeof result === 'object') delete result._method;
const durationMs = Math.round(performance.now() - start);
sendToBroker({ type: 'tool_result', callId, result, durationMs, method } satisfies ExtToolResult);
reportToolDone(callId, name, result, undefined, durationMs, method);
} catch (e: unknown) {
const error = e instanceof Error ? e.message : String(e);
const durationMs = Math.round(performance.now() - start);
sendToBroker({ type: 'tool_result', callId, result: null, error, durationMs, method } satisfies ExtToolResult);
reportToolDone(callId, name, null, error, durationMs, method);
}
}
/** Ergebnis ans Panel — Bilder gehen getrennt, damit tool_done klein bleibt. */
function reportToolDone(
callId: string, name: string, result: any, error: string | undefined,
durationMs: number, method: InteractionMethod,
): void {
if (result && typeof result === 'object' && typeof result.image === 'string') {
sendToPanel({ type: 'screenshot', callId, dataUrl: result.image });
const { image, ...rest } = result;
sendToPanel({ type: 'tool_done', callId, name, result: { ...rest, image: '[siehe Vorschau]' }, error, durationMs, method });
return;
}
sendToPanel({ type: 'tool_done', callId, name, result, error, durationMs, method });
}
/** Sendet den aktuellen Target-Tab ans Panel. */
async function notifyTargetTab(): Promise<void> {
if (targetTabId === null) {
sendToPanel({ type: 'target_tab', tabId: null });
return;
}
try {
const tab = await chrome.tabs.get(targetTabId);
sendToPanel({ type: 'target_tab', tabId: targetTabId, title: tab.title, url: tab.url });
} catch {
sendToPanel({ type: 'target_tab', tabId: null });
}
}
/**
* DER Engpass. Jeder Antrieb geht hier durch, also greift das Risiko-Gate überall.
* Bei riskMode='off' (Standard) kostet das Gate nichts und führt sofort aus.
*/
async function runTool(name: string, input: Record<string, unknown>): Promise<any> {
// Gedächtnis braucht keinen Tab und kein Risiko-Gate — es fasst die Seite nicht an.
if (name === 'memory_save') {
const text = String(input.text ?? '').trim();
if (!text) return errorResult('NO_TEXT', 'text erforderlich', false);
const note = await memoryLocalSave(
text, String(input.type ?? 'fakt'),
Array.isArray(input.tags) ? (input.tags as string[]) : [],
);
return note
? { saved: note.id, type: note.type, text: note.text }
: errorResult('MEMORY_UNAVAILABLE',
'Der lokale Agent ist nicht erreichbar — die Notiz wurde NICHT gespeichert. '
+ 'Arbeite ohne sie weiter.', false);
}
if (name === 'memory_search') {
const notes = await memoryLocalSearch(
String(input.query ?? ''), Number(input.limit) || 8,
);
return { count: notes.length, notes };
}
const tabId = await currentTabId();
const { allowed, verdict } = await gate(tabId, name, input, (v) => askUser(v, name));
if (!allowed) {
return {
error: {
code: 'USER_DECLINED',
message: `Vom Nutzer abgelehnt: ${verdict.reason}`,
retryable: false,
},
_method: 'synthetic',
};
}
const result = await dispatchTool(name, input);
// Pace: kurze Pause nach jeder Aktion, damit Seiten rendern können und der
// Agent nicht hektisch wirkt. Screenshots und reine Lese-Tools brauchen keine Pause.
const readOnly = name === 'browser_screenshot' || name === 'browser_read_page'
|| name === 'browser_read_network' || name === 'browser_read_console'
|| name === 'browser_tabs_list' || name === 'browser_find';
if (!readOnly) {
const cfg = await loadConfig();
const pace = cfg.paceMs ?? TOOL_PACE_MS;
if (pace > 0) await new Promise(r => setTimeout(r, pace));
}
return result;
}
/** Fragt den Nutzer. Ohne offenes Panel wird gehandelt — Autonomie hat Vorrang. */
function askUser(verdict: RiskVerdict, toolName: string): Promise<boolean> {
if (!panelPort) return Promise.resolve(true);
const requestId = crypto.randomUUID();
sendToPanel({ type: 'risk_request', requestId, name: toolName, verdict, timeoutMs: RISK_TIMEOUT_MS });
return new Promise<boolean>((resolve) => {
const timer = setTimeout(() => {
pendingRisk.delete(requestId);
resolve(true); // Zeitablauf = ausführen, siehe RISK_TIMEOUT_MS
}, RISK_TIMEOUT_MS);
pendingRisk.set(requestId, (approved) => {
clearTimeout(timer);
pendingRisk.delete(requestId);
resolve(approved);
});
});
}
// ─── Helfer ───────────────────────────────────────────────────────────────────
async function getActiveTab(): Promise<chrome.tabs.Tab> {
// Wenn ein Target-Tab gesetzt ist, diesen verwenden (Hintergrundarbeit).
if (targetTabId !== null) {
try {
const tab = await chrome.tabs.get(targetTabId);
if (tab && tab.id) return tab;
} catch {
// Tab existiert nicht mehr — zurückfallen auf aktiven Tab.
targetTabId = null;
}
}
const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
if (!tab || !tab.id) throw new ToolError('NO_ACTIVE_TAB', 'Kein aktiver Tab gefunden', false);
// Beim ersten Aufruf den Target-Tab setzen.
targetTabId = tab.id!;
notifyTargetTab();
return tab;
}
async function currentTabId(): Promise<number | undefined> {
if (targetTabId !== null) {
try {
await chrome.tabs.get(targetTabId);
return targetTabId;
} catch {
targetTabId = null;
}
}
try {
const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
return tab?.id;
} catch {
return undefined;
}
}
async function execInTab<T>(tabId: number, func: (...args: any[]) => T, args: any[] = []): Promise<T> {
const results = await chrome.scripting.executeScript({ target: { tabId }, func: func as any, args });
return results?.[0]?.result as T;
}
function resolveTarget(input: Record<string, unknown>): string | null {
if (input.ref_id) return `[data-nexus-ref="${input.ref_id}"]`;
if (input.selector) return input.selector as string;
return null;
}
class ToolError extends Error {
code: string;
retryable: boolean;
constructor(code: string, message: string, retryable: boolean) {
super(message);
this.code = code;
this.retryable = retryable;
}
}
function errorResult(code: string, message: string, retryable: boolean): any {
return { error: { code, message, retryable }, _method: 'synthetic' };
}
/** Liest Position und Kennung eines Elements im Viewport. Magnet-Logik: wenn der
* exakte Selektor nichts findet, wird per Textsuche das nächstliegende interaktive
* Element gesucht (übernommen aus browser-pilot). */
async function elementBox(tabId: number, selector: string): Promise<{ x: number; y: number; tag: string; text: string } | null> {
return await execInTab(tabId, (sel: string) => {
let el = document.querySelector(sel) as HTMLElement | null;
// Magnet-Fallback: Textsuche auf interaktiven Elementen
if (!el) {
const searchText = sel.replace(/[^a-zA-Z0-9äöüÄÖÜß ]/g, '').toLowerCase().trim();
if (searchText) {
const interactives = Array.from(document.querySelectorAll(
'a, button, input, select, textarea, [role="button"], [onclick], [tabindex]:not([tabindex="-1"]), summary, label'
));
for (const candidate of interactives) {
const cText = ((candidate as HTMLElement).innerText || (candidate as HTMLInputElement).value ||
candidate.getAttribute('aria-label') || '').toLowerCase();
if (cText.includes(searchText) || searchText.includes(cText.trim())) {
el = candidate as HTMLElement;
break;
}
}
}
}
if (!el) return null;
el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' as ScrollBehavior });
const r = el.getBoundingClientRect();
if (r.width === 0 && r.height === 0) return null;
return {
x: r.x + r.width / 2,
y: r.y + r.height / 2,
tag: el.tagName.toLowerCase(),
text: ((el as HTMLElement).innerText || (el as HTMLInputElement).value || '').trim().slice(0, 80),
};
}, [selector]);
}
/**
* Entscheidet je nach inputMode, ob eine echte (CDP) oder synthetische Eingabe erzeugt wird.
* 'auto' versucht CDP und fällt bei fehlender Permission oder belegtem Debugger zurück.
*/
async function wantsTrusted(tabId: number): Promise<boolean> {
const cfg = await loadConfig();
if (cfg.inputMode === 'synthetic') return false;
if (!cdp.cdpAvailable()) return false;
if (cfg.inputMode === 'trusted') return true;
try {
await cdp.attach(tabId);
return true;
} catch {
return false;
}
}
// ─── Interaktion ──────────────────────────────────────────────────────────────
async function clickElement(tabId: number, selector: string, forceTrusted: boolean): Promise<any> {
const trusted = forceTrusted || await wantsTrusted(tabId);
if (trusted) {
const box = await elementBox(tabId, selector);
if (!box) throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden oder unsichtbar: ${selector}`, true);
try {
await cdp.attach(tabId);
await cdp.mouseClick(tabId, box.x, box.y);
notifyCursor(tabId, box.x, box.y, 'left');
return { clicked: selector, tag: box.tag, text: box.text, _method: 'cdp' };
} catch (e: any) {
// Kein harter Fehler: lieber synthetisch klicken als gar nicht handeln.
console.warn('[Nexus] CDP-Klick fehlgeschlagen, weiche auf synthetisch aus:', e?.message);
}
}
const result = await execInTab(tabId, (sel: string) => {
let el = document.querySelector(sel) as HTMLElement | null;
// Magnet-Fallback: Textsuche auf interaktiven Elementen
if (!el) {
const searchText = sel.replace(/[^a-zA-Z0-9äöüÄÖÜß ]/g, '').toLowerCase().trim();
if (searchText) {
const interactives = Array.from(document.querySelectorAll(
'a, button, input, select, textarea, [role="button"], [onclick], [tabindex]:not([tabindex="-1"]), summary, label'
));
for (const candidate of interactives) {
const cText = ((candidate as HTMLElement).innerText || (candidate as HTMLInputElement).value ||
candidate.getAttribute('aria-label') || '').toLowerCase();
if (cText.includes(searchText) || searchText.includes(cText.trim())) {
el = candidate as HTMLElement;
break;
}
}
}
}
if (!el) return { error: 'NOT_FOUND' };
el.scrollIntoView({ block: 'center', behavior: 'instant' as ScrollBehavior });
// Vollständige Maus-Event-Sequenz für maximale Kompatibilität
const rect = el.getBoundingClientRect();
const x = rect.left + rect.width / 2;
const y = rect.top + rect.height / 2;
const eventOpts: MouseEventInit = {
bubbles: true, cancelable: true, view: window,
clientX: x, clientY: y, button: 0, buttons: 1,
};
el.dispatchEvent(new PointerEvent('pointerover', eventOpts));
el.dispatchEvent(new PointerEvent('pointerenter', { ...eventOpts, bubbles: false }));
el.dispatchEvent(new MouseEvent('mouseover', eventOpts));
el.dispatchEvent(new MouseEvent('mouseenter', { ...eventOpts, bubbles: false }));
el.dispatchEvent(new PointerEvent('pointerdown', eventOpts));
el.dispatchEvent(new MouseEvent('mousedown', eventOpts));
(el as HTMLElement).focus?.();
el.dispatchEvent(new PointerEvent('pointerup', eventOpts));
el.dispatchEvent(new MouseEvent('mouseup', eventOpts));
el.dispatchEvent(new MouseEvent('click', eventOpts));
return { clicked: sel, tag: el.tagName.toLowerCase(), text: (el.innerText || '').trim().slice(0, 80), coordinates: { x: Math.round(x), y: Math.round(y) } };
}, [selector]);
if (!result || (result as any).error === 'NOT_FOUND') {
throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
}
return { ...result, _method: 'synthetic' };
}
/**
* Riegel gegen versehentliches Absenden von Mails.
*
* "Nur Entwurf" als Prompt-Regel ist eine Bitte — ein Modell kann sie ignorieren,
* und eine verschickte Mail holt niemand zurück. Deshalb hier hart, unabhängig
* vom Risikomodus. Greift nur, wenn BEIDES zutrifft: die Schaltfläche heißt nach
* Senden, UND die Seite ist erkennbar eine Mail-Oberfläche. Ein "Senden"-Knopf in
* einem Kontaktformular bleibt damit bedienbar.
*
* Grenze, die man kennen muss: browser_computer klickt auf Pixelkoordinaten. Was
* dort getroffen wird, ist vorher nicht bekannt — dieser Riegel sieht es nicht.
*/
async function mailSendBlock(tabId: number, selector: string): Promise<any | null> {
const cfg = await loadConfig();
if (!cfg.blockMailSend) return null;
const hit = await execInTab(tabId, (sel: string) => {
const el = document.querySelector(sel) as HTMLElement | null;
if (!el) return null;
const label = [
el.getAttribute('aria-label'), el.getAttribute('title'),
(el as HTMLInputElement).value, el.innerText || el.textContent,
].filter(Boolean).join(' ').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 120);
// Wortgrenzen: "Absender", "Gesendet" oder "Nachsenden" sind keine Sende-Knöpfe.
const isSend = /(^|\s)(senden|send|absenden|abschicken|versenden|jetzt senden|send now)(\s|$)/.test(label);
if (!isSend) return null;
const host = location.hostname.toLowerCase();
const mailHost = /(outlook|office|mail|owa|gmail|zimbra|roundcube)/.test(host)
|| /(outlook|owa|mail)/.test(location.pathname.toLowerCase());
// Oder ein erkennbares Verfassen-Formular auf der Seite.
const compose = !!document.querySelector(
'[aria-label*="Nachrichtentext" i], [aria-label*="message body" i], '
+ '[aria-label*="Betreff" i], [aria-label*="subject" i], [name="subject"]',
);
if (!mailHost && !compose) return null;
return { label, host };
}, [selector]);
if (!hit) return null;
return errorResult(
'MAIL_SEND_BLOCKED',
`Absenden ist gesperrt: "${hit.label}" auf ${hit.host}. Du legst nur Entwürfe an — `
+ 'das Verschicken macht der Nutzer selbst. Speichere den Entwurf und melde, dass er '
+ 'bereitliegt. (Abschaltbar in den Einstellungen: "Senden blockieren".)',
false,
);
}
/**
* Beschreibt, WO der Text wirklich gelandet ist — gemeintes und tatsächlich
* fokussiertes Element im Vergleich.
*
* Der CDP-Pfad fokussiert über einen Klick auf die zuvor gemessene Box. Rendert
* die Seite zwischen Messung und Klick neu (in SPAs der Normalfall), landet der
* Klick woanders und der Text im falschen Feld. Die frühere Rücklese-Prüfung las
* den Wert des GEMEINTEN Elements und meldete deshalb Erfolg, während der Text
* im Betreff stand. Genau dieser blinde Fleck hat in Outlook ganze Mails in die
* Betreffzeile geschrieben.
*/
async function describeTypeTarget(tabId: number, selector: string): Promise<any> {
return await execInTab(tabId, (sel: string) => {
const describe = (el: any) => {
if (!el || !el.tagName) return null;
// aria-labelledby zuerst: in Outlook Web wird der Nachrichtentext genau so
// beschriftet, und ohne diese Auflösung sieht er aus wie das Betreff-Feld.
let label = el.getAttribute?.('aria-label') || '';
const by = el.getAttribute?.('aria-labelledby');
if (!label && by) {
label = by.split(/\s+/)
.map((id: string) => document.getElementById(id)?.textContent || '')
.join(' ').trim();
}
if (!label) {
label = (el.labels?.[0]?.textContent || el.getAttribute?.('placeholder')
|| el.getAttribute?.('name') || '').trim();
}
return {
ref: el.getAttribute?.('data-nexus-ref') || undefined,
tag: String(el.tagName).toLowerCase(),
type: el.type || undefined,
role: el.getAttribute?.('role') || undefined,
label: label.slice(0, 80) || undefined,
editable: el.isContentEditable || undefined,
value: String(el.value ?? el.textContent ?? '').slice(0, 200),
};
};
const intended = document.querySelector(sel);
const active = document.activeElement;
return { intended: describe(intended), active: describe(active), same: !!intended && intended === active };
}, [selector]);
}
/** Ergebnis einer Eingabe, das den Fehlgriff sichtbar macht statt ihn zu verschlucken. */
function typeResult(text: string, selector: string, check: any, submit: boolean, method: string): any {
const landed = check?.active ?? check?.intended ?? null;
const out: any = {
typed: text.length,
selector,
typed_into: landed,
value_after: landed?.value ?? '',
submitted: submit,
_method: method,
};
if (check && !check.same) {
// Laut und maschinenlesbar: das Modell soll das korrigieren, nicht weiterschreiben.
out.target_mismatch = true;
out.intended_target = check.intended;
out.warning = 'Der Text ist NICHT im gemeinten Element gelandet, sondern in '
+ `"${landed?.label || landed?.tag || 'unbekannt'}". Inhalt dort prüfen, ggf. leeren `
+ 'und mit dem richtigen ref_id erneut tippen.';
}
return out;
}
async function typeInElement(
tabId: number, selector: string, text: string, clear: boolean, submit: boolean,
): Promise<any> {
const trusted = await wantsTrusted(tabId);
if (trusted) {
const box = await elementBox(tabId, selector);
if (!box) throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
try {
await cdp.attach(tabId);
// Fokus über einen echten Klick — so verhält es sich wie bei einem Menschen.
await cdp.mouseClick(tabId, box.x, box.y);
notifyCursor(tabId, box.x, box.y, 'left');
if (clear) {
await cdp.pressKey(tabId, 'a', { modifiers: ['ctrl'] });
await cdp.pressKey(tabId, 'Delete');
}
await cdp.typeText(tabId, text);
// VOR dem Absenden prüfen: Enter verschiebt den Fokus oft legitim, danach
// wäre der Vergleich wertlos.
const check = await describeTypeTarget(tabId, selector);
if (submit) await cdp.pressKey(tabId, 'Enter');
return typeResult(text, selector, check, submit, 'cdp');
} catch (e: any) {
console.warn('[Nexus] CDP-Eingabe fehlgeschlagen, weiche auf synthetisch aus:', e?.message);
}
}
const result = await execInTab(tabId, (sel: string, txt: string, clr: boolean, sub: boolean) => {
const el = document.querySelector(sel) as HTMLInputElement | HTMLTextAreaElement | null;
if (!el) return { error: 'NOT_FOUND' };
el.focus();
// Der native Setter muss zum konkreten Elementtyp passen. Nimmt man immer den von
// HTMLInputElement, wirft .call() auf ein <textarea> "Illegal invocation".
const proto = el instanceof HTMLTextAreaElement
? window.HTMLTextAreaElement.prototype
: window.HTMLInputElement.prototype;
const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
const isEditable = (el as HTMLElement).isContentEditable;
if (isEditable) {
if (clr) (el as HTMLElement).textContent = '';
(el as HTMLElement).textContent = ((el as HTMLElement).textContent || '') + txt;
el.dispatchEvent(new InputEvent('input', { bubbles: true }));
} else {
const next = (clr ? '' : el.value) + txt;
if (setter) setter.call(el, next);
else el.value = next;
el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true }));
}
if (sub) {
el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }));
el.dispatchEvent(new KeyboardEvent('keyup', { key: 'Enter', bubbles: true }));
const form = (el as HTMLElement).closest('form');
if (form) form.requestSubmit?.();
}
return { ok: true };
}, [selector, text, clear, submit]);
if (!result || (result as any).error === 'NOT_FOUND') {
throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
}
// Auch hier zurücklesen: der Wert kann von der Seite normalisiert oder
// zurückgesetzt worden sein (Masken, kontrollierte React-Felder).
const check = await describeTypeTarget(tabId, selector);
return typeResult(text, selector, check, submit, 'synthetic');
}
/** Schickt die Zeigerposition an das Overlay im Content-Script. Fehler sind belanglos. */
function notifyCursor(tabId: number, x: number, y: number, click?: 'left' | 'right' | 'double'): void {
chrome.tabs.sendMessage(tabId, { type: 'NEXUS_CURSOR', x, y, click }).catch(() => {});
}
/** Innenschimmer an/aus — zeigt, in welchem Tab der Agent gerade arbeitet. */
async function notifyGlow(on: boolean): Promise<void> {
const cfg = await loadConfig();
if (cfg.pageOverlay !== 'glow' && cfg.pageOverlay !== 'both') return;
const id = await currentTabId().catch(() => undefined);
if (typeof id !== 'number') return;
chrome.tabs.sendMessage(id, { type: 'NEXUS_GLOW', on }).catch(() => {});
}
/**
* Hebt das Element hervor, das gerade drankommt. Die Box wird ohnehin für den
* Klick gemessen — das Overlay kostet also keinen zusätzlichen Seitenzugriff.
*/
async function notifySpotlight(tabId: number, selector: string, label: string): Promise<void> {
const cfg = await loadConfig();
if (cfg.pageOverlay !== 'spotlight' && cfg.pageOverlay !== 'both') return;
try {
const rect = await execInTab(tabId, (sel: string) => {
const el = document.querySelector(sel) as HTMLElement | null;
if (!el) return null;
const r = el.getBoundingClientRect();
if (!r.width && !r.height) return null;
return { x: r.x, y: r.y, w: r.width, h: r.height };
}, [selector]);
if (rect) chrome.tabs.sendMessage(tabId, { type: 'NEXUS_SPOTLIGHT', rect, label }).catch(() => {});
} catch {
/* Dekoration — nie den Lauf stören. */
}
}
// ─── Tool-Dispatch ────────────────────────────────────────────────────────────
async function dispatchTool(name: string, input: Record<string, unknown>): Promise<any> {
const needsTab = name !== 'browser_tabs_list' && name !== 'browser_tabs_create';
const tab = needsTab ? await getActiveTab() : null;
const tabId = tab?.id;
switch (name) {
// ── Navigation ──
case 'browser_navigate': {
const url = input.url as string;
if (input.newTab) {
const newTab = await chrome.tabs.create({ url, active: false });
targetTabId = newTab.id!;
notifyTargetTab();
return { tabId: newTab.id, url, _method: 'synthetic' };
}
await chrome.tabs.update(tabId!, { url });
notifyTargetTab();
return { tabId, url, _method: 'synthetic' };
}
case 'browser_go_back':
await chrome.tabs.goBack(tabId!);
return { ok: true, _method: 'synthetic' };
case 'browser_go_forward':
await chrome.tabs.goForward(tabId!);
return { ok: true, _method: 'synthetic' };
case 'browser_reload':
await chrome.tabs.reload(tabId!);
return { ok: true, _method: 'synthetic' };
// ── Tabs ──
case 'browser_tabs_list': {
const tabs = await chrome.tabs.query({});
return {
tabs: tabs.map(t => ({ id: t.id, title: t.title, url: t.url, active: t.active })),
_method: 'synthetic',
};
}
case 'browser_tabs_create': {
// Im Hintergrund erstellen — kein Vordergrund-Wechsel für den Nutzer.
const newTab = await chrome.tabs.create({ url: input.url as string, active: false });
targetTabId = newTab.id!;
notifyTargetTab();
return { tabId: newTab.id, url: input.url, _method: 'synthetic' };
}
case 'browser_tabs_select': {
const tid = input.tabId as number;
// Nur den internen Target-Tab setzen — KEIN Vordergrund-Wechsel.
// Der Agent arbeitet im Hintergrund; der Nutzer soll seinen aktiven Tab
// behalten. Nur wenn der Nutzer explizit `focus: true` übergibt, wird
// der Tab sichtbar gemacht.
if (input.focus) {
await chrome.tabs.update(tid, { active: true });
const t = await chrome.tabs.get(tid);
await chrome.windows.update(t.windowId!, { focused: true });
}
targetTabId = tid;
notifyTargetTab();
return { ok: true, tabId: tid, _method: 'synthetic' };
}
case 'browser_tabs_close': {
const tid = input.tabId as number;
await chrome.tabs.remove(tid);
cleanupTab(tid);
// Wenn der Agent-Tab geschlossen wurde, Kontext zurücksetzen
if (targetTabId === tid) targetTabId = null;
return { ok: true, tabId: tid, _method: 'synthetic' };
}
// ── Menschenähnliche Bedienung ──
case 'browser_computer': {
const result = await runComputer(tabId!, input);
if (typeof result?.scale === 'number') lastScale.set(tabId!, result.scale);
return result;
}
case 'browser_click': {
const selector = resolveTarget(input);
if (!selector) return errorResult('NO_TARGET', 'selector oder ref_id erforderlich', false);
const blocked = await mailSendBlock(tabId!, selector);
if (blocked) return blocked;
await notifySpotlight(tabId!, selector, 'klickt');
return await clickElement(tabId!, selector, !!input.trusted);
}
case 'browser_type': {
// Kein Raten mehr auf den Fokus. Genau diese Zeile hat in Outlook ganze
// Mails in die Betreffzeile geschrieben: nach "Neue Nachricht" liegt der
// Fokus im Betreff, und ein browser_type ohne Ziel traf ihn.
// Bewusstes Tippen ins fokussierte Feld bleibt möglich — aber nur explizit.
const selector = resolveTarget(input)
|| (input.use_focus ? 'input:focus, textarea:focus, [contenteditable]:focus' : '');
if (!selector) {
return errorResult(
'NO_TARGET',
'ref_id oder selector erforderlich. Ohne Ziel würde der Text in das gerade '
+ 'fokussierte Feld laufen — in Formularen ist das meist das falsche. '
+ 'Ziel mit browser_find suchen. Nur wenn du den Fokus selbst gesetzt hast '
+ 'und ihn wirklich meinst: use_focus=true.',
false,
);
}
await notifySpotlight(tabId!, selector, 'tippt');
return await typeInElement(
tabId!, selector, input.text as string, input.clear !== false, !!input.submit,
);
}
case 'browser_key': {
const key = input.key as string;
const modifiers = (input.modifiers as any[]) || [];
if (await wantsTrusted(tabId!)) {
try {
await cdp.attach(tabId!);
await cdp.pressKey(tabId!, key, { modifiers });
return { pressed: key, modifiers, _method: 'cdp' };
} catch (e: any) {
console.warn('[Nexus] CDP-Taste fehlgeschlagen, weiche aus:', e?.message);
}
}
const result = await execInTab(tabId!, (k: string, mods: string[]) => {
const opts: KeyboardEventInit = {
key: k, bubbles: true, cancelable: true,
ctrlKey: mods.includes('ctrl'), altKey: mods.includes('alt'),
shiftKey: mods.includes('shift'), metaKey: mods.includes('meta'),
};
const target = document.activeElement || document.body;
target.dispatchEvent(new KeyboardEvent('keydown', opts));
target.dispatchEvent(new KeyboardEvent('keyup', opts));
return { pressed: k, modifiers: mods };
}, [key, modifiers]);
return { ...result, _method: 'synthetic' };
}
case 'browser_scroll': {
const direction = (input.direction as string) || 'down';
const amount = (input.amount as number) || 500;
const selector = (input.selector as string) || null;
if (!selector && await wantsTrusted(tabId!)) {
try {
await cdp.attach(tabId!);
const m = await cdp.getLayoutMetrics(tabId!);
const cx = m.viewport.w / 2, cy = m.viewport.h / 2;
const dx = direction === 'right' ? amount : direction === 'left' ? -amount : 0;
const dy = direction === 'down' ? amount : direction === 'up' ? -amount : 0;
await cdp.mouseWheel(tabId!, cx, cy, dx, dy);
return { scrolled: direction, amount, _method: 'cdp' };
} catch (e: any) {
console.warn('[Nexus] CDP-Scroll fehlgeschlagen, weiche aus:', e?.message);
}
}
await execInTab(tabId!, (dir: string, amt: number, sel: string | null) => {
const target: any = sel ? document.querySelector(sel) : window;
if (!target) return;
const opts = { behavior: 'instant' as ScrollBehavior };
if (dir === 'down') target.scrollBy({ top: amt, ...opts });
else if (dir === 'up') target.scrollBy({ top: -amt, ...opts });
else if (dir === 'right') target.scrollBy({ left: amt, ...opts });
else if (dir === 'left') target.scrollBy({ left: -amt, ...opts });
}, [direction, amount, selector]);
return { scrolled: direction, amount, _method: 'synthetic' };
}
case 'browser_hover': {
const selector = resolveTarget(input);
if (!selector) return errorResult('NO_TARGET', 'selector oder ref_id erforderlich', false);
const box = await elementBox(tabId!, selector);
if (!box) throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
if (await wantsTrusted(tabId!)) {
try {
await cdp.attach(tabId!);
await cdp.mouseMove(tabId!, box.x, box.y);
notifyCursor(tabId!, box.x, box.y);
return { hovered: selector, tag: box.tag, _method: 'cdp' };
} catch { /* weiter synthetisch */ }
}
await execInTab(tabId!, (sel: string) => {
const el = document.querySelector(sel) as HTMLElement | null;
if (!el) return;
el.dispatchEvent(new MouseEvent('mouseover', { bubbles: true }));
el.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true }));
}, [selector]);
return { hovered: selector, tag: box.tag, _method: 'synthetic' };
}
case 'browser_drag': {
const srcSel = (input.sourceSelector as string) || (input.source_ref_id ? `[data-nexus-ref="${input.source_ref_id}"]` : '');
const tgtSel = (input.targetSelector as string) || (input.target_ref_id ? `[data-nexus-ref="${input.target_ref_id}"]` : '');
if (!srcSel || !tgtSel) return errorResult('NO_TARGET', 'sourceSelector und targetSelector erforderlich', false);
const from = await elementBox(tabId!, srcSel);
const to = await elementBox(tabId!, tgtSel);
if (!from) throw new ToolError('ELEMENT_NOT_FOUND', `Quelle nicht gefunden: ${srcSel}`, true);
if (!to) throw new ToolError('ELEMENT_NOT_FOUND', `Ziel nicht gefunden: ${tgtSel}`, true);
if (await wantsTrusted(tabId!)) {
try {
await cdp.attach(tabId!);
await cdp.mouseDrag(tabId!, { x: from.x, y: from.y }, { x: to.x, y: to.y });
notifyCursor(tabId!, to.x, to.y);
return { dragged: srcSel, target: tgtSel, _method: 'cdp' };
} catch (e: any) {
console.warn('[Nexus] CDP-Drag fehlgeschlagen, weiche aus:', e?.message);
}
}
// Synthetisches Drag&Drop funktioniert nur bei HTML5-DnD, nicht bei Maus-basiertem.
const result = await execInTab(tabId!, (src: string, tgt: string) => {
const s = document.querySelector(src) as HTMLElement | null;
const t = document.querySelector(tgt) as HTMLElement | null;
if (!s || !t) return { error: 'NOT_FOUND' };
const dt = new DataTransfer();
s.dispatchEvent(new DragEvent('dragstart', { dataTransfer: dt, bubbles: true }));
t.dispatchEvent(new DragEvent('dragover', { dataTransfer: dt, bubbles: true }));
t.dispatchEvent(new DragEvent('drop', { dataTransfer: dt, bubbles: true }));
s.dispatchEvent(new DragEvent('dragend', { dataTransfer: dt, bubbles: true }));
return { dragged: src, target: tgt };
}, [srcSel, tgtSel]);
return { ...result, _method: 'synthetic' };
}
// ── Beobachtung ──
case 'browser_screenshot': {
const fullPage = !!input.fullPage;
const maxWidth = (input.maxWidth as number) || 1400;
if (cdp.cdpAvailable()) {
try {
await cdp.attach(tabId!);
const shot = await cdp.captureScreenshot(tabId!, { format: 'jpeg', quality: 70, fullPage, maxWidth });
lastScale.set(tabId!, shot.scale);
return {
image: shot.dataUrl, format: 'jpeg', width: shot.width, height: shot.height,
scale: shot.scale, fullPage, _method: 'cdp',
};
} catch (e: any) {
console.warn('[Nexus] CDP-Screenshot fehlgeschlagen, weiche aus:', e?.message);
}
}
const dataUrl = await chrome.tabs.captureVisibleTab(tab!.windowId!, { format: 'jpeg', quality: 70 });
lastScale.set(tabId!, 1);
return { image: dataUrl, format: 'jpeg', scale: 1, fullPage: false, _method: 'synthetic' };
}
case 'browser_zoom': {
await chrome.tabs.setZoom(tabId!, input.level as number);
return { zoom: input.level, _method: 'synthetic' };
}
case 'browser_resize_window': {
const width = input.width as number;
const height = input.height as number;
const win = await chrome.windows.get(tab!.windowId!);
await chrome.windows.update(win.id!, { width, height, state: 'normal' });
return { width, height, _method: 'synthetic' };
}
case 'browser_get_page_info': {
const info = await execInTab(tabId!, () => ({
title: document.title,
url: location.href,
viewport: { w: window.innerWidth, h: window.innerHeight },
scroll: { x: window.scrollX, y: window.scrollY },
docHeight: document.documentElement.scrollHeight,
readyState: document.readyState,
}));
return { ...info, tabId, _method: 'synthetic' };
}
case 'browser_get_text': {
const selector = (input.selector as string) || null;
const maxLength = (input.maxLength as number) || 8000;
const result = await execInTab(tabId!, (sel: string | null, maxLen: number) => {
const el = sel ? document.querySelector(sel) : document.body;
if (!el) return { error: 'NOT_FOUND' };
const text = (el as HTMLElement).innerText || '';
return { text: text.slice(0, maxLen), truncated: text.length > maxLen, length: text.length };
}, [selector, maxLength]);
if ((result as any)?.error === 'NOT_FOUND') {
throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
}
return { ...result, _method: 'synthetic' };
}
case 'browser_form_input': {
const selector = resolveTarget(input);
if (!selector) return errorResult('NO_TARGET', 'selector oder ref_id erforderlich', false);
const value = String(input.value ?? '');
const result = await execInTab(tabId!, (sel: string, val: string) => {
const el = document.querySelector(sel) as HTMLInputElement | HTMLSelectElement | HTMLTextAreaElement | null;
if (!el) return { error: 'NOT_FOUND' };
const tag = el.tagName.toLowerCase();
const type = ((el as HTMLInputElement).type || '').toLowerCase();
if (tag === 'select') {
const sel2 = el as HTMLSelectElement;
const opt = Array.from(sel2.options).find(o => o.value === val || o.text.trim() === val.trim());
if (!opt) return { error: 'OPTION_NOT_FOUND', message: `Option "${val}" nicht gefunden` };
sel2.value = opt.value;
sel2.dispatchEvent(new Event('input', { bubbles: true }));
sel2.dispatchEvent(new Event('change', { bubbles: true }));
return { set: opt.text, value: opt.value, type: 'select' };
}
if (type === 'checkbox' || type === 'radio') {
const checked = val === 'true' || val === '1' || val === 'on';
(el as HTMLInputElement).checked = checked;
el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true }));
return { set: String(checked), type };
}
el.focus();
// Setter passend zum Elementtyp wählen — sonst "Illegal invocation" bei <textarea>.
const proto = el instanceof HTMLTextAreaElement
? window.HTMLTextAreaElement.prototype
: window.HTMLInputElement.prototype;
const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
if ((el as HTMLElement).isContentEditable) {
(el as HTMLElement).textContent = val;
} else if (setter) {
setter.call(el, val);
} else {
(el as HTMLInputElement).value = val;
}
el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true }));
return { set: val.slice(0, 100), type: type || tag };
}, [selector, value]);
if ((result as any)?.error === 'NOT_FOUND') throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
if ((result as any)?.error === 'OPTION_NOT_FOUND') throw new ToolError('OPTION_NOT_FOUND', (result as any).message, true);
return { ...result, _method: 'synthetic' };
}
case 'browser_select':
return await dispatchTool('browser_form_input', input);
case 'browser_file_upload': {
const selector = resolveTarget(input);
if (!selector) return errorResult('NO_TARGET', 'selector oder ref_id erforderlich', false);
const files = input.files as string[];
if (!Array.isArray(files) || files.length === 0) {
return errorResult('NO_FILES', 'files muss mindestens einen absoluten Pfad enthalten', false);
}
if (!cdp.cdpAvailable()) {
return errorResult('CDP_REQUIRED', 'Datei-Upload benötigt die debugger-Permission', false);
}
await cdp.attach(tabId!);
await cdp.setFileInputFiles(tabId!, selector, files);
return { uploaded: files, selector, _method: 'cdp' };
}
// ── Diagnose ──
case 'browser_read_console': {
const level = (input.level as string) || 'all';
const limit = (input.limit as number) || 50;
let logs = [...consoleLogs];
if (level !== 'all') logs = logs.filter(l => l.level === level);
if (input.filter) {
const rx = new RegExp(input.filter as string, 'i');
logs = logs.filter(l => rx.test(l.text));
}
logs = logs.slice(-limit);
if (input.clear) consoleLogs.length = 0;
return { logs, count: logs.length, _method: 'synthetic' };
}
case 'browser_read_network': {
if (!cdp.cdpAvailable()) {
return errorResult('CDP_REQUIRED', 'Netzwerkmitschnitt benötigt die debugger-Permission', false);
}
await cdp.attach(tabId!);
await cdp.enableNetworkCapture(tabId!);
const entries = cdp.getNetworkRequests(tabId!, {
filter: input.filter as string | undefined,
method: input.method as string | undefined,
limit: (input.limit as number) || 50,
includeBody: !!input.includeBody,
});
return { requests: entries, count: entries.length, _method: 'cdp' };
}
case 'browser_execute_js': {
const code = input.code as string;
// MAIN world: nur so sind window-Globals der Anwendung (React, Angular, App-State) sichtbar.
if (cdp.cdpAvailable()) {
try {
await cdp.attach(tabId!);
// Runtime.evaluate wertet einen Ausdruck aus — ein `return` waere dort ein
// Syntaxfehler. Der Code laeuft deshalb in einer async-IIFE: damit gilt dieselbe
// Semantik wie beim frueheren new Function(js), und `await` ist zusaetzlich moeglich.
const wrapped = `(async () => {\n${code}\n})()`;
const value = await cdp.evaluateInMainWorld(tabId!, wrapped, { awaitPromise: true });
return { value, world: 'MAIN', _method: 'cdp' };
} catch (e: any) {
return { error: { code: 'JS_ERROR', message: e?.message || String(e), retryable: false }, _method: 'cdp' };
}
}
const results = await chrome.scripting.executeScript({
target: { tabId: tabId! },
// as any: die chrome-types-Überladung erkennt parametrisierte func nicht zuverlässig.
func: ((js: string) => {
try {
const r = new Function(js)();
return { value: JSON.parse(JSON.stringify(r ?? null)) };
} catch (e: any) { return { error: e.message }; }
}) as any,
args: [code],
});
const res = results?.[0]?.result as any;
if (res?.error) return { error: { code: 'JS_ERROR', message: res.error, retryable: false }, _method: 'synthetic' };
return { value: res?.value, world: 'ISOLATED', _method: 'synthetic' };
}
case 'browser_wait': {
const selector = input.selector as string | undefined;
const timeout = Math.min((input.timeout as number) || 5000, 120_000);
if (!selector) {
await new Promise(r => setTimeout(r, timeout));
return { waited: timeout, _method: 'synthetic' };
}
const deadline = Date.now() + timeout;
const visible = input.visible !== false;
// Im Service Worker pollen statt in der Seite: überlebt Navigationen während des Wartens.
while (Date.now() < deadline) {
const found = await execInTab(tabId!, (sel: string, vis: boolean) => {
const el = document.querySelector(sel) as HTMLElement | null;
if (!el) return false;
return !vis || (el.offsetWidth > 0 && el.offsetHeight > 0);
}, [selector, visible]).catch(() => false);
if (found) return { found: true, selector, _method: 'synthetic' };
await new Promise(r => setTimeout(r, 150));
}
return { found: false, timeout: true, selector, _method: 'synthetic' };
}
case 'browser_highlight': {
const selector = resolveTarget(input) || (input.selector as string);
const color = (input.color as string) || '#ff6b35';
const duration = (input.duration as number) || 3000;
const result = await execInTab(tabId!, (sel: string, col: string, dur: number) => {
const el = document.querySelector(sel) as HTMLElement | null;
if (!el) return { error: 'NOT_FOUND' };
const prev = el.style.outline;
el.style.outline = `3px solid ${col}`;
el.style.outlineOffset = '2px';
setTimeout(() => { el.style.outline = prev; el.style.outlineOffset = ''; }, dur);
return { highlighted: sel };
}, [selector, color, duration]);
if ((result as any)?.error === 'NOT_FOUND') throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
return { ...result, _method: 'synthetic' };
}
// ── Seitenstruktur ──
case 'browser_read_page':
return await readPage(tabId!, input);
case 'browser_find':
return await findElements(tabId!, input);
// ── Mehrere Aktionen in einem Aufruf ──
case 'browser_batch': {
const actions = (input.actions as Array<{ name: string; input?: Record<string, unknown> }>) || [];
const stopOnError = input.stopOnError !== false;
const results: any[] = [];
for (const action of actions) {
try {
// Über runTool, damit auch Teilaktionen durchs Risiko-Gate gehen.
const r = await runTool(action.name, action.input || {});
results.push({ name: action.name, ok: !r?.error, result: r });
if (r?.error && stopOnError) break;
} catch (e: any) {
results.push({ name: action.name, ok: false, error: e?.message || String(e) });
if (stopOnError) break;
}
}
return { results, count: results.length, _method: 'synthetic' };
}
default:
return errorResult('UNKNOWN_TOOL', `Tool '${name}' ist nicht implementiert`, false);
}
}
// ─── read_page: A11y-Baum mit stabilen Refs ───────────────────────────────────
async function readPage(tabId: number, input: Record<string, unknown>): Promise<any> {
const opts = {
viewportOnly: input.viewportOnly !== false,
maxDepth: (input.maxDepth as number) || 12,
maxTokens: (input.maxTokens as number) || 6000,
includeHidden: !!input.includeHidden,
selector: (input.selector as string) || null,
};
const tree = await execInTab(tabId, (options: any) => {
const INTERACTIVE_ROLES = new Set([
'button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'listbox',
'menuitem', 'tab', 'switch', 'slider', 'spinbutton', 'searchbox', 'option',
'menuitemcheckbox', 'menuitemradio',
]);
const INTERACTIVE_TAGS = new Set(['a', 'button', 'input', 'select', 'textarea', 'details', 'summary']);
const LANDMARK_ROLES = new Set(['banner', 'navigation', 'main', 'complementary', 'contentinfo', 'search', 'form', 'region']);
const SKIP_TAGS = new Set(['script', 'style', 'noscript', 'svg', 'path', 'template']);
let refCounter = 0;
let tokenEstimate = 0;
const TOKEN_PER_CHAR = 0.25;
// Alte Refs entfernen, damit ref_ids nicht über Läufe hinweg kollidieren.
document.querySelectorAll('[data-nexus-ref]').forEach(el => el.removeAttribute('data-nexus-ref'));
function isVisible(el: HTMLElement): boolean {
if (options.includeHidden) return true;
const style = getComputedStyle(el);
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false;
if (el.getAttribute('aria-hidden') === 'true') return false;
return true;
}
function isInViewport(el: HTMLElement): boolean {
if (!options.viewportOnly) return true;
const r = el.getBoundingClientRect();
return r.bottom > 0 && r.top < window.innerHeight && r.right > 0 && r.left < window.innerWidth;
}
function getRole(el: HTMLElement): string {
const explicit = el.getAttribute('role');
if (explicit) return explicit;
const tag = el.tagName.toLowerCase();
if (tag === 'a' && el.hasAttribute('href')) return 'link';
if (tag === 'button') return 'button';
if (tag === 'input') {
const type = (el as HTMLInputElement).type;
// Submit-Knöpfe MÜSSEN von Textfeldern unterscheidbar sein — sonst kann die
// Risikoerkennung "Formular absenden" nicht erkennen.
if (type === 'submit' || type === 'button' || type === 'reset' || type === 'image') return 'button';
if (type === 'checkbox') return 'checkbox';
if (type === 'radio') return 'radio';
if (type === 'range') return 'slider';
if (type === 'file') return 'fileupload';
if (type === 'search') return 'searchbox';
return 'textbox';
}
if (tag === 'select') return 'combobox';
if (tag === 'textarea') return 'textbox';
if (tag === 'nav') return 'navigation';
if (tag === 'main') return 'main';
if (tag === 'header') return 'banner';
if (tag === 'footer') return 'contentinfo';
if (tag === 'aside') return 'complementary';
if (tag === 'form') return 'form';
return '';
}
function getLabel(el: HTMLElement): string {
const aria = el.getAttribute('aria-label');
if (aria) return aria;
const labelledBy = el.getAttribute('aria-labelledby');
if (labelledBy) {
// aria-labelledby ist eine LISTE von IDs. Mit getElementById(labelledBy)
// schlug jede Mehrfach-Referenz fehl — und genau so beschriftet Outlook
// Web den Nachrichtentext. Ohne Beschriftung sah er für das Modell aus
// wie das Betreff-Feld.
const text = labelledBy.split(/\s+/)
.map(id => document.getElementById(id)?.textContent || '')
.join(' ').trim();
if (text) return text.slice(0, 80);
}
if ((el as HTMLInputElement).labels?.length) {
return ((el as HTMLInputElement).labels![0].textContent || '').trim().slice(0, 80);
}
return el.getAttribute('title') || el.getAttribute('placeholder') || el.getAttribute('name') || '';
}
function serialize(el: HTMLElement, depth: number): any {
if (depth > options.maxDepth || tokenEstimate > options.maxTokens) return null;
if (SKIP_TAGS.has(el.tagName.toLowerCase())) return null;
if (!isVisible(el)) return null;
const tag = el.tagName.toLowerCase();
const role = getRole(el);
// isContentEditable zählt mit: Rich-Text-Felder (Mail-Text, Editoren) sind
// oft ein <div> ohne role und ohne tabindex — ohne diesen Zweig fehlen sie
// im Baum, und das Modell tippt mangels Alternative ins Betreff-Feld.
const isInteractive = INTERACTIVE_ROLES.has(role) || INTERACTIVE_TAGS.has(tag) ||
el.isContentEditable ||
el.hasAttribute('onclick') || el.hasAttribute('tabindex');
const isLandmark = LANDMARK_ROLES.has(role);
if (!isInteractive && !isLandmark && !isInViewport(el)) {
const children: any[] = [];
for (const child of Array.from(el.children)) {
if (tokenEstimate > options.maxTokens) break;
const c = serialize(child as HTMLElement, depth + 1);
if (c) children.push(c);
}
return children.length === 1 ? children[0] : children.length ? { children } : null;
}
const node: any = {};
if (isInteractive) {
const ref = `r${++refCounter}`;
el.setAttribute('data-nexus-ref', ref);
node.ref = ref;
// Koordinaten mitgeben, damit das Modell nahtlos auf browser_computer wechseln kann.
const r = el.getBoundingClientRect();
if (r.width || r.height) {
node.rect = { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) };
node.center = [Math.round(r.x + r.width / 2), Math.round(r.y + r.height / 2)];
}
}
if (role) node.role = role;
node.tag = tag;
// type ist für die Risikoerkennung entscheidend (submit vs. text).
if (tag === 'input' || tag === 'button') {
const t = (el as HTMLInputElement).type;
if (t) node.type = t;
}
const label = getLabel(el);
if (label) node.label = label.slice(0, 80);
if (el.children.length === 0 || isInteractive) {
const text = (el.textContent || '').trim().replace(/\s+/g, ' ');
if (text && text.length <= 120) node.text = text;
else if (text) node.text = text.slice(0, 100) + '…';
}
// Für das Modell der entscheidende Unterschied zwischen Betreff (<input>)
// und Nachrichtentext (contenteditable) — beide melden role="textbox".
if (el.isContentEditable) {
node.editable = true;
const val = (el.textContent || '').trim();
if (val) node.value = val.slice(0, 60);
}
if (tag === 'input' || tag === 'textarea' || tag === 'select') {
const val = (el as HTMLInputElement).value;
if (val) node.value = val.slice(0, 60);
if (typeof (el as HTMLInputElement).checked === 'boolean') node.checked = (el as HTMLInputElement).checked;
if ((el as HTMLInputElement).disabled) node.disabled = true;
if ((el as HTMLInputElement).required) node.required = true;
}
if (tag === 'a') {
const href = el.getAttribute('href');
if (href && href !== '#') node.href = href.slice(0, 120);
}
if (tag === 'form') {
const action = el.getAttribute('action');
if (action) node.action = action.slice(0, 120);
node.method = (el.getAttribute('method') || 'get').toLowerCase();
}
tokenEstimate += JSON.stringify(node).length * TOKEN_PER_CHAR;
if (el.children.length) {
const children: any[] = [];
for (const child of Array.from(el.children)) {
if (tokenEstimate > options.maxTokens) break;
const c = serialize(child as HTMLElement, depth + 1);
if (c) children.push(c);
}
if (children.length) node.children = children;
}
return node;
}
const root = options.selector
? document.querySelector(options.selector) as HTMLElement
: document.body;
if (!root) return { error: 'Wurzelelement nicht gefunden' };
const serialized = serialize(root, 0);
return {
tree: serialized,
meta: {
url: location.href,
title: document.title,
refCount: refCounter,
tokenEstimate: Math.round(tokenEstimate),
truncated: tokenEstimate > options.maxTokens,
viewport: { w: window.innerWidth, h: window.innerHeight },
},
};
}, [opts]);
return { ...tree, _method: 'synthetic' };
}
// ─── find: Element per natürlicher Beschreibung ───────────────────────────────
async function findElements(tabId: number, input: Record<string, unknown>): Promise<any> {
const query = String(input.query || '').trim();
if (!query) return errorResult('NO_QUERY', 'query erforderlich', false);
const limit = (input.limit as number) || 5;
const matches = await execInTab(tabId, (q: string, lim: number) => {
const terms = q.toLowerCase().split(/\s+/).filter(t => t.length > 1);
const SEL = 'a,button,input,select,textarea,summary,[role],[onclick],[tabindex]';
const candidates = Array.from(document.querySelectorAll(SEL)) as HTMLElement[];
// Alte Refs bleiben gültig, neue nur für Treffer ohne Ref vergeben.
let counter = document.querySelectorAll('[data-nexus-ref]').length;
const scored = candidates.map(el => {
const style = getComputedStyle(el);
if (style.display === 'none' || style.visibility === 'hidden') return null;
const r = el.getBoundingClientRect();
if (!r.width && !r.height) return null;
const haystack = [
el.getAttribute('aria-label'), el.getAttribute('title'), el.getAttribute('placeholder'),
el.getAttribute('name'), el.getAttribute('value'), el.id,
(el as HTMLInputElement).labels?.[0]?.textContent,
el.innerText || el.textContent,
].filter(Boolean).join(' ').toLowerCase().replace(/\s+/g, ' ');
if (!haystack) return null;
let score = 0;
for (const t of terms) {
if (haystack.includes(t)) score += 2;
// Wortanfang zählt stärker als ein Treffer irgendwo in der Mitte.
if (new RegExp('\\b' + t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).test(haystack)) score += 1;
}
if (!score) return null;
// Kürzere Beschriftungen sind meist die genauere Entsprechung.
score += Math.max(0, 3 - haystack.length / 60);
// Sichtbares im Viewport bevorzugen.
if (r.top >= 0 && r.top < window.innerHeight) score += 2;
let ref = el.getAttribute('data-nexus-ref');
if (!ref) { ref = `r${++counter}`; el.setAttribute('data-nexus-ref', ref); }
return {
ref_id: ref,
tag: el.tagName.toLowerCase(),
type: (el as HTMLInputElement).type || undefined,
role: el.getAttribute('role') || undefined,
// value zaehlt mit: <input type="submit" value="Bestellen"> hat kein innerText,
// der Treffer waere sonst ohne jede Beschriftung und fuer das Modell wertlos.
text: (el.innerText || el.textContent || (el as HTMLInputElement).value || '')
.trim().replace(/\s+/g, ' ').slice(0, 100),
label: el.getAttribute('aria-label') || el.getAttribute('placeholder') || undefined,
href: el.getAttribute('href') || undefined,
center: [Math.round(r.x + r.width / 2), Math.round(r.y + r.height / 2)],
rect: { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) },
score: Math.round(score * 10) / 10,
};
}).filter(Boolean) as any[];
scored.sort((a, b) => b.score - a.score);
return scored.slice(0, lim);
}, [query, limit]);
return { query, matches, count: matches.length, _method: 'synthetic' };
}
// ─── Eingebauter Agent-Loop ───────────────────────────────────────────────────
/**
* Gedächtnis-Werkzeuge für den direct-Modus.
*
* Sie stehen NICHT in tools/schema/, weil der Broker-Pfad sie über
* agent/memory_tools.py schon hat — dort registriert sie die Python-Seite. Hier
* wären sie sonst doppelt. Beide Wege schreiben in denselben lokalen Speicher.
*/
const MEMORY_TOOL_DEFS: LlmToolDef[] = [
{
name: 'memory_save',
description: 'Merkt sich etwas über den Nutzer dauerhaft — Präferenz, Schreibstil, '
+ 'Fakt oder laufendes Projekt. Nutze es, wenn er eine Vorliebe äußert, dich '
+ 'korrigiert oder etwas über sich sagt, das später wieder zählt. Nicht für '
+ 'Arbeitsergebnisse.',
inputSchema: {
type: 'object',
properties: {
text: { type: 'string', description: 'Die Notiz, ein Satz.' },
type: { type: 'string', enum: ['praeferenz', 'schreibstil', 'fakt', 'projekt'] },
tags: { type: 'array', items: { type: 'string' } },
},
required: ['text'],
},
},
{
name: 'memory_search',
description: 'Sucht in den Notizen über den Nutzer. Die wichtigsten stehen schon im '
+ 'Kontext — nutze es gezielt, etwa für den Schreibstil vor einer Mail.',
inputSchema: {
type: 'object',
properties: {
query: { type: 'string', description: 'Wonach du suchst.' },
limit: { type: 'integer' },
},
required: ['query'],
},
},
];
function toolDefs(): LlmToolDef[] {
return [
...BROWSER_TOOL_SCHEMAS.map(s => ({
name: s.name,
description: s.description,
inputSchema: s.inputSchema,
})),
...MEMORY_TOOL_DEFS,
];
}
async function startBuiltinRun(
text: string,
maxRounds?: number,
isContinue = false,
images?: { mediaType: string; data: string; name?: string }[],
): Promise<void> {
runStartedAt = Date.now();
setRunBadge(true);
notifyGlow(true);
try {
await runAgentTurn(text, {
images,
// URL des Ziel-Tabs: entscheidet, ob das Mercedes-Wissen mitgeschickt wird.
currentUrl: async () => {
const id = await currentTabId().catch(() => undefined);
if (typeof id !== 'number') return undefined;
return (await chrome.tabs.get(id).catch(() => null))?.url;
},
runTool: (name, input) => runTool(name, input),
tools: toolDefs(),
emit: (msg) => {
// Screenshots aus tool_done herauslösen, damit das Panel keine Base64-Wand rendert.
if (msg?.type === 'tool_done') {
reportToolDone(msg.callId, msg.name, msg.result, msg.error, msg.durationMs, msg.method);
return;
}
sendToPanel(msg);
},
maxRounds,
isContinue,
});
} finally {
runStartedAt = 0;
setRunBadge(false);
notifyGlow(false);
}
}
// ─── Panel-Kommunikation ──────────────────────────────────────────────────────
/**
* Ereignispuffer — damit ein Lauf das Panel überlebt.
*
* Chrome haelt den Service Worker am Leben, das Panel aber nicht. Vorher stand
* hier nur `if (!panelPort) return;`: der Agent arbeitete nach dem Schliessen
* weiter und redete gegen eine Wand, beim Oeffnen war alles fort. Jetzt bekommt
* jedes Ereignis eine Sequenznummer und landet im Puffer; das Panel holt beim
* Anmelden alles ab seinem letzten Stand nach.
*/
const EVENT_BUFFER_MAX = 2000;
let eventSeq = 0;
const eventBuffer: { seq: number; msg: any }[] = [];
/** Beginn des laufenden Auftrags, fuer "Lauf aktiv seit N min". */
let runStartedAt = 0;
function sendToPanel(msg: object): void {
const stamped: any = { ...msg, _seq: ++eventSeq };
eventBuffer.push({ seq: stamped._seq, msg: stamped });
if (eventBuffer.length > EVENT_BUFFER_MAX) {
eventBuffer.splice(0, eventBuffer.length - EVENT_BUFFER_MAX);
}
if (!panelPort) return;
try { panelPort.postMessage(stamped); } catch { panelPort = null; }
}
/** Badge am Extension-Symbol: ein aktiver Lauf ist sichtbar, ohne die Sidebar zu oeffnen. */
function setRunBadge(active: boolean): void {
try {
chrome.action?.setBadgeText({ text: active ? '●' : '' });
chrome.action?.setBadgeBackgroundColor({ color: '#3fb27f' });
} catch { /* Badge ist Beiwerk */ }
}
chrome.runtime.onConnect.addListener((port) => {
if (port.name !== 'nexus-panel') return;
panelPort = port;
sendToPanel({ type: 'connection_status', connected: ws?.readyState === WebSocket.OPEN });
port.onMessage.addListener(async (msg: PanelMessage) => {
switch (msg.type) {
case 'user_message': {
const cfg = await loadConfig();
if (cfg.driveMode === 'direct') {
if (isAgentRunning()) {
// Lauf läuft — Nachricht in Queue puffern
queueMessage(msg.text);
sendToPanel({ type: 'log', level: 'info', text: `📥 Nachricht gepuffert — wird nach aktuellem Turn ausgeführt.` });
} else {
await startBuiltinRun(msg.text, msg.maxRounds ?? cfg.maxRounds, msg.isContinue, msg.images);
}
} else {
// Der Broker-Kanal kennt keine Bilder — lieber sagen als still schlucken.
if (msg.images?.length) {
sendToPanel({ type: 'log', level: 'warn',
text: `${msg.images.length} Bild(er) ignoriert: Bildanhänge gehen nur im Antrieb "direct".` });
}
sendToBroker({ type: 'user_message', text: msg.text, maxRounds: msg.maxRounds ?? cfg.maxRounds } satisfies ExtUserMessage);
}
break;
}
case 'sync': {
// Nachliefern statt neu erzeugen: der Puffer ist die Wahrheit.
const from = typeof msg.from === 'number' ? msg.from : 0;
const missed = eventBuffer.filter(e => e.seq > from);
if (panelPort) {
try {
panelPort.postMessage({
type: 'sync_start', count: missed.length, running: isAgentRunning(),
latest: eventSeq, startedAt: runStartedAt || undefined,
});
// Direkt posten, nicht ueber sendToPanel — sonst landet alles erneut im Puffer.
for (const e of missed) panelPort.postMessage(e.msg);
} catch { panelPort = null; }
}
break;
}
case 'abort':
abortAgent();
sendToBroker({ type: 'abort' } satisfies ExtAbort);
break;
case 'approval_response': {
// Kann von unserem Risiko-Gate ODER vom Broker stammen.
const resolver = pendingRisk.get(msg.requestId);
if (resolver) resolver(msg.approved);
else sendToBroker({ type: 'approval_response', requestId: msg.requestId, approved: msg.approved } satisfies ExtApprovalResponse);
break;
}
case 'connect': connect(); break;
case 'disconnect': ws?.close(); break;
case 'set_config': {
const next = await patchConfig(msg.patch);
if (msg.patch.brokerUrl) { ws?.close(); setTimeout(connect, 300); }
sendToPanel({ type: 'state', config: next, connected: ws?.readyState === WebSocket.OPEN, running: isAgentRunning(), version: EXTENSION_VERSION });
break;
}
case 'get_state': {
const cfg = await loadConfig();
sendToPanel({ type: 'state', config: cfg, connected: ws?.readyState === WebSocket.OPEN, running: isAgentRunning(), version: EXTENSION_VERSION });
break;
}
case 'run_tool': {
const callId = crypto.randomUUID();
sendToPanel({ type: 'tool_executing', callId, name: msg.name, input: msg.input });
const started = performance.now();
try {
const result = await runTool(msg.name, msg.input);
const method = (result?._method as InteractionMethod) || 'synthetic';
if (result && typeof result === 'object') delete result._method;
reportToolDone(callId, msg.name, result, undefined, Math.round(performance.now() - started), method);
} catch (e: any) {
reportToolDone(callId, msg.name, null, e?.message || String(e), Math.round(performance.now() - started), 'synthetic');
}
break;
}
case 'new_conversation':
case 'reset_agent':
resetConversation();
targetTabId = null; // Neuer Auftrag, neuer Tab-Kontext
break;
}
});
port.onDisconnect.addListener(() => { panelPort = null; });
});
// ─── Content-Script-Meldungen ─────────────────────────────────────────────────
chrome.runtime.onMessage.addListener((msg, sender): undefined => {
if (msg?.type === 'CONSOLE_LOG' || msg?.type === 'PAGE_ERROR') {
consoleLogs.push({
level: msg.type === 'PAGE_ERROR' ? 'error' : msg.level,
text: msg.text,
ts: Date.now(),
url: sender.tab?.url,
});
if (consoleLogs.length > MAX_CONSOLE_LOGS) consoleLogs.shift();
}
});
// ─── Aufräumen ────────────────────────────────────────────────────────────────
function cleanupTab(tabId: number): void {
lastScale.delete(tabId);
cdp.detach(tabId).catch(() => {});
}
chrome.tabs.onRemoved.addListener(cleanupTab);
chrome.runtime.onSuspend?.addListener(() => { cdp.detachAll().catch(() => {}); });
// ─── Start ────────────────────────────────────────────────────────────────────
chrome.alarms.create('keepAlive', { periodInMinutes: 0.5 });
chrome.alarms.onAlarm.addListener(() => { /* hält den Worker wach */ });
chrome.action.onClicked.addListener(async (tab) => {
await chrome.sidePanel.open({ windowId: tab.windowId! });
});
chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true }).catch(() => {});
/**
* Prüf-Oberfläche für automatisierte Tests.
*
* Sie liegt im globalThis des Service Workers und ist damit ausschließlich aus dem
* Worker-Kontext erreichbar — über die DevTools oder CDP. Webseiten und andere
* Extensions kommen hier nicht heran, es entsteht also keine Angriffsfläche.
*/
(globalThis as any).__nexus = {
version: EXTENSION_VERSION,
runTool,
dispatchTool,
cdp,
loadConfig,
patchConfig,
};
loadConfig().then((cfg) => {
if (cfg.autoConnect && cfg.driveMode === 'broker') connect();
console.log(`[Nexus] Browser Pilot ${EXTENSION_VERSION} bereit — CDP ${cdp.cdpAvailable() ? 'verfügbar' : 'NICHT verfügbar'}, Risikomodus: ${cfg.riskMode}`);
});
export { resetConversation };