Nexus gateway (measured 2026-08-07, not assumed):
- Gemini speaks Google-GenAI (/v1beta/models/{id}:streamGenerateContent, header
api-key), NOT the Azure-OpenAI path — that returned 404 "no Route matched" and
was the cause of the reported failures. New agent/gemini_bridge.py translates
Bedrock Converse <-> Gemini in both directions.
- Only gemini-3.6-flash is subscribed; 2.5-flash/2.5-pro/3.1-flash-lite give 403,
every other name 404. Catalog corrected.
- Four Gemini rules, each previously an HTTP 400, now covered by tests:
thought signatures are mandatory, they belong to the TURN (not the individual
call), functionResponse turns must be homogeneous, arrays need `items`.
- Prompt caching is NOT available: cachePoint is accepted and ignored.
System prompt:
- Was an f-string; a code sample containing braces broke build_system_prompt at
request time (CLI and web both 500, import stayed green). Now a plain template
with __TOKEN__ placeholders. Regression guards in tests/test_system_prompt.py.
CLI:
- `agent resume` now prints the stored transcript. The history was always loaded
into the model context, only the terminal stayed empty.
Memory (new, all three surfaces):
- agent/memory.py stores notes about the user in one local file, written
atomically; memory.cnull.net remains an optional mirror that can never fail a
write. Tools memory_save/search/forget, injected into the prompt with a budget.
HTTP surface /api/memory for the extension.
Browser extension (agent/extension, first commit of the source):
- driveMode 'direct' talks to Nexus without the Python broker: Claude via
Bedrock converse, GPT via Azure-OpenAI, Gemini via Google-GenAI.
- browser_type no longer guesses the focused element — that wrote whole mails
into Outlook's subject line. Read-back now reports where the text actually
landed, so a mis-target is visible instead of silent.
- aria-labelledby is resolved across all ids (it is a list); contenteditable is
interactive and marked editable. Without this, subject and message body look
identical to the model.
- Hard block against sending mail, independent of riskMode.
- Runs survive the panel: events are buffered and replayed by sequence number.
- Image input (paste, file, drag&drop), on-page glow/spotlight, memory tools.
Cost: fixed tokens per round 11434 -> 6540 (-43%) by trimming tool schemas,
dropping gateway docs from the browser prompt and sending site knowledge only
where it applies.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1760 lines
72 KiB
TypeScript
1760 lines
72 KiB
TypeScript
/**
|
||
* Nexus Browser Pilot – Background Service Worker
|
||
*
|
||
* Der Service Worker ist die einzige Stelle, an der Browser-Tools wirklich laufen.
|
||
* Alle drei Antriebe münden in denselben Engpass runTool():
|
||
*
|
||
* Broker (Python, WebSocket) ─┐
|
||
* MCP-Bridge (WebSocket) ─┼─→ runTool() → Risiko-Gate → dispatchTool()
|
||
* Eingebauter Loop (agent.ts)─┘
|
||
*
|
||
* Dadurch gilt der Risikomodus für jeden Antrieb, nicht nur für den Chat im Panel.
|
||
* Der Agent-Loop lebt bewusst hier und nicht im Side Panel: ein geschlossenes Panel
|
||
* darf einen laufenden Auftrag nicht abbrechen.
|
||
*/
|
||
|
||
import type {
|
||
BrokerMessage,
|
||
ExtHello,
|
||
ExtToolResult,
|
||
ExtUserMessage,
|
||
ExtAbort,
|
||
ExtApprovalResponse,
|
||
PanelMessage,
|
||
RiskVerdict,
|
||
InteractionMethod,
|
||
} from '../shared/protocol';
|
||
import {
|
||
loadConfig,
|
||
patchConfig,
|
||
EXTENSION_VERSION,
|
||
RECONNECT_INTERVAL_MS,
|
||
RECONNECT_MAX_ATTEMPTS,
|
||
type ExtensionConfig,
|
||
} from '../shared/config';
|
||
import { BROWSER_TOOL_SCHEMAS } from '../shared/tool-schemas.generated';
|
||
import * as cdp from './cdp';
|
||
import { runComputer } from './computer';
|
||
import { gate, setScaleProvider } from './risk';
|
||
import { localSave as memoryLocalSave, localSearch as memoryLocalSearch } from './memory';
|
||
import { runAgentTurn, abortAgent, isAgentRunning, resetConversation, queueMessage } from './agent';
|
||
import type { LlmToolDef } from './llm';
|
||
|
||
// ─── State ────────────────────────────────────────────────────────────────────
|
||
|
||
let ws: WebSocket | null = null;
|
||
let reconnectAttempts = 0;
|
||
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||
let sessionSecret = '';
|
||
let panelPort: chrome.runtime.Port | null = null;
|
||
|
||
/**
|
||
* Target-Tab: Der Agent arbeitet auf diesem Tab, unabhängig davon, welcher Tab
|
||
* gerade im Vordergrund ist. Wird beim ersten Tool-Aufruf auf den aktiven Tab
|
||
* gesetzt und bleibt dort, bis der Nutzer explizit wechselt (browser_tabs_select)
|
||
* oder eine neue Konversation startet.
|
||
*/
|
||
let targetTabId: number | null = null;
|
||
|
||
/**
|
||
* Pause zwischen Tool-Aufrufen in ms. Macht den Agent weniger hektisch und gibt
|
||
* Seiten Zeit zum Rendern. Konfigurierbar über die Config.
|
||
*/
|
||
const TOOL_PACE_MS = 600;
|
||
|
||
/** Offene Risiko-Rückfragen: requestId → Auflöser. */
|
||
const pendingRisk = new Map<string, (approved: boolean) => void>();
|
||
|
||
/**
|
||
* Wartezeit für eine Risiko-Rückfrage. Läuft sie ab, wird AUSGEFÜHRT, nicht abgebrochen.
|
||
* Der Nutzer hat Autonomie verlangt; Schweigen darf keinen stillen Abbruch bedeuten.
|
||
* Nur eine ausdrückliche Ablehnung stoppt.
|
||
*/
|
||
const RISK_TIMEOUT_MS = 120_000;
|
||
|
||
/** Konsolenpuffer, gefüllt aus dem Content-Script. */
|
||
const consoleLogs: Array<{ level: string; text: string; ts: number; url?: string }> = [];
|
||
const MAX_CONSOLE_LOGS = 300;
|
||
|
||
/** Letzter Screenshot-Maßstab je Tab — für die Koordinaten-Rückrechnung. */
|
||
const lastScale = new Map<number, number>();
|
||
|
||
// Die Risiko-Engine muss Modell-Koordinaten in CSS-Pixel zurückrechnen können,
|
||
// bevor sie per elementFromPoint nachsieht, worauf da eigentlich geklickt wird.
|
||
setScaleProvider((tabId) => lastScale.get(tabId) ?? 1);
|
||
|
||
// ─── Session Secret ───────────────────────────────────────────────────────────
|
||
|
||
async function getOrCreateSecret(): Promise<string> {
|
||
const stored = await chrome.storage.session.get(['nexus_secret']);
|
||
if (stored.nexus_secret) return stored.nexus_secret;
|
||
const secret = crypto.randomUUID();
|
||
await chrome.storage.session.set({ nexus_secret: secret });
|
||
return secret;
|
||
}
|
||
|
||
// ─── WebSocket zum Antrieb (Broker oder MCP-Bridge) ───────────────────────────
|
||
|
||
async function connect(): Promise<void> {
|
||
if (ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) return;
|
||
|
||
const config = await loadConfig();
|
||
sessionSecret = await getOrCreateSecret();
|
||
|
||
try {
|
||
ws = new WebSocket(config.brokerUrl);
|
||
} catch (e) {
|
||
console.error('[Nexus] WS connect error:', e);
|
||
scheduleReconnect();
|
||
return;
|
||
}
|
||
|
||
ws.onopen = () => {
|
||
reconnectAttempts = 0;
|
||
const hello: ExtHello = { type: 'hello', version: EXTENSION_VERSION, secret: sessionSecret };
|
||
ws!.send(JSON.stringify(hello));
|
||
sendToPanel({ type: 'connection_status', connected: true });
|
||
};
|
||
|
||
ws.onmessage = (event) => {
|
||
try {
|
||
handleBrokerMessage(JSON.parse(event.data as string) as BrokerMessage);
|
||
} catch (e) {
|
||
console.error('[Nexus] Failed to parse broker message:', e);
|
||
}
|
||
};
|
||
|
||
ws.onclose = () => {
|
||
ws = null;
|
||
sendToPanel({ type: 'connection_status', connected: false });
|
||
scheduleReconnect();
|
||
};
|
||
|
||
ws.onerror = () => { /* onclose folgt */ };
|
||
}
|
||
|
||
function scheduleReconnect(): void {
|
||
if (reconnectTimer) return;
|
||
if (reconnectAttempts >= RECONNECT_MAX_ATTEMPTS) {
|
||
sendToPanel({ type: 'connection_status', connected: false, maxRetries: true });
|
||
return;
|
||
}
|
||
reconnectAttempts++;
|
||
const delay = Math.min(RECONNECT_INTERVAL_MS * reconnectAttempts, 30_000);
|
||
reconnectTimer = setTimeout(() => { reconnectTimer = null; connect(); }, delay);
|
||
}
|
||
|
||
function sendToBroker(msg: object): void {
|
||
if (ws && ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(msg));
|
||
}
|
||
|
||
async function handleBrokerMessage(msg: BrokerMessage): Promise<void> {
|
||
switch (msg.type) {
|
||
case 'welcome':
|
||
case 'text_delta':
|
||
case 'round':
|
||
case 'done':
|
||
case 'error':
|
||
case 'approval_request':
|
||
case 'tool_result_event':
|
||
case 'mcp_status':
|
||
sendToPanel(msg);
|
||
break;
|
||
case 'tool_call':
|
||
await executeForBroker(msg.callId, msg.name, msg.input);
|
||
break;
|
||
case 'ping':
|
||
sendToBroker({ type: 'pong', ts: msg.ts });
|
||
break;
|
||
}
|
||
}
|
||
|
||
// ─── Tool-Ausführung ──────────────────────────────────────────────────────────
|
||
|
||
async function executeForBroker(callId: string, name: string, input: Record<string, unknown>): Promise<void> {
|
||
const start = performance.now();
|
||
let method: InteractionMethod = 'synthetic';
|
||
|
||
try {
|
||
sendToPanel({ type: 'tool_executing', callId, name, input });
|
||
const result = await runTool(name, input);
|
||
method = (result?._method as InteractionMethod) || 'synthetic';
|
||
if (result && typeof result === 'object') delete result._method;
|
||
|
||
const durationMs = Math.round(performance.now() - start);
|
||
sendToBroker({ type: 'tool_result', callId, result, durationMs, method } satisfies ExtToolResult);
|
||
reportToolDone(callId, name, result, undefined, durationMs, method);
|
||
} catch (e: unknown) {
|
||
const error = e instanceof Error ? e.message : String(e);
|
||
const durationMs = Math.round(performance.now() - start);
|
||
sendToBroker({ type: 'tool_result', callId, result: null, error, durationMs, method } satisfies ExtToolResult);
|
||
reportToolDone(callId, name, null, error, durationMs, method);
|
||
}
|
||
}
|
||
|
||
/** Ergebnis ans Panel — Bilder gehen getrennt, damit tool_done klein bleibt. */
|
||
function reportToolDone(
|
||
callId: string, name: string, result: any, error: string | undefined,
|
||
durationMs: number, method: InteractionMethod,
|
||
): void {
|
||
if (result && typeof result === 'object' && typeof result.image === 'string') {
|
||
sendToPanel({ type: 'screenshot', callId, dataUrl: result.image });
|
||
const { image, ...rest } = result;
|
||
sendToPanel({ type: 'tool_done', callId, name, result: { ...rest, image: '[siehe Vorschau]' }, error, durationMs, method });
|
||
return;
|
||
}
|
||
sendToPanel({ type: 'tool_done', callId, name, result, error, durationMs, method });
|
||
}
|
||
|
||
/** Sendet den aktuellen Target-Tab ans Panel. */
|
||
async function notifyTargetTab(): Promise<void> {
|
||
if (targetTabId === null) {
|
||
sendToPanel({ type: 'target_tab', tabId: null });
|
||
return;
|
||
}
|
||
try {
|
||
const tab = await chrome.tabs.get(targetTabId);
|
||
sendToPanel({ type: 'target_tab', tabId: targetTabId, title: tab.title, url: tab.url });
|
||
} catch {
|
||
sendToPanel({ type: 'target_tab', tabId: null });
|
||
}
|
||
}
|
||
|
||
/**
|
||
* DER Engpass. Jeder Antrieb geht hier durch, also greift das Risiko-Gate überall.
|
||
* Bei riskMode='off' (Standard) kostet das Gate nichts und führt sofort aus.
|
||
*/
|
||
async function runTool(name: string, input: Record<string, unknown>): Promise<any> {
|
||
// Gedächtnis braucht keinen Tab und kein Risiko-Gate — es fasst die Seite nicht an.
|
||
if (name === 'memory_save') {
|
||
const text = String(input.text ?? '').trim();
|
||
if (!text) return errorResult('NO_TEXT', 'text erforderlich', false);
|
||
const note = await memoryLocalSave(
|
||
text, String(input.type ?? 'fakt'),
|
||
Array.isArray(input.tags) ? (input.tags as string[]) : [],
|
||
);
|
||
return note
|
||
? { saved: note.id, type: note.type, text: note.text }
|
||
: errorResult('MEMORY_UNAVAILABLE',
|
||
'Der lokale Agent ist nicht erreichbar — die Notiz wurde NICHT gespeichert. '
|
||
+ 'Arbeite ohne sie weiter.', false);
|
||
}
|
||
if (name === 'memory_search') {
|
||
const notes = await memoryLocalSearch(
|
||
String(input.query ?? ''), Number(input.limit) || 8,
|
||
);
|
||
return { count: notes.length, notes };
|
||
}
|
||
|
||
const tabId = await currentTabId();
|
||
const { allowed, verdict } = await gate(tabId, name, input, (v) => askUser(v, name));
|
||
|
||
if (!allowed) {
|
||
return {
|
||
error: {
|
||
code: 'USER_DECLINED',
|
||
message: `Vom Nutzer abgelehnt: ${verdict.reason}`,
|
||
retryable: false,
|
||
},
|
||
_method: 'synthetic',
|
||
};
|
||
}
|
||
|
||
const result = await dispatchTool(name, input);
|
||
|
||
// Pace: kurze Pause nach jeder Aktion, damit Seiten rendern können und der
|
||
// Agent nicht hektisch wirkt. Screenshots und reine Lese-Tools brauchen keine Pause.
|
||
const readOnly = name === 'browser_screenshot' || name === 'browser_read_page'
|
||
|| name === 'browser_read_network' || name === 'browser_read_console'
|
||
|| name === 'browser_tabs_list' || name === 'browser_find';
|
||
if (!readOnly) {
|
||
const cfg = await loadConfig();
|
||
const pace = cfg.paceMs ?? TOOL_PACE_MS;
|
||
if (pace > 0) await new Promise(r => setTimeout(r, pace));
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
/** Fragt den Nutzer. Ohne offenes Panel wird gehandelt — Autonomie hat Vorrang. */
|
||
function askUser(verdict: RiskVerdict, toolName: string): Promise<boolean> {
|
||
if (!panelPort) return Promise.resolve(true);
|
||
|
||
const requestId = crypto.randomUUID();
|
||
sendToPanel({ type: 'risk_request', requestId, name: toolName, verdict, timeoutMs: RISK_TIMEOUT_MS });
|
||
|
||
return new Promise<boolean>((resolve) => {
|
||
const timer = setTimeout(() => {
|
||
pendingRisk.delete(requestId);
|
||
resolve(true); // Zeitablauf = ausführen, siehe RISK_TIMEOUT_MS
|
||
}, RISK_TIMEOUT_MS);
|
||
|
||
pendingRisk.set(requestId, (approved) => {
|
||
clearTimeout(timer);
|
||
pendingRisk.delete(requestId);
|
||
resolve(approved);
|
||
});
|
||
});
|
||
}
|
||
|
||
// ─── Helfer ───────────────────────────────────────────────────────────────────
|
||
|
||
async function getActiveTab(): Promise<chrome.tabs.Tab> {
|
||
// Wenn ein Target-Tab gesetzt ist, diesen verwenden (Hintergrundarbeit).
|
||
if (targetTabId !== null) {
|
||
try {
|
||
const tab = await chrome.tabs.get(targetTabId);
|
||
if (tab && tab.id) return tab;
|
||
} catch {
|
||
// Tab existiert nicht mehr — zurückfallen auf aktiven Tab.
|
||
targetTabId = null;
|
||
}
|
||
}
|
||
const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
|
||
if (!tab || !tab.id) throw new ToolError('NO_ACTIVE_TAB', 'Kein aktiver Tab gefunden', false);
|
||
// Beim ersten Aufruf den Target-Tab setzen.
|
||
targetTabId = tab.id!;
|
||
notifyTargetTab();
|
||
return tab;
|
||
}
|
||
|
||
async function currentTabId(): Promise<number | undefined> {
|
||
if (targetTabId !== null) {
|
||
try {
|
||
await chrome.tabs.get(targetTabId);
|
||
return targetTabId;
|
||
} catch {
|
||
targetTabId = null;
|
||
}
|
||
}
|
||
try {
|
||
const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
|
||
return tab?.id;
|
||
} catch {
|
||
return undefined;
|
||
}
|
||
}
|
||
|
||
async function execInTab<T>(tabId: number, func: (...args: any[]) => T, args: any[] = []): Promise<T> {
|
||
const results = await chrome.scripting.executeScript({ target: { tabId }, func: func as any, args });
|
||
return results?.[0]?.result as T;
|
||
}
|
||
|
||
function resolveTarget(input: Record<string, unknown>): string | null {
|
||
if (input.ref_id) return `[data-nexus-ref="${input.ref_id}"]`;
|
||
if (input.selector) return input.selector as string;
|
||
return null;
|
||
}
|
||
|
||
class ToolError extends Error {
|
||
code: string;
|
||
retryable: boolean;
|
||
constructor(code: string, message: string, retryable: boolean) {
|
||
super(message);
|
||
this.code = code;
|
||
this.retryable = retryable;
|
||
}
|
||
}
|
||
|
||
function errorResult(code: string, message: string, retryable: boolean): any {
|
||
return { error: { code, message, retryable }, _method: 'synthetic' };
|
||
}
|
||
|
||
/** Liest Position und Kennung eines Elements im Viewport. Magnet-Logik: wenn der
|
||
* exakte Selektor nichts findet, wird per Textsuche das nächstliegende interaktive
|
||
* Element gesucht (übernommen aus browser-pilot). */
|
||
async function elementBox(tabId: number, selector: string): Promise<{ x: number; y: number; tag: string; text: string } | null> {
|
||
return await execInTab(tabId, (sel: string) => {
|
||
let el = document.querySelector(sel) as HTMLElement | null;
|
||
|
||
// Magnet-Fallback: Textsuche auf interaktiven Elementen
|
||
if (!el) {
|
||
const searchText = sel.replace(/[^a-zA-Z0-9äöüÄÖÜß ]/g, '').toLowerCase().trim();
|
||
if (searchText) {
|
||
const interactives = Array.from(document.querySelectorAll(
|
||
'a, button, input, select, textarea, [role="button"], [onclick], [tabindex]:not([tabindex="-1"]), summary, label'
|
||
));
|
||
for (const candidate of interactives) {
|
||
const cText = ((candidate as HTMLElement).innerText || (candidate as HTMLInputElement).value ||
|
||
candidate.getAttribute('aria-label') || '').toLowerCase();
|
||
if (cText.includes(searchText) || searchText.includes(cText.trim())) {
|
||
el = candidate as HTMLElement;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!el) return null;
|
||
el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' as ScrollBehavior });
|
||
const r = el.getBoundingClientRect();
|
||
if (r.width === 0 && r.height === 0) return null;
|
||
return {
|
||
x: r.x + r.width / 2,
|
||
y: r.y + r.height / 2,
|
||
tag: el.tagName.toLowerCase(),
|
||
text: ((el as HTMLElement).innerText || (el as HTMLInputElement).value || '').trim().slice(0, 80),
|
||
};
|
||
}, [selector]);
|
||
}
|
||
|
||
/**
|
||
* Entscheidet je nach inputMode, ob eine echte (CDP) oder synthetische Eingabe erzeugt wird.
|
||
* 'auto' versucht CDP und fällt bei fehlender Permission oder belegtem Debugger zurück.
|
||
*/
|
||
async function wantsTrusted(tabId: number): Promise<boolean> {
|
||
const cfg = await loadConfig();
|
||
if (cfg.inputMode === 'synthetic') return false;
|
||
if (!cdp.cdpAvailable()) return false;
|
||
if (cfg.inputMode === 'trusted') return true;
|
||
try {
|
||
await cdp.attach(tabId);
|
||
return true;
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
// ─── Interaktion ──────────────────────────────────────────────────────────────
|
||
|
||
async function clickElement(tabId: number, selector: string, forceTrusted: boolean): Promise<any> {
|
||
const trusted = forceTrusted || await wantsTrusted(tabId);
|
||
|
||
if (trusted) {
|
||
const box = await elementBox(tabId, selector);
|
||
if (!box) throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden oder unsichtbar: ${selector}`, true);
|
||
try {
|
||
await cdp.attach(tabId);
|
||
await cdp.mouseClick(tabId, box.x, box.y);
|
||
notifyCursor(tabId, box.x, box.y, 'left');
|
||
return { clicked: selector, tag: box.tag, text: box.text, _method: 'cdp' };
|
||
} catch (e: any) {
|
||
// Kein harter Fehler: lieber synthetisch klicken als gar nicht handeln.
|
||
console.warn('[Nexus] CDP-Klick fehlgeschlagen, weiche auf synthetisch aus:', e?.message);
|
||
}
|
||
}
|
||
|
||
const result = await execInTab(tabId, (sel: string) => {
|
||
let el = document.querySelector(sel) as HTMLElement | null;
|
||
|
||
// Magnet-Fallback: Textsuche auf interaktiven Elementen
|
||
if (!el) {
|
||
const searchText = sel.replace(/[^a-zA-Z0-9äöüÄÖÜß ]/g, '').toLowerCase().trim();
|
||
if (searchText) {
|
||
const interactives = Array.from(document.querySelectorAll(
|
||
'a, button, input, select, textarea, [role="button"], [onclick], [tabindex]:not([tabindex="-1"]), summary, label'
|
||
));
|
||
for (const candidate of interactives) {
|
||
const cText = ((candidate as HTMLElement).innerText || (candidate as HTMLInputElement).value ||
|
||
candidate.getAttribute('aria-label') || '').toLowerCase();
|
||
if (cText.includes(searchText) || searchText.includes(cText.trim())) {
|
||
el = candidate as HTMLElement;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!el) return { error: 'NOT_FOUND' };
|
||
el.scrollIntoView({ block: 'center', behavior: 'instant' as ScrollBehavior });
|
||
|
||
// Vollständige Maus-Event-Sequenz für maximale Kompatibilität
|
||
const rect = el.getBoundingClientRect();
|
||
const x = rect.left + rect.width / 2;
|
||
const y = rect.top + rect.height / 2;
|
||
const eventOpts: MouseEventInit = {
|
||
bubbles: true, cancelable: true, view: window,
|
||
clientX: x, clientY: y, button: 0, buttons: 1,
|
||
};
|
||
el.dispatchEvent(new PointerEvent('pointerover', eventOpts));
|
||
el.dispatchEvent(new PointerEvent('pointerenter', { ...eventOpts, bubbles: false }));
|
||
el.dispatchEvent(new MouseEvent('mouseover', eventOpts));
|
||
el.dispatchEvent(new MouseEvent('mouseenter', { ...eventOpts, bubbles: false }));
|
||
el.dispatchEvent(new PointerEvent('pointerdown', eventOpts));
|
||
el.dispatchEvent(new MouseEvent('mousedown', eventOpts));
|
||
(el as HTMLElement).focus?.();
|
||
el.dispatchEvent(new PointerEvent('pointerup', eventOpts));
|
||
el.dispatchEvent(new MouseEvent('mouseup', eventOpts));
|
||
el.dispatchEvent(new MouseEvent('click', eventOpts));
|
||
|
||
return { clicked: sel, tag: el.tagName.toLowerCase(), text: (el.innerText || '').trim().slice(0, 80), coordinates: { x: Math.round(x), y: Math.round(y) } };
|
||
}, [selector]);
|
||
|
||
if (!result || (result as any).error === 'NOT_FOUND') {
|
||
throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
|
||
}
|
||
return { ...result, _method: 'synthetic' };
|
||
}
|
||
|
||
/**
|
||
* Riegel gegen versehentliches Absenden von Mails.
|
||
*
|
||
* "Nur Entwurf" als Prompt-Regel ist eine Bitte — ein Modell kann sie ignorieren,
|
||
* und eine verschickte Mail holt niemand zurück. Deshalb hier hart, unabhängig
|
||
* vom Risikomodus. Greift nur, wenn BEIDES zutrifft: die Schaltfläche heißt nach
|
||
* Senden, UND die Seite ist erkennbar eine Mail-Oberfläche. Ein "Senden"-Knopf in
|
||
* einem Kontaktformular bleibt damit bedienbar.
|
||
*
|
||
* Grenze, die man kennen muss: browser_computer klickt auf Pixelkoordinaten. Was
|
||
* dort getroffen wird, ist vorher nicht bekannt — dieser Riegel sieht es nicht.
|
||
*/
|
||
async function mailSendBlock(tabId: number, selector: string): Promise<any | null> {
|
||
const cfg = await loadConfig();
|
||
if (!cfg.blockMailSend) return null;
|
||
|
||
const hit = await execInTab(tabId, (sel: string) => {
|
||
const el = document.querySelector(sel) as HTMLElement | null;
|
||
if (!el) return null;
|
||
|
||
const label = [
|
||
el.getAttribute('aria-label'), el.getAttribute('title'),
|
||
(el as HTMLInputElement).value, el.innerText || el.textContent,
|
||
].filter(Boolean).join(' ').trim().toLowerCase().replace(/\s+/g, ' ').slice(0, 120);
|
||
|
||
// Wortgrenzen: "Absender", "Gesendet" oder "Nachsenden" sind keine Sende-Knöpfe.
|
||
const isSend = /(^|\s)(senden|send|absenden|abschicken|versenden|jetzt senden|send now)(\s|$)/.test(label);
|
||
if (!isSend) return null;
|
||
|
||
const host = location.hostname.toLowerCase();
|
||
const mailHost = /(outlook|office|mail|owa|gmail|zimbra|roundcube)/.test(host)
|
||
|| /(outlook|owa|mail)/.test(location.pathname.toLowerCase());
|
||
// Oder ein erkennbares Verfassen-Formular auf der Seite.
|
||
const compose = !!document.querySelector(
|
||
'[aria-label*="Nachrichtentext" i], [aria-label*="message body" i], '
|
||
+ '[aria-label*="Betreff" i], [aria-label*="subject" i], [name="subject"]',
|
||
);
|
||
if (!mailHost && !compose) return null;
|
||
|
||
return { label, host };
|
||
}, [selector]);
|
||
|
||
if (!hit) return null;
|
||
|
||
return errorResult(
|
||
'MAIL_SEND_BLOCKED',
|
||
`Absenden ist gesperrt: "${hit.label}" auf ${hit.host}. Du legst nur Entwürfe an — `
|
||
+ 'das Verschicken macht der Nutzer selbst. Speichere den Entwurf und melde, dass er '
|
||
+ 'bereitliegt. (Abschaltbar in den Einstellungen: "Senden blockieren".)',
|
||
false,
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Beschreibt, WO der Text wirklich gelandet ist — gemeintes und tatsächlich
|
||
* fokussiertes Element im Vergleich.
|
||
*
|
||
* Der CDP-Pfad fokussiert über einen Klick auf die zuvor gemessene Box. Rendert
|
||
* die Seite zwischen Messung und Klick neu (in SPAs der Normalfall), landet der
|
||
* Klick woanders und der Text im falschen Feld. Die frühere Rücklese-Prüfung las
|
||
* den Wert des GEMEINTEN Elements und meldete deshalb Erfolg, während der Text
|
||
* im Betreff stand. Genau dieser blinde Fleck hat in Outlook ganze Mails in die
|
||
* Betreffzeile geschrieben.
|
||
*/
|
||
async function describeTypeTarget(tabId: number, selector: string): Promise<any> {
|
||
return await execInTab(tabId, (sel: string) => {
|
||
const describe = (el: any) => {
|
||
if (!el || !el.tagName) return null;
|
||
// aria-labelledby zuerst: in Outlook Web wird der Nachrichtentext genau so
|
||
// beschriftet, und ohne diese Auflösung sieht er aus wie das Betreff-Feld.
|
||
let label = el.getAttribute?.('aria-label') || '';
|
||
const by = el.getAttribute?.('aria-labelledby');
|
||
if (!label && by) {
|
||
label = by.split(/\s+/)
|
||
.map((id: string) => document.getElementById(id)?.textContent || '')
|
||
.join(' ').trim();
|
||
}
|
||
if (!label) {
|
||
label = (el.labels?.[0]?.textContent || el.getAttribute?.('placeholder')
|
||
|| el.getAttribute?.('name') || '').trim();
|
||
}
|
||
return {
|
||
ref: el.getAttribute?.('data-nexus-ref') || undefined,
|
||
tag: String(el.tagName).toLowerCase(),
|
||
type: el.type || undefined,
|
||
role: el.getAttribute?.('role') || undefined,
|
||
label: label.slice(0, 80) || undefined,
|
||
editable: el.isContentEditable || undefined,
|
||
value: String(el.value ?? el.textContent ?? '').slice(0, 200),
|
||
};
|
||
};
|
||
const intended = document.querySelector(sel);
|
||
const active = document.activeElement;
|
||
return { intended: describe(intended), active: describe(active), same: !!intended && intended === active };
|
||
}, [selector]);
|
||
}
|
||
|
||
/** Ergebnis einer Eingabe, das den Fehlgriff sichtbar macht statt ihn zu verschlucken. */
|
||
function typeResult(text: string, selector: string, check: any, submit: boolean, method: string): any {
|
||
const landed = check?.active ?? check?.intended ?? null;
|
||
const out: any = {
|
||
typed: text.length,
|
||
selector,
|
||
typed_into: landed,
|
||
value_after: landed?.value ?? '',
|
||
submitted: submit,
|
||
_method: method,
|
||
};
|
||
if (check && !check.same) {
|
||
// Laut und maschinenlesbar: das Modell soll das korrigieren, nicht weiterschreiben.
|
||
out.target_mismatch = true;
|
||
out.intended_target = check.intended;
|
||
out.warning = 'Der Text ist NICHT im gemeinten Element gelandet, sondern in '
|
||
+ `"${landed?.label || landed?.tag || 'unbekannt'}". Inhalt dort prüfen, ggf. leeren `
|
||
+ 'und mit dem richtigen ref_id erneut tippen.';
|
||
}
|
||
return out;
|
||
}
|
||
|
||
async function typeInElement(
|
||
tabId: number, selector: string, text: string, clear: boolean, submit: boolean,
|
||
): Promise<any> {
|
||
const trusted = await wantsTrusted(tabId);
|
||
|
||
if (trusted) {
|
||
const box = await elementBox(tabId, selector);
|
||
if (!box) throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
|
||
try {
|
||
await cdp.attach(tabId);
|
||
// Fokus über einen echten Klick — so verhält es sich wie bei einem Menschen.
|
||
await cdp.mouseClick(tabId, box.x, box.y);
|
||
notifyCursor(tabId, box.x, box.y, 'left');
|
||
if (clear) {
|
||
await cdp.pressKey(tabId, 'a', { modifiers: ['ctrl'] });
|
||
await cdp.pressKey(tabId, 'Delete');
|
||
}
|
||
await cdp.typeText(tabId, text);
|
||
// VOR dem Absenden prüfen: Enter verschiebt den Fokus oft legitim, danach
|
||
// wäre der Vergleich wertlos.
|
||
const check = await describeTypeTarget(tabId, selector);
|
||
if (submit) await cdp.pressKey(tabId, 'Enter');
|
||
return typeResult(text, selector, check, submit, 'cdp');
|
||
} catch (e: any) {
|
||
console.warn('[Nexus] CDP-Eingabe fehlgeschlagen, weiche auf synthetisch aus:', e?.message);
|
||
}
|
||
}
|
||
|
||
const result = await execInTab(tabId, (sel: string, txt: string, clr: boolean, sub: boolean) => {
|
||
const el = document.querySelector(sel) as HTMLInputElement | HTMLTextAreaElement | null;
|
||
if (!el) return { error: 'NOT_FOUND' };
|
||
el.focus();
|
||
|
||
// Der native Setter muss zum konkreten Elementtyp passen. Nimmt man immer den von
|
||
// HTMLInputElement, wirft .call() auf ein <textarea> "Illegal invocation".
|
||
const proto = el instanceof HTMLTextAreaElement
|
||
? window.HTMLTextAreaElement.prototype
|
||
: window.HTMLInputElement.prototype;
|
||
const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
|
||
|
||
const isEditable = (el as HTMLElement).isContentEditable;
|
||
if (isEditable) {
|
||
if (clr) (el as HTMLElement).textContent = '';
|
||
(el as HTMLElement).textContent = ((el as HTMLElement).textContent || '') + txt;
|
||
el.dispatchEvent(new InputEvent('input', { bubbles: true }));
|
||
} else {
|
||
const next = (clr ? '' : el.value) + txt;
|
||
if (setter) setter.call(el, next);
|
||
else el.value = next;
|
||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||
}
|
||
|
||
if (sub) {
|
||
el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }));
|
||
el.dispatchEvent(new KeyboardEvent('keyup', { key: 'Enter', bubbles: true }));
|
||
const form = (el as HTMLElement).closest('form');
|
||
if (form) form.requestSubmit?.();
|
||
}
|
||
return { ok: true };
|
||
}, [selector, text, clear, submit]);
|
||
|
||
if (!result || (result as any).error === 'NOT_FOUND') {
|
||
throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
|
||
}
|
||
// Auch hier zurücklesen: der Wert kann von der Seite normalisiert oder
|
||
// zurückgesetzt worden sein (Masken, kontrollierte React-Felder).
|
||
const check = await describeTypeTarget(tabId, selector);
|
||
return typeResult(text, selector, check, submit, 'synthetic');
|
||
}
|
||
|
||
/** Schickt die Zeigerposition an das Overlay im Content-Script. Fehler sind belanglos. */
|
||
function notifyCursor(tabId: number, x: number, y: number, click?: 'left' | 'right' | 'double'): void {
|
||
chrome.tabs.sendMessage(tabId, { type: 'NEXUS_CURSOR', x, y, click }).catch(() => {});
|
||
}
|
||
|
||
/** Innenschimmer an/aus — zeigt, in welchem Tab der Agent gerade arbeitet. */
|
||
async function notifyGlow(on: boolean): Promise<void> {
|
||
const cfg = await loadConfig();
|
||
if (cfg.pageOverlay !== 'glow' && cfg.pageOverlay !== 'both') return;
|
||
const id = await currentTabId().catch(() => undefined);
|
||
if (typeof id !== 'number') return;
|
||
chrome.tabs.sendMessage(id, { type: 'NEXUS_GLOW', on }).catch(() => {});
|
||
}
|
||
|
||
/**
|
||
* Hebt das Element hervor, das gerade drankommt. Die Box wird ohnehin für den
|
||
* Klick gemessen — das Overlay kostet also keinen zusätzlichen Seitenzugriff.
|
||
*/
|
||
async function notifySpotlight(tabId: number, selector: string, label: string): Promise<void> {
|
||
const cfg = await loadConfig();
|
||
if (cfg.pageOverlay !== 'spotlight' && cfg.pageOverlay !== 'both') return;
|
||
try {
|
||
const rect = await execInTab(tabId, (sel: string) => {
|
||
const el = document.querySelector(sel) as HTMLElement | null;
|
||
if (!el) return null;
|
||
const r = el.getBoundingClientRect();
|
||
if (!r.width && !r.height) return null;
|
||
return { x: r.x, y: r.y, w: r.width, h: r.height };
|
||
}, [selector]);
|
||
if (rect) chrome.tabs.sendMessage(tabId, { type: 'NEXUS_SPOTLIGHT', rect, label }).catch(() => {});
|
||
} catch {
|
||
/* Dekoration — nie den Lauf stören. */
|
||
}
|
||
}
|
||
|
||
// ─── Tool-Dispatch ────────────────────────────────────────────────────────────
|
||
|
||
async function dispatchTool(name: string, input: Record<string, unknown>): Promise<any> {
|
||
const needsTab = name !== 'browser_tabs_list' && name !== 'browser_tabs_create';
|
||
const tab = needsTab ? await getActiveTab() : null;
|
||
const tabId = tab?.id;
|
||
|
||
switch (name) {
|
||
// ── Navigation ──
|
||
case 'browser_navigate': {
|
||
const url = input.url as string;
|
||
if (input.newTab) {
|
||
const newTab = await chrome.tabs.create({ url, active: false });
|
||
targetTabId = newTab.id!;
|
||
notifyTargetTab();
|
||
return { tabId: newTab.id, url, _method: 'synthetic' };
|
||
}
|
||
await chrome.tabs.update(tabId!, { url });
|
||
notifyTargetTab();
|
||
return { tabId, url, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_go_back':
|
||
await chrome.tabs.goBack(tabId!);
|
||
return { ok: true, _method: 'synthetic' };
|
||
|
||
case 'browser_go_forward':
|
||
await chrome.tabs.goForward(tabId!);
|
||
return { ok: true, _method: 'synthetic' };
|
||
|
||
case 'browser_reload':
|
||
await chrome.tabs.reload(tabId!);
|
||
return { ok: true, _method: 'synthetic' };
|
||
|
||
// ── Tabs ──
|
||
case 'browser_tabs_list': {
|
||
const tabs = await chrome.tabs.query({});
|
||
return {
|
||
tabs: tabs.map(t => ({ id: t.id, title: t.title, url: t.url, active: t.active })),
|
||
_method: 'synthetic',
|
||
};
|
||
}
|
||
|
||
case 'browser_tabs_create': {
|
||
// Im Hintergrund erstellen — kein Vordergrund-Wechsel für den Nutzer.
|
||
const newTab = await chrome.tabs.create({ url: input.url as string, active: false });
|
||
targetTabId = newTab.id!;
|
||
notifyTargetTab();
|
||
return { tabId: newTab.id, url: input.url, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_tabs_select': {
|
||
const tid = input.tabId as number;
|
||
// Nur den internen Target-Tab setzen — KEIN Vordergrund-Wechsel.
|
||
// Der Agent arbeitet im Hintergrund; der Nutzer soll seinen aktiven Tab
|
||
// behalten. Nur wenn der Nutzer explizit `focus: true` übergibt, wird
|
||
// der Tab sichtbar gemacht.
|
||
if (input.focus) {
|
||
await chrome.tabs.update(tid, { active: true });
|
||
const t = await chrome.tabs.get(tid);
|
||
await chrome.windows.update(t.windowId!, { focused: true });
|
||
}
|
||
targetTabId = tid;
|
||
notifyTargetTab();
|
||
return { ok: true, tabId: tid, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_tabs_close': {
|
||
const tid = input.tabId as number;
|
||
await chrome.tabs.remove(tid);
|
||
cleanupTab(tid);
|
||
// Wenn der Agent-Tab geschlossen wurde, Kontext zurücksetzen
|
||
if (targetTabId === tid) targetTabId = null;
|
||
return { ok: true, tabId: tid, _method: 'synthetic' };
|
||
}
|
||
|
||
// ── Menschenähnliche Bedienung ──
|
||
case 'browser_computer': {
|
||
const result = await runComputer(tabId!, input);
|
||
if (typeof result?.scale === 'number') lastScale.set(tabId!, result.scale);
|
||
return result;
|
||
}
|
||
|
||
case 'browser_click': {
|
||
const selector = resolveTarget(input);
|
||
if (!selector) return errorResult('NO_TARGET', 'selector oder ref_id erforderlich', false);
|
||
const blocked = await mailSendBlock(tabId!, selector);
|
||
if (blocked) return blocked;
|
||
await notifySpotlight(tabId!, selector, 'klickt');
|
||
return await clickElement(tabId!, selector, !!input.trusted);
|
||
}
|
||
|
||
case 'browser_type': {
|
||
// Kein Raten mehr auf den Fokus. Genau diese Zeile hat in Outlook ganze
|
||
// Mails in die Betreffzeile geschrieben: nach "Neue Nachricht" liegt der
|
||
// Fokus im Betreff, und ein browser_type ohne Ziel traf ihn.
|
||
// Bewusstes Tippen ins fokussierte Feld bleibt möglich — aber nur explizit.
|
||
const selector = resolveTarget(input)
|
||
|| (input.use_focus ? 'input:focus, textarea:focus, [contenteditable]:focus' : '');
|
||
if (!selector) {
|
||
return errorResult(
|
||
'NO_TARGET',
|
||
'ref_id oder selector erforderlich. Ohne Ziel würde der Text in das gerade '
|
||
+ 'fokussierte Feld laufen — in Formularen ist das meist das falsche. '
|
||
+ 'Ziel mit browser_find suchen. Nur wenn du den Fokus selbst gesetzt hast '
|
||
+ 'und ihn wirklich meinst: use_focus=true.',
|
||
false,
|
||
);
|
||
}
|
||
await notifySpotlight(tabId!, selector, 'tippt');
|
||
return await typeInElement(
|
||
tabId!, selector, input.text as string, input.clear !== false, !!input.submit,
|
||
);
|
||
}
|
||
|
||
case 'browser_key': {
|
||
const key = input.key as string;
|
||
const modifiers = (input.modifiers as any[]) || [];
|
||
if (await wantsTrusted(tabId!)) {
|
||
try {
|
||
await cdp.attach(tabId!);
|
||
await cdp.pressKey(tabId!, key, { modifiers });
|
||
return { pressed: key, modifiers, _method: 'cdp' };
|
||
} catch (e: any) {
|
||
console.warn('[Nexus] CDP-Taste fehlgeschlagen, weiche aus:', e?.message);
|
||
}
|
||
}
|
||
const result = await execInTab(tabId!, (k: string, mods: string[]) => {
|
||
const opts: KeyboardEventInit = {
|
||
key: k, bubbles: true, cancelable: true,
|
||
ctrlKey: mods.includes('ctrl'), altKey: mods.includes('alt'),
|
||
shiftKey: mods.includes('shift'), metaKey: mods.includes('meta'),
|
||
};
|
||
const target = document.activeElement || document.body;
|
||
target.dispatchEvent(new KeyboardEvent('keydown', opts));
|
||
target.dispatchEvent(new KeyboardEvent('keyup', opts));
|
||
return { pressed: k, modifiers: mods };
|
||
}, [key, modifiers]);
|
||
return { ...result, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_scroll': {
|
||
const direction = (input.direction as string) || 'down';
|
||
const amount = (input.amount as number) || 500;
|
||
const selector = (input.selector as string) || null;
|
||
|
||
if (!selector && await wantsTrusted(tabId!)) {
|
||
try {
|
||
await cdp.attach(tabId!);
|
||
const m = await cdp.getLayoutMetrics(tabId!);
|
||
const cx = m.viewport.w / 2, cy = m.viewport.h / 2;
|
||
const dx = direction === 'right' ? amount : direction === 'left' ? -amount : 0;
|
||
const dy = direction === 'down' ? amount : direction === 'up' ? -amount : 0;
|
||
await cdp.mouseWheel(tabId!, cx, cy, dx, dy);
|
||
return { scrolled: direction, amount, _method: 'cdp' };
|
||
} catch (e: any) {
|
||
console.warn('[Nexus] CDP-Scroll fehlgeschlagen, weiche aus:', e?.message);
|
||
}
|
||
}
|
||
|
||
await execInTab(tabId!, (dir: string, amt: number, sel: string | null) => {
|
||
const target: any = sel ? document.querySelector(sel) : window;
|
||
if (!target) return;
|
||
const opts = { behavior: 'instant' as ScrollBehavior };
|
||
if (dir === 'down') target.scrollBy({ top: amt, ...opts });
|
||
else if (dir === 'up') target.scrollBy({ top: -amt, ...opts });
|
||
else if (dir === 'right') target.scrollBy({ left: amt, ...opts });
|
||
else if (dir === 'left') target.scrollBy({ left: -amt, ...opts });
|
||
}, [direction, amount, selector]);
|
||
return { scrolled: direction, amount, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_hover': {
|
||
const selector = resolveTarget(input);
|
||
if (!selector) return errorResult('NO_TARGET', 'selector oder ref_id erforderlich', false);
|
||
const box = await elementBox(tabId!, selector);
|
||
if (!box) throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
|
||
|
||
if (await wantsTrusted(tabId!)) {
|
||
try {
|
||
await cdp.attach(tabId!);
|
||
await cdp.mouseMove(tabId!, box.x, box.y);
|
||
notifyCursor(tabId!, box.x, box.y);
|
||
return { hovered: selector, tag: box.tag, _method: 'cdp' };
|
||
} catch { /* weiter synthetisch */ }
|
||
}
|
||
await execInTab(tabId!, (sel: string) => {
|
||
const el = document.querySelector(sel) as HTMLElement | null;
|
||
if (!el) return;
|
||
el.dispatchEvent(new MouseEvent('mouseover', { bubbles: true }));
|
||
el.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true }));
|
||
}, [selector]);
|
||
return { hovered: selector, tag: box.tag, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_drag': {
|
||
const srcSel = (input.sourceSelector as string) || (input.source_ref_id ? `[data-nexus-ref="${input.source_ref_id}"]` : '');
|
||
const tgtSel = (input.targetSelector as string) || (input.target_ref_id ? `[data-nexus-ref="${input.target_ref_id}"]` : '');
|
||
if (!srcSel || !tgtSel) return errorResult('NO_TARGET', 'sourceSelector und targetSelector erforderlich', false);
|
||
|
||
const from = await elementBox(tabId!, srcSel);
|
||
const to = await elementBox(tabId!, tgtSel);
|
||
if (!from) throw new ToolError('ELEMENT_NOT_FOUND', `Quelle nicht gefunden: ${srcSel}`, true);
|
||
if (!to) throw new ToolError('ELEMENT_NOT_FOUND', `Ziel nicht gefunden: ${tgtSel}`, true);
|
||
|
||
if (await wantsTrusted(tabId!)) {
|
||
try {
|
||
await cdp.attach(tabId!);
|
||
await cdp.mouseDrag(tabId!, { x: from.x, y: from.y }, { x: to.x, y: to.y });
|
||
notifyCursor(tabId!, to.x, to.y);
|
||
return { dragged: srcSel, target: tgtSel, _method: 'cdp' };
|
||
} catch (e: any) {
|
||
console.warn('[Nexus] CDP-Drag fehlgeschlagen, weiche aus:', e?.message);
|
||
}
|
||
}
|
||
|
||
// Synthetisches Drag&Drop funktioniert nur bei HTML5-DnD, nicht bei Maus-basiertem.
|
||
const result = await execInTab(tabId!, (src: string, tgt: string) => {
|
||
const s = document.querySelector(src) as HTMLElement | null;
|
||
const t = document.querySelector(tgt) as HTMLElement | null;
|
||
if (!s || !t) return { error: 'NOT_FOUND' };
|
||
const dt = new DataTransfer();
|
||
s.dispatchEvent(new DragEvent('dragstart', { dataTransfer: dt, bubbles: true }));
|
||
t.dispatchEvent(new DragEvent('dragover', { dataTransfer: dt, bubbles: true }));
|
||
t.dispatchEvent(new DragEvent('drop', { dataTransfer: dt, bubbles: true }));
|
||
s.dispatchEvent(new DragEvent('dragend', { dataTransfer: dt, bubbles: true }));
|
||
return { dragged: src, target: tgt };
|
||
}, [srcSel, tgtSel]);
|
||
return { ...result, _method: 'synthetic' };
|
||
}
|
||
|
||
// ── Beobachtung ──
|
||
case 'browser_screenshot': {
|
||
const fullPage = !!input.fullPage;
|
||
const maxWidth = (input.maxWidth as number) || 1400;
|
||
if (cdp.cdpAvailable()) {
|
||
try {
|
||
await cdp.attach(tabId!);
|
||
const shot = await cdp.captureScreenshot(tabId!, { format: 'jpeg', quality: 70, fullPage, maxWidth });
|
||
lastScale.set(tabId!, shot.scale);
|
||
return {
|
||
image: shot.dataUrl, format: 'jpeg', width: shot.width, height: shot.height,
|
||
scale: shot.scale, fullPage, _method: 'cdp',
|
||
};
|
||
} catch (e: any) {
|
||
console.warn('[Nexus] CDP-Screenshot fehlgeschlagen, weiche aus:', e?.message);
|
||
}
|
||
}
|
||
const dataUrl = await chrome.tabs.captureVisibleTab(tab!.windowId!, { format: 'jpeg', quality: 70 });
|
||
lastScale.set(tabId!, 1);
|
||
return { image: dataUrl, format: 'jpeg', scale: 1, fullPage: false, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_zoom': {
|
||
await chrome.tabs.setZoom(tabId!, input.level as number);
|
||
return { zoom: input.level, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_resize_window': {
|
||
const width = input.width as number;
|
||
const height = input.height as number;
|
||
const win = await chrome.windows.get(tab!.windowId!);
|
||
await chrome.windows.update(win.id!, { width, height, state: 'normal' });
|
||
return { width, height, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_get_page_info': {
|
||
const info = await execInTab(tabId!, () => ({
|
||
title: document.title,
|
||
url: location.href,
|
||
viewport: { w: window.innerWidth, h: window.innerHeight },
|
||
scroll: { x: window.scrollX, y: window.scrollY },
|
||
docHeight: document.documentElement.scrollHeight,
|
||
readyState: document.readyState,
|
||
}));
|
||
return { ...info, tabId, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_get_text': {
|
||
const selector = (input.selector as string) || null;
|
||
const maxLength = (input.maxLength as number) || 8000;
|
||
const result = await execInTab(tabId!, (sel: string | null, maxLen: number) => {
|
||
const el = sel ? document.querySelector(sel) : document.body;
|
||
if (!el) return { error: 'NOT_FOUND' };
|
||
const text = (el as HTMLElement).innerText || '';
|
||
return { text: text.slice(0, maxLen), truncated: text.length > maxLen, length: text.length };
|
||
}, [selector, maxLength]);
|
||
if ((result as any)?.error === 'NOT_FOUND') {
|
||
throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
|
||
}
|
||
return { ...result, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_form_input': {
|
||
const selector = resolveTarget(input);
|
||
if (!selector) return errorResult('NO_TARGET', 'selector oder ref_id erforderlich', false);
|
||
const value = String(input.value ?? '');
|
||
const result = await execInTab(tabId!, (sel: string, val: string) => {
|
||
const el = document.querySelector(sel) as HTMLInputElement | HTMLSelectElement | HTMLTextAreaElement | null;
|
||
if (!el) return { error: 'NOT_FOUND' };
|
||
const tag = el.tagName.toLowerCase();
|
||
const type = ((el as HTMLInputElement).type || '').toLowerCase();
|
||
|
||
if (tag === 'select') {
|
||
const sel2 = el as HTMLSelectElement;
|
||
const opt = Array.from(sel2.options).find(o => o.value === val || o.text.trim() === val.trim());
|
||
if (!opt) return { error: 'OPTION_NOT_FOUND', message: `Option "${val}" nicht gefunden` };
|
||
sel2.value = opt.value;
|
||
sel2.dispatchEvent(new Event('input', { bubbles: true }));
|
||
sel2.dispatchEvent(new Event('change', { bubbles: true }));
|
||
return { set: opt.text, value: opt.value, type: 'select' };
|
||
}
|
||
|
||
if (type === 'checkbox' || type === 'radio') {
|
||
const checked = val === 'true' || val === '1' || val === 'on';
|
||
(el as HTMLInputElement).checked = checked;
|
||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||
return { set: String(checked), type };
|
||
}
|
||
|
||
el.focus();
|
||
// Setter passend zum Elementtyp wählen — sonst "Illegal invocation" bei <textarea>.
|
||
const proto = el instanceof HTMLTextAreaElement
|
||
? window.HTMLTextAreaElement.prototype
|
||
: window.HTMLInputElement.prototype;
|
||
const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
|
||
if ((el as HTMLElement).isContentEditable) {
|
||
(el as HTMLElement).textContent = val;
|
||
} else if (setter) {
|
||
setter.call(el, val);
|
||
} else {
|
||
(el as HTMLInputElement).value = val;
|
||
}
|
||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||
return { set: val.slice(0, 100), type: type || tag };
|
||
}, [selector, value]);
|
||
|
||
if ((result as any)?.error === 'NOT_FOUND') throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
|
||
if ((result as any)?.error === 'OPTION_NOT_FOUND') throw new ToolError('OPTION_NOT_FOUND', (result as any).message, true);
|
||
return { ...result, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_select':
|
||
return await dispatchTool('browser_form_input', input);
|
||
|
||
case 'browser_file_upload': {
|
||
const selector = resolveTarget(input);
|
||
if (!selector) return errorResult('NO_TARGET', 'selector oder ref_id erforderlich', false);
|
||
const files = input.files as string[];
|
||
if (!Array.isArray(files) || files.length === 0) {
|
||
return errorResult('NO_FILES', 'files muss mindestens einen absoluten Pfad enthalten', false);
|
||
}
|
||
if (!cdp.cdpAvailable()) {
|
||
return errorResult('CDP_REQUIRED', 'Datei-Upload benötigt die debugger-Permission', false);
|
||
}
|
||
await cdp.attach(tabId!);
|
||
await cdp.setFileInputFiles(tabId!, selector, files);
|
||
return { uploaded: files, selector, _method: 'cdp' };
|
||
}
|
||
|
||
// ── Diagnose ──
|
||
case 'browser_read_console': {
|
||
const level = (input.level as string) || 'all';
|
||
const limit = (input.limit as number) || 50;
|
||
let logs = [...consoleLogs];
|
||
if (level !== 'all') logs = logs.filter(l => l.level === level);
|
||
if (input.filter) {
|
||
const rx = new RegExp(input.filter as string, 'i');
|
||
logs = logs.filter(l => rx.test(l.text));
|
||
}
|
||
logs = logs.slice(-limit);
|
||
if (input.clear) consoleLogs.length = 0;
|
||
return { logs, count: logs.length, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_read_network': {
|
||
if (!cdp.cdpAvailable()) {
|
||
return errorResult('CDP_REQUIRED', 'Netzwerkmitschnitt benötigt die debugger-Permission', false);
|
||
}
|
||
await cdp.attach(tabId!);
|
||
await cdp.enableNetworkCapture(tabId!);
|
||
const entries = cdp.getNetworkRequests(tabId!, {
|
||
filter: input.filter as string | undefined,
|
||
method: input.method as string | undefined,
|
||
limit: (input.limit as number) || 50,
|
||
includeBody: !!input.includeBody,
|
||
});
|
||
return { requests: entries, count: entries.length, _method: 'cdp' };
|
||
}
|
||
|
||
case 'browser_execute_js': {
|
||
const code = input.code as string;
|
||
// MAIN world: nur so sind window-Globals der Anwendung (React, Angular, App-State) sichtbar.
|
||
if (cdp.cdpAvailable()) {
|
||
try {
|
||
await cdp.attach(tabId!);
|
||
// Runtime.evaluate wertet einen Ausdruck aus — ein `return` waere dort ein
|
||
// Syntaxfehler. Der Code laeuft deshalb in einer async-IIFE: damit gilt dieselbe
|
||
// Semantik wie beim frueheren new Function(js), und `await` ist zusaetzlich moeglich.
|
||
const wrapped = `(async () => {\n${code}\n})()`;
|
||
const value = await cdp.evaluateInMainWorld(tabId!, wrapped, { awaitPromise: true });
|
||
return { value, world: 'MAIN', _method: 'cdp' };
|
||
} catch (e: any) {
|
||
return { error: { code: 'JS_ERROR', message: e?.message || String(e), retryable: false }, _method: 'cdp' };
|
||
}
|
||
}
|
||
const results = await chrome.scripting.executeScript({
|
||
target: { tabId: tabId! },
|
||
// as any: die chrome-types-Überladung erkennt parametrisierte func nicht zuverlässig.
|
||
func: ((js: string) => {
|
||
try {
|
||
const r = new Function(js)();
|
||
return { value: JSON.parse(JSON.stringify(r ?? null)) };
|
||
} catch (e: any) { return { error: e.message }; }
|
||
}) as any,
|
||
args: [code],
|
||
});
|
||
const res = results?.[0]?.result as any;
|
||
if (res?.error) return { error: { code: 'JS_ERROR', message: res.error, retryable: false }, _method: 'synthetic' };
|
||
return { value: res?.value, world: 'ISOLATED', _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_wait': {
|
||
const selector = input.selector as string | undefined;
|
||
const timeout = Math.min((input.timeout as number) || 5000, 120_000);
|
||
if (!selector) {
|
||
await new Promise(r => setTimeout(r, timeout));
|
||
return { waited: timeout, _method: 'synthetic' };
|
||
}
|
||
const deadline = Date.now() + timeout;
|
||
const visible = input.visible !== false;
|
||
// Im Service Worker pollen statt in der Seite: überlebt Navigationen während des Wartens.
|
||
while (Date.now() < deadline) {
|
||
const found = await execInTab(tabId!, (sel: string, vis: boolean) => {
|
||
const el = document.querySelector(sel) as HTMLElement | null;
|
||
if (!el) return false;
|
||
return !vis || (el.offsetWidth > 0 && el.offsetHeight > 0);
|
||
}, [selector, visible]).catch(() => false);
|
||
if (found) return { found: true, selector, _method: 'synthetic' };
|
||
await new Promise(r => setTimeout(r, 150));
|
||
}
|
||
return { found: false, timeout: true, selector, _method: 'synthetic' };
|
||
}
|
||
|
||
case 'browser_highlight': {
|
||
const selector = resolveTarget(input) || (input.selector as string);
|
||
const color = (input.color as string) || '#ff6b35';
|
||
const duration = (input.duration as number) || 3000;
|
||
const result = await execInTab(tabId!, (sel: string, col: string, dur: number) => {
|
||
const el = document.querySelector(sel) as HTMLElement | null;
|
||
if (!el) return { error: 'NOT_FOUND' };
|
||
const prev = el.style.outline;
|
||
el.style.outline = `3px solid ${col}`;
|
||
el.style.outlineOffset = '2px';
|
||
setTimeout(() => { el.style.outline = prev; el.style.outlineOffset = ''; }, dur);
|
||
return { highlighted: sel };
|
||
}, [selector, color, duration]);
|
||
if ((result as any)?.error === 'NOT_FOUND') throw new ToolError('ELEMENT_NOT_FOUND', `Element nicht gefunden: ${selector}`, true);
|
||
return { ...result, _method: 'synthetic' };
|
||
}
|
||
|
||
// ── Seitenstruktur ──
|
||
case 'browser_read_page':
|
||
return await readPage(tabId!, input);
|
||
|
||
case 'browser_find':
|
||
return await findElements(tabId!, input);
|
||
|
||
// ── Mehrere Aktionen in einem Aufruf ──
|
||
case 'browser_batch': {
|
||
const actions = (input.actions as Array<{ name: string; input?: Record<string, unknown> }>) || [];
|
||
const stopOnError = input.stopOnError !== false;
|
||
const results: any[] = [];
|
||
for (const action of actions) {
|
||
try {
|
||
// Über runTool, damit auch Teilaktionen durchs Risiko-Gate gehen.
|
||
const r = await runTool(action.name, action.input || {});
|
||
results.push({ name: action.name, ok: !r?.error, result: r });
|
||
if (r?.error && stopOnError) break;
|
||
} catch (e: any) {
|
||
results.push({ name: action.name, ok: false, error: e?.message || String(e) });
|
||
if (stopOnError) break;
|
||
}
|
||
}
|
||
return { results, count: results.length, _method: 'synthetic' };
|
||
}
|
||
|
||
default:
|
||
return errorResult('UNKNOWN_TOOL', `Tool '${name}' ist nicht implementiert`, false);
|
||
}
|
||
}
|
||
|
||
// ─── read_page: A11y-Baum mit stabilen Refs ───────────────────────────────────
|
||
|
||
async function readPage(tabId: number, input: Record<string, unknown>): Promise<any> {
|
||
const opts = {
|
||
viewportOnly: input.viewportOnly !== false,
|
||
maxDepth: (input.maxDepth as number) || 12,
|
||
maxTokens: (input.maxTokens as number) || 6000,
|
||
includeHidden: !!input.includeHidden,
|
||
selector: (input.selector as string) || null,
|
||
};
|
||
|
||
const tree = await execInTab(tabId, (options: any) => {
|
||
const INTERACTIVE_ROLES = new Set([
|
||
'button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'listbox',
|
||
'menuitem', 'tab', 'switch', 'slider', 'spinbutton', 'searchbox', 'option',
|
||
'menuitemcheckbox', 'menuitemradio',
|
||
]);
|
||
const INTERACTIVE_TAGS = new Set(['a', 'button', 'input', 'select', 'textarea', 'details', 'summary']);
|
||
const LANDMARK_ROLES = new Set(['banner', 'navigation', 'main', 'complementary', 'contentinfo', 'search', 'form', 'region']);
|
||
const SKIP_TAGS = new Set(['script', 'style', 'noscript', 'svg', 'path', 'template']);
|
||
|
||
let refCounter = 0;
|
||
let tokenEstimate = 0;
|
||
const TOKEN_PER_CHAR = 0.25;
|
||
|
||
// Alte Refs entfernen, damit ref_ids nicht über Läufe hinweg kollidieren.
|
||
document.querySelectorAll('[data-nexus-ref]').forEach(el => el.removeAttribute('data-nexus-ref'));
|
||
|
||
function isVisible(el: HTMLElement): boolean {
|
||
if (options.includeHidden) return true;
|
||
const style = getComputedStyle(el);
|
||
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false;
|
||
if (el.getAttribute('aria-hidden') === 'true') return false;
|
||
return true;
|
||
}
|
||
|
||
function isInViewport(el: HTMLElement): boolean {
|
||
if (!options.viewportOnly) return true;
|
||
const r = el.getBoundingClientRect();
|
||
return r.bottom > 0 && r.top < window.innerHeight && r.right > 0 && r.left < window.innerWidth;
|
||
}
|
||
|
||
function getRole(el: HTMLElement): string {
|
||
const explicit = el.getAttribute('role');
|
||
if (explicit) return explicit;
|
||
const tag = el.tagName.toLowerCase();
|
||
if (tag === 'a' && el.hasAttribute('href')) return 'link';
|
||
if (tag === 'button') return 'button';
|
||
if (tag === 'input') {
|
||
const type = (el as HTMLInputElement).type;
|
||
// Submit-Knöpfe MÜSSEN von Textfeldern unterscheidbar sein — sonst kann die
|
||
// Risikoerkennung "Formular absenden" nicht erkennen.
|
||
if (type === 'submit' || type === 'button' || type === 'reset' || type === 'image') return 'button';
|
||
if (type === 'checkbox') return 'checkbox';
|
||
if (type === 'radio') return 'radio';
|
||
if (type === 'range') return 'slider';
|
||
if (type === 'file') return 'fileupload';
|
||
if (type === 'search') return 'searchbox';
|
||
return 'textbox';
|
||
}
|
||
if (tag === 'select') return 'combobox';
|
||
if (tag === 'textarea') return 'textbox';
|
||
if (tag === 'nav') return 'navigation';
|
||
if (tag === 'main') return 'main';
|
||
if (tag === 'header') return 'banner';
|
||
if (tag === 'footer') return 'contentinfo';
|
||
if (tag === 'aside') return 'complementary';
|
||
if (tag === 'form') return 'form';
|
||
return '';
|
||
}
|
||
|
||
function getLabel(el: HTMLElement): string {
|
||
const aria = el.getAttribute('aria-label');
|
||
if (aria) return aria;
|
||
const labelledBy = el.getAttribute('aria-labelledby');
|
||
if (labelledBy) {
|
||
// aria-labelledby ist eine LISTE von IDs. Mit getElementById(labelledBy)
|
||
// schlug jede Mehrfach-Referenz fehl — und genau so beschriftet Outlook
|
||
// Web den Nachrichtentext. Ohne Beschriftung sah er für das Modell aus
|
||
// wie das Betreff-Feld.
|
||
const text = labelledBy.split(/\s+/)
|
||
.map(id => document.getElementById(id)?.textContent || '')
|
||
.join(' ').trim();
|
||
if (text) return text.slice(0, 80);
|
||
}
|
||
if ((el as HTMLInputElement).labels?.length) {
|
||
return ((el as HTMLInputElement).labels![0].textContent || '').trim().slice(0, 80);
|
||
}
|
||
return el.getAttribute('title') || el.getAttribute('placeholder') || el.getAttribute('name') || '';
|
||
}
|
||
|
||
function serialize(el: HTMLElement, depth: number): any {
|
||
if (depth > options.maxDepth || tokenEstimate > options.maxTokens) return null;
|
||
if (SKIP_TAGS.has(el.tagName.toLowerCase())) return null;
|
||
if (!isVisible(el)) return null;
|
||
|
||
const tag = el.tagName.toLowerCase();
|
||
const role = getRole(el);
|
||
// isContentEditable zählt mit: Rich-Text-Felder (Mail-Text, Editoren) sind
|
||
// oft ein <div> ohne role und ohne tabindex — ohne diesen Zweig fehlen sie
|
||
// im Baum, und das Modell tippt mangels Alternative ins Betreff-Feld.
|
||
const isInteractive = INTERACTIVE_ROLES.has(role) || INTERACTIVE_TAGS.has(tag) ||
|
||
el.isContentEditable ||
|
||
el.hasAttribute('onclick') || el.hasAttribute('tabindex');
|
||
const isLandmark = LANDMARK_ROLES.has(role);
|
||
|
||
if (!isInteractive && !isLandmark && !isInViewport(el)) {
|
||
const children: any[] = [];
|
||
for (const child of Array.from(el.children)) {
|
||
if (tokenEstimate > options.maxTokens) break;
|
||
const c = serialize(child as HTMLElement, depth + 1);
|
||
if (c) children.push(c);
|
||
}
|
||
return children.length === 1 ? children[0] : children.length ? { children } : null;
|
||
}
|
||
|
||
const node: any = {};
|
||
if (isInteractive) {
|
||
const ref = `r${++refCounter}`;
|
||
el.setAttribute('data-nexus-ref', ref);
|
||
node.ref = ref;
|
||
// Koordinaten mitgeben, damit das Modell nahtlos auf browser_computer wechseln kann.
|
||
const r = el.getBoundingClientRect();
|
||
if (r.width || r.height) {
|
||
node.rect = { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) };
|
||
node.center = [Math.round(r.x + r.width / 2), Math.round(r.y + r.height / 2)];
|
||
}
|
||
}
|
||
|
||
if (role) node.role = role;
|
||
node.tag = tag;
|
||
|
||
// type ist für die Risikoerkennung entscheidend (submit vs. text).
|
||
if (tag === 'input' || tag === 'button') {
|
||
const t = (el as HTMLInputElement).type;
|
||
if (t) node.type = t;
|
||
}
|
||
|
||
const label = getLabel(el);
|
||
if (label) node.label = label.slice(0, 80);
|
||
|
||
if (el.children.length === 0 || isInteractive) {
|
||
const text = (el.textContent || '').trim().replace(/\s+/g, ' ');
|
||
if (text && text.length <= 120) node.text = text;
|
||
else if (text) node.text = text.slice(0, 100) + '…';
|
||
}
|
||
|
||
// Für das Modell der entscheidende Unterschied zwischen Betreff (<input>)
|
||
// und Nachrichtentext (contenteditable) — beide melden role="textbox".
|
||
if (el.isContentEditable) {
|
||
node.editable = true;
|
||
const val = (el.textContent || '').trim();
|
||
if (val) node.value = val.slice(0, 60);
|
||
}
|
||
|
||
if (tag === 'input' || tag === 'textarea' || tag === 'select') {
|
||
const val = (el as HTMLInputElement).value;
|
||
if (val) node.value = val.slice(0, 60);
|
||
if (typeof (el as HTMLInputElement).checked === 'boolean') node.checked = (el as HTMLInputElement).checked;
|
||
if ((el as HTMLInputElement).disabled) node.disabled = true;
|
||
if ((el as HTMLInputElement).required) node.required = true;
|
||
}
|
||
|
||
if (tag === 'a') {
|
||
const href = el.getAttribute('href');
|
||
if (href && href !== '#') node.href = href.slice(0, 120);
|
||
}
|
||
if (tag === 'form') {
|
||
const action = el.getAttribute('action');
|
||
if (action) node.action = action.slice(0, 120);
|
||
node.method = (el.getAttribute('method') || 'get').toLowerCase();
|
||
}
|
||
|
||
tokenEstimate += JSON.stringify(node).length * TOKEN_PER_CHAR;
|
||
|
||
if (el.children.length) {
|
||
const children: any[] = [];
|
||
for (const child of Array.from(el.children)) {
|
||
if (tokenEstimate > options.maxTokens) break;
|
||
const c = serialize(child as HTMLElement, depth + 1);
|
||
if (c) children.push(c);
|
||
}
|
||
if (children.length) node.children = children;
|
||
}
|
||
return node;
|
||
}
|
||
|
||
const root = options.selector
|
||
? document.querySelector(options.selector) as HTMLElement
|
||
: document.body;
|
||
if (!root) return { error: 'Wurzelelement nicht gefunden' };
|
||
|
||
const serialized = serialize(root, 0);
|
||
return {
|
||
tree: serialized,
|
||
meta: {
|
||
url: location.href,
|
||
title: document.title,
|
||
refCount: refCounter,
|
||
tokenEstimate: Math.round(tokenEstimate),
|
||
truncated: tokenEstimate > options.maxTokens,
|
||
viewport: { w: window.innerWidth, h: window.innerHeight },
|
||
},
|
||
};
|
||
}, [opts]);
|
||
|
||
return { ...tree, _method: 'synthetic' };
|
||
}
|
||
|
||
// ─── find: Element per natürlicher Beschreibung ───────────────────────────────
|
||
|
||
async function findElements(tabId: number, input: Record<string, unknown>): Promise<any> {
|
||
const query = String(input.query || '').trim();
|
||
if (!query) return errorResult('NO_QUERY', 'query erforderlich', false);
|
||
const limit = (input.limit as number) || 5;
|
||
|
||
const matches = await execInTab(tabId, (q: string, lim: number) => {
|
||
const terms = q.toLowerCase().split(/\s+/).filter(t => t.length > 1);
|
||
const SEL = 'a,button,input,select,textarea,summary,[role],[onclick],[tabindex]';
|
||
const candidates = Array.from(document.querySelectorAll(SEL)) as HTMLElement[];
|
||
|
||
// Alte Refs bleiben gültig, neue nur für Treffer ohne Ref vergeben.
|
||
let counter = document.querySelectorAll('[data-nexus-ref]').length;
|
||
|
||
const scored = candidates.map(el => {
|
||
const style = getComputedStyle(el);
|
||
if (style.display === 'none' || style.visibility === 'hidden') return null;
|
||
const r = el.getBoundingClientRect();
|
||
if (!r.width && !r.height) return null;
|
||
|
||
const haystack = [
|
||
el.getAttribute('aria-label'), el.getAttribute('title'), el.getAttribute('placeholder'),
|
||
el.getAttribute('name'), el.getAttribute('value'), el.id,
|
||
(el as HTMLInputElement).labels?.[0]?.textContent,
|
||
el.innerText || el.textContent,
|
||
].filter(Boolean).join(' ').toLowerCase().replace(/\s+/g, ' ');
|
||
|
||
if (!haystack) return null;
|
||
let score = 0;
|
||
for (const t of terms) {
|
||
if (haystack.includes(t)) score += 2;
|
||
// Wortanfang zählt stärker als ein Treffer irgendwo in der Mitte.
|
||
if (new RegExp('\\b' + t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).test(haystack)) score += 1;
|
||
}
|
||
if (!score) return null;
|
||
|
||
// Kürzere Beschriftungen sind meist die genauere Entsprechung.
|
||
score += Math.max(0, 3 - haystack.length / 60);
|
||
// Sichtbares im Viewport bevorzugen.
|
||
if (r.top >= 0 && r.top < window.innerHeight) score += 2;
|
||
|
||
let ref = el.getAttribute('data-nexus-ref');
|
||
if (!ref) { ref = `r${++counter}`; el.setAttribute('data-nexus-ref', ref); }
|
||
|
||
return {
|
||
ref_id: ref,
|
||
tag: el.tagName.toLowerCase(),
|
||
type: (el as HTMLInputElement).type || undefined,
|
||
role: el.getAttribute('role') || undefined,
|
||
// value zaehlt mit: <input type="submit" value="Bestellen"> hat kein innerText,
|
||
// der Treffer waere sonst ohne jede Beschriftung und fuer das Modell wertlos.
|
||
text: (el.innerText || el.textContent || (el as HTMLInputElement).value || '')
|
||
.trim().replace(/\s+/g, ' ').slice(0, 100),
|
||
label: el.getAttribute('aria-label') || el.getAttribute('placeholder') || undefined,
|
||
href: el.getAttribute('href') || undefined,
|
||
center: [Math.round(r.x + r.width / 2), Math.round(r.y + r.height / 2)],
|
||
rect: { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) },
|
||
score: Math.round(score * 10) / 10,
|
||
};
|
||
}).filter(Boolean) as any[];
|
||
|
||
scored.sort((a, b) => b.score - a.score);
|
||
return scored.slice(0, lim);
|
||
}, [query, limit]);
|
||
|
||
return { query, matches, count: matches.length, _method: 'synthetic' };
|
||
}
|
||
|
||
// ─── Eingebauter Agent-Loop ───────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Gedächtnis-Werkzeuge für den direct-Modus.
|
||
*
|
||
* Sie stehen NICHT in tools/schema/, weil der Broker-Pfad sie über
|
||
* agent/memory_tools.py schon hat — dort registriert sie die Python-Seite. Hier
|
||
* wären sie sonst doppelt. Beide Wege schreiben in denselben lokalen Speicher.
|
||
*/
|
||
const MEMORY_TOOL_DEFS: LlmToolDef[] = [
|
||
{
|
||
name: 'memory_save',
|
||
description: 'Merkt sich etwas über den Nutzer dauerhaft — Präferenz, Schreibstil, '
|
||
+ 'Fakt oder laufendes Projekt. Nutze es, wenn er eine Vorliebe äußert, dich '
|
||
+ 'korrigiert oder etwas über sich sagt, das später wieder zählt. Nicht für '
|
||
+ 'Arbeitsergebnisse.',
|
||
inputSchema: {
|
||
type: 'object',
|
||
properties: {
|
||
text: { type: 'string', description: 'Die Notiz, ein Satz.' },
|
||
type: { type: 'string', enum: ['praeferenz', 'schreibstil', 'fakt', 'projekt'] },
|
||
tags: { type: 'array', items: { type: 'string' } },
|
||
},
|
||
required: ['text'],
|
||
},
|
||
},
|
||
{
|
||
name: 'memory_search',
|
||
description: 'Sucht in den Notizen über den Nutzer. Die wichtigsten stehen schon im '
|
||
+ 'Kontext — nutze es gezielt, etwa für den Schreibstil vor einer Mail.',
|
||
inputSchema: {
|
||
type: 'object',
|
||
properties: {
|
||
query: { type: 'string', description: 'Wonach du suchst.' },
|
||
limit: { type: 'integer' },
|
||
},
|
||
required: ['query'],
|
||
},
|
||
},
|
||
];
|
||
|
||
function toolDefs(): LlmToolDef[] {
|
||
return [
|
||
...BROWSER_TOOL_SCHEMAS.map(s => ({
|
||
name: s.name,
|
||
description: s.description,
|
||
inputSchema: s.inputSchema,
|
||
})),
|
||
...MEMORY_TOOL_DEFS,
|
||
];
|
||
}
|
||
|
||
async function startBuiltinRun(
|
||
text: string,
|
||
maxRounds?: number,
|
||
isContinue = false,
|
||
images?: { mediaType: string; data: string; name?: string }[],
|
||
): Promise<void> {
|
||
runStartedAt = Date.now();
|
||
setRunBadge(true);
|
||
notifyGlow(true);
|
||
try {
|
||
await runAgentTurn(text, {
|
||
images,
|
||
// URL des Ziel-Tabs: entscheidet, ob das Mercedes-Wissen mitgeschickt wird.
|
||
currentUrl: async () => {
|
||
const id = await currentTabId().catch(() => undefined);
|
||
if (typeof id !== 'number') return undefined;
|
||
return (await chrome.tabs.get(id).catch(() => null))?.url;
|
||
},
|
||
runTool: (name, input) => runTool(name, input),
|
||
tools: toolDefs(),
|
||
emit: (msg) => {
|
||
// Screenshots aus tool_done herauslösen, damit das Panel keine Base64-Wand rendert.
|
||
if (msg?.type === 'tool_done') {
|
||
reportToolDone(msg.callId, msg.name, msg.result, msg.error, msg.durationMs, msg.method);
|
||
return;
|
||
}
|
||
sendToPanel(msg);
|
||
},
|
||
maxRounds,
|
||
isContinue,
|
||
});
|
||
} finally {
|
||
runStartedAt = 0;
|
||
setRunBadge(false);
|
||
notifyGlow(false);
|
||
}
|
||
}
|
||
|
||
// ─── Panel-Kommunikation ──────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Ereignispuffer — damit ein Lauf das Panel überlebt.
|
||
*
|
||
* Chrome haelt den Service Worker am Leben, das Panel aber nicht. Vorher stand
|
||
* hier nur `if (!panelPort) return;`: der Agent arbeitete nach dem Schliessen
|
||
* weiter und redete gegen eine Wand, beim Oeffnen war alles fort. Jetzt bekommt
|
||
* jedes Ereignis eine Sequenznummer und landet im Puffer; das Panel holt beim
|
||
* Anmelden alles ab seinem letzten Stand nach.
|
||
*/
|
||
const EVENT_BUFFER_MAX = 2000;
|
||
let eventSeq = 0;
|
||
const eventBuffer: { seq: number; msg: any }[] = [];
|
||
/** Beginn des laufenden Auftrags, fuer "Lauf aktiv seit N min". */
|
||
let runStartedAt = 0;
|
||
|
||
function sendToPanel(msg: object): void {
|
||
const stamped: any = { ...msg, _seq: ++eventSeq };
|
||
eventBuffer.push({ seq: stamped._seq, msg: stamped });
|
||
if (eventBuffer.length > EVENT_BUFFER_MAX) {
|
||
eventBuffer.splice(0, eventBuffer.length - EVENT_BUFFER_MAX);
|
||
}
|
||
if (!panelPort) return;
|
||
try { panelPort.postMessage(stamped); } catch { panelPort = null; }
|
||
}
|
||
|
||
/** Badge am Extension-Symbol: ein aktiver Lauf ist sichtbar, ohne die Sidebar zu oeffnen. */
|
||
function setRunBadge(active: boolean): void {
|
||
try {
|
||
chrome.action?.setBadgeText({ text: active ? '●' : '' });
|
||
chrome.action?.setBadgeBackgroundColor({ color: '#3fb27f' });
|
||
} catch { /* Badge ist Beiwerk */ }
|
||
}
|
||
|
||
chrome.runtime.onConnect.addListener((port) => {
|
||
if (port.name !== 'nexus-panel') return;
|
||
panelPort = port;
|
||
sendToPanel({ type: 'connection_status', connected: ws?.readyState === WebSocket.OPEN });
|
||
|
||
port.onMessage.addListener(async (msg: PanelMessage) => {
|
||
switch (msg.type) {
|
||
case 'user_message': {
|
||
const cfg = await loadConfig();
|
||
if (cfg.driveMode === 'direct') {
|
||
if (isAgentRunning()) {
|
||
// Lauf läuft — Nachricht in Queue puffern
|
||
queueMessage(msg.text);
|
||
sendToPanel({ type: 'log', level: 'info', text: `📥 Nachricht gepuffert — wird nach aktuellem Turn ausgeführt.` });
|
||
} else {
|
||
await startBuiltinRun(msg.text, msg.maxRounds ?? cfg.maxRounds, msg.isContinue, msg.images);
|
||
}
|
||
} else {
|
||
// Der Broker-Kanal kennt keine Bilder — lieber sagen als still schlucken.
|
||
if (msg.images?.length) {
|
||
sendToPanel({ type: 'log', level: 'warn',
|
||
text: `${msg.images.length} Bild(er) ignoriert: Bildanhänge gehen nur im Antrieb "direct".` });
|
||
}
|
||
sendToBroker({ type: 'user_message', text: msg.text, maxRounds: msg.maxRounds ?? cfg.maxRounds } satisfies ExtUserMessage);
|
||
}
|
||
break;
|
||
}
|
||
case 'sync': {
|
||
// Nachliefern statt neu erzeugen: der Puffer ist die Wahrheit.
|
||
const from = typeof msg.from === 'number' ? msg.from : 0;
|
||
const missed = eventBuffer.filter(e => e.seq > from);
|
||
if (panelPort) {
|
||
try {
|
||
panelPort.postMessage({
|
||
type: 'sync_start', count: missed.length, running: isAgentRunning(),
|
||
latest: eventSeq, startedAt: runStartedAt || undefined,
|
||
});
|
||
// Direkt posten, nicht ueber sendToPanel — sonst landet alles erneut im Puffer.
|
||
for (const e of missed) panelPort.postMessage(e.msg);
|
||
} catch { panelPort = null; }
|
||
}
|
||
break;
|
||
}
|
||
|
||
case 'abort':
|
||
abortAgent();
|
||
sendToBroker({ type: 'abort' } satisfies ExtAbort);
|
||
break;
|
||
case 'approval_response': {
|
||
// Kann von unserem Risiko-Gate ODER vom Broker stammen.
|
||
const resolver = pendingRisk.get(msg.requestId);
|
||
if (resolver) resolver(msg.approved);
|
||
else sendToBroker({ type: 'approval_response', requestId: msg.requestId, approved: msg.approved } satisfies ExtApprovalResponse);
|
||
break;
|
||
}
|
||
case 'connect': connect(); break;
|
||
case 'disconnect': ws?.close(); break;
|
||
case 'set_config': {
|
||
const next = await patchConfig(msg.patch);
|
||
if (msg.patch.brokerUrl) { ws?.close(); setTimeout(connect, 300); }
|
||
sendToPanel({ type: 'state', config: next, connected: ws?.readyState === WebSocket.OPEN, running: isAgentRunning(), version: EXTENSION_VERSION });
|
||
break;
|
||
}
|
||
case 'get_state': {
|
||
const cfg = await loadConfig();
|
||
sendToPanel({ type: 'state', config: cfg, connected: ws?.readyState === WebSocket.OPEN, running: isAgentRunning(), version: EXTENSION_VERSION });
|
||
break;
|
||
}
|
||
case 'run_tool': {
|
||
const callId = crypto.randomUUID();
|
||
sendToPanel({ type: 'tool_executing', callId, name: msg.name, input: msg.input });
|
||
const started = performance.now();
|
||
try {
|
||
const result = await runTool(msg.name, msg.input);
|
||
const method = (result?._method as InteractionMethod) || 'synthetic';
|
||
if (result && typeof result === 'object') delete result._method;
|
||
reportToolDone(callId, msg.name, result, undefined, Math.round(performance.now() - started), method);
|
||
} catch (e: any) {
|
||
reportToolDone(callId, msg.name, null, e?.message || String(e), Math.round(performance.now() - started), 'synthetic');
|
||
}
|
||
break;
|
||
}
|
||
case 'new_conversation':
|
||
case 'reset_agent':
|
||
resetConversation();
|
||
targetTabId = null; // Neuer Auftrag, neuer Tab-Kontext
|
||
break;
|
||
}
|
||
});
|
||
|
||
port.onDisconnect.addListener(() => { panelPort = null; });
|
||
});
|
||
|
||
// ─── Content-Script-Meldungen ─────────────────────────────────────────────────
|
||
|
||
chrome.runtime.onMessage.addListener((msg, sender): undefined => {
|
||
if (msg?.type === 'CONSOLE_LOG' || msg?.type === 'PAGE_ERROR') {
|
||
consoleLogs.push({
|
||
level: msg.type === 'PAGE_ERROR' ? 'error' : msg.level,
|
||
text: msg.text,
|
||
ts: Date.now(),
|
||
url: sender.tab?.url,
|
||
});
|
||
if (consoleLogs.length > MAX_CONSOLE_LOGS) consoleLogs.shift();
|
||
}
|
||
});
|
||
|
||
// ─── Aufräumen ────────────────────────────────────────────────────────────────
|
||
|
||
function cleanupTab(tabId: number): void {
|
||
lastScale.delete(tabId);
|
||
cdp.detach(tabId).catch(() => {});
|
||
}
|
||
|
||
chrome.tabs.onRemoved.addListener(cleanupTab);
|
||
chrome.runtime.onSuspend?.addListener(() => { cdp.detachAll().catch(() => {}); });
|
||
|
||
// ─── Start ────────────────────────────────────────────────────────────────────
|
||
|
||
chrome.alarms.create('keepAlive', { periodInMinutes: 0.5 });
|
||
chrome.alarms.onAlarm.addListener(() => { /* hält den Worker wach */ });
|
||
|
||
chrome.action.onClicked.addListener(async (tab) => {
|
||
await chrome.sidePanel.open({ windowId: tab.windowId! });
|
||
});
|
||
chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true }).catch(() => {});
|
||
|
||
/**
|
||
* Prüf-Oberfläche für automatisierte Tests.
|
||
*
|
||
* Sie liegt im globalThis des Service Workers und ist damit ausschließlich aus dem
|
||
* Worker-Kontext erreichbar — über die DevTools oder CDP. Webseiten und andere
|
||
* Extensions kommen hier nicht heran, es entsteht also keine Angriffsfläche.
|
||
*/
|
||
(globalThis as any).__nexus = {
|
||
version: EXTENSION_VERSION,
|
||
runTool,
|
||
dispatchTool,
|
||
cdp,
|
||
loadConfig,
|
||
patchConfig,
|
||
};
|
||
|
||
loadConfig().then((cfg) => {
|
||
if (cfg.autoConnect && cfg.driveMode === 'broker') connect();
|
||
console.log(`[Nexus] Browser Pilot ${EXTENSION_VERSION} bereit — CDP ${cdp.cdpAvailable() ? 'verfügbar' : 'NICHT verfügbar'}, Risikomodus: ${cfg.riskMode}`);
|
||
});
|
||
|
||
export { resetConversation };
|