feat: Gemini bridge, browser extension hardening, shared memory

Nexus gateway (measured 2026-08-07, not assumed):
- Gemini speaks Google-GenAI (/v1beta/models/{id}:streamGenerateContent, header
  api-key), NOT the Azure-OpenAI path — that returned 404 "no Route matched" and
  was the cause of the reported failures. New agent/gemini_bridge.py translates
  Bedrock Converse <-> Gemini in both directions.
- Only gemini-3.6-flash is subscribed; 2.5-flash/2.5-pro/3.1-flash-lite give 403,
  every other name 404. Catalog corrected.
- Four Gemini rules, each previously an HTTP 400, now covered by tests:
  thought signatures are mandatory, they belong to the TURN (not the individual
  call), functionResponse turns must be homogeneous, arrays need `items`.
- Prompt caching is NOT available: cachePoint is accepted and ignored.

System prompt:
- Was an f-string; a code sample containing braces broke build_system_prompt at
  request time (CLI and web both 500, import stayed green). Now a plain template
  with __TOKEN__ placeholders. Regression guards in tests/test_system_prompt.py.

CLI:
- `agent resume` now prints the stored transcript. The history was always loaded
  into the model context, only the terminal stayed empty.

Memory (new, all three surfaces):
- agent/memory.py stores notes about the user in one local file, written
  atomically; memory.cnull.net remains an optional mirror that can never fail a
  write. Tools memory_save/search/forget, injected into the prompt with a budget.
  HTTP surface /api/memory for the extension.

Browser extension (agent/extension, first commit of the source):
- driveMode 'direct' talks to Nexus without the Python broker: Claude via
  Bedrock converse, GPT via Azure-OpenAI, Gemini via Google-GenAI.
- browser_type no longer guesses the focused element — that wrote whole mails
  into Outlook's subject line. Read-back now reports where the text actually
  landed, so a mis-target is visible instead of silent.
- aria-labelledby is resolved across all ids (it is a list); contenteditable is
  interactive and marked editable. Without this, subject and message body look
  identical to the model.
- Hard block against sending mail, independent of riskMode.
- Runs survive the panel: events are buffered and replayed by sequence number.
- Image input (paste, file, drag&drop), on-page glow/spotlight, memory tools.

Cost: fixed tokens per round 11434 -> 6540 (-43%) by trimming tool schemas,
dropping gateway docs from the browser prompt and sending site knowledge only
where it applies.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-07 17:31:26 +02:00
co-authored by Claude Opus 5
commit f943261c3c
30 changed files with 13575 additions and 0 deletions
+115
View File
@@ -0,0 +1,115 @@
/**
* Auto-generated browser tool schemas and types.
* DO NOT EDIT — generated by tools/generate_tool_defs.py from tools/schema/*.json.
*/
export interface BrowserToolSchema {
name: string;
description: string;
requires_confirmation: boolean;
inputSchema: Record<string, unknown>;
}
export const BROWSER_TOOL_SCHEMAS: BrowserToolSchema[] = [
{"name": "browser_batch", "description": "Runs several browser tools in one call, in order, returning all results. Saves one model round-trip per step — the dominant cost of form filling and navigation. Use when the whole sequence is known in advance. Do NOT use when a later step depends on what an earlier one returns.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"actions": {"type": "array", "description": "Steps in order: [{name, input}, ...].", "items": {"type": "object", "properties": {"name": {"type": "string", "description": "Name of the browser tool to run, e.g. 'browser_click', 'browser_type', 'browser_computer'. Must be one of the available browser tools, and not 'browser_batch'."}, "input": {"type": "object", "description": "Argument object for that tool — exactly what you would pass in a single call. Omit or use {} for tools without arguments."}}, "required": ["name"]}}, "stopOnError": {"type": "boolean", "description": "Stop at the first failing step (default true)."}}, "required": ["actions"]}},
{"name": "browser_click", "description": "Click an element addressed by ref_id or CSS selector.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector of the element to click, e.g. 'button[type=submit]' or '#login'."}, "ref_id": {"type": "string", "description": "Stable ref_id from browser_read_page or browser_find."}, "trusted": {"type": "boolean", "description": "Force real CDP events for this one call even if the extension is configured for synthetic…"}}}},
{"name": "browser_computer", "description": "Operates mouse and keyboard at pixel coordinates via CDP (isTrusted events). Workflow: action='screenshot' first, read coordinates off that image, then act. Use only for canvas, maps, drag&drop and anything that ignores selectors — for normal HTML browser_find + browser_click are cheaper and more reliable.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"action": {"type": "string", "enum": ["screenshot", "mouse_move", "left_click", "right_click", "middle_click", "double_click", "triple_click", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", "type", "key", "hold_key", "wait", "cursor_position"], "description": "What to do. Pixel coordinates come from a screenshot."}, "coordinate": {"type": "array", "items": {"type": "integer"}, "minItems": 2, "maxItems": 2, "description": "[x, y] in screenshot pixels."}, "start_coordinate": {"type": "array", "items": {"type": "integer"}, "minItems": 2, "maxItems": 2, "description": "[x, y] where a drag begins."}, "text": {"type": "string", "description": "Text to type, or key name for key presses (e.g. 'Return', 'ctrl+a')."}, "scroll_direction": {"type": "string", "enum": ["up", "down", "left", "right"], "description": "up | down | left | right"}, "scroll_amount": {"type": "integer", "description": "Number of wheel clicks."}, "duration": {"type": "number", "description": "Seconds to hold or wait."}, "modifiers": {"type": "array", "items": {"type": "string", "enum": ["ctrl", "alt", "shift", "meta"]}, "description": "Held modifier keys, e.g. ['ctrl','shift']."}}, "required": ["action"]}},
{"name": "browser_drag", "description": "Drag one element onto another with the real mouse: press the left button over the centre of the source, move to the centre of the target in several intermediate steps, and release there.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"sourceSelector": {"type": "string", "description": "CSS selector of the element to pick up. The drag starts at its centre."}, "targetSelector": {"type": "string", "description": "CSS selector of the drop target. The button is released at its centre."}}, "required": ["sourceSelector", "targetSelector"]}},
{"name": "browser_execute_js", "description": "Run JavaScript in the page's MAIN world and return the result. Same realm as the site's own scripts, so you can reach the application's window globals — frameworks, state stores, config objects, jQuery, data layers — which an isolated content script cannot see.", "requires_confirmation": true, "inputSchema": {"type": "object", "properties": {"code": {"type": "string", "description": "JavaScript source to evaluate in the page. The last expression's value is returned, e.g."}}, "required": ["code"]}},
{"name": "browser_file_upload", "description": "Attaches local files to an <input type=\"file\"> via CDP, as if picked in the OS dialog, and fires a change event. Paths must be absolute on the machine running the browser. Never try to open the native file dialog by clicking — it cannot be operated.", "requires_confirmation": true, "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector of the file input."}, "ref_id": {"type": "string", "description": "ref_id of the file input. Preferred."}, "files": {"type": "array", "items": {"type": "string"}, "description": "Absolute paths on the browser machine."}}, "required": ["files"]}},
{"name": "browser_find", "description": "Finds elements by plain-language description instead of a CSS selector. Matches label text, accessible name, placeholder, aria-label, title, role and type, ranked by fit. Returns ref_id (for browser_click/type/select), role, text, tag and coordinates. Start here instead of reading the whole page.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"query": {"type": "string", "description": "What you are looking for, in plain language."}, "limit": {"type": "integer", "description": "Maximum number of candidates (default 5)."}}, "required": ["query"]}},
{"name": "browser_form_input", "description": "Fill a form field: handles input, textarea, select, checkbox, radio. For select elements, matches by value or visible text.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector of the form element"}, "ref_id": {"type": "string", "description": "Stable ref_id from read_page"}, "value": {"type": "string", "description": "Value to set (for select: option value or text; for checkbox/radio: 'true'/'false')"}}, "required": ["value"]}},
{"name": "browser_get_page_info", "description": "Get current page URL, title, viewport dimensions, scroll position, and document height.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {}}},
{"name": "browser_get_text", "description": "Get the text content of the page or a specific element (Readability-style extraction).", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector (default: body)"}, "maxLength": {"type": "number", "description": "Max characters to return (default 8000)"}}}},
{"name": "browser_go_back", "description": "Navigate back in the active tab's history.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {}}},
{"name": "browser_go_forward", "description": "Navigate forward in the active tab's history.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {}}},
{"name": "browser_highlight", "description": "Visually highlight an element on the page (for debugging/demonstration).", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector of the element to highlight"}, "color": {"type": "string", "description": "Outline color (default '#ff6b35')"}, "duration": {"type": "number", "description": "Highlight duration in ms (default 3000)"}}, "required": ["selector"]}},
{"name": "browser_hover", "description": "Move the real mouse pointer onto an element and leave it there.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector of the element to hover, e.g. 'nav .menu-item:first-child'."}, "ref_id": {"type": "string", "description": "Stable ref_id from browser_read_page or browser_find."}}}},
{"name": "browser_key", "description": "Press a single key, optionally with modifiers, as a real keystroke through the DevTools Protocol — trusted keydown/keyup with the correct key code, so keyboard shortcuts, form submission via Enter and focus traversal via Tab all behave as they would for a human.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"key": {"type": "string", "description": "Key name in DOM KeyboardEvent.key notation, e.g."}, "modifiers": {"type": "array", "items": {"type": "string", "enum": ["ctrl", "alt", "shift", "meta"]}, "description": "Modifier keys held while the key is pressed, e.g."}}, "required": ["key"]}},
{"name": "browser_navigate", "description": "Navigate the active tab to a URL, or open a URL in a new tab.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"url": {"type": "string", "description": "Target URL to navigate to"}, "newTab": {"type": "boolean", "description": "If true, open in a new tab instead of the active one"}}, "required": ["url"]}},
{"name": "browser_read_console", "description": "Read captured console logs (log, warn, error, info) from the active page.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"clear": {"type": "boolean", "description": "Clear the log buffer after reading (default false)"}, "level": {"type": "string", "enum": ["all", "log", "warn", "error", "info"], "description": "Filter by log level (default 'all')"}, "limit": {"type": "number", "description": "Max entries to return (default 50)"}}}},
{"name": "browser_read_network", "description": "Read the network requests the active page has made. Captured live through the DevTools Protocol — the same data the Network tab of DevTools shows: URL, HTTP method, status code, resource type, timing and size.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"filter": {"type": "string", "description": "Only return requests whose URL contains this substring, e.g. '/api/' or 'graphql'."}, "method": {"type": "string", "description": "Only return requests with this HTTP method, e.g. 'GET', 'POST', 'PUT', 'DELETE'."}, "limit": {"type": "integer", "description": "Maximum number of entries to return, most recent first (default 50)."}, "includeBody": {"type": "boolean", "description": "Include the response body of matching requests (default false)."}}}},
{"name": "browser_read_page", "description": "Reads the accessibility tree of the page: a compact view of what is on screen and interactive. Cheaper and more precise than a screenshot for normal HTML. Every interactive node carries a ref_id — pass it to browser_click/type/select instead of guessing a selector. Fields report label, role and editable.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"viewportOnly": {"type": "boolean", "description": "Only what is currently visible (default true)."}, "maxDepth": {"type": "number", "description": "Maximum nesting depth."}, "maxTokens": {"type": "number", "description": "Budget; the tree is cut off when reached."}, "includeHidden": {"type": "boolean", "description": "Include invisible elements (default false)."}, "selector": {"type": "string", "description": "Read only this subtree, e.g. a compose form."}}}},
{"name": "browser_reload", "description": "Reload the current page in the active tab.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {}}},
{"name": "browser_resize_window", "description": "Resize the browser window holding the active tab. Changes the viewport, so every pixel coordinate from earlier screenshots becomes invalid — take a fresh screenshot afterwards.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"width": {"type": "integer", "description": "Window width in pixels, e.g. 1280 for a desktop layout or 390 to force a mobile layout."}, "height": {"type": "integer", "description": "Window height in pixels, e.g. 900. Omit to keep the current height."}}}},
{"name": "browser_screenshot", "description": "Captures a JPEG of the page as an image you can look at. Read pixel coordinates off it for browser_computer; origin (0,0) is top-left, x grows right, y grows down. Viewport only by default — exactly the area browser_computer can reach. Expensive in tokens: prefer browser_read_page for normal HTML.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"fullPage": {"type": "boolean", "description": "Whole page instead of the viewport. Coordinates then no longer match browser_computer."}, "maxWidth": {"type": "integer", "description": "Scale down to this width in pixels."}}}},
{"name": "browser_scroll", "description": "Scroll the page, or a specific scrollable element, by a pixel amount.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"direction": {"type": "string", "enum": ["up", "down", "left", "right"], "description": "Scroll direction."}, "amount": {"type": "number", "description": "Pixels to scroll (default 500)."}, "selector": {"type": "string", "description": "CSS selector of the scrollable element (default: the page itself)."}}, "required": ["direction"]}},
{"name": "browser_select", "description": "Select an option in a <select> element by value or visible text.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector of the <select> element"}, "ref_id": {"type": "string", "description": "Stable ref_id from read_page"}, "value": {"type": "string", "description": "Option value or visible text to select"}}, "required": ["value"]}},
{"name": "browser_tabs_close", "description": "Close a tab by its ID. Get IDs from browser_tabs_list. Use it to tidy up tabs you opened while working; closing a tab does not affect the pages themselves.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"tabId": {"type": "number", "description": "The ID of the tab to close, as reported by browser_tabs_list."}}, "required": ["tabId"]}},
{"name": "browser_tabs_create", "description": "Open a new tab with the given URL.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"url": {"type": "string", "description": "URL to open in the new tab"}}, "required": ["url"]}},
{"name": "browser_tabs_list", "description": "List all open browser tabs with their IDs, titles, URLs, and active state.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {}}},
{"name": "browser_tabs_select", "description": "Switch to a tab by its ID (makes it the active tab).", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"tabId": {"type": "number", "description": "The tab ID to activate"}}, "required": ["tabId"]}},
{"name": "browser_type", "description": "Types text into an input, textarea or contenteditable as real CDP keystrokes, so masks, validation and framework state react as for a human. A target is REQUIRED (ref_id or selector) — without one the text would land in whatever happens to be focused, which in forms is usually the wrong field. The result reports typed_into (where the text actually went) and sets target_mismatch=true if that is not the element you asked for.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector of the field."}, "ref_id": {"type": "string", "description": "ref_id from browser_find or browser_read_page. Preferred; wins over selector."}, "text": {"type": "string", "description": "Text to type, character by character."}, "clear": {"type": "boolean", "description": "Clear the field first (default true)."}, "submit": {"type": "boolean", "description": "Press Enter afterwards (default false). Submits most forms."}, "use_focus": {"type": "boolean", "description": "Type into the focused element without a target. Only when you set that focus yourself and mean it."}}, "required": ["text"]}},
{"name": "browser_wait", "description": "Wait for an element to appear or a timeout to elapse.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"selector": {"type": "string", "description": "CSS selector to wait for (if omitted, just waits for timeout)"}, "timeout": {"type": "number", "description": "Max milliseconds to wait (default 5000)"}, "visible": {"type": "boolean", "description": "Wait until element is visible, not just present (default true)"}}}},
{"name": "browser_zoom", "description": "Set the zoom level of the active tab.", "requires_confirmation": false, "inputSchema": {"type": "object", "properties": {"level": {"type": "number", "description": "Zoom factor (1.0 = 100%, 1.5 = 150%, 0.5 = 50%)"}}, "required": ["level"]}},
];
export const BROWSER_TOOL_NAMES = [
"browser_batch",
"browser_click",
"browser_computer",
"browser_drag",
"browser_execute_js",
"browser_file_upload",
"browser_find",
"browser_form_input",
"browser_get_page_info",
"browser_get_text",
"browser_go_back",
"browser_go_forward",
"browser_highlight",
"browser_hover",
"browser_key",
"browser_navigate",
"browser_read_console",
"browser_read_network",
"browser_read_page",
"browser_reload",
"browser_resize_window",
"browser_screenshot",
"browser_scroll",
"browser_select",
"browser_tabs_close",
"browser_tabs_create",
"browser_tabs_list",
"browser_tabs_select",
"browser_type",
"browser_wait",
"browser_zoom",
] as const;
export type BrowserToolName = typeof BROWSER_TOOL_NAMES[number];
export const REQUIRES_CONFIRMATION: Record<string, boolean> = {
"browser_batch": false,
"browser_click": false,
"browser_computer": false,
"browser_drag": false,
"browser_execute_js": true,
"browser_file_upload": true,
"browser_find": false,
"browser_form_input": false,
"browser_get_page_info": false,
"browser_get_text": false,
"browser_go_back": false,
"browser_go_forward": false,
"browser_highlight": false,
"browser_hover": false,
"browser_key": false,
"browser_navigate": false,
"browser_read_console": false,
"browser_read_network": false,
"browser_read_page": false,
"browser_reload": false,
"browser_resize_window": false,
"browser_screenshot": false,
"browser_scroll": false,
"browser_select": false,
"browser_tabs_close": false,
"browser_tabs_create": false,
"browser_tabs_list": false,
"browser_tabs_select": false,
"browser_type": false,
"browser_wait": false,
"browser_zoom": false,
};