From 0f1df92f7eb1a778fe0ed3f79b54ebac023cb890 Mon Sep 17 00:00:00 2001 From: David McKay Date: Sat, 26 Sep 2026 16:54:53 -0700 Subject: [PATCH 1/2] Show browser ownership controls in chat and Computer sidebar --- CHANGELOG.md | 7 + .../components/computer/computer-controls.tsx | 100 ++++++ .../components/computer/computer-panel.tsx | 21 ++ app/src/components/computer/computer-view.tsx | 208 +++-------- app/src/lib/computers/use-control.ts | 149 ++++++++ app/src/routes/_authed/_app/bot.tsx | 174 +++++---- .../_authed/_app/channel/$channelId.tsx | 76 +--- app/tests/bot-route-default-agent.fixture.tsx | 30 +- app/tests/bot-route-default-agent.test.tsx | 118 +++++- app/tests/computer-control-button.test.tsx | 338 ++++++++++++++++++ 10 files changed, 913 insertions(+), 308 deletions(-) create mode 100644 app/src/components/computer/computer-controls.tsx create mode 100644 app/src/components/computer/computer-panel.tsx create mode 100644 app/src/lib/computers/use-control.ts create mode 100644 app/tests/computer-control-button.test.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d9f52d1..a8aa3da88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### Browser controls are visible in chat and the Computer sidebar + +Channel chats and standalone Bot chats now have a labeled Computer button and always-visible +Take control or Hand back controls. The same ownership state is shown in the chat, live Computer +sidebar, and full-size viewer. Standalone Bot chats can open the current live browser alongside +the conversation without losing the selected Bot or chat history. + ### Browser challenges can be handed to a person without losing the Bot's page Bots pause for actionable browser challenges and resume from a fresh page snapshot after an explicit diff --git a/app/src/components/computer/computer-controls.tsx b/app/src/components/computer/computer-controls.tsx new file mode 100644 index 000000000..ec9208dc8 --- /dev/null +++ b/app/src/components/computer/computer-controls.tsx @@ -0,0 +1,100 @@ +import { useEffect, useRef } from "react"; +import { IconDeviceDesktop } from "@tabler/icons-react"; +import { Button } from "@/components/ui/button"; +import { useComputerControl } from "@/lib/computers/use-control"; + +/** The same ownership action appears in chat, the computer sidebar, and the full-size viewer. */ +export function ComputerControlButton({ + computerId, + onTakeControl, +}: { + computerId: string; + onTakeControl?: () => void; +}) { + const { control, busy, problem, change } = useComputerControl(computerId); + const human = control?.holder === "human"; + return ( +
+ + {control?.transitioning ? ( + + Finishing the current action… + + ) : null} + {problem ? ( + + {problem} + + ) : null} +
+ ); +} + +export function ComputerChatControls({ + computerId, + open, + onOpenChange, +}: { + computerId?: string; + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const { control } = useComputerControl(computerId ?? "", Boolean(computerId)); + const needsYou = Boolean( + control && + (control.requested || + control.holder === "human" || + control.request?.status === "interrupted" || + control.secretWanted !== undefined), + ); + const promptKey = needsYou + ? `${computerId}:${control?.secretWanted ?? control?.request?.id ?? "human"}` + : null; + const shownPrompt = useRef(null); + useEffect(() => { + if (shownPrompt.current === promptKey) return; + shownPrompt.current = promptKey; + // Surface each new prompt once; closing the Computer remains a real dismissal. + if (promptKey) onOpenChange(true); + }, [promptKey, onOpenChange]); + return ( +
+ + {computerId ? ( + onOpenChange(true)} + /> + ) : null} +
+ ); +} diff --git a/app/src/components/computer/computer-panel.tsx b/app/src/components/computer/computer-panel.tsx new file mode 100644 index 000000000..d9dc71509 --- /dev/null +++ b/app/src/components/computer/computer-panel.tsx @@ -0,0 +1,21 @@ +import { ActivityLog } from "./activity-log"; +import { ComputerView } from "./computer-view"; + +/** The current live computer, separate from screenshots kept in historical tool results. */ +export function ComputerViewPanel({ + agentId, + name, +}: { + agentId: string; + name?: string; +}) { + return ( +
+ +
+

Activity

+ +
+
+ ); +} diff --git a/app/src/components/computer/computer-view.tsx b/app/src/components/computer/computer-view.tsx index f3b960f02..21d9ac616 100644 --- a/app/src/components/computer/computer-view.tsx +++ b/app/src/components/computer/computer-view.tsx @@ -1,13 +1,8 @@ import { useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; -import { - cancelControl, - type ControlState, - readControl, - releaseControl, - supplySecret, - takeControl, -} from "@/lib/computers/control"; +import { supplySecret } from "@/lib/computers/control"; +import { useComputerControl } from "@/lib/computers/use-control"; +import { ComputerControlButton } from "./computer-controls"; import { readPageFrame, readScreenshot, @@ -227,69 +222,6 @@ export function ComputerView({ const [expanded, setExpanded] = useState(false); const [streamProblem, setStreamProblem] = useState(null); const [retryKey, setRetryKey] = useState(0); - const [controlProblem, setControlProblem] = useState(null); - const [changingControl, setChangingControl] = useState(false); - const [control, setControl] = useState(null); - /** Held only until it is sent. Never lifted into a URL, a log, or anything that outlives this form. */ - const [secret, setSecret] = useState(""); - const [secretProblem, setSecretProblem] = useState(null); - const [sendingSecret, setSendingSecret] = useState(false); - const pageVisible = usePageVisible(); - const [previewRef, previewIntersecting] = useElementVisible(); - const driving = control?.holder === "human" && !control.transitioning; - /** Read by the polling loop without restarting it on control changes. */ - const drivingRef = useRef(false); - drivingRef.current = driving; - - const changeControl = async (action: "take" | "release" | "cancel") => { - if (changingControl) return; - const requestId = control?.request?.id; - if (action !== "take" && !requestId) { - setControlProblem( - "The handoff request could not be identified. Retry the connection.", - ); - return; - } - setChangingControl(true); - setControlProblem(null); - try { - const state = - action === "take" - ? await takeControl( - computerId, - control?.requested ? requestId : undefined, - ) - : requestId - ? action === "release" - ? await releaseControl(computerId, requestId) - : await cancelControl(computerId, requestId) - : null; - if (!state) - setControlProblem( - "Control could not be changed. Check the connection and retry.", - ); - else { - setControl(state); - if (action === "take") setExpanded(true); - } - } catch { - setControlProblem( - "The computer could not be reached. Retry when it reconnects.", - ); - } finally { - setChangingControl(false); - } - }; - const handBack = () => changeControl("release"); - /** Secret prompts keep the screen live even though the human does not hold the wheel. */ - const secretPending = Boolean(control?.secretWanted); - const secretPendingRef = useRef(false); - secretPendingRef.current = secretPending; - // Held in a ref so a slow response cannot overwrite a newer frame after the component moved on. - const generation = useRef(0); - /** Force a short watch window after non-Bot actions such as secret entry. */ - const watchUntil = useRef(0); - /** * A finished turn is history, and history is not polled. * @@ -320,6 +252,32 @@ export function ComputerView({ const [, setFrameArrived] = useState(0); const settled = !active && (finished || Boolean(knownPage)); + const { + control, + busy: changingControl, + change: changeControl, + refresh: refreshControl, + } = useComputerControl(computerId, !settled); + /** Held only until it is sent. Never lifted into a URL, a log, or anything that outlives this form. */ + const [secret, setSecret] = useState(""); + const [secretProblem, setSecretProblem] = useState(null); + const [sendingSecret, setSendingSecret] = useState(false); + const pageVisible = usePageVisible(); + const [previewRef, previewIntersecting] = useElementVisible(); + const driving = control?.holder === "human" && !control.transitioning; + /** Read by the polling loop without restarting it on control changes. */ + const drivingRef = useRef(false); + drivingRef.current = driving; + + /** Secret prompts keep the screen live even though the human does not hold the wheel. */ + const secretPending = Boolean(control?.secretWanted); + const secretPendingRef = useRef(false); + secretPendingRef.current = secretPending; + // Held in a ref so a slow response cannot overwrite a newer frame after the component moved on. + const generation = useRef(0); + /** Force a short watch window after non-Bot actions such as secret entry. */ + const watchUntil = useRef(0); + const visualVisible = pageVisible && (expanded || previewIntersecting); /* @@ -411,32 +369,6 @@ export function ComputerView({ }; }, [computerId, active, intervalMs, secretPending, settled, visualVisible]); - /** Poll control state independently from screenshot polling so help/secret prompts surface. */ - useEffect(() => { - if (settled) return; - let live = true; - let timer: ReturnType; - const tick = async () => { - const state = await readControl(computerId).catch(() => null); - if (!live) return; - setControl(state); - if (state) - setControlProblem((previous) => - previous?.startsWith("Reconnecting:") ? null : previous, - ); - if (!state) - setControlProblem( - "Reconnecting: browser ownership could not be checked. Input is paused.", - ); - timer = setTimeout(tick, 1000); - }; - void tick(); - return () => { - live = false; - clearTimeout(timer); - }; - }, [computerId, settled]); - // Input forwarding lives in LiveScreen on the socket. // Escape is bound to the window so it works regardless of overlay focus. useEffect(() => { @@ -550,30 +482,13 @@ export function ComputerView({ )} - {/* - * The Bot ASKING for the wheel, which is not the same thing as a person wanting it. - * - * The standing "who is driving" prose and the everyday Take control button live in the - * full-size view, where there is a page big enough to drive. This row is the exception: a - * request is an exceptional state with a reason attached, it is the one moment the screen is - * waiting on a person rather than the other way round, and making them open the full-size - * view to find out what was wanted would hide the reason behind a click. Taking the wheel - * from here opens that view, because driving is what they are being asked to do. - */} + {/* The request reason appears above the always-available ownership controls. */} {!driving && !settled && control?.requested ? (
The assistant needs you.{" "} {control.reason} -
) : null} @@ -583,7 +498,7 @@ export function ComputerView({ type="button" className="underline" disabled={changingControl} - onClick={() => void changeControl("cancel")} + onClick={() => void changeControl?.("cancel")} > Cancel request @@ -600,11 +515,6 @@ export function ComputerView({ screen and take control again to check it.

) : null} - {!settled && controlProblem ? ( -

- {controlProblem} -

- ) : null} {/* Secret values go directly to the page path and are never included in the conversation. Audit records that a secret was supplied, not the value. @@ -622,8 +532,7 @@ export function ComputerView({ // Clear even on failure so plaintext is not left in the DOM. setSecret(""); setSecretProblem(result.ok ? null : (result.error ?? null)); - const state = await readControl(computerId); - if (state) setControl(state); + await refreshControl?.(); }} >