From f15b1aedb1be85dbf60f5cd14b1ca13be2aa2d3f Mon Sep 17 00:00:00 2001 From: David Kaya Date: Thu, 26 Mar 2026 17:45:27 +0100 Subject: [PATCH] feat: add ask_user interactive user input to frontend Wire the sidecar user-input-requested protocol event through the main process, IPC layer, and renderer UI so agents can ask the user interactive questions with choices and freeform input. - Add UserInputRequestedEvent and ResolveUserInputCommand to sidecar protocol types - Add resolveUserInput method to SidecarClient and onUserInput callback to runTurn - Create PendingUserInputRecord domain type and add pendingUserInput to SessionRecord - Add handleUserInputRequested and resolveSessionUserInput to AryxAppService with handle management - Register sessions:resolve-user-input IPC channel with preload bridge - Create UserInputBanner component with choice buttons and freeform input - Integrate UserInputBanner into ChatPane with header indicator Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/main/AryxAppService.ts | 98 +++++++++++++++ src/main/ipc/registerIpcHandlers.ts | 4 + src/main/sidecar/runTurnPending.ts | 8 +- src/main/sidecar/sidecarProcess.ts | 36 +++++- src/preload/index.ts | 1 + src/renderer/App.tsx | 3 + src/renderer/components/ChatPane.tsx | 54 +++++++-- .../components/chat/UserInputBanner.tsx | 113 ++++++++++++++++++ src/shared/contracts/channels.ts | 1 + src/shared/contracts/ipc.ts | 8 ++ src/shared/contracts/sidecar.ts | 24 +++- src/shared/domain/session.ts | 2 + src/shared/domain/userInput.ts | 14 +++ tests/main/runTurnPending.test.ts | 2 + 14 files changed, 356 insertions(+), 12 deletions(-) create mode 100644 src/renderer/components/chat/UserInputBanner.tsx create mode 100644 src/shared/domain/userInput.ts diff --git a/src/main/AryxAppService.ts b/src/main/AryxAppService.ts index 1d252d1..251a147 100644 --- a/src/main/AryxAppService.ts +++ b/src/main/AryxAppService.ts @@ -10,6 +10,7 @@ import type { RunTurnToolingConfig, SidecarCapabilities, TurnDeltaEvent, + UserInputRequestedEvent, } from '@shared/contracts/sidecar'; import { buildAvailableModelCatalog, @@ -120,6 +121,12 @@ type PendingApprovalHandle = { resolve: (decision: ApprovalDecision) => void | Promise; }; +type PendingUserInputHandle = { + sessionId: string; + requestId: string; + resolve: (answer: string, wasFreeform: boolean) => void | Promise; +}; + type DiscoveredToolingResolution = 'accept' | 'dismiss'; function isBuiltinPattern(patternId: string): boolean { @@ -147,6 +154,7 @@ export class AryxAppService extends EventEmitter { private readonly gitService = new GitService(); private readonly configScanner = new ConfigScannerRegistry(); private readonly pendingApprovalHandles = new Map(); + private readonly pendingUserInputHandles = new Map(); private workspace?: WorkspaceState; private sidecarCapabilities?: SidecarCapabilities; private sidecarCapabilitiesPromise?: Promise; @@ -619,6 +627,10 @@ export class AryxAppService extends EventEmitter { await this.handleApprovalRequested(workspace, session.id, requestId, event, (decision) => this.sidecar.resolveApproval(event.approvalId, decision)); }, + async (event) => { + await this.handleUserInputRequested(workspace, session.id, requestId, event, (answer, wasFreeform) => + this.sidecar.resolveUserInput(event.userInputId, answer, wasFreeform)); + }, ); await this.awaitFinalResponseApproval(workspace, session.id, requestId, effectivePattern, responseMessages); @@ -742,6 +754,59 @@ export class AryxAppService extends EventEmitter { return result; } + async resolveSessionUserInput( + sessionId: string, + userInputId: string, + answer: string, + wasFreeform: boolean, + ): Promise { + const workspace = await this.loadWorkspace(); + const session = this.requireSession(workspace, sessionId); + const pending = session.pendingUserInput; + if (!pending || pending.id !== userInputId) { + throw new Error(`User input "${userInputId}" is not pending for session "${sessionId}".`); + } + + const handle = this.pendingUserInputHandles.get(userInputId); + if (!handle || handle.sessionId !== sessionId) { + throw new Error(`User input "${userInputId}" is no longer active. Restart the run and try again.`); + } + + const answeredAt = nowIso(); + session.pendingUserInput = { + ...pending, + status: 'answered', + answer, + answeredAt, + }; + session.updatedAt = answeredAt; + + const result = await this.persistAndBroadcast(workspace); + this.pendingUserInputHandles.delete(userInputId); + + try { + await Promise.resolve(handle.resolve(answer, wasFreeform)); + session.pendingUserInput = undefined; + await this.persistAndBroadcast(workspace); + } catch (error) { + session.status = 'error'; + session.lastError = error instanceof Error ? error.message : String(error); + session.updatedAt = nowIso(); + + this.emitSessionEvent({ + sessionId, + kind: 'error', + occurredAt: session.updatedAt, + error: session.lastError, + }); + + await this.persistAndBroadcast(workspace); + throw error; + } + + return result; + } + async updateSessionModelConfig( sessionId: string, model: string, @@ -1114,6 +1179,7 @@ export class AryxAppService extends EventEmitter { const completedAt = nowIso(); session.status = 'idle'; session.lastError = undefined; + session.pendingUserInput = undefined; session.updatedAt = completedAt; const completedRun = this.updateSessionRun(session, requestId, (run) => completeSessionRunRecord(run, completedAt)); @@ -1144,6 +1210,7 @@ export class AryxAppService extends EventEmitter { const cancelledAt = nowIso(); session.status = 'idle'; session.lastError = undefined; + session.pendingUserInput = undefined; session.updatedAt = cancelledAt; const cancelledRun = this.updateSessionRun(session, requestId, (run) => cancelSessionRunRecord(run, cancelledAt)); @@ -1187,6 +1254,37 @@ export class AryxAppService extends EventEmitter { } } + private async handleUserInputRequested( + workspace: WorkspaceState, + sessionId: string, + _requestId: string, + event: UserInputRequestedEvent, + resolve: (answer: string, wasFreeform: boolean) => void | Promise, + ): Promise { + const session = this.requireSession(workspace, sessionId); + const requestedAt = nowIso(); + + session.pendingUserInput = { + id: event.userInputId, + status: 'pending', + agentId: event.agentId, + agentName: event.agentName, + question: event.question, + choices: event.choices, + allowFreeform: event.allowFreeform ?? true, + requestedAt, + }; + session.updatedAt = requestedAt; + + this.pendingUserInputHandles.set(event.userInputId, { + sessionId, + requestId: _requestId, + resolve, + }); + + await this.persistAndBroadcast(workspace); + } + private createPendingApprovalFromSidecarEvent(event: ApprovalRequestedEvent): PendingApprovalRecord { return { id: event.approvalId, diff --git a/src/main/ipc/registerIpcHandlers.ts b/src/main/ipc/registerIpcHandlers.ts index 29c54b0..28807a7 100644 --- a/src/main/ipc/registerIpcHandlers.ts +++ b/src/main/ipc/registerIpcHandlers.ts @@ -11,6 +11,7 @@ import type { RenameSessionInput, RescanProjectConfigsInput, ResolveSessionApprovalInput, + ResolveSessionUserInputInput, SaveLspProfileInput, SaveMcpServerInput, SavePatternInput, @@ -110,6 +111,9 @@ export function registerIpcHandlers(window: BrowserWindow, service: AryxAppServi ipcMain.handle(ipcChannels.resolveSessionApproval, (_event, input: ResolveSessionApprovalInput) => service.resolveSessionApproval(input.sessionId, input.approvalId, input.decision), ); + ipcMain.handle(ipcChannels.resolveSessionUserInput, (_event, input: ResolveSessionUserInputInput) => + service.resolveSessionUserInput(input.sessionId, input.userInputId, input.answer, input.wasFreeform), + ); ipcMain.handle( ipcChannels.updateSessionModelConfig, (_event, input: UpdateSessionModelConfigInput) => diff --git a/src/main/sidecar/runTurnPending.ts b/src/main/sidecar/runTurnPending.ts index 0b1477d..4f17115 100644 --- a/src/main/sidecar/runTurnPending.ts +++ b/src/main/sidecar/runTurnPending.ts @@ -1,4 +1,9 @@ -import type { AgentActivityEvent, ApprovalRequestedEvent, TurnDeltaEvent } from '@shared/contracts/sidecar'; +import type { + AgentActivityEvent, + ApprovalRequestedEvent, + TurnDeltaEvent, + UserInputRequestedEvent, +} from '@shared/contracts/sidecar'; import type { ChatMessageRecord } from '@shared/domain/session'; export interface RunTurnPendingCommand { @@ -8,6 +13,7 @@ export interface RunTurnPendingCommand { onDelta: (event: TurnDeltaEvent) => void | Promise; onActivity: (event: AgentActivityEvent) => void | Promise; onApproval: (event: ApprovalRequestedEvent) => void | Promise; + onUserInput: (event: UserInputRequestedEvent) => void | Promise; errored: boolean; } diff --git a/src/main/sidecar/sidecarProcess.ts b/src/main/sidecar/sidecarProcess.ts index fff3a06..5bf75d8 100644 --- a/src/main/sidecar/sidecarProcess.ts +++ b/src/main/sidecar/sidecarProcess.ts @@ -9,6 +9,7 @@ import type { SidecarCapabilities, SidecarEvent, TurnDeltaEvent, + UserInputRequestedEvent, ValidatePatternCommand, RunTurnCommand, } from '@shared/contracts/sidecar'; @@ -44,6 +45,12 @@ type PendingCommand = resolve: () => void; reject: (error: Error) => void; }) + | ({ + processId: number; + kind: 'resolve-user-input'; + resolve: () => void; + reject: (error: Error) => void; + }) | ({ processId: number; kind: 'cancel-turn'; @@ -94,8 +101,19 @@ export class SidecarClient { onDelta: (event: TurnDeltaEvent) => void | Promise, onActivity: (event: AgentActivityEvent) => void | Promise, onApproval: (event: ApprovalRequestedEvent) => void | Promise, + onUserInput: (event: UserInputRequestedEvent) => void | Promise, ): Promise { - return this.dispatch(command, onDelta, onActivity, onApproval); + return this.dispatch(command, onDelta, onActivity, onApproval, onUserInput); + } + + async resolveUserInput(userInputId: string, answer: string, wasFreeform: boolean): Promise { + return this.dispatch({ + type: 'resolve-user-input', + requestId: `user-input-${Date.now()}`, + userInputId, + answer, + wasFreeform, + }); } async resolveApproval(approvalId: string, decision: ApprovalDecision): Promise { @@ -199,6 +217,7 @@ export class SidecarClient { onDelta?: (event: TurnDeltaEvent) => void | Promise, onActivity?: (event: AgentActivityEvent) => void | Promise, onApproval?: (event: ApprovalRequestedEvent) => void | Promise, + onUserInput?: (event: UserInputRequestedEvent) => void | Promise, ): Promise { const state = await this.ensureProcess(); @@ -212,6 +231,7 @@ export class SidecarClient { onDelta: onDelta ?? (() => undefined), onActivity: onActivity ?? (() => undefined), onApproval: onApproval ?? (() => undefined), + onUserInput: onUserInput ?? (() => undefined), errored: false, }); } else if (command.type === 'validate-pattern') { @@ -228,6 +248,13 @@ export class SidecarClient { resolve: resolve as () => void, reject, }); + } else if (command.type === 'resolve-user-input') { + this.pending.set(command.requestId, { + processId: state.id, + kind: 'resolve-user-input', + resolve: resolve as () => void, + reject, + }); } else if (command.type === 'cancel-turn') { this.pending.set(command.requestId, { processId: state.id, @@ -297,6 +324,11 @@ export class SidecarClient { this.invokeRunTurnHandler(event.requestId, pending, () => pending.onApproval(event)); } return; + case 'user-input-requested': + if (pending.kind === 'run-turn' && shouldHandleRunTurnEvent(pending)) { + this.invokeRunTurnHandler(event.requestId, pending, () => pending.onUserInput(event)); + } + return; case 'turn-complete': if (pending.kind === 'run-turn') { if (shouldHandleRunTurnEvent(pending)) { @@ -318,7 +350,7 @@ export class SidecarClient { this.pending.delete(event.requestId); return; case 'command-complete': - if (pending.kind === 'resolve-approval' || pending.kind === 'cancel-turn') { + if (pending.kind === 'resolve-approval' || pending.kind === 'resolve-user-input' || pending.kind === 'cancel-turn') { pending.resolve(); this.pending.delete(event.requestId); } else if (pending.kind !== 'run-turn' || pending.errored) { diff --git a/src/preload/index.ts b/src/preload/index.ts index e3ee00b..d685022 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -36,6 +36,7 @@ const api: ElectronApi = { sendSessionMessage: (input) => ipcRenderer.invoke(ipcChannels.sendSessionMessage, input), cancelSessionTurn: (input) => ipcRenderer.invoke(ipcChannels.cancelSessionTurn, input), resolveSessionApproval: (input) => ipcRenderer.invoke(ipcChannels.resolveSessionApproval, input), + resolveSessionUserInput: (input) => ipcRenderer.invoke(ipcChannels.resolveSessionUserInput, input), updateSessionModelConfig: (input) => ipcRenderer.invoke(ipcChannels.updateSessionModelConfig, input), querySessions: (input) => ipcRenderer.invoke(ipcChannels.querySessions, input), diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index 7f4db28..9e47935 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -250,6 +250,9 @@ export default function App() { onResolveApproval={(approvalId, decision) => api.resolveSessionApproval({ sessionId: selectedSession.id, approvalId, decision }) } + onResolveUserInput={(userInputId, answer, wasFreeform) => + api.resolveSessionUserInput({ sessionId: selectedSession.id, userInputId, answer, wasFreeform }) + } onUpdateSessionModelConfig={(config) => api.updateSessionModelConfig({ sessionId: selectedSession.id, diff --git a/src/renderer/components/ChatPane.tsx b/src/renderer/components/ChatPane.tsx index 923d8c6..951555e 100644 --- a/src/renderer/components/ChatPane.tsx +++ b/src/renderer/components/ChatPane.tsx @@ -1,9 +1,10 @@ import { useEffect, useMemo, useRef, useState } from 'react'; -import { AlertCircle, ArrowUp, Bot, Circle, GitBranch, Loader2, ShieldAlert, Square, User } from 'lucide-react'; +import { AlertCircle, ArrowUp, Bot, Circle, GitBranch, Loader2, MessageCircleQuestion, ShieldAlert, Square, User } from 'lucide-react'; import { MarkdownContent } from '@renderer/components/MarkdownContent'; import { MarkdownComposer, type MarkdownComposerHandle } from '@renderer/components/MarkdownComposer'; import { ApprovalBanner, QueuedApprovalsList } from '@renderer/components/chat/ApprovalBanner'; +import { UserInputBanner } from '@renderer/components/chat/UserInputBanner'; import { InlineApprovalPill, InlineModelPill, InlineThinkingPill, InlineToolsPill } from '@renderer/components/chat/InlinePills'; import { ThinkingDots } from '@renderer/components/chat/ThinkingDots'; import { getAssistantMessagePhase } from '@renderer/lib/messagePhase'; @@ -36,6 +37,7 @@ interface ChatPaneProps { onSend: (content: string) => Promise; onCancelTurn?: () => void; onResolveApproval?: (approvalId: string, decision: ApprovalDecision) => Promise; + onResolveUserInput?: (userInputId: string, answer: string, wasFreeform: boolean) => Promise; onUpdateSessionModelConfig?: (config: { model: string; reasoningEffort?: ReasoningEffort; @@ -54,6 +56,7 @@ export function ChatPane({ onSend, onCancelTurn, onResolveApproval, + onResolveUserInput, onUpdateSessionModelConfig, onUpdateSessionTooling, onUpdateSessionApprovalSettings, @@ -62,6 +65,7 @@ export function ChatPane({ const [configError, setConfigError] = useState(); const [approvalError, setApprovalError] = useState(); const [isResolvingApproval, setIsResolvingApproval] = useState(false); + const [isSubmittingUserInput, setIsSubmittingUserInput] = useState(false); const [isUpdatingSessionModelConfig, setIsUpdatingSessionModelConfig] = useState(false); const transcriptRef = useRef(null); const composerRef = useRef(null); @@ -70,6 +74,7 @@ export function ChatPane({ const pendingApproval = session.pendingApproval?.status === 'pending' ? session.pendingApproval : undefined; const queuedApprovals = (session.pendingApprovalQueue ?? []).filter((a) => a.status === 'pending'); const totalPendingCount = (pendingApproval ? 1 : 0) + queuedApprovals.length; + const pendingUserInput = session.pendingUserInput?.status === 'pending' ? session.pendingUserInput : undefined; const isScratchpad = isScratchpadProject(project); const isSingleAgent = pattern.agents.length === 1; const primaryAgent = pattern.agents[0]; @@ -158,6 +163,20 @@ export function ChatPane({ } } + async function handleResolveUserInput(answer: string, wasFreeform: boolean) { + if (!pendingUserInput || !onResolveUserInput || isSubmittingUserInput) return; + + setIsSubmittingUserInput(true); + + try { + await onResolveUserInput(pendingUserInput.id, answer, wasFreeform); + } catch { + // User input errors are non-critical; the turn will fail and show the error status + } finally { + setIsSubmittingUserInput(false); + } + } + return (
{/* Header — extra top padding clears the title bar overlay zone */} @@ -194,14 +213,20 @@ export function ChatPane({ )}
)} - {isSessionBusy && !pendingApproval && } + {pendingUserInput && !pendingApproval && ( +
+ + Awaiting your input +
+ )} + {isSessionBusy && !pendingApproval && !pendingUserInput && } {session.status === 'error' && (
Error
)} - {session.status === 'idle' && !pendingApproval && session.messages.length > 0 && ( + {session.status === 'idle' && !pendingApproval && !pendingUserInput && session.messages.length > 0 && ( {session.messages.length} message{session.messages.length === 1 ? '' : 's'} @@ -339,6 +364,17 @@ export function ChatPane({ )} + {/* Pending user input banner */} + {pendingUserInput && ( +
+ void handleResolveUserInput(answer, wasFreeform)} + userInput={pendingUserInput} + /> +
+ )} + {/* Session config pills — tools/approval left, model/reasoning right */} {isSingleAgent && (
@@ -426,11 +462,13 @@ export function ChatPane({ placeholder={ pendingApproval ? 'Awaiting approval...' - : isSessionBusy - ? 'Waiting for response...' - : isUpdatingSessionModelConfig - ? 'Saving model settings...' - : 'Message...' + : pendingUserInput + ? 'Awaiting your input above...' + : isSessionBusy + ? 'Waiting for response...' + : isUpdatingSessionModelConfig + ? 'Saving model settings...' + : 'Message...' } > + ))} +
+ )} + + {/* Freeform input */} + {userInput.allowFreeform && ( +
+ setFreeformText(e.target.value)} + onKeyDown={handleKeyDown} + placeholder={hasChoices ? 'Or type your own answer…' : 'Type your answer…'} + type="text" + value={freeformText} + /> + +
+ )} + + ); +} diff --git a/src/shared/contracts/channels.ts b/src/shared/contracts/channels.ts index 16ad698..c6e3c1b 100644 --- a/src/shared/contracts/channels.ts +++ b/src/shared/contracts/channels.ts @@ -26,6 +26,7 @@ export const ipcChannels = { sendSessionMessage: 'sessions:send-message', cancelSessionTurn: 'sessions:cancel-turn', resolveSessionApproval: 'sessions:resolve-approval', + resolveSessionUserInput: 'sessions:resolve-user-input', querySessions: 'sessions:query', updateSessionModelConfig: 'sessions:update-model-config', selectProject: 'selection:project', diff --git a/src/shared/contracts/ipc.ts b/src/shared/contracts/ipc.ts index 6b4bd57..79ba60b 100644 --- a/src/shared/contracts/ipc.ts +++ b/src/shared/contracts/ipc.ts @@ -36,6 +36,13 @@ export interface ResolveSessionApprovalInput { decision: ApprovalDecision; } +export interface ResolveSessionUserInputInput { + sessionId: string; + userInputId: string; + answer: string; + wasFreeform: boolean; +} + export interface UpdateSessionModelConfigInput { sessionId: string; model: string; @@ -126,6 +133,7 @@ export interface ElectronApi { sendSessionMessage(input: SendSessionMessageInput): Promise; cancelSessionTurn(input: CancelSessionTurnInput): Promise; resolveSessionApproval(input: ResolveSessionApprovalInput): Promise; + resolveSessionUserInput(input: ResolveSessionUserInputInput): Promise; updateSessionModelConfig(input: UpdateSessionModelConfigInput): Promise; querySessions(input: QuerySessionsInput): Promise; selectProject(projectId?: string): Promise; diff --git a/src/shared/contracts/sidecar.ts b/src/shared/contracts/sidecar.ts index a51b5ff..2adeba2 100644 --- a/src/shared/contracts/sidecar.ts +++ b/src/shared/contracts/sidecar.ts @@ -92,12 +92,21 @@ export interface ResolveApprovalCommand { decision: ApprovalDecision; } +export interface ResolveUserInputCommand { + type: 'resolve-user-input'; + requestId: string; + userInputId: string; + answer: string; + wasFreeform: boolean; +} + export type SidecarCommand = | DescribeCapabilitiesCommand | ValidatePatternCommand | RunTurnCommand | CancelTurnCommand - | ResolveApprovalCommand; + | ResolveApprovalCommand + | ResolveUserInputCommand; export interface RunTurnLocalMcpServerConfig { id: string; @@ -220,6 +229,18 @@ export interface ApprovalRequestedEvent { permissionDetail?: PermissionDetail; } +export interface UserInputRequestedEvent { + type: 'user-input-requested'; + requestId: string; + sessionId: string; + userInputId: string; + agentId?: string; + agentName?: string; + question: string; + choices?: string[]; + allowFreeform?: boolean; +} + export interface CommandErrorEvent { type: 'command-error'; requestId: string; @@ -238,5 +259,6 @@ export type SidecarEvent = | TurnCompleteEvent | AgentActivityEvent | ApprovalRequestedEvent + | UserInputRequestedEvent | CommandErrorEvent | CommandCompleteEvent; diff --git a/src/shared/domain/session.ts b/src/shared/domain/session.ts index 1b38844..f54dcc7 100644 --- a/src/shared/domain/session.ts +++ b/src/shared/domain/session.ts @@ -11,6 +11,7 @@ import { type SessionApprovalSettings, } from '@shared/domain/approval'; import type { SessionRunRecord } from '@shared/domain/runTimeline'; +import type { PendingUserInputRecord } from '@shared/domain/userInput'; export type ChatRole = 'system' | 'user' | 'assistant'; export type SessionStatus = 'idle' | 'running' | 'error'; @@ -48,6 +49,7 @@ export interface SessionRecord { approvalSettings?: SessionApprovalSettings; pendingApproval?: PendingApprovalRecord; pendingApprovalQueue?: PendingApprovalRecord[]; + pendingUserInput?: PendingUserInputRecord; runs: SessionRunRecord[]; } diff --git a/src/shared/domain/userInput.ts b/src/shared/domain/userInput.ts new file mode 100644 index 0000000..0673b80 --- /dev/null +++ b/src/shared/domain/userInput.ts @@ -0,0 +1,14 @@ +export type UserInputStatus = 'pending' | 'answered'; + +export interface PendingUserInputRecord { + id: string; + status: UserInputStatus; + agentId?: string; + agentName?: string; + question: string; + choices?: string[]; + allowFreeform: boolean; + requestedAt: string; + answer?: string; + answeredAt?: string; +} diff --git a/tests/main/runTurnPending.test.ts b/tests/main/runTurnPending.test.ts index a0642f5..f472da2 100644 --- a/tests/main/runTurnPending.test.ts +++ b/tests/main/runTurnPending.test.ts @@ -16,6 +16,7 @@ describe('run turn pending helpers', () => { onDelta: () => undefined, onActivity: () => undefined, onApproval: () => undefined, + onUserInput: () => undefined, errored: false, }; @@ -38,6 +39,7 @@ describe('run turn pending helpers', () => { onDelta: () => undefined, onActivity: () => undefined, onApproval: () => undefined, + onUserInput: () => undefined, errored: false, };