feat: add handoff workflow checkpoint recovery

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
David Kaya
2026-04-01 19:28:05 +02:00
co-authored by Copilot
parent 1ceb3d5669
commit 13bcc44f1a
10 changed files with 761 additions and 24 deletions
+3
View File
@@ -225,6 +225,7 @@ The protocol also carries **turn-scoped lifecycle events** alongside output delt
- **Session usage events**: current token count and context-window limit from `session.usage_info` for context-bar rendering
- **Pending-messages-modified events**: emitted when mid-turn steering changes the pending message queue
- **Workflow diagnostic events**: normalized warnings and errors from Agent Framework (`WorkflowWarningEvent`, `WorkflowErrorEvent`, `ExecutorFailedEvent`) with optional executor or subworkflow metadata for richer debugging surfaces
- **Workflow checkpoint events**: emitted at Agent Framework superstep boundaries with workflow session ID, checkpoint ID, step number, and checkpoint-store path so the main process can prepare crash-recovery state
These events flow through a single `onTurnScopedEvent` callback on the `runTurn` command, avoiding per-event-type callback proliferation. The main process maps each event to a `SessionEventRecord` and pushes it to the renderer, where lightweight state maps (activity, usage, turn-event log) consume them without touching the persisted workspace.
@@ -236,6 +237,8 @@ For project-backed sessions, the sidecar also discovers GitHub Copilot CLI hook
The `run-turn` command now also carries a project-instruction payload derived from scanned repo customization files. The main process composes that payload from repo-level instruction files and merges enabled discovered custom agent profiles into the primary pattern agent's Copilot configuration before sending the command across the stdio boundary. The sidecar then folds those project instructions into the final SDK system message alongside the agent's own instructions and runtime guidance.
For handoff workflows, the sidecar now also enables Agent Framework JSON checkpointing backed by a per-turn filesystem store under local app data. Each saved checkpoint is surfaced to the main process, which pairs the durable Agent Framework checkpoint with an in-memory rollback snapshot of `session.messages` and the active run timeline events. If the sidecar child process exits unexpectedly during the same app lifetime, Aryx restores the latest snapshot, clears pending approval/user-input/MCP-auth state for that run, and retries the `run-turn` request once with `resumeFromCheckpoint`. Checkpoint directories are deleted after the turn completes, cancels, or fails. This recovery path is intentionally scoped to same-app sidecar restarts; full app-restart workflow rehydration would require durable rollback snapshots in addition to the Agent Framework checkpoint payloads.
## Security model
Security in this system is mostly about **desktop trust boundaries**.
@@ -188,6 +188,7 @@ public sealed class RunTurnCommandDto : SidecarCommandEnvelope
public PatternDefinitionDto Pattern { get; init; } = new();
public IReadOnlyList<ChatMessageDto> Messages { get; init; } = [];
public RunTurnToolingConfigDto? Tooling { get; init; }
public WorkflowCheckpointResumeDto? ResumeFromCheckpoint { get; init; }
}
public sealed class CancelTurnCommandDto : SidecarCommandEnvelope
@@ -488,6 +489,15 @@ public sealed class PendingMessagesModifiedEventDto : SidecarEventDto
public string? AgentName { get; init; }
}
public sealed class WorkflowCheckpointSavedEventDto : SidecarEventDto
{
public string SessionId { get; init; } = string.Empty;
public string WorkflowSessionId { get; init; } = string.Empty;
public string CheckpointId { get; init; } = string.Empty;
public string StorePath { get; init; } = string.Empty;
public int StepNumber { get; init; }
}
public sealed class WorkflowDiagnosticEventDto : SidecarEventDto
{
public string SessionId { get; init; } = string.Empty;
@@ -604,6 +614,13 @@ public sealed class ExitPlanModeRequestedEventDto : SidecarEventDto
public string? RecommendedAction { get; init; }
}
public sealed class WorkflowCheckpointResumeDto
{
public string WorkflowSessionId { get; init; } = string.Empty;
public string CheckpointId { get; init; } = string.Empty;
public string StorePath { get; init; } = string.Empty;
}
public sealed class CommandErrorEventDto : SidecarEventDto
{
public string Message { get; init; } = string.Empty;
@@ -1,7 +1,9 @@
using System.IO;
using System.Linq;
using Aryx.AgentHost.Contracts;
using GitHub.Copilot.SDK;
using Microsoft.Agents.AI.Workflows;
using Microsoft.Agents.AI.Workflows.Checkpointing;
using Microsoft.Extensions.AI;
namespace Aryx.AgentHost.Services;
@@ -81,7 +83,16 @@ public sealed class CopilotWorkflowRunner : ITurnWorkflowRunner
List<ChatMessage> inputMessages = command.Messages.Select(WorkflowTranscriptProjector.ToChatMessage).ToList();
WorkflowTranscriptProjector.AttachMessageMode(inputMessages, command.MessageMode);
await using StreamingRun run = await InProcessExecution.RunStreamingAsync(workflow, inputMessages).ConfigureAwait(false);
using FileSystemJsonCheckpointStore? checkpointStore = CreateCheckpointStore(command);
CheckpointManager? checkpointManager = checkpointStore is not null
? CheckpointManager.CreateJson(checkpointStore)
: null;
await using StreamingRun run = await OpenWorkflowRunAsync(
command,
workflow,
inputMessages,
checkpointManager).ConfigureAwait(false);
await run.TrySendMessageAsync(new TurnToken(emitEvents: true)).ConfigureAwait(false);
await foreach (WorkflowEvent evt in run.WatchStreamAsync(runCancellation.Token).ConfigureAwait(false))
@@ -120,6 +131,62 @@ public sealed class CopilotWorkflowRunner : ITurnWorkflowRunner
}
}
internal static FileSystemJsonCheckpointStore? CreateCheckpointStore(RunTurnCommandDto command)
{
if (!ShouldEnableWorkflowCheckpointing(command))
{
return null;
}
DirectoryInfo checkpointDirectory = new(GetCheckpointStorePath(command));
return new FileSystemJsonCheckpointStore(checkpointDirectory);
}
internal static bool ShouldEnableWorkflowCheckpointing(RunTurnCommandDto command)
{
ArgumentNullException.ThrowIfNull(command);
return string.Equals(command.Pattern.Mode, "handoff", StringComparison.OrdinalIgnoreCase);
}
internal static string GetCheckpointStorePath(RunTurnCommandDto command)
{
ArgumentNullException.ThrowIfNull(command);
if (!string.IsNullOrWhiteSpace(command.ResumeFromCheckpoint?.StorePath))
{
return command.ResumeFromCheckpoint.StorePath;
}
string localAppData = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
return Path.Combine(localAppData, "Aryx", "workflow-checkpoints", command.SessionId, command.RequestId);
}
private static ValueTask<StreamingRun> OpenWorkflowRunAsync(
RunTurnCommandDto command,
Workflow workflow,
IReadOnlyList<ChatMessage> inputMessages,
CheckpointManager? checkpointManager)
{
if (checkpointManager is not null && command.ResumeFromCheckpoint is { } resumeFromCheckpoint)
{
return InProcessExecution.ResumeStreamingAsync(
workflow,
new CheckpointInfo(resumeFromCheckpoint.WorkflowSessionId, resumeFromCheckpoint.CheckpointId),
checkpointManager);
}
if (checkpointManager is not null)
{
return InProcessExecution.RunStreamingAsync(
workflow,
inputMessages.ToList(),
checkpointManager,
sessionId: command.RequestId);
}
return InProcessExecution.RunStreamingAsync(workflow, inputMessages.ToList());
}
internal static void ConfigureHookLifecycleEventSuppression(
CopilotTurnExecutionState state,
CopilotAgentBundle bundle)
@@ -211,6 +278,12 @@ public sealed class CopilotWorkflowRunner : ITurnWorkflowRunner
return false;
}
if (TryCreateWorkflowCheckpointSavedEvent(command, evt, out WorkflowCheckpointSavedEventDto? checkpointSaved))
{
await onEvent(checkpointSaved).ConfigureAwait(false);
return false;
}
if (TryCreateWorkflowDiagnosticEvent(command, evt, state, out WorkflowDiagnosticEventDto? diagnostic))
{
await onEvent(diagnostic).ConfigureAwait(false);
@@ -347,6 +420,33 @@ public sealed class CopilotWorkflowRunner : ITurnWorkflowRunner
}
}
internal static bool TryCreateWorkflowCheckpointSavedEvent(
RunTurnCommandDto command,
WorkflowEvent evt,
out WorkflowCheckpointSavedEventDto checkpointSaved)
{
checkpointSaved = default!;
if (!ShouldEnableWorkflowCheckpointing(command)
|| evt is not SuperStepCompletedEvent superStepCompleted
|| superStepCompleted.CompletionInfo?.Checkpoint is not CheckpointInfo checkpoint)
{
return false;
}
checkpointSaved = new WorkflowCheckpointSavedEventDto
{
Type = "workflow-checkpoint-saved",
RequestId = command.RequestId,
SessionId = command.SessionId,
WorkflowSessionId = checkpoint.SessionId,
CheckpointId = checkpoint.CheckpointId,
StorePath = GetCheckpointStorePath(command),
StepNumber = superStepCompleted.StepNumber,
};
return true;
}
private static bool TryCreateWorkflowDiagnosticEvent(
RunTurnCommandDto command,
WorkflowEvent evt,
@@ -1,3 +1,4 @@
using System.IO;
using System.Reflection;
using System.Runtime.CompilerServices;
using Aryx.AgentHost.Contracts;
@@ -798,6 +799,50 @@ public sealed class CopilotWorkflowRunnerTests
});
}
[Fact]
public async Task HandleWorkflowEventAsync_EmitsWorkflowCheckpointSavedEvent()
{
RunTurnCommandDto command = CreateHandoffCommand();
CopilotTurnExecutionState state = new(command);
List<WorkflowCheckpointSavedEventDto> checkpoints = [];
MethodInfo handleWorkflowEvent = typeof(CopilotWorkflowRunner).GetMethod(
"HandleWorkflowEventAsync",
BindingFlags.NonPublic | BindingFlags.Static)!;
Task<bool> handleTask = (Task<bool>)handleWorkflowEvent.Invoke(
null,
[
command,
new SuperStepCompletedEvent(
3,
new SuperStepCompletionInfo([])
{
Checkpoint = new CheckpointInfo(command.RequestId, "checkpoint-1"),
}),
Array.Empty<ChatMessage>(),
state,
(Func<TurnDeltaEventDto, Task>)(_ => Task.CompletedTask),
(Func<SidecarEventDto, Task>)(sidecarEvent =>
{
checkpoints.Add(Assert.IsType<WorkflowCheckpointSavedEventDto>(sidecarEvent));
return Task.CompletedTask;
}),
])!;
bool shouldEndTurn = await handleTask;
Assert.False(shouldEndTurn);
WorkflowCheckpointSavedEventDto checkpoint = Assert.Single(checkpoints);
Assert.Equal("workflow-checkpoint-saved", checkpoint.Type);
Assert.Equal(command.SessionId, checkpoint.SessionId);
Assert.Equal(command.RequestId, checkpoint.WorkflowSessionId);
Assert.Equal("checkpoint-1", checkpoint.CheckpointId);
Assert.Equal(3, checkpoint.StepNumber);
Assert.EndsWith(
Path.Combine("Aryx", "workflow-checkpoints", command.SessionId, command.RequestId),
checkpoint.StorePath);
}
[Fact]
public async Task HandleWorkflowEventAsync_EmitsExecutorFailedDiagnostic()
{
+173 -16
View File
@@ -12,10 +12,13 @@ import type {
McpOauthRequiredEvent,
MessageReclassifiedEvent,
RunTurnCustomAgentConfig,
RunTurnCommand,
RunTurnToolingConfig,
SidecarCapabilities,
UserInputRequestedEvent,
TurnDeltaEvent,
WorkflowCheckpointResume,
WorkflowCheckpointSavedEvent,
} from '@shared/contracts/sidecar';
import type { TurnScopedEvent } from '@main/sidecar/runTurnPending';
import {
@@ -104,6 +107,7 @@ import {
upsertRunApprovalEvent,
upsertRunMessageEvent,
upsertSessionRunRecord,
type RunTimelineEventRecord,
type SessionRunRecord,
} from '@shared/domain/runTimeline';
import {
@@ -170,6 +174,15 @@ type PendingUserInputHandle = {
resolve: (answer: string, wasFreeform: boolean) => void | Promise<void>;
};
type WorkflowCheckpointRecoveryState = {
workflowSessionId: string;
checkpointId: string;
storePath: string;
stepNumber: number;
sessionMessages: ChatMessageRecord[];
runEvents: RunTimelineEventRecord[];
};
type DiscoveredToolingResolution = 'accept' | 'dismiss';
function isBuiltinPattern(patternId: string): boolean {
@@ -190,6 +203,16 @@ function isSidecarStoppedBeforeCompletionError(error: unknown): error is Error {
return error instanceof Error && error.message === SIDECAR_STOPPED_BEFORE_COMPLETION_MESSAGE;
}
function isUnexpectedSidecarTerminationError(error: unknown): error is Error {
if (!(error instanceof Error)) {
return false;
}
const { message } = error;
return isSidecarStoppedBeforeCompletionError(error)
|| message.startsWith('The .NET sidecar exited unexpectedly with code ');
}
const INTERRUPTED_RUN_ERROR =
'This session was interrupted because Aryx restarted while a run was in progress.';
const INTERRUPTED_APPROVAL_ERROR =
@@ -208,6 +231,7 @@ export class AryxAppService extends EventEmitter<AppServiceEvents> {
private readonly ptyManager = new PtyManager();
private readonly pendingApprovalHandles = new Map<string, PendingApprovalHandle>();
private readonly pendingUserInputHandles = new Map<string, PendingUserInputHandle>();
private readonly workflowCheckpointRecoveries = new Map<string, WorkflowCheckpointRecoveryState>();
private workspace?: WorkspaceState;
private sidecarCapabilities?: SidecarCapabilities;
private sidecarCapabilitiesPromise?: Promise<SidecarCapabilities>;
@@ -1407,21 +1431,29 @@ export class AryxAppService extends EventEmitter<AppServiceEvents> {
});
try {
const responseMessages = await this.sidecar.runTurn(
{
type: 'run-turn',
requestId,
sessionId: session.id,
projectPath: runWorkingDirectory,
workspaceKind,
mode: session.interactionMode ?? 'interactive',
messageMode,
projectInstructions,
pattern: effectivePattern,
messages: session.messages,
attachments: attachments?.length ? attachments : undefined,
tooling: this.buildRunTurnToolingConfig(workspace, session),
},
const createRunTurnCommand = (
resumeFromCheckpoint?: WorkflowCheckpointResume,
): RunTurnCommand => ({
type: 'run-turn',
requestId,
sessionId: session.id,
projectPath: runWorkingDirectory,
workspaceKind,
mode: session.interactionMode ?? 'interactive',
messageMode,
projectInstructions,
pattern: effectivePattern,
messages: session.messages,
attachments: attachments?.length ? attachments : undefined,
tooling: this.buildRunTurnToolingConfig(workspace, session),
resumeFromCheckpoint,
});
const responseMessages = await this.runSidecarTurnWithCheckpointRecovery(
workspace,
session,
requestId,
createRunTurnCommand,
async (event) => {
await this.applyTurnDelta(workspace, session.id, requestId, event);
},
@@ -1459,6 +1491,7 @@ export class AryxAppService extends EventEmitter<AppServiceEvents> {
}
}
await this.persistAndBroadcast(workspace);
await this.cleanupWorkflowCheckpointRecovery(requestId);
if (workspaceKind === 'project') {
this.scheduleProjectGitRefresh(project.id);
}
@@ -1472,6 +1505,7 @@ export class AryxAppService extends EventEmitter<AppServiceEvents> {
}
}
await this.persistAndBroadcast(workspace);
await this.cleanupWorkflowCheckpointRecovery(requestId);
if (workspaceKind === 'project') {
this.scheduleProjectGitRefresh(project.id);
}
@@ -1504,6 +1538,7 @@ export class AryxAppService extends EventEmitter<AppServiceEvents> {
}
await this.persistAndBroadcast(workspace);
await this.cleanupWorkflowCheckpointRecovery(requestId);
if (workspaceKind === 'project') {
this.scheduleProjectGitRefresh(project.id);
}
@@ -2421,13 +2456,22 @@ export class AryxAppService extends EventEmitter<AppServiceEvents> {
}
private handleTurnScopedEvent(
_workspace: WorkspaceState,
workspace: WorkspaceState,
sessionId: string,
event: TurnScopedEvent,
): void {
const occurredAt = nowIso();
switch (event.type) {
case 'workflow-checkpoint-saved': {
const session = this.requireSession(workspace, sessionId);
const run = session.runs.find((candidate) => candidate.requestId === event.requestId);
if (run) {
this.recordWorkflowCheckpointRecovery(session, run, event);
}
return;
}
case 'subagent-event':
this.emitSessionEvent({
sessionId,
@@ -2540,6 +2584,119 @@ export class AryxAppService extends EventEmitter<AppServiceEvents> {
}
}
private async runSidecarTurnWithCheckpointRecovery(
workspace: WorkspaceState,
session: SessionRecord,
requestId: string,
createCommand: (resumeFromCheckpoint?: WorkflowCheckpointResume) => RunTurnCommand,
onDelta: (event: TurnDeltaEvent) => void | Promise<void>,
onActivity: (event: AgentActivityEvent) => void | Promise<void>,
onApproval: (event: ApprovalRequestedEvent) => void | Promise<void>,
onUserInput: (event: UserInputRequestedEvent) => void | Promise<void>,
onMcpOAuthRequired: (event: McpOauthRequiredEvent) => void | Promise<void>,
onExitPlanMode: (event: ExitPlanModeRequestedEvent) => void | Promise<void>,
onMessageReclassified: (event: MessageReclassifiedEvent) => void | Promise<void>,
onTurnScopedEvent: (event: TurnScopedEvent) => void | Promise<void>,
): Promise<ChatMessageRecord[]> {
const invokeTurn = (resumeFromCheckpoint?: WorkflowCheckpointResume) => this.sidecar.runTurn(
createCommand(resumeFromCheckpoint),
onDelta,
onActivity,
onApproval,
onUserInput,
onMcpOAuthRequired,
onExitPlanMode,
onMessageReclassified,
onTurnScopedEvent,
);
try {
return await invokeTurn();
} catch (error) {
const recovery = this.workflowCheckpointRecoveries.get(requestId);
if (!isUnexpectedSidecarTerminationError(error) || !recovery) {
throw error;
}
const restoredRun = this.restoreWorkflowCheckpointRecovery(session, requestId, recovery);
await this.persistAndBroadcast(workspace);
if (restoredRun) {
this.emitRunUpdated(session.id, session.updatedAt, restoredRun);
}
return invokeTurn({
workflowSessionId: recovery.workflowSessionId,
checkpointId: recovery.checkpointId,
storePath: recovery.storePath,
});
}
}
private recordWorkflowCheckpointRecovery(
session: SessionRecord,
run: SessionRunRecord,
event: WorkflowCheckpointSavedEvent,
): void {
this.workflowCheckpointRecoveries.set(event.requestId, {
workflowSessionId: event.workflowSessionId,
checkpointId: event.checkpointId,
storePath: event.storePath,
stepNumber: event.stepNumber,
sessionMessages: structuredClone(session.messages),
runEvents: structuredClone(run.events),
});
}
private restoreWorkflowCheckpointRecovery(
session: SessionRecord,
requestId: string,
recovery: WorkflowCheckpointRecoveryState,
): SessionRunRecord | undefined {
session.messages = structuredClone(recovery.sessionMessages);
session.status = 'running';
session.lastError = undefined;
session.updatedAt = nowIso();
this.clearPendingRunState(session, requestId);
return this.updateSessionRun(session, requestId, (run) => ({
...run,
events: structuredClone(recovery.runEvents),
}));
}
private clearPendingRunState(session: SessionRecord, requestId: string): void {
this.setSessionPendingApprovalState(session, {});
session.pendingUserInput = undefined;
session.pendingPlanReview = undefined;
session.pendingMcpAuth = undefined;
for (const [approvalId, handle] of this.pendingApprovalHandles.entries()) {
if (handle.sessionId === session.id && handle.requestId === requestId) {
this.pendingApprovalHandles.delete(approvalId);
}
}
for (const [userInputId, handle] of this.pendingUserInputHandles.entries()) {
if (handle.sessionId === session.id && handle.requestId === requestId) {
this.pendingUserInputHandles.delete(userInputId);
}
}
}
private async cleanupWorkflowCheckpointRecovery(requestId: string): Promise<void> {
const recovery = this.workflowCheckpointRecoveries.get(requestId);
this.workflowCheckpointRecoveries.delete(requestId);
if (!recovery) {
return;
}
try {
await rm(recovery.storePath, { recursive: true, force: true });
} catch (error) {
console.warn('[aryx workflow-checkpoint] Failed to clean checkpoint store:', error);
}
}
private createPendingApprovalFromSidecarEvent(event: ApprovalRequestedEvent): PendingApprovalRecord {
return {
id: event.approvalId,
+2
View File
@@ -12,6 +12,7 @@ import type {
SessionUsageEvent,
SessionCompactionEvent,
PendingMessagesModifiedEvent,
WorkflowCheckpointSavedEvent,
AssistantUsageEvent,
AssistantIntentEvent,
ReasoningDeltaEvent,
@@ -26,6 +27,7 @@ export type TurnScopedEvent =
| SessionUsageEvent
| SessionCompactionEvent
| PendingMessagesModifiedEvent
| WorkflowCheckpointSavedEvent
| AssistantUsageEvent
| AssistantIntentEvent
| ReasoningDeltaEvent
+7 -6
View File
@@ -452,12 +452,13 @@ export class SidecarClient {
case 'skill-invoked':
case 'hook-lifecycle':
case 'session-usage':
case 'session-compaction':
case 'pending-messages-modified':
case 'workflow-diagnostic':
case 'assistant-usage':
case 'assistant-intent':
case 'reasoning-delta':
case 'session-compaction':
case 'pending-messages-modified':
case 'workflow-checkpoint-saved':
case 'workflow-diagnostic':
case 'assistant-usage':
case 'assistant-intent':
case 'reasoning-delta':
if (pending.kind === 'run-turn' && shouldHandleRunTurnEvent(pending)) {
this.invokeRunTurnHandler(event.requestId, pending, () => pending.onTurnScopedEvent(event));
}
+18
View File
@@ -72,6 +72,12 @@ export interface ValidatePatternCommand {
export type InteractionMode = 'interactive' | 'plan';
export type MessageMode = 'enqueue' | 'immediate';
export interface WorkflowCheckpointResume {
workflowSessionId: string;
checkpointId: string;
storePath: string;
}
export interface RunTurnCommand {
type: 'run-turn';
requestId: string;
@@ -85,6 +91,7 @@ export interface RunTurnCommand {
messages: ChatMessageRecord[];
attachments?: ChatMessageAttachment[];
tooling?: RunTurnToolingConfig;
resumeFromCheckpoint?: WorkflowCheckpointResume;
}
export interface CancelTurnCommand {
@@ -384,6 +391,16 @@ export interface PendingMessagesModifiedEvent {
agentName?: string;
}
export interface WorkflowCheckpointSavedEvent {
type: 'workflow-checkpoint-saved';
requestId: string;
sessionId: string;
workflowSessionId: string;
checkpointId: string;
storePath: string;
stepNumber: number;
}
export type WorkflowDiagnosticSeverity = 'warning' | 'error';
export type WorkflowDiagnosticKind =
| 'workflow-warning'
@@ -585,6 +602,7 @@ export type SidecarEvent =
| SessionUsageEvent
| SessionCompactionEvent
| PendingMessagesModifiedEvent
| WorkflowCheckpointSavedEvent
| WorkflowDiagnosticEvent
| ApprovalRequestedEvent
| UserInputRequestedEvent
@@ -0,0 +1,297 @@
import { describe, expect, mock, test } from 'bun:test';
import type { RunTurnCommand, WorkflowCheckpointSavedEvent, WorkflowCheckpointResume } from '@shared/contracts/sidecar';
import { SCRATCHPAD_PROJECT_ID } from '@shared/domain/project';
import type { ChatMessageRecord, SessionRecord } from '@shared/domain/session';
import {
createSessionRunRecord,
type RunTimelineEventRecord,
type SessionRunRecord,
} from '@shared/domain/runTimeline';
mock.module('electron', () => {
const electronMock = {
app: {
isPackaged: false,
getAppPath: () => 'C:\\workspace\\personal\\repositories\\aryx',
getPath: () => 'C:\\workspace\\personal\\repositories\\aryx\\tests\\fixtures',
},
dialog: {
showOpenDialog: async () => ({ canceled: true, filePaths: [] }),
},
shell: {
openPath: async () => '',
},
};
return {
...electronMock,
default: electronMock,
};
});
mock.module('keytar', () => ({
default: {
getPassword: async () => null,
setPassword: async () => undefined,
deletePassword: async () => false,
},
}));
const { AryxAppService } = await import('@main/AryxAppService');
describe('AryxAppService workflow checkpointing', () => {
test('records workflow checkpoint recovery snapshots from turn-scoped events', async () => {
const service = new AryxAppService();
const { session, run } = createRunningSession();
const checkpointEvent: WorkflowCheckpointSavedEvent = {
type: 'workflow-checkpoint-saved',
requestId: run.requestId,
sessionId: session.id,
workflowSessionId: run.requestId,
checkpointId: 'checkpoint-1',
storePath: 'C:\\Users\\tester\\AppData\\Local\\Aryx\\workflow-checkpoints\\session-1\\turn-1',
stepNumber: 2,
};
const internals = service as unknown as {
workflowCheckpointRecoveries: Map<string, unknown>;
handleTurnScopedEvent: (
workspace: { sessions: SessionRecord[] },
sessionId: string,
event: WorkflowCheckpointSavedEvent,
) => void | Promise<void>;
};
await internals.handleTurnScopedEvent({ sessions: [session] }, session.id, checkpointEvent);
expect(internals.workflowCheckpointRecoveries.get(run.requestId)).toEqual({
workflowSessionId: run.requestId,
checkpointId: 'checkpoint-1',
storePath: checkpointEvent.storePath,
stepNumber: 2,
sessionMessages: session.messages,
runEvents: run.events,
});
});
test('retries a checkpointed turn with resume metadata after sidecar exit', async () => {
const service = new AryxAppService();
const { session, run } = createRunningSession();
const workspace = { sessions: [session] };
const checkpointRecovery = {
workflowSessionId: run.requestId,
checkpointId: 'checkpoint-7',
storePath: 'C:\\Users\\tester\\AppData\\Local\\Aryx\\workflow-checkpoints\\session-1\\turn-1',
stepNumber: 7,
sessionMessages: structuredClone(session.messages),
runEvents: structuredClone(run.events),
};
const invocations: RunTurnCommand[] = [];
session.messages.push({
id: 'msg-partial',
role: 'assistant',
authorName: 'Primary',
content: 'Partial output after the checkpoint.',
createdAt: '2026-04-01T12:00:05.000Z',
pending: true,
});
run.events = [
...run.events,
{
id: 'run-event-extra',
kind: 'message',
occurredAt: '2026-04-01T12:00:05.000Z',
status: 'running',
messageId: 'msg-partial',
content: 'Partial output after the checkpoint.',
} satisfies RunTimelineEventRecord,
];
session.pendingUserInput = {
id: 'user-input-1',
status: 'pending',
requestedAt: '2026-04-01T12:00:05.000Z',
question: 'Need more detail?',
choices: ['Yes', 'No'],
allowFreeform: true,
};
(
service as unknown as {
workflowCheckpointRecoveries: Map<string, unknown>;
sidecar: {
runTurn: (
command: RunTurnCommand,
) => Promise<ChatMessageRecord[]>;
};
persistAndBroadcast: (workspace: unknown) => Promise<void>;
emitRunUpdated: (sessionId: string, occurredAt: string, run: SessionRunRecord) => void;
runSidecarTurnWithCheckpointRecovery: (
workspace: unknown,
session: SessionRecord,
requestId: string,
createCommand: (resumeFromCheckpoint?: WorkflowCheckpointResume) => RunTurnCommand,
onDelta: () => Promise<void>,
onActivity: () => Promise<void>,
onApproval: () => Promise<void>,
onUserInput: () => Promise<void>,
onMcpOAuthRequired: () => Promise<void>,
onExitPlanMode: () => Promise<void>,
onMessageReclassified: () => Promise<void>,
onTurnScopedEvent: () => Promise<void>,
) => Promise<ChatMessageRecord[]>;
}
).workflowCheckpointRecoveries.set(run.requestId, checkpointRecovery);
(
service as unknown as {
sidecar: {
runTurn: (command: RunTurnCommand) => Promise<ChatMessageRecord[]>;
};
}
).sidecar = {
runTurn: async (command: RunTurnCommand) => {
invocations.push(structuredClone(command));
if (invocations.length === 1) {
throw new Error('The .NET sidecar exited unexpectedly with code 1.');
}
return [];
},
};
(
service as unknown as {
persistAndBroadcast: (workspace: unknown) => Promise<void>;
emitRunUpdated: (sessionId: string, occurredAt: string, run: SessionRunRecord) => void;
}
).persistAndBroadcast = async () => undefined;
(
service as unknown as {
emitRunUpdated: (sessionId: string, occurredAt: string, run: SessionRunRecord) => void;
}
).emitRunUpdated = () => undefined;
const result = await (
service as unknown as {
runSidecarTurnWithCheckpointRecovery: (
workspace: unknown,
session: SessionRecord,
requestId: string,
createCommand: (resumeFromCheckpoint?: WorkflowCheckpointResume) => RunTurnCommand,
onDelta: () => Promise<void>,
onActivity: () => Promise<void>,
onApproval: () => Promise<void>,
onUserInput: () => Promise<void>,
onMcpOAuthRequired: () => Promise<void>,
onExitPlanMode: () => Promise<void>,
onMessageReclassified: () => Promise<void>,
onTurnScopedEvent: () => Promise<void>,
) => Promise<ChatMessageRecord[]>;
}
).runSidecarTurnWithCheckpointRecovery(
workspace,
session,
run.requestId,
(resumeFromCheckpoint?: WorkflowCheckpointResume): RunTurnCommand => ({
type: 'run-turn',
requestId: run.requestId,
sessionId: session.id,
projectPath: 'C:\\scratchpad',
workspaceKind: 'scratchpad',
mode: 'interactive',
messageMode: 'enqueue',
pattern: createPattern(),
messages: session.messages,
resumeFromCheckpoint,
}),
async () => undefined,
async () => undefined,
async () => undefined,
async () => undefined,
async () => undefined,
async () => undefined,
async () => undefined,
async () => undefined,
);
expect(result).toEqual([]);
expect(invocations).toHaveLength(2);
expect(invocations[0]?.resumeFromCheckpoint).toBeUndefined();
expect(invocations[1]?.resumeFromCheckpoint).toEqual({
workflowSessionId: run.requestId,
checkpointId: 'checkpoint-7',
storePath: checkpointRecovery.storePath,
});
expect(invocations[1]?.messages).toEqual(checkpointRecovery.sessionMessages);
expect(session.messages).toEqual(checkpointRecovery.sessionMessages);
expect(session.pendingUserInput).toBeUndefined();
expect(session.runs[0]?.events).toEqual(checkpointRecovery.runEvents);
});
});
function createRunningSession(): { session: SessionRecord; run: SessionRunRecord } {
const pattern = createPattern();
const run = createSessionRunRecord({
requestId: 'turn-1',
project: {
id: SCRATCHPAD_PROJECT_ID,
path: 'C:\\scratchpad',
},
workingDirectory: 'C:\\scratchpad',
workspaceKind: 'scratchpad',
pattern,
triggerMessageId: 'msg-user-1',
startedAt: '2026-04-01T12:00:00.000Z',
});
const session: SessionRecord = {
id: 'session-1',
projectId: SCRATCHPAD_PROJECT_ID,
patternId: pattern.id,
title: 'Checkpoint session',
createdAt: '2026-04-01T12:00:00.000Z',
updatedAt: '2026-04-01T12:00:00.000Z',
status: 'running',
messages: [
{
id: 'msg-user-1',
role: 'user',
authorName: 'You',
content: 'Continue the workflow.',
createdAt: '2026-04-01T12:00:00.000Z',
},
{
id: 'msg-assistant-1',
role: 'assistant',
authorName: 'Primary',
content: 'Working on it.',
createdAt: '2026-04-01T12:00:01.000Z',
pending: true,
},
],
runs: [run],
};
return { session, run };
}
function createPattern() {
return {
id: 'pattern-handoff',
name: 'Checkpointing flow',
description: '',
mode: 'handoff' as const,
availability: 'available' as const,
maxIterations: 4,
agents: [
{
id: 'agent-1',
name: 'Primary',
description: '',
instructions: 'Help with the request.',
model: 'gpt-5.4',
},
],
createdAt: '2026-04-01T00:00:00.000Z',
updatedAt: '2026-04-01T00:00:00.000Z',
};
}
+98 -1
View File
@@ -2,7 +2,12 @@ import { EventEmitter } from 'node:events';
import { describe, expect, mock, test } from 'bun:test';
import type { RunTurnCommand, SidecarCapabilities, WorkflowDiagnosticEvent } from '@shared/contracts/sidecar';
import type {
RunTurnCommand,
SidecarCapabilities,
WorkflowCheckpointSavedEvent,
WorkflowDiagnosticEvent,
} from '@shared/contracts/sidecar';
class FakeReadableStream extends EventEmitter {
setEncoding(_encoding: BufferEncoding): void {}
@@ -238,4 +243,96 @@ describe('SidecarClient', () => {
spawnedProcesses[0]!.completeExit();
await dispose;
});
test('routes workflow checkpoint events through the turn-scoped callback', async () => {
spawnedProcesses.length = 0;
const client = new SidecarClient();
const checkpoints: WorkflowCheckpointSavedEvent[] = [];
const command: RunTurnCommand = {
type: 'run-turn',
requestId: 'turn-1',
sessionId: 'session-1',
projectPath: 'C:\\workspace\\project',
pattern: {
id: 'pattern-1',
name: 'Handoff',
description: '',
mode: 'handoff',
availability: 'available',
maxIterations: 1,
agents: [
{
id: 'agent-1',
name: 'Primary',
description: '',
instructions: 'Help with the request.',
model: 'gpt-5.4',
},
],
createdAt: '2026-04-01T00:00:00.000Z',
updatedAt: '2026-04-01T00:00:00.000Z',
},
messages: [],
};
const turn = client.runTurn(
command,
async () => undefined,
async () => undefined,
async () => undefined,
async () => undefined,
async () => undefined,
async () => undefined,
async () => undefined,
async (event) => {
if (event.type === 'workflow-checkpoint-saved') {
checkpoints.push(event);
}
},
);
await Promise.resolve();
expect(spawnedProcesses).toHaveLength(1);
spawnedProcesses[0]!.emitStdout(
`${JSON.stringify({
type: 'workflow-checkpoint-saved',
requestId: command.requestId,
sessionId: command.sessionId,
workflowSessionId: 'turn-1',
checkpointId: 'checkpoint-1',
storePath: 'C:\\Users\\tester\\AppData\\Local\\Aryx\\workflow-checkpoints\\session-1\\turn-1',
stepNumber: 2,
} satisfies WorkflowCheckpointSavedEvent)}\n`,
);
spawnedProcesses[0]!.emitStdout(
`${JSON.stringify({
type: 'turn-complete',
requestId: command.requestId,
sessionId: command.sessionId,
messages: [],
cancelled: false,
})}\n`,
);
spawnedProcesses[0]!.emitStdout(
`${JSON.stringify({
type: 'command-complete',
requestId: command.requestId,
})}\n`,
);
await expect(turn).resolves.toEqual([]);
expect(checkpoints).toEqual([
expect.objectContaining({
type: 'workflow-checkpoint-saved',
workflowSessionId: 'turn-1',
checkpointId: 'checkpoint-1',
stepNumber: 2,
}),
]);
const dispose = client.dispose();
spawnedProcesses[0]!.completeExit();
await dispose;
});
});