feat: full Copilot SDK feature parity — custom agents, hooks, image input, skills, steering, session persistence

Backend (sidecar):
- Extended ProtocolModels with DTOs for custom agents, hooks, skills,
  infinite sessions, session lifecycle, and 9 new event types
- Added CopilotManagedSessionIds for stable SDK session ID mapping
- Added CopilotSessionManager/ICopilotSessionManager for session lifecycle
- Added CopilotSessionHooks for hook registration
- Added CopilotMessageOptionsMetadata for mid-turn steering
- Extended CopilotAgentBundle to wire custom agents, hooks, skills,
  infinite sessions, and stable session IDs
- Extended CopilotTurnExecutionState to project 13 new SDK event types
- Widened ITurnWorkflowRunner callback to accept SidecarEventDto
- Added list/delete/disconnect session commands to SidecarProtocolHost
- Added AryxCopilotAgentMessageOptionsTests (14 new tests, 142 total)

Frontend (renderer + main + shared):
- Added ChatMessageAttachment type and helpers (attachment.ts)
- Extended sidecar contracts with MessageMode, 3 new command types,
  9 new event types, and agent/session config DTOs
- Extended SessionEventRecord with 6 new event kinds and ~20 fields
- Added PatternAgentCopilotConfig to pattern domain
- Added attachments support to ChatMessageRecord
- Updated sidecar client with session lifecycle methods and
  turn-scoped event routing via onTurnScopedEvent callback
- Updated main process: handleTurnScopedEvent(), deleteSession(),
  steering bypass for mid-turn messages, attachment passthrough
- Added deleteSession IPC handler and preload binding
- Added TurnEventLog state tracker with format/apply/prune helpers
- ChatPane: always-enabled composer, steering indicator, attachment
  picker with preview, image thumbnails in message history,
  context-usage bar, amber steer mode for send button
- ActivityPanel: turn events section with sub-agent, hook, skill,
  and compaction event rendering
- Sidebar: delete session action in context menu
- App.tsx: wired sessionUsage, turnEventLogs, and deleteSession

Documentation:
- AGENTS.md: added glob safety rule for node_modules
- README.md: added steering, image input, and richer observability
- ARCHITECTURE.md: added turn-scoped events, steering, and attachments
- Website: added steering and image input feature cards, updated
  live visibility and session cards

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
David Kaya
2026-03-28 12:28:20 +01:00
co-authored by Copilot
parent 0c2973c599
commit f1fa52f9c3
40 changed files with 2515 additions and 92 deletions
@@ -0,0 +1,81 @@
using Aryx.AgentHost.Contracts;
using Aryx.AgentHost.Services;
using GitHub.Copilot.SDK;
using Microsoft.Extensions.AI;
namespace Aryx.AgentHost.Tests;
public sealed class AryxCopilotAgentMessageOptionsTests
{
[Fact]
public async Task ProcessMessageAttachmentsAsync_MapsProtocolAttachmentsAndMessageMode()
{
ChatMessage message = new(ChatRole.User, "Please inspect these images.");
message.Contents.Add(new AIContent
{
RawRepresentation = new ChatMessageAttachmentDto
{
Type = "file",
Path = @"C:\workspace\project\assets\diagram.png",
DisplayName = "diagram.png",
},
});
message.Contents.Add(new AIContent
{
RawRepresentation = new ChatMessageAttachmentDto
{
Type = "blob",
Data = "QUJDRA==",
MimeType = "image/png",
DisplayName = "clipboard.png",
},
});
message.Contents.Add(new AIContent
{
RawRepresentation = new CopilotMessageOptionsMetadata("immediate"),
});
(List<UserMessageDataAttachmentsItem>? attachments, string? messageMode, string? tempDir) =
await AryxCopilotAgent.ProcessMessageAttachmentsAsync([message], CancellationToken.None);
Assert.Equal("immediate", messageMode);
Assert.Null(tempDir);
Assert.NotNull(attachments);
Assert.Collection(
attachments!,
first =>
{
UserMessageDataAttachmentsItemFile file = Assert.IsType<UserMessageDataAttachmentsItemFile>(first);
Assert.Equal(@"C:\workspace\project\assets\diagram.png", file.Path);
Assert.Equal("diagram.png", file.DisplayName);
},
second =>
{
UserMessageDataAttachmentsItemBlob blob = Assert.IsType<UserMessageDataAttachmentsItemBlob>(second);
Assert.Equal("QUJDRA==", blob.Data);
Assert.Equal("image/png", blob.MimeType);
Assert.Equal("clipboard.png", blob.DisplayName);
});
}
[Fact]
public async Task ProcessMessageAttachmentsAsync_RejectsRelativeFileAttachments()
{
ChatMessage message = new(ChatRole.User, "Inspect this file.");
message.Contents.Add(new AIContent
{
RawRepresentation = new ChatMessageAttachmentDto
{
Type = "file",
Path = "relative\\image.png",
},
});
InvalidOperationException error = await Assert.ThrowsAsync<InvalidOperationException>(() =>
AryxCopilotAgent.ProcessMessageAttachmentsAsync([message], CancellationToken.None));
Assert.Contains("absolute", error.Message, StringComparison.OrdinalIgnoreCase);
}
}
@@ -1,5 +1,6 @@
using System.Reflection;
using System.Text.Json;
using Aryx.AgentHost.Contracts;
using GitHub.Copilot.SDK;
using Aryx.AgentHost.Services;
using Microsoft.Agents.AI;
@@ -143,6 +144,122 @@ public sealed class CopilotAgentBundleTests
Assert.Equal("handoff_to_reviewer", single.Name);
}
[Fact]
public void CreateCustomAgents_MapsSdkCustomAgentConfiguration()
{
List<CustomAgentConfig> customAgents = Assert.IsType<List<CustomAgentConfig>>(CopilotAgentBundle.CreateCustomAgents(
[
new RunTurnCustomAgentConfigDto
{
Name = "designer",
DisplayName = "Designer",
Description = "Design specialist",
Tools = ["view", "glob"],
Prompt = "Focus on UX design.",
Infer = true,
McpServers =
[
new RunTurnMcpServerConfigDto
{
Id = "designer-mcp",
Name = "Designer MCP",
Transport = "local",
Command = "node",
Args = ["designer.js"],
},
],
},
]));
CustomAgentConfig customAgent = Assert.Single(customAgents);
Assert.Equal("designer", customAgent.Name);
Assert.Equal("Designer", customAgent.DisplayName);
Assert.Equal("Design specialist", customAgent.Description);
Assert.Equal(["view", "glob"], customAgent.Tools);
Assert.Equal("Focus on UX design.", customAgent.Prompt);
Assert.True(customAgent.Infer);
KeyValuePair<string, object> mcpServer = Assert.Single(customAgent.McpServers!);
Assert.Equal("Designer MCP", mcpServer.Key);
McpLocalServerConfig localServer = Assert.IsType<McpLocalServerConfig>(mcpServer.Value);
Assert.Equal("node", localServer.Command);
Assert.Equal(["designer.js"], localServer.Args);
}
[Fact]
public void CreateInfiniteSessions_MapsSdkInfiniteSessionConfiguration()
{
InfiniteSessionConfig config = Assert.IsType<InfiniteSessionConfig>(CopilotAgentBundle.CreateInfiniteSessions(
new RunTurnInfiniteSessionsConfigDto
{
Enabled = true,
BackgroundCompactionThreshold = 0.75,
BufferExhaustionThreshold = 0.9,
}));
Assert.True(config.Enabled);
Assert.Equal(0.75, config.BackgroundCompactionThreshold);
Assert.Equal(0.9, config.BufferExhaustionThreshold);
}
[Fact]
public async Task CopilotSessionHooks_Create_UsesApprovalPolicyForPreToolUse()
{
RunTurnCommandDto command = new()
{
RequestId = "turn-1",
SessionId = "session-1",
Pattern = new PatternDefinitionDto
{
Id = "pattern-1",
Name = "Pattern",
Mode = "single",
Availability = "available",
ApprovalPolicy = new ApprovalPolicyDto
{
Rules =
[
new ApprovalCheckpointRuleDto
{
Kind = "tool-call",
AgentIds = ["agent-1"],
},
],
},
Agents =
[
new PatternAgentDefinitionDto
{
Id = "agent-1",
Name = "Primary",
Model = "gpt-5.4",
Instructions = "Help.",
},
],
},
};
SessionHooks hooks = CopilotSessionHooks.Create(command, command.Pattern.Agents[0]);
PreToolUseHookOutput? decision = await hooks.OnPreToolUse!(
new PreToolUseHookInput
{
ToolName = "view",
},
null!);
Assert.Equal("ask", decision?.PermissionDecision);
}
[Fact]
public void CopilotManagedSessionIds_BuildsAndParsesStableIds()
{
string sessionId = CopilotManagedSessionIds.Build("session-1", "agent-ux");
Assert.True(CopilotManagedSessionIds.TryParse(sessionId, out string aryxSessionId, out string agentId));
Assert.Equal("session-1", aryxSessionId);
Assert.Equal("agent-ux", agentId);
}
private static AIFunction CreateTool()
{
ToolTarget target = new();
@@ -50,7 +50,7 @@ public sealed class CopilotTurnExecutionStateTests
}
"""));
AgentActivityEventDto activity = Assert.Single(state.DrainPendingActivityEvents());
AgentActivityEventDto activity = Assert.Single(state.DrainPendingEvents().OfType<AgentActivityEventDto>());
Assert.Equal("thinking", activity.ActivityType);
Assert.Equal("agent-1", activity.AgentId);
Assert.Equal("Primary", activity.AgentName);
@@ -104,13 +104,13 @@ public sealed class CopilotTurnExecutionStateTests
}
"""));
List<AgentActivityEventDto> activities = [.. state.DrainPendingActivityEvents()];
List<AgentActivityEventDto> activities = [.. state.DrainPendingEvents().OfType<AgentActivityEventDto>()];
await state.EmitThinkingIfNeeded(
new AgentIdentity("agent-1", "Primary"),
activity =>
sidecarEvent =>
{
activities.Add(activity);
activities.Add(Assert.IsType<AgentActivityEventDto>(sidecarEvent));
return Task.CompletedTask;
});
@@ -144,6 +144,157 @@ public sealed class CopilotTurnExecutionStateTests
Assert.Empty(secondDrain);
}
[Fact]
public void ObserveSessionEvent_SubagentStarted_QueuesSubagentEvent()
{
RunTurnCommandDto command = CreateCommand();
CopilotTurnExecutionState state = new(command);
state.ObserveSessionEvent(
command.Pattern.Agents[0],
SessionEvent.FromJson(
"""
{
"type": "subagent.started",
"data": {
"toolCallId": "tool-call-1",
"agentName": "designer",
"agentDisplayName": "Designer",
"agentDescription": "Design specialist"
},
"id": "44444444-4444-4444-4444-444444444444",
"timestamp": "2026-03-27T00:00:00Z"
}
"""));
SubagentEventDto evt = Assert.Single(state.DrainPendingEvents().OfType<SubagentEventDto>());
Assert.Equal("started", evt.EventKind);
Assert.Equal("tool-call-1", evt.ToolCallId);
Assert.Equal("designer", evt.CustomAgentName);
Assert.Equal("Designer", evt.CustomAgentDisplayName);
}
[Fact]
public void ObserveSessionEvent_SkillInvoked_QueuesSkillEvent()
{
RunTurnCommandDto command = CreateCommand();
CopilotTurnExecutionState state = new(command);
state.ObserveSessionEvent(
command.Pattern.Agents[0],
SessionEvent.FromJson(
"""
{
"type": "skill.invoked",
"data": {
"name": "reviewer",
"path": "C:\\skills\\reviewer\\SKILL.md",
"content": "# Reviewer",
"allowedTools": ["view"],
"pluginName": "aryx-plugin",
"pluginVersion": "1.0.0"
},
"id": "55555555-5555-5555-5555-555555555555",
"timestamp": "2026-03-27T00:00:00Z"
}
"""));
SkillInvokedEventDto evt = Assert.Single(state.DrainPendingEvents().OfType<SkillInvokedEventDto>());
Assert.Equal("reviewer", evt.SkillName);
Assert.Equal(@"C:\skills\reviewer\SKILL.md", evt.Path);
Assert.Equal(["view"], evt.AllowedTools);
Assert.Equal("aryx-plugin", evt.PluginName);
}
[Fact]
public void ObserveSessionEvent_HookStart_QueuesHookLifecycleEvent()
{
RunTurnCommandDto command = CreateCommand();
CopilotTurnExecutionState state = new(command);
state.ObserveSessionEvent(
command.Pattern.Agents[0],
SessionEvent.FromJson(
"""
{
"type": "hook.start",
"data": {
"hookInvocationId": "hook-1",
"hookType": "postToolUse",
"input": {
"toolName": "view"
}
},
"id": "66666666-6666-6666-6666-666666666666",
"timestamp": "2026-03-27T00:00:00Z"
}
"""));
HookLifecycleEventDto evt = Assert.Single(state.DrainPendingEvents().OfType<HookLifecycleEventDto>());
Assert.Equal("start", evt.Phase);
Assert.Equal("postToolUse", evt.HookType);
Assert.Equal("hook-1", evt.HookInvocationId);
Assert.NotNull(evt.Input);
}
[Fact]
public void ObserveSessionEvent_SessionCompactionComplete_QueuesCompactionEvent()
{
RunTurnCommandDto command = CreateCommand();
CopilotTurnExecutionState state = new(command);
state.ObserveSessionEvent(
command.Pattern.Agents[0],
SessionEvent.FromJson(
"""
{
"type": "session.compaction_complete",
"data": {
"success": true,
"preCompactionTokens": 1000,
"postCompactionTokens": 400,
"messagesRemoved": 8,
"tokensRemoved": 600,
"summaryContent": "Compacted summary",
"checkpointNumber": 2,
"checkpointPath": "C:\\Users\\me\\.copilot\\session-state\\checkpoint-2.json"
},
"id": "77777777-7777-7777-7777-777777777777",
"timestamp": "2026-03-27T00:00:00Z"
}
"""));
SessionCompactionEventDto evt = Assert.Single(state.DrainPendingEvents().OfType<SessionCompactionEventDto>());
Assert.Equal("complete", evt.Phase);
Assert.True(evt.Success);
Assert.Equal(1000, evt.PreCompactionTokens);
Assert.Equal(400, evt.PostCompactionTokens);
Assert.Equal("Compacted summary", evt.SummaryContent);
}
[Fact]
public void ObserveSessionEvent_PendingMessagesModified_QueuesPendingMessageSignal()
{
RunTurnCommandDto command = CreateCommand();
CopilotTurnExecutionState state = new(command);
state.ObserveSessionEvent(
command.Pattern.Agents[0],
SessionEvent.FromJson(
"""
{
"type": "pending_messages.modified",
"data": {},
"id": "88888888-8888-8888-8888-888888888888",
"timestamp": "2026-03-27T00:00:00Z"
}
"""));
PendingMessagesModifiedEventDto evt = Assert.Single(state.DrainPendingEvents().OfType<PendingMessagesModifiedEventDto>());
Assert.Equal("session-1", evt.SessionId);
Assert.Equal("agent-1", evt.AgentId);
}
private static RunTurnCommandDto CreateCommand()
{
return new RunTurnCommandDto
@@ -649,7 +649,7 @@ public sealed class CopilotWorkflowRunnerTests
Assert.Equal("agent-handoff-ux", observedAgent.AgentId);
Assert.Equal("UX Specialist", observedAgent.AgentName);
Assert.Equal("agent-handoff-ux", state.ActiveAgent?.AgentId);
AgentActivityEventDto activity = Assert.Single(state.DrainPendingActivityEvents());
AgentActivityEventDto activity = Assert.Single(state.DrainPendingEvents().OfType<AgentActivityEventDto>());
Assert.Equal("thinking", activity.ActivityType);
Assert.Equal("agent-handoff-ux", activity.AgentId);
}
@@ -675,7 +675,7 @@ public sealed class CopilotWorkflowRunnerTests
Assert.Equal("agent-handoff-ux", state.ActiveAgent?.AgentId);
Assert.Equal("UX Specialist", state.ActiveAgent?.AgentName);
AgentActivityEventDto activity = Assert.Single(state.DrainPendingActivityEvents());
AgentActivityEventDto activity = Assert.Single(state.DrainPendingEvents().OfType<AgentActivityEventDto>());
Assert.Equal("thinking", activity.ActivityType);
Assert.Equal("agent-handoff-ux", activity.AgentId);
}
@@ -699,7 +699,7 @@ public sealed class CopilotWorkflowRunnerTests
"timestamp": "2026-03-27T00:00:00Z"
}
"""));
_ = state.DrainPendingActivityEvents();
_ = state.DrainPendingEvents();
RequestInfoEvent requestInfo = CreateRequestInfoEvent(
CreateHandoffTarget("agent-handoff-ux", "UX Specialist"));
List<AgentActivityEventDto> activities = [];
@@ -715,9 +715,9 @@ public sealed class CopilotWorkflowRunnerTests
Array.Empty<ChatMessage>(),
state,
(Func<TurnDeltaEventDto, Task>)(_ => Task.CompletedTask),
(Func<AgentActivityEventDto, Task>)(activity =>
(Func<SidecarEventDto, Task>)(sidecarEvent =>
{
activities.Add(activity);
activities.Add(Assert.IsType<AgentActivityEventDto>(sidecarEvent));
return Task.CompletedTask;
}),
])!;
@@ -780,6 +780,109 @@ public sealed class SidecarProtocolHostTests
Assert.False(string.IsNullOrWhiteSpace(diagnostics.CheckedAt));
}
[Fact]
public async Task ListSessionsCommand_ReturnsSessionsListedEvent()
{
SidecarProtocolHost host = new(
new PatternValidator(),
sessionManager: new FakeSessionManager
{
Sessions =
[
new CopilotSessionInfoDto
{
CopilotSessionId = "aryx::session-1::agent-1",
ManagedByAryx = true,
SessionId = "session-1",
AgentId = "agent-1",
Summary = "Review session",
},
],
});
IReadOnlyList<JsonElement> events = await RunHostAsync(
new ListSessionsCommandDto
{
Type = "list-sessions",
RequestId = "list-1",
},
host);
JsonElement listedEvent = AssertSingleEvent(events, "sessions-listed", "list-1");
JsonElement session = Assert.Single(listedEvent.GetProperty("sessions").EnumerateArray());
Assert.Equal("aryx::session-1::agent-1", session.GetProperty("copilotSessionId").GetString());
Assert.Equal("session-1", session.GetProperty("sessionId").GetString());
}
[Fact]
public async Task DeleteSessionCommand_ReturnsDeletedSessionsEvent()
{
FakeSessionManager sessionManager = new()
{
DeletedSessions =
[
new CopilotSessionInfoDto
{
CopilotSessionId = "aryx::session-1::agent-1",
ManagedByAryx = true,
SessionId = "session-1",
AgentId = "agent-1",
},
],
};
SidecarProtocolHost host = new(
new PatternValidator(),
sessionManager: sessionManager);
IReadOnlyList<JsonElement> events = await RunHostAsync(
new DeleteSessionCommandDto
{
Type = "delete-session",
RequestId = "delete-1",
SessionId = "session-1",
},
host);
JsonElement deletedEvent = AssertSingleEvent(events, "sessions-deleted", "delete-1");
JsonElement session = Assert.Single(deletedEvent.GetProperty("sessions").EnumerateArray());
Assert.Equal("session-1", deletedEvent.GetProperty("sessionId").GetString());
Assert.Equal("aryx::session-1::agent-1", session.GetProperty("copilotSessionId").GetString());
Assert.Equal("session-1", sessionManager.DeletedAryxSessionId);
}
[Fact]
public async Task DisconnectSessionCommand_CancelsActiveTurnsForSession()
{
FakeWorkflowRunner runner = new(async (command, onDelta, onActivity, onApproval, onUserInput, onMcpOAuthRequired, onExitPlanMode, cancellationToken) =>
{
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
return [];
});
SidecarProtocolHost host = new(new PatternValidator(), runner);
IReadOnlyList<JsonElement> events = await RunHostAsync(
[
CreateRunTurnCommand(requestId: "turn-1", sessionId: "session-1"),
new DisconnectSessionCommandDto
{
Type = "disconnect-session",
RequestId = "disconnect-1",
SessionId = "session-1",
},
],
host);
JsonElement disconnectedEvent = AssertSingleEvent(events, "session-disconnected", "disconnect-1");
string[] cancelledRequestIds = disconnectedEvent.GetProperty("cancelledRequestIds")
.EnumerateArray()
.Select(value => value.GetString() ?? string.Empty)
.ToArray();
Assert.Equal(["turn-1"], cancelledRequestIds);
JsonElement turnComplete = AssertSingleEvent(events, "turn-complete", "turn-1");
Assert.True(turnComplete.GetProperty("cancelled").GetBoolean());
}
private static async Task<IReadOnlyList<JsonElement>> RunHostAsync(
object command,
SidecarProtocolHost? host = null)
@@ -949,7 +1052,7 @@ public sealed class SidecarProtocolHostTests
private readonly Func<
RunTurnCommandDto,
Func<TurnDeltaEventDto, Task>,
Func<AgentActivityEventDto, Task>,
Func<SidecarEventDto, Task>,
Func<ApprovalRequestedEventDto, Task>,
Func<UserInputRequestedEventDto, Task>,
Func<McpOauthRequiredEventDto, Task>,
@@ -963,7 +1066,7 @@ public sealed class SidecarProtocolHostTests
Func<
RunTurnCommandDto,
Func<TurnDeltaEventDto, Task>,
Func<AgentActivityEventDto, Task>,
Func<SidecarEventDto, Task>,
Func<ApprovalRequestedEventDto, Task>,
Func<UserInputRequestedEventDto, Task>,
Func<McpOauthRequiredEventDto, Task>,
@@ -981,7 +1084,7 @@ public sealed class SidecarProtocolHostTests
public Task<IReadOnlyList<ChatMessageDto>> RunTurnAsync(
RunTurnCommandDto command,
Func<TurnDeltaEventDto, Task> onDelta,
Func<AgentActivityEventDto, Task> onActivity,
Func<SidecarEventDto, Task> onActivity,
Func<ApprovalRequestedEventDto, Task> onApproval,
Func<UserInputRequestedEventDto, Task> onUserInput,
Func<McpOauthRequiredEventDto, Task> onMcpOAuthRequired,
@@ -1005,4 +1108,32 @@ public sealed class SidecarProtocolHostTests
return _resolveUserInputHandler(command, cancellationToken);
}
}
private sealed class FakeSessionManager : ICopilotSessionManager
{
public IReadOnlyList<CopilotSessionInfoDto> Sessions { get; init; } = [];
public IReadOnlyList<CopilotSessionInfoDto> DeletedSessions { get; init; } = [];
public string? DeletedAryxSessionId { get; private set; }
public string? DeletedCopilotSessionId { get; private set; }
public Task<IReadOnlyList<CopilotSessionInfoDto>> ListSessionsAsync(
CopilotSessionListFilterDto? filter,
CancellationToken cancellationToken)
{
return Task.FromResult(Sessions);
}
public Task<IReadOnlyList<CopilotSessionInfoDto>> DeleteSessionsAsync(
string? aryxSessionId,
string? copilotSessionId,
CancellationToken cancellationToken)
{
DeletedAryxSessionId = aryxSessionId;
DeletedCopilotSessionId = copilotSessionId;
return Task.FromResult(DeletedSessions);
}
}
}