Files
AX-Copilot-Codex/src/AxCopilot.Tests/Services/AgentLoopCodeQualityTests.cs
lacvet 8cb08576d5 AX Agent 도구·스킬 정합성 재구성 및 실행 품질 보강
변경 목적:
- AX Agent의 도구 이름, 내부 설정, 스킬 정책, 실행 루프 사이의 불일치를 줄이고 전체 동작 품질을 높인다.
- claw-code 수준의 일관된 동작 품질을 참고하되 AX 구조에 맞는 고유한 카탈로그·정규화 레이어로 재구성한다.

핵심 수정사항:
- 도구 canonical id, legacy alias, 탭 노출, 설정 카테고리, read-only 분류를 중앙 카탈로그로 통합했다.
- ToolRegistry, AgentLoopService, 병렬 실행 분류, 권한 처리, 훅 처리, 스킬 allowed-tools 해석이 같은 이름 체계를 사용하도록 정리했다.
- Agent 설정/일반 설정/도움말의 도구 카드와 훅 편집기, 스킬 설명을 현재 런타임 구조에 맞게 갱신했다.
- 컨텍스트 압축, intent gate, spawn agents, session learning, model prompt adapter, workspace context 관련 변경과 테스트 추가를 함께 반영했다.
- 문서 이력과 비교/로드맵 문서를 최신 상태로 갱신했다.

검증 결과:
- dotnet build src/AxCopilot/AxCopilot.csproj -c Release -v minimal -p:OutputPath=bin\verify_toolcat\ -p:IntermediateOutputPath=obj\verify_toolcat\ : 경고 0 / 오류 0
- dotnet test src/AxCopilot.Tests/AxCopilot.Tests.csproj -c Release -v minimal --filter AgentToolCatalogTests -p:OutputPath=bin\verify_toolcat_tests\ -p:IntermediateOutputPath=obj\verify_toolcat_tests\ : 통과 8
2026-04-14 17:52:46 +09:00

2154 lines
71 KiB
C#

using System.Reflection;
using System.IO;
using System.Text.Json;
using AxCopilot.Models;
using AxCopilot.Services;
using AxCopilot.Services.Agent;
using FluentAssertions;
using Xunit;
namespace AxCopilot.Tests.Services;
public class AgentLoopCodeQualityTests
{
[Fact]
public void IsHighImpactCodeModification_ReturnsTrueForServiceFile()
{
var result = ToolResult.Ok("updated", @"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs");
var highImpact = InvokePrivateStatic<bool>(
"IsHighImpactCodeModification",
"Code",
"file_edit",
result);
highImpact.Should().BeTrue();
}
[Theory]
[InlineData("fix this error", "Code", "bugfix")]
[InlineData("add a new feature", "Code", "feature")]
[InlineData("refactor this flow", "Code", "refactor")]
[InlineData("review this code", "Code", "review")]
[InlineData("write a report", "Cowork", "docs")]
public void ClassifyTaskType_ReturnsExpectedType(string query, string tab, string expected)
{
var taskType = AgentLoopService.ClassifyTaskType(query, tab);
taskType.Should().Be(expected);
}
[Fact]
public void BuildTaskTypeGuidanceMessage_IncludesToolOrderHints()
{
var bugfix = AgentLoopService.BuildTaskTypeGuidanceMessage("bugfix");
var refactor = AgentLoopService.BuildTaskTypeGuidanceMessage("refactor");
bugfix.Should().Contain("Preferred tool order");
bugfix.Should().Contain("build_run/test_loop");
refactor.Should().Contain("git_tool(diff)");
}
[Fact]
public void BuildCodeQualityFollowUpPrompt_IncludesBaselineAndTestReviewForProductionCode()
{
var result = ToolResult.Ok("updated", @"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs");
var prompt = InvokePrivateStatic<string>(
"BuildCodeQualityFollowUpPrompt",
"file_edit",
result,
false,
false,
"bugfix");
prompt.Should().Contain("baseline build/test");
prompt.Should().Contain("grep 또는 glob");
prompt.Should().Contain("build_run");
prompt.Should().Contain("테스트 부재 사실");
prompt.Should().Contain("Task type: bugfix");
}
[Fact]
public void BuildCodeQualityFollowUpPrompt_StrengthensHighImpactVerification()
{
var result = ToolResult.Ok("updated", @"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs");
var prompt = InvokePrivateStatic<string>(
"BuildCodeQualityFollowUpPrompt",
"file_edit",
result,
true,
true,
"refactor");
prompt.Should().Contain("spawn_agent");
prompt.Should().Contain("build_run");
prompt.Should().Contain("grep 또는 glob");
prompt.Should().Contain("테스트 부재 사실");
prompt.Should().Contain("영향 범위가 넓을 가능성");
prompt.Should().Contain("Task type: refactor");
}
[Fact]
public void HasCodeVerificationEvidenceAfterLastModification_RequiresReferenceSearchForHighImpact()
{
var messages =
new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("file_read"),
CreateToolResult("git_tool"),
CreateToolResult("build_run")
};
var hasEvidence = InvokePrivateStatic<bool>(
"HasCodeVerificationEvidenceAfterLastModification",
messages,
true);
hasEvidence.Should().BeFalse();
}
[Fact]
public void HasCodeVerificationEvidenceAfterLastModification_AcceptsReferenceSearchAndExecutionForHighImpact()
{
var messages =
new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("file_read"),
CreateToolResult("grep"),
CreateToolResult("build_run")
};
var hasEvidence = InvokePrivateStatic<bool>(
"HasCodeVerificationEvidenceAfterLastModification",
messages,
true);
hasEvidence.Should().BeTrue();
}
[Fact]
public void HasCodeVerificationEvidenceAfterLastModification_AcceptsDelegatedInvestigationForHighImpact()
{
var messages =
new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("file_read"),
CreateToolResult("wait_agents"),
CreateToolResult("build_run")
};
var hasEvidence = InvokePrivateStatic<bool>(
"HasCodeVerificationEvidenceAfterLastModification",
messages,
true);
hasEvidence.Should().BeTrue();
}
[Fact]
public void HasCodeVerificationEvidenceAfterLastModification_AcceptsInspectionOnlyForNormalChange()
{
var messages =
new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("file_read")
};
var hasEvidence = InvokePrivateStatic<bool>(
"HasCodeVerificationEvidenceAfterLastModification",
messages,
false);
hasEvidence.Should().BeTrue();
}
[Fact]
public void BuildFailureInvestigationPrompt_IncludesHighImpactChecks()
{
var prompt = InvokePrivateStatic<string>(
"BuildFailureInvestigationPrompt",
"build_run",
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
true,
"bugfix");
prompt.Should().Contain("OrderService.cs");
prompt.Should().Contain("spawn_agent");
prompt.Should().Contain("build/test");
prompt.Should().Contain("테스트 부재 사실");
prompt.Should().Contain("symptom is no longer reproducible");
}
[Fact]
public void BuildFailureInvestigationPrompt_ChangesByTaskType()
{
var featurePrompt = InvokePrivateStatic<string>(
"BuildFailureInvestigationPrompt",
"build_run",
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
false,
"feature");
var refactorPrompt = InvokePrivateStatic<string>(
"BuildFailureInvestigationPrompt",
"build_run",
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
false,
"refactor");
featurePrompt.Should().Contain("feature path and caller linkage");
refactorPrompt.Should().Contain("behavior-compatible");
}
[Fact]
public void BuildFailureReflectionMessage_IncludesFallbackSequenceForBuildFailures()
{
var result = ToolResult.Fail("build failed");
var message = InvokePrivateStatic<string>(
"BuildFailureReflectionMessage",
"build_run",
result,
2,
3,
"bugfix");
message.Should().Contain("Fallback sequence");
message.Should().Contain("file_read -> grep/glob -> git_tool(diff)");
message.Should().Contain("repro/root-cause");
}
[Fact]
public void BuildFailureNextToolPriorityPrompt_IncludesOrderedPriority()
{
var prompt = InvokePrivateStatic<string>(
"BuildFailureNextToolPriorityPrompt",
"build_run",
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
true,
"feature");
prompt.Should().Contain("[System:NextToolPriority]");
prompt.Should().Contain("file_read -> grep/glob -> git_tool(diff) -> file_edit -> build_run/test_loop");
prompt.Should().Contain("작업유형: feature");
prompt.Should().Contain("고영향 변경");
}
[Fact]
public void ComputeAdaptiveMaxRetry_AdjustsByTaskType()
{
InvokePrivateStatic<int>(
"ComputeAdaptiveMaxRetry",
3,
"bugfix").Should().Be(4);
InvokePrivateStatic<int>(
"ComputeAdaptiveMaxRetry",
3,
"feature").Should().Be(3);
InvokePrivateStatic<int>(
"ComputeAdaptiveMaxRetry",
3,
"refactor").Should().Be(2);
}
[Fact]
public void ComputeQualityAwareMaxRetry_AdjustsByRecentQuality()
{
InvokePrivateStatic<int>(
"ComputeQualityAwareMaxRetry",
3,
0.30,
"feature").Should().Be(4);
InvokePrivateStatic<int>(
"ComputeQualityAwareMaxRetry",
3,
0.90,
"feature").Should().Be(2);
InvokePrivateStatic<int>(
"ComputeQualityAwareMaxRetry",
3,
0.90,
"bugfix").Should().Be(3);
}
[Fact]
public void HasSufficientFinalReportEvidence_RequiresRiskForHighImpact()
{
var weakReport = "변경: 서비스 파일 수정. 검증: build/test 통과.";
var strongReport = "Changed service file. Verified by build and test. Feature flow path reviewed. Caller reference impact scope checked. Remaining risk: integration follow-up needed.";
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
weakReport,
"feature",
true).Should().BeFalse();
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
strongReport,
"feature",
true).Should().BeTrue();
}
[Fact]
public void HasSufficientFinalReportEvidence_BugfixRequiresTaskSpecificSignals()
{
var generic = "Changed file and build/test verified.";
var specific = "Changed file and build/test verified. Root cause was null path; fix applied; repro steps now pass.";
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
generic,
"bugfix",
false).Should().BeFalse();
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
specific,
"bugfix",
false).Should().BeTrue();
}
[Fact]
public void TryParseApprovedPlanDecision_ParsesEditedSteps()
{
var decision = $"{AgentLoopService.ApprovedPlanDecisionPrefix}\n1. 파일 구조 점검\n2. 테스트 실행";
var ok = AgentLoopService.TryParseApprovedPlanDecision(decision, out var planText, out var steps);
ok.Should().BeTrue();
steps.Should().Equal("파일 구조 점검", "테스트 실행");
planText.Should().Contain("1. 파일 구조 점검");
planText.Should().Contain("2. 테스트 실행");
}
[Fact]
public void TryParseApprovedPlanDecision_ReturnsFalseForNormalDecision()
{
var ok = AgentLoopService.TryParseApprovedPlanDecision("승인", out var planText, out var steps);
ok.Should().BeFalse();
planText.Should().BeEmpty();
steps.Should().BeEmpty();
}
[Fact]
public void HasSufficientFinalReportEvidence_FeatureRequiresBehaviorAndImpactSignals()
{
var missingBehavior = "Changed files and build/test verified. caller references checked.";
var strong = "Changed files and build/test verified. New input/output flow added and caller impact scope checked via references.";
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
missingBehavior,
"feature",
false).Should().BeFalse();
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
strong,
"feature",
false).Should().BeTrue();
}
[Fact]
public void BuildFinalReportQualityPrompt_IncludesTaskSpecificGuidance()
{
var prompt = InvokePrivateStatic<string>(
"BuildFinalReportQualityPrompt",
"bugfix",
true);
prompt.Should().Contain("무엇을 변경했는지");
prompt.Should().Contain("build/test/검증 근거");
prompt.Should().Contain("bug fix");
prompt.Should().Contain("남은 리스크");
}
[Fact]
public void BuildFailurePatternGuidance_ReturnsEmptyWhenNoPatterns()
{
var guidance = AgentLoopService.BuildFailurePatternGuidance(Array.Empty<string>(), "bugfix");
guidance.Should().BeEmpty();
}
[Fact]
public void BuildFailurePatternGuidance_IncludesTaskSpecificFocusAndPatterns()
{
var guidance = AgentLoopService.BuildFailurePatternGuidance(
new[]
{
"code-failure | task:bugfix | tool:build_run | retry:1/3 | error:CS1002",
"code-failure | task:bugfix | tool:test_loop | retry:2/3 | error:NRE"
},
"bugfix");
guidance.Should().Contain("[System:FailurePatterns]");
guidance.Should().Contain("reproduction");
guidance.Should().Contain("CS1002");
guidance.Should().Contain("NRE");
}
[Fact]
public void UnknownAndDisallowedRecoveryPrompts_ShouldGuideToolSearch()
{
var unknownPrompt = InvokePrivateStatic<string>(
"BuildUnknownToolRecoveryPrompt",
"read_file_typo",
new[] { "file_read", "file_edit", "tool_search" });
var disallowedPrompt = InvokePrivateStatic<string>(
"BuildDisallowedToolRecoveryPrompt",
"shell_exec",
new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "file_read", "tool_search" },
new[] { "file_read", "tool_search" });
unknownPrompt.Should().Contain("tool_search");
disallowedPrompt.Should().Contain("tool_search");
}
[Fact]
public void IsFailurePatternForTaskType_MatchesCaseInsensitiveTaskToken()
{
var matched = InvokePrivateStatic<bool>(
"IsFailurePatternForTaskType",
"code-failure | task:BugFix | tool:build_run | error:CS1002",
"bugfix");
var notMatched = InvokePrivateStatic<bool>(
"IsFailurePatternForTaskType",
"code-failure | task:feature | tool:build_run | error:CS1002",
"bugfix");
matched.Should().BeTrue();
notMatched.Should().BeFalse();
}
[Fact]
public void IsFailurePatternForTaskType_HandlesSpacedAndShuffledTokens()
{
var matched = InvokePrivateStatic<bool>(
"IsFailurePatternForTaskType",
"code-failure | retry: 2 / 3 | impact:high | tool:build_run | task: bugfix | error:CS1002",
"bugfix");
matched.Should().BeTrue();
}
[Fact]
public void ExtractRetryCurrent_ReturnsZeroForMalformedPattern()
{
var retry = InvokePrivateStatic<int>(
"ExtractRetryCurrent",
"code-failure | task:bugfix | retry:abc | error:CS1002");
retry.Should().Be(0);
}
[Fact]
public void BuildFailureMemoryContent_SanitizesPipeAndNewlineInErrorAndPath()
{
var result = ToolResult.Ok("line1|line2\nline3");
var content = InvokePrivateStatic<string>(
"BuildFailureMemoryContent",
"build_run",
result,
"bugfix",
@"E:\AX Copilot - Codex\src\A|B\Foo.cs",
2,
3);
content.Should().Contain("task:bugfix");
content.Should().Contain("retry:2/3");
content.Should().NotContain("|line2\n");
content.Should().Contain("line1/line2 line3");
content.Should().Contain(@"file:E:\AX Copilot - Codex\src\A/B\Foo.cs");
}
[Fact]
public void BuildFailureMemoryContent_NormalizesToolNameDelimiters()
{
var result = ToolResult.Ok("err");
var content = InvokePrivateStatic<string>(
"BuildFailureMemoryContent",
"build|run",
result,
"feature",
null,
1,
3);
content.Should().Contain("tool:build/run");
}
[Fact]
public void SelectTopFailurePatterns_PrioritizesExactTaskAndHighRetry()
{
var now = DateTime.Now;
var entries = new List<MemoryEntry>
{
new()
{
Type = "correction",
Content = "code-failure | task:feature | tool:build_run | retry:3/3 | impact:high | error:CS1002",
LastUsedAt = now,
UseCount = 4,
Relevance = 0.2
},
new()
{
Type = "correction",
Content = "code-failure | task:bugfix | tool:build_run | retry:1/3 | error:NRE",
LastUsedAt = now.AddDays(-2),
UseCount = 1,
Relevance = 0.8
},
new()
{
Type = "correction",
Content = "code-failure | task:general | tool:test_loop | retry:3/3 | error:timeout",
LastUsedAt = now,
UseCount = 5,
Relevance = 0.9
}
};
var selected = AgentLoopService.SelectTopFailurePatterns(entries, "feature", 2);
selected.Should().HaveCount(2);
selected[0].Should().Contain("task:feature");
}
[Fact]
public void SelectTopFailurePatterns_PrioritizesRecentPatternsWhenTaskMatches()
{
var now = DateTime.Now;
var entries = new List<MemoryEntry>
{
new()
{
Type = "correction",
Content = "code-failure | task:bugfix | tool:build_run | retry:1/3 | error:old",
LastUsedAt = now.AddDays(-40),
UseCount = 10,
Relevance = 0.7
},
new()
{
Type = "correction",
Content = "code-failure | task:bugfix | tool:build_run | retry:1/3 | error:new",
LastUsedAt = now.AddHours(-5),
UseCount = 1,
Relevance = 0.4
}
};
var selected = AgentLoopService.SelectTopFailurePatterns(entries, "bugfix", 1);
selected.Should().HaveCount(1);
selected[0].Should().Contain("error:new");
}
[Fact]
public void SelectTopFailurePatterns_DeduplicatesSameRootCauseSignature()
{
var now = DateTime.Now;
var entries = new List<MemoryEntry>
{
new()
{
Type = "correction",
Content = "code-failure | task:bugfix | tool:build_run | retry:1/3 | error:CS1002 token missing",
LastUsedAt = now.AddHours(-1),
UseCount = 1,
Relevance = 0.6
},
new()
{
Type = "correction",
Content = "code-failure | task:bugfix | tool:build_run | retry:3/3 | error:CS1002 token missing",
LastUsedAt = now,
UseCount = 2,
Relevance = 0.4
},
new()
{
Type = "correction",
Content = "code-failure | task:bugfix | tool:test_loop | retry:1/3 | error:NRE",
LastUsedAt = now,
UseCount = 1,
Relevance = 0.4
}
};
var selected = AgentLoopService.SelectTopFailurePatterns(entries, "bugfix", 3);
selected.Should().HaveCount(2);
selected.Count(x => x.Contains("CS1002", StringComparison.OrdinalIgnoreCase)).Should().Be(1);
selected.Should().Contain(x => x.Contains("retry:3/3", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void BuildToolCallSignature_IncludesToolAndCanonicalInput()
{
var input = JsonDocument.Parse("""{"path":"src/A.cs","line":10}""").RootElement.Clone();
var call = new ContentBlock
{
Type = "tool_use",
ToolName = "file_edit",
ToolId = "t1",
ToolInput = input
};
var signature = InvokePrivateStatic<string>("BuildToolCallSignature", call);
signature.Should().StartWith("file_edit|");
signature.Should().Contain("\"path\":\"src/A.cs\"");
}
[Fact]
public void ShouldBlockRepeatedFailedCall_ReturnsTrueWhenThresholdReached()
{
var shouldBlock = InvokePrivateStatic<bool>(
"ShouldBlockRepeatedFailedCall",
"sig",
"sig",
3,
3);
var shouldNotBlock = InvokePrivateStatic<bool>(
"ShouldBlockRepeatedFailedCall",
"sig-a",
"sig-b",
5,
3);
shouldBlock.Should().BeTrue();
shouldNotBlock.Should().BeFalse();
}
[Fact]
public void CreateParallelExecutionPlan_DisabledFlagKeepsSequentialOnly()
{
var calls = new List<ContentBlock>
{
new() { Type = "tool_use", ToolName = "file_read", ToolId = "t1", ToolInput = JsonDocument.Parse("""{"path":"a.txt"}""").RootElement.Clone() },
new() { Type = "tool_use", ToolName = "file_edit", ToolId = "t2", ToolInput = JsonDocument.Parse("""{"path":"a.txt","old":"a","new":"b"}""").RootElement.Clone() }
};
var plan = InvokePrivateStatic<(bool ShouldRun, List<ContentBlock> ParallelBatch, List<ContentBlock> SequentialBatch)>(
"CreateParallelExecutionPlan",
false,
calls,
0);
plan.ShouldRun.Should().BeFalse();
plan.ParallelBatch.Should().BeEmpty();
plan.SequentialBatch.Should().HaveCount(2);
}
[Fact]
public void CreateParallelExecutionPlan_UsesOnlyReadOnlyPrefixForParallelBatch()
{
var calls = new List<ContentBlock>
{
new() { Type = "tool_use", ToolName = "file_read", ToolId = "t1", ToolInput = JsonDocument.Parse("""{"path":"a.txt"}""").RootElement.Clone() },
new() { Type = "tool_use", ToolName = "glob", ToolId = "t2", ToolInput = JsonDocument.Parse("""{"pattern":"*.cs"}""").RootElement.Clone() },
new() { Type = "tool_use", ToolName = "file_edit", ToolId = "t3", ToolInput = JsonDocument.Parse("""{"path":"a.txt","old":"a","new":"b"}""").RootElement.Clone() },
new() { Type = "tool_use", ToolName = "file_read", ToolId = "t4", ToolInput = JsonDocument.Parse("""{"path":"b.txt"}""").RootElement.Clone() }
};
var plan = InvokePrivateStatic<(bool ShouldRun, List<ContentBlock> ParallelBatch, List<ContentBlock> SequentialBatch)>(
"CreateParallelExecutionPlan",
true,
calls,
0);
plan.ShouldRun.Should().BeTrue();
plan.ParallelBatch.Select(x => x.ToolId).Should().Equal("t1", "t2");
plan.SequentialBatch.Select(x => x.ToolId).Should().Equal("t3", "t4");
}
[Fact]
public void CreateParallelExecutionPlan_RecognizesAliasReadOnlyToolInPrefix()
{
var calls = new List<ContentBlock>
{
new() { Type = "tool_use", ToolName = "Read", ToolId = "t1", ToolInput = JsonDocument.Parse("""{"path":"a.txt"}""").RootElement.Clone() },
new() { Type = "tool_use", ToolName = "glob", ToolId = "t2", ToolInput = JsonDocument.Parse("""{"pattern":"*.cs"}""").RootElement.Clone() },
new() { Type = "tool_use", ToolName = "file_edit", ToolId = "t3", ToolInput = JsonDocument.Parse("""{"path":"a.txt","old":"a","new":"b"}""").RootElement.Clone() }
};
var plan = InvokePrivateStatic<(bool ShouldRun, List<ContentBlock> ParallelBatch, List<ContentBlock> SequentialBatch)>(
"CreateParallelExecutionPlan",
true,
calls,
0);
plan.ShouldRun.Should().BeTrue();
plan.ParallelBatch.Select(x => x.ToolId).Should().Equal("t1", "t2");
plan.SequentialBatch.Select(x => x.ToolId).Should().Equal("t3");
}
[Fact]
public void ComputeFailureTransitionState_IncrementsRepeatAndConsecutiveOnSameSignature()
{
var state = InvokePrivateStatic<(string? LastFailedToolSignature, int RepeatedFailedToolSignatureCount, int ConsecutiveErrors, bool CanRetry)>(
"ComputeFailureTransitionState",
"sig-a",
"sig-a",
2,
1,
3);
state.LastFailedToolSignature.Should().Be("sig-a");
state.RepeatedFailedToolSignatureCount.Should().Be(3);
state.ConsecutiveErrors.Should().Be(2);
state.CanRetry.Should().BeTrue();
}
[Fact]
public void ComputeFailureTransitionState_ResetsRepeatOnDifferentSignatureAndDetectsRetryLimit()
{
var state = InvokePrivateStatic<(string? LastFailedToolSignature, int RepeatedFailedToolSignatureCount, int ConsecutiveErrors, bool CanRetry)>(
"ComputeFailureTransitionState",
"sig-new",
"sig-old",
4,
2,
2);
state.LastFailedToolSignature.Should().Be("sig-new");
state.RepeatedFailedToolSignatureCount.Should().Be(1);
state.ConsecutiveErrors.Should().Be(3);
state.CanRetry.Should().BeFalse();
}
[Fact]
public void IsNonRetriableToolFailure_DetectsPermissionAndSchemaFailures()
{
var permissionDenied = ToolResult.Fail("Permission denied while writing file");
var invalidArgs = ToolResult.Fail("schema validation failed: missing path");
var transient = ToolResult.Fail("timeout while executing command");
InvokePrivateStatic<bool>("IsNonRetriableToolFailure", permissionDenied).Should().BeTrue();
InvokePrivateStatic<bool>("IsNonRetriableToolFailure", invalidArgs).Should().BeTrue();
InvokePrivateStatic<bool>("IsNonRetriableToolFailure", transient).Should().BeFalse();
}
[Fact]
public void IsNonRetriableToolFailure_DetectsUnknownToolFailures()
{
var unknown = ToolResult.Fail("알 수 없는 도구: x_tool");
InvokePrivateStatic<bool>("IsNonRetriableToolFailure", unknown).Should().BeTrue();
}
[Fact]
public void ShouldBlockNoProgressReadOnlyLoop_BlocksAfterThresholdForReadOnlyTools()
{
var blocked = InvokePrivateStatic<bool>(
"ShouldBlockNoProgressReadOnlyLoop",
"file_read",
4);
var notYet = InvokePrivateStatic<bool>(
"ShouldBlockNoProgressReadOnlyLoop",
"file_read",
3);
blocked.Should().BeTrue();
notYet.Should().BeFalse();
}
[Fact]
public void ShouldBlockNoProgressReadOnlyLoop_DoesNotBlockWriteTools()
{
var blocked = InvokePrivateStatic<bool>(
"ShouldBlockNoProgressReadOnlyLoop",
"file_edit",
8);
blocked.Should().BeFalse();
}
[Fact]
public void ShouldRequestDocumentArtifact_RequiresDocsTaskAndMissingArtifact()
{
var tempFile = Path.GetTempFileName();
var shouldRequest = InvokePrivateStatic<bool>(
"ShouldRequestDocumentArtifact",
"docs",
null,
0);
var noRequestWithArtifact = InvokePrivateStatic<bool>(
"ShouldRequestDocumentArtifact",
"docs",
tempFile,
0);
var noRequestAfterRetries = InvokePrivateStatic<bool>(
"ShouldRequestDocumentArtifact",
"docs",
null,
2);
var noRequestForCode = InvokePrivateStatic<bool>(
"ShouldRequestDocumentArtifact",
"bugfix",
null,
0);
shouldRequest.Should().BeTrue();
noRequestWithArtifact.Should().BeFalse();
noRequestAfterRetries.Should().BeFalse();
noRequestForCode.Should().BeFalse();
File.Delete(tempFile);
}
[Fact]
public void HasMaterializedArtifact_ReturnsTrueOnlyForExistingFiles()
{
var tempFile = Path.GetTempFileName();
try
{
InvokePrivateStatic<bool>("HasMaterializedArtifact", tempFile).Should().BeTrue();
InvokePrivateStatic<bool>("HasMaterializedArtifact", tempFile + ".missing").Should().BeFalse();
InvokePrivateStatic<bool>("HasMaterializedArtifact", (object?)null).Should().BeFalse();
}
finally
{
if (File.Exists(tempFile))
File.Delete(tempFile);
}
}
[Fact]
public void HasDocumentVerificationEvidenceAfterLastArtifact_ReturnsFalseWithoutReadAfterCreate()
{
var messages = new List<ChatMessage>
{
CreateToolResult("html_create"),
CreateToolResult("grep_tool")
};
var hasVerification = InvokePrivateStatic<bool>(
"HasDocumentVerificationEvidenceAfterLastArtifact",
messages);
hasVerification.Should().BeFalse();
}
[Fact]
public void HasDocumentVerificationEvidenceAfterLastArtifact_ReturnsTrueWithReadAfterCreate()
{
var messages = new List<ChatMessage>
{
CreateToolResult("html_create"),
CreateToolResult("file_read")
};
var hasVerification = InvokePrivateStatic<bool>(
"HasDocumentVerificationEvidenceAfterLastArtifact",
messages);
hasVerification.Should().BeTrue();
}
[Fact]
public void HasSufficientFinalReportEvidence_DocsRequiresVerificationEvidenceAfterArtifact()
{
var response = "Changed report file and verified structure with file_read. File: report.html";
var withoutVerification = new List<ChatMessage>
{
CreateToolResult("html_create")
};
var withVerification = new List<ChatMessage>
{
CreateToolResult("html_create"),
CreateToolResult("file_read")
};
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
response,
"docs",
false,
withoutVerification).Should().BeTrue();
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
response,
"docs",
false,
withVerification).Should().BeTrue();
}
[Fact]
public void HasExplicitDocumentVerificationToolMention_DetectsReadVerificationKeywords()
{
var withTool = InvokePrivateStatic<bool>(
"HasExplicitDocumentVerificationToolMention",
"검증은 file_read로 수행했습니다.");
var withKorean = InvokePrivateStatic<bool>(
"HasExplicitDocumentVerificationToolMention",
"문서 내용을 다시 확인 후 문제 없음으로 판단했습니다.");
var withoutHint = InvokePrivateStatic<bool>(
"HasExplicitDocumentVerificationToolMention",
"문서 생성 완료. report.html 저장.");
withTool.Should().BeTrue();
withKorean.Should().BeTrue();
withoutHint.Should().BeFalse();
}
[Fact]
public void HasExplicitDiffToolMention_DetectsDiffSignals()
{
var withGitTool = InvokePrivateStatic<bool>(
"HasExplicitDiffToolMention",
"검증은 git_tool과 git diff 기준으로 확인했습니다.");
var withGenericDiff = InvokePrivateStatic<bool>(
"HasExplicitDiffToolMention",
"Changed files diff를 검토했습니다.");
var withoutDiff = InvokePrivateStatic<bool>(
"HasExplicitDiffToolMention",
"파일 확인 및 테스트 검증 완료.");
withGitTool.Should().BeTrue();
withGenericDiff.Should().BeTrue();
withoutDiff.Should().BeFalse();
}
[Fact]
public void HasExplicitVerificationToolMention_DetectsCommonExecutionCommands()
{
var withTool = InvokePrivateStatic<bool>(
"HasExplicitVerificationToolMention",
"Verified with build_run and test_loop. dotnet test passed.");
var withCommand = InvokePrivateStatic<bool>(
"HasExplicitVerificationToolMention",
"Validation: dotnet build succeeded.");
var withoutTool = InvokePrivateStatic<bool>(
"HasExplicitVerificationToolMention",
"Verified behavior and checked logs.");
withTool.Should().BeTrue();
withCommand.Should().BeTrue();
withoutTool.Should().BeFalse();
}
[Fact]
public void HasExplicitFileLikeMention_DetectsCommonSourcePaths()
{
var withFile = InvokePrivateStatic<bool>(
"HasExplicitFileLikeMention",
"Changed src/AxCopilot/Services/AgentLoopService.cs and verified build.");
var withWindowsPath = InvokePrivateStatic<bool>(
"HasExplicitFileLikeMention",
@"Updated E:\AX Copilot - Codex\src\AxCopilot\Views\ChatWindow.xaml.");
var withoutFile = InvokePrivateStatic<bool>(
"HasExplicitFileLikeMention",
"Changed the flow and verified tests.");
withFile.Should().BeTrue();
withWindowsPath.Should().BeTrue();
withoutFile.Should().BeFalse();
}
[Fact]
public void UpdateConsecutiveReadOnlySuccessTools_IncrementsForReadOnlyAndResetsForWrite()
{
var next = InvokePrivateStatic<int>(
"UpdateConsecutiveReadOnlySuccessTools",
2,
"file_read",
true);
var resetByWrite = InvokePrivateStatic<int>(
"UpdateConsecutiveReadOnlySuccessTools",
5,
"file_edit",
true);
next.Should().Be(3);
resetByWrite.Should().Be(0);
}
[Fact]
public void UpdateConsecutiveReadOnlySuccessTools_ResetsOnFailure()
{
var reset = InvokePrivateStatic<int>(
"UpdateConsecutiveReadOnlySuccessTools",
4,
"file_read",
false);
reset.Should().Be(0);
}
[Fact]
public void ShouldTriggerNoProgressExecutionRecovery_MatchesThresholdAndRetryLimit()
{
var shouldTrigger = InvokePrivateStatic<bool>(
"ShouldTriggerNoProgressExecutionRecovery",
8,
0);
var belowThreshold = InvokePrivateStatic<bool>(
"ShouldTriggerNoProgressExecutionRecovery",
7,
0);
var overRetry = InvokePrivateStatic<bool>(
"ShouldTriggerNoProgressExecutionRecovery",
10,
2);
shouldTrigger.Should().BeTrue();
belowThreshold.Should().BeFalse();
overRetry.Should().BeFalse();
}
[Fact]
public void ShouldAbortNoProgressExecution_RequiresHighCountAfterRecoveryRetries()
{
var shouldAbort = InvokePrivateStatic<bool>(
"ShouldAbortNoProgressExecution",
12,
2);
var notEnoughCount = InvokePrivateStatic<bool>(
"ShouldAbortNoProgressExecution",
11,
3);
var notEnoughRetry = InvokePrivateStatic<bool>(
"ShouldAbortNoProgressExecution",
20,
1);
shouldAbort.Should().BeTrue();
notEnoughCount.Should().BeFalse();
notEnoughRetry.Should().BeFalse();
}
[Fact]
public void GetReadOnlySignatureLoopThreshold_UsesEnvAndClamp()
{
var original = Environment.GetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD");
try
{
Environment.SetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD", "1");
InvokePrivateStatic<int>("GetReadOnlySignatureLoopThreshold").Should().Be(2);
Environment.SetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD", "9");
InvokePrivateStatic<int>("GetReadOnlySignatureLoopThreshold").Should().Be(9);
Environment.SetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD", "999");
InvokePrivateStatic<int>("GetReadOnlySignatureLoopThreshold").Should().Be(12);
}
finally
{
Environment.SetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD", original);
}
}
[Fact]
public void GetNoProgressRecoveryMaxRetries_UsesDefaultWhenEnvMissing()
{
var original = Environment.GetEnvironmentVariable("AXCOPILOT_NOPROGRESS_RECOVERY_MAX_RETRIES");
try
{
Environment.SetEnvironmentVariable("AXCOPILOT_NOPROGRESS_RECOVERY_MAX_RETRIES", null);
InvokePrivateStatic<int>("GetNoProgressRecoveryMaxRetries").Should().Be(2);
}
finally
{
Environment.SetEnvironmentVariable("AXCOPILOT_NOPROGRESS_RECOVERY_MAX_RETRIES", original);
}
}
[Fact]
public void ResolveThresholdValue_ReturnsDefaultWhenInputInvalid()
{
InvokePrivateStatic<int>(
"ResolveThresholdValue",
"abc",
7,
2,
10).Should().Be(7);
}
[Fact]
public void ResolveThresholdValue_AppliesClamp()
{
InvokePrivateStatic<int>(
"ResolveThresholdValue",
"1",
7,
2,
10).Should().Be(2);
InvokePrivateStatic<int>(
"ResolveThresholdValue",
"99",
7,
2,
10).Should().Be(10);
}
[Fact]
public void ResolveConfiguredOrEnvThresholdValue_PrefersConfiguredValue()
{
var resolved = InvokePrivateStatic<int>(
"ResolveConfiguredOrEnvThresholdValue",
9,
"4",
7,
2,
10);
resolved.Should().Be(9);
}
[Fact]
public void ResolveConfiguredOrEnvThresholdValue_UsesEnvThenDefault()
{
var fromEnv = InvokePrivateStatic<int>(
"ResolveConfiguredOrEnvThresholdValue",
null,
"15",
7,
2,
10);
var fromDefault = InvokePrivateStatic<int>(
"ResolveConfiguredOrEnvThresholdValue",
null,
"invalid",
7,
2,
10);
fromEnv.Should().Be(10);
fromDefault.Should().Be(7);
}
[Fact]
public void ResolveToolExecutionTimeoutMs_PrioritizesConfiguredValue()
{
var timeout = InvokePrivateStatic<int>(
"ResolveToolExecutionTimeoutMs",
45000,
"120000");
timeout.Should().Be(45000);
}
[Fact]
public void ResolveToolExecutionTimeoutMs_UsesEnvWhenConfiguredIsZero()
{
var timeout = InvokePrivateStatic<int>(
"ResolveToolExecutionTimeoutMs",
0,
"120000");
timeout.Should().Be(120000);
}
[Fact]
public void ResolveToolExecutionTimeoutMs_UsesDefaultWhenBothMissing()
{
var timeout = InvokePrivateStatic<int>(
"ResolveToolExecutionTimeoutMs",
0,
"not-a-number");
timeout.Should().Be(90000);
}
[Fact]
public void ResolveNoToolCallResponseThreshold_UsesDefaultAndClamps()
{
AgentLoopRuntimeThresholds.ResolveNoToolCallResponseThreshold(null).Should().Be(2);
AgentLoopRuntimeThresholds.ResolveNoToolCallResponseThreshold("0").Should().Be(1);
AgentLoopRuntimeThresholds.ResolveNoToolCallResponseThreshold("99").Should().Be(6);
}
[Fact]
public void ResolveNoToolCallRecoveryMaxRetries_UsesDefaultAndClamps()
{
AgentLoopRuntimeThresholds.ResolveNoToolCallRecoveryMaxRetries(null).Should().Be(3);
AgentLoopRuntimeThresholds.ResolveNoToolCallRecoveryMaxRetries("-1").Should().Be(0);
AgentLoopRuntimeThresholds.ResolveNoToolCallRecoveryMaxRetries("99").Should().Be(6);
}
[Fact]
public void ResolvePlanExecutionRetryMax_UsesDefaultAndClamps()
{
AgentLoopRuntimeThresholds.ResolvePlanExecutionRetryMax(null).Should().Be(2);
AgentLoopRuntimeThresholds.ResolvePlanExecutionRetryMax("-5").Should().Be(0);
AgentLoopRuntimeThresholds.ResolvePlanExecutionRetryMax("10").Should().Be(6);
}
[Fact]
public void ResolveTerminalEvidenceGateMaxRetries_UsesDefaultAndClamps()
{
AgentLoopRuntimeThresholds.ResolveTerminalEvidenceGateMaxRetries(null).Should().Be(1);
AgentLoopRuntimeThresholds.ResolveTerminalEvidenceGateMaxRetries("-2").Should().Be(0);
AgentLoopRuntimeThresholds.ResolveTerminalEvidenceGateMaxRetries("9").Should().Be(3);
}
[Fact]
public void ShouldSkipTerminalEvidenceGateForAnalysisQuery_MatchesAnalysisKeywords()
{
var policy = TaskTypePolicy.FromTaskType("feature");
InvokePrivateStatic<bool>(
"ShouldSkipTerminalEvidenceGateForAnalysisQuery",
"코드 분석 결과 설명해줘",
policy).Should().BeTrue();
InvokePrivateStatic<bool>(
"ShouldSkipTerminalEvidenceGateForAnalysisQuery",
"OrderService 파일을 수정해줘",
policy).Should().BeFalse();
}
[Fact]
public void HasAnySuccessfulProgressToolResult_DetectsMutatingOrExecutionSuccess()
{
var readOnlyMessages = new List<ChatMessage>
{
CreateToolResult("file_read", "ok")
};
var progressMessages = new List<ChatMessage>
{
CreateToolResult("file_edit", "updated"),
CreateToolResult("build_run", "Build succeeded")
};
InvokePrivateStatic<bool>(
"HasAnySuccessfulProgressToolResult",
readOnlyMessages).Should().BeFalse();
InvokePrivateStatic<bool>(
"HasAnySuccessfulProgressToolResult",
progressMessages).Should().BeTrue();
}
[Fact]
public void UpdateConsecutiveNonMutatingSuccessTools_ResetsOnMutatingTools()
{
var incremented = InvokePrivateStatic<int>(
"UpdateConsecutiveNonMutatingSuccessTools",
3,
"file_read",
true);
var resetByMutation = InvokePrivateStatic<int>(
"UpdateConsecutiveNonMutatingSuccessTools",
6,
"file_edit",
true);
incremented.Should().Be(4);
resetByMutation.Should().Be(0);
}
[Fact]
public void BuildIterationLimitFallbackResponse_IncludesArtifactAndMetrics()
{
var messages = new List<ChatMessage>
{
CreateToolResult("build_run"),
new() { Role = "assistant", Content = "updated src/AxCopilot/Services/AgentLoopService.cs" }
};
var policy = TaskTypePolicy.FromTaskType("docs");
var response = InvokePrivateStatic<string>(
"BuildIterationLimitFallbackResponse",
25,
policy,
12,
8,
4,
@"E:\AX Copilot - Codex\out\report.html",
new List<string> { "file_read", "html_create" },
new Dictionary<string, int> { ["file_read"] = 2, ["html_create"] = 1 },
messages);
response.Should().Contain("최대 반복 횟수");
response.Should().Contain("도구 호출: 총 12회");
response.Should().Contain("E:\\AX Copilot - Codex\\out\\report.html");
response.Should().Contain("html_create");
}
[Fact]
public void BuildVerificationReport_UsesStructuredFormatAndPassMetadata()
{
var report = InvokePrivateStatic<string>(
"BuildVerificationReport",
"draft.md",
@"E:\AX Copilot - Codex\out\draft.md",
false,
"검증 통과\n핵심 근거: markdown 구조 정상",
false,
2);
report.Should().Contain("[VerificationReport]");
report.Should().Contain("- Status: PASS");
report.Should().Contain("- Type: document");
report.Should().Contain(@"- Target: E:\AX Copilot - Codex\out\draft.md");
report.Should().Contain("- ReadTools: 2");
report.Should().Contain("- Highlights:");
}
[Fact]
public void BuildVerificationReport_UsesFallbackTargetAndFailMetadata()
{
var report = InvokePrivateStatic<string>(
"BuildVerificationReport",
"OrderService.cs",
null,
true,
"오류 발견: null 검사 누락",
true,
4);
report.Should().Contain("- Status: FAIL");
report.Should().Contain("- Type: code");
report.Should().Contain("- Target: OrderService.cs");
report.Should().Contain("- ReadTools: 4");
}
[Fact]
public void BuildTopFailureSummary_OrdersByCountAndLimitsToTopThree()
{
var histogram = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase)
{
["build_run"] = 4,
["file_edit"] = 7,
["grep_tool"] = 3,
["test_loop"] = 7,
["file_read"] = 1,
};
var summary = InvokePrivateStatic<string>(
"BuildTopFailureSummary",
histogram);
summary.Should().Be("file_edit(7), test_loop(7), build_run(4)");
}
[Fact]
public void BuildNoProgressAbortResponse_IncludesRetrySummaryAndNextAction()
{
var policy = TaskTypePolicy.FromTaskType("bugfix");
var response = InvokePrivateStatic<string>(
"BuildNoProgressAbortResponse",
policy,
14,
2,
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
new List<string> { "file_read", "grep_tool", "build_run" });
response.Should().Contain("비진행 상태");
response.Should().Contain("연속 비진행 호출: 14");
response.Should().Contain("복구 시도 횟수: 2");
response.Should().Contain("OrderService.cs");
}
[Fact]
public void HasDiffEvidenceAfterLastModification_ReturnsFalseWithoutGitDiffAfterEdit()
{
var messages = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("file_read")
};
var hasDiff = InvokePrivateStatic<bool>(
"HasDiffEvidenceAfterLastModification",
messages);
hasDiff.Should().BeFalse();
}
[Fact]
public void HasDiffEvidenceAfterLastModification_ReturnsTrueWithGitDiffAfterEdit()
{
var messages = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("git_tool", "diff --git a/src/A.cs b/src/A.cs\n@@ -1,2 +1,2 @@")
};
var hasDiff = InvokePrivateStatic<bool>(
"HasDiffEvidenceAfterLastModification",
messages);
hasDiff.Should().BeTrue();
}
[Fact]
public void HasDiffEvidenceAfterLastModification_ReturnsFalseForGitStatusOnly()
{
var messages = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("git_tool", "On branch main\nnothing to commit, working tree clean")
};
var hasDiff = InvokePrivateStatic<bool>(
"HasDiffEvidenceAfterLastModification",
messages);
hasDiff.Should().BeFalse();
}
[Fact]
public void HasBuildOrTestEvidenceAfterLastModification_ReturnsFalseWhenExecutionIsBeforeEdit()
{
var messages = new List<ChatMessage>
{
CreateToolResult("build_run"),
CreateToolResult("file_edit")
};
var hasRecentExecution = InvokePrivateStatic<bool>(
"HasBuildOrTestEvidenceAfterLastModification",
messages);
hasRecentExecution.Should().BeFalse();
}
[Fact]
public void HasBuildOrTestEvidenceAfterLastModification_ReturnsTrueWhenExecutionIsAfterEdit()
{
var messages = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("test_loop")
};
var hasRecentExecution = InvokePrivateStatic<bool>(
"HasBuildOrTestEvidenceAfterLastModification",
messages);
hasRecentExecution.Should().BeTrue();
}
[Fact]
public void HasBuildOrTestEvidenceAfterLastModification_IgnoresFailedExecution()
{
var messages = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("build_run", "Build failed with error CS1002")
};
var hasRecentExecution = InvokePrivateStatic<bool>(
"HasBuildOrTestEvidenceAfterLastModification",
messages);
hasRecentExecution.Should().BeFalse();
}
[Fact]
public void IsSuccessfulBuildOrTestResult_DetectsSuccessAndFailureSignals()
{
var success = InvokePrivateStatic<bool>(
"IsSuccessfulBuildOrTestResult",
"Build succeeded. Tests passed.");
var failure = InvokePrivateStatic<bool>(
"IsSuccessfulBuildOrTestResult",
"빌드 실패: 오류가 발생했습니다.");
success.Should().BeTrue();
failure.Should().BeFalse();
}
[Fact]
public void HasAnyBuildOrTestAttempt_DetectsAttemptsRegardlessOfOutcome()
{
var messages = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("build_run", "Build failed with error")
};
var hasAttempt = InvokePrivateStatic<bool>(
"HasAnyBuildOrTestAttempt",
messages);
hasAttempt.Should().BeTrue();
}
[Fact]
public void HasSuccessfulBuildAndTestAfterLastModification_RequiresBothSuccessfulSignals()
{
var onlyBuild = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("build_run", "Build succeeded")
};
var buildAndTest = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("build_run", "Build succeeded"),
CreateToolResult("test_loop", "Tests passed")
};
InvokePrivateStatic<bool>(
"HasSuccessfulBuildAndTestAfterLastModification",
onlyBuild).Should().BeFalse();
InvokePrivateStatic<bool>(
"HasSuccessfulBuildAndTestAfterLastModification",
buildAndTest).Should().BeTrue();
}
[Fact]
public void HasSufficientFinalReportEvidence_HighImpactRequiresBothBuildAndTestSignals()
{
var response = "Changed src/AxCopilot/Services/OrderService.cs. build_run success with output logs. " +
"Feature flow path validated and caller impact scope reviewed. Remaining risk documented.";
var onlyBuild = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("build_run", "Build succeeded")
};
var buildAndTest = new List<ChatMessage>
{
CreateToolResult("file_edit"),
CreateToolResult("build_run", "Build succeeded"),
CreateToolResult("test_loop", "Tests passed")
};
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
response,
"feature",
true,
onlyBuild).Should().BeFalse();
InvokePrivateStatic<bool>(
"HasSufficientFinalReportEvidence",
response + " test_loop success.",
"feature",
true,
buildAndTest).Should().BeTrue();
}
[Fact]
public void BuildRecentExecutionEvidencePrompt_IncludesExecutionGuidance()
{
var policy = TaskTypePolicy.FromTaskType("feature");
var prompt = InvokePrivateStatic<string>(
"BuildRecentExecutionEvidencePrompt",
policy);
prompt.Should().Contain("RecentExecutionGate");
prompt.Should().Contain("build_run");
prompt.Should().Contain("test_loop");
}
[Fact]
public void BuildExecutionSuccessGatePrompt_IncludesSuccessGuidance()
{
var policy = TaskTypePolicy.FromTaskType("bugfix");
var prompt = InvokePrivateStatic<string>(
"BuildExecutionSuccessGatePrompt",
policy);
prompt.Should().Contain("ExecutionSuccessGate");
prompt.Should().Contain("build_run");
prompt.Should().Contain("test_loop");
}
[Fact]
public void BuildUnknownToolRecoveryPrompt_ContainsFallbackToolList()
{
var prompt = InvokePrivateStatic<string>(
"BuildUnknownToolRecoveryPrompt",
"write_file",
new List<string> { "file_write", "file_read", "glob", "grep_tool" });
prompt.Should().Contain("UnknownToolRecovery");
prompt.Should().Contain("write_file");
prompt.Should().Contain("file_write");
}
[Fact]
public void BuildUnknownToolRecoveryPrompt_IncludesAliasHintWhenAvailable()
{
var prompt = InvokePrivateStatic<string>(
"BuildUnknownToolRecoveryPrompt",
"Read",
new List<string> { "file_read", "file_write", "glob" });
prompt.Should().Contain("자동 매핑 후보");
prompt.Should().Contain("Read");
prompt.Should().Contain("file_read");
}
[Fact]
public void BuildUnknownToolLoopAbortResponse_ContainsAbortSummary()
{
var response = InvokePrivateStatic<string>(
"BuildUnknownToolLoopAbortResponse",
"bad_tool",
3,
new List<string> { "file_read", "file_write", "glob" });
response.Should().Contain("중단");
response.Should().Contain("bad_tool");
response.Should().Contain("반복 횟수: 3");
}
[Fact]
public void BuildDisallowedToolRecoveryPrompt_ContainsPolicyAndActiveToolHints()
{
var prompt = InvokePrivateStatic<string>(
"BuildDisallowedToolRecoveryPrompt",
"write_file",
new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "file_read", "file_write" },
new List<string> { "file_read", "file_write", "glob" });
prompt.Should().Contain("DisallowedToolRecovery");
prompt.Should().Contain("write_file");
prompt.Should().Contain("file_read");
prompt.Should().Contain("file_write");
}
[Fact]
public void BuildDisallowedToolLoopAbortResponse_ContainsAbortSummary()
{
var response = InvokePrivateStatic<string>(
"BuildDisallowedToolLoopAbortResponse",
"delete_all",
3,
new List<string> { "file_read", "file_write", "glob" });
response.Should().Contain("중단");
response.Should().Contain("delete_all");
response.Should().Contain("반복 횟수: 3");
}
[Fact]
public void BuildUnknownToolLoopAbortResponse_ShouldGuideToolSearch()
{
var response = InvokePrivateStatic<string>(
"BuildUnknownToolLoopAbortResponse",
"unknown_exec",
4,
new List<string> { "tool_search", "file_read", "process" });
response.Should().Contain("tool_search");
response.Should().Contain("unknown_exec");
}
[Fact]
public void BuildDisallowedToolLoopAbortResponse_ShouldGuideToolSearch()
{
var response = InvokePrivateStatic<string>(
"BuildDisallowedToolLoopAbortResponse",
"dangerous_exec",
5,
new List<string> { "tool_search", "file_read", "file_edit" });
response.Should().Contain("tool_search");
response.Should().Contain("dangerous_exec");
}
[Fact]
public void MixedRecoverySignals_ShouldRemainConsistentAcrossUnknownDisallowedAndNoProgress()
{
var unknownPrompt = InvokePrivateStatic<string>(
"BuildUnknownToolRecoveryPrompt",
"bad_tool",
new List<string> { "tool_search", "file_read", "glob" });
var disallowedPrompt = InvokePrivateStatic<string>(
"BuildDisallowedToolRecoveryPrompt",
"unsafe_tool",
new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "tool_search", "file_read" },
new List<string> { "tool_search", "file_read" });
unknownPrompt.Should().Contain("tool_search");
disallowedPrompt.Should().Contain("tool_search");
unknownPrompt.Should().Contain("bad_tool");
disallowedPrompt.Should().Contain("unsafe_tool");
}
[Fact]
public void ResolveRequestedToolName_MapsCommonAliasWhenTargetIsActive()
{
var resolved = InvokePrivateStatic<string>(
"ResolveRequestedToolName",
"Read",
new List<string> { "file_read", "file_write", "process" });
resolved.Should().Be("file_read");
}
[Fact]
public void GetRuntimeHooksForCall_AppliesTimingAndToolFilter()
{
var hooks = new List<AgentHookEntry>
{
new() { Name = "lint-pre", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
new() { Name = "verify-post", ToolName = "*", Timing = "post", ScriptPath = "x.cmd", Enabled = true },
};
var runtimeOverrides = CreateRuntimeOverridesForHookFilter("lint-pre@pre@file_edit");
var filtered = InvokePrivateStatic<IReadOnlyList<AgentHookEntry>>(
"GetRuntimeHooksForCall",
hooks.AsReadOnly(),
runtimeOverrides,
"file_edit",
"pre");
filtered.Select(h => h.Name).Should().Equal("lint-pre");
}
[Fact]
public void GetRuntimeHooksForCall_ReturnsEmptyWhenFilterDoesNotMatch()
{
var hooks = new List<AgentHookEntry>
{
new() { Name = "lint-pre", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
};
var runtimeOverrides = CreateRuntimeOverridesForHookFilter("lint-pre@pre@file_write");
var filtered = InvokePrivateStatic<IReadOnlyList<AgentHookEntry>>(
"GetRuntimeHooksForCall",
hooks.AsReadOnly(),
runtimeOverrides,
"file_edit",
"pre");
filtered.Should().BeEmpty();
}
[Fact]
public void GetRuntimeHooksForCall_ExcludeRuleOverridesWildcardAllow()
{
var hooks = new List<AgentHookEntry>
{
new() { Name = "hook-a", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
new() { Name = "hook-b", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
};
var runtimeOverrides = CreateRuntimeOverridesForHookFilter("hook-a@*@*, !hook-b@pre@math_eval");
var filtered = InvokePrivateStatic<IReadOnlyList<AgentHookEntry>>(
"GetRuntimeHooksForCall",
hooks.AsReadOnly(),
runtimeOverrides,
"math_eval",
"pre");
filtered.Select(h => h.Name).Should().Equal("hook-a");
}
[Fact]
public void GetRuntimeHooksForCall_MoreSpecificRuleWins()
{
var hooks = new List<AgentHookEntry>
{
new() { Name = "hook-a", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
};
var runtimeOverrides = CreateRuntimeOverridesForHookFilter("hook-a@*@*, !hook-a@pre@math_eval");
var filtered = InvokePrivateStatic<IReadOnlyList<AgentHookEntry>>(
"GetRuntimeHooksForCall",
hooks.AsReadOnly(),
runtimeOverrides,
"math_eval",
"pre");
filtered.Should().BeEmpty();
}
[Fact]
public void ResolveRequestedToolName_UsesNormalizedNameMatchBeforeUnknown()
{
var resolved = InvokePrivateStatic<string>(
"ResolveRequestedToolName",
"grep-tool",
new List<string> { "file_read", "grep_tool", "glob" });
resolved.Should().Be("grep_tool");
}
[Fact]
public void ResolveRequestedToolName_ReturnsOriginalWhenAliasTargetIsUnavailable()
{
var resolved = InvokePrivateStatic<string>(
"ResolveRequestedToolName",
"Write",
new List<string> { "file_read", "glob" });
resolved.Should().Be("Write");
}
[Fact]
public void ResolveRequestedToolName_MapsRgAliasToGrep()
{
var resolved = InvokePrivateStatic<string>(
"ResolveRequestedToolName",
"rg",
new List<string> { "file_read", "grep", "glob" });
resolved.Should().Be("grep");
}
[Fact]
public void ResolveRequestedToolName_MapsCodeSearchAliasToSearchCodebase()
{
var resolved = InvokePrivateStatic<string>(
"ResolveRequestedToolName",
"code_search",
new List<string> { "search_codebase", "file_read" });
resolved.Should().Be("search_codebase");
}
[Fact]
public void ShouldRunPostToolVerification_MatchesTabAndToolTypeRules()
{
var codeTrue = InvokePrivateStatic<bool>(
"ShouldRunPostToolVerification",
"Code",
"file_edit",
true,
true,
false);
var codeFalse = InvokePrivateStatic<bool>(
"ShouldRunPostToolVerification",
"Code",
"document_plan",
true,
true,
true);
var coworkTrue = InvokePrivateStatic<bool>(
"ShouldRunPostToolVerification",
"Cowork",
"html_create",
true,
false,
true);
var failedFalse = InvokePrivateStatic<bool>(
"ShouldRunPostToolVerification",
"Code",
"file_edit",
false,
true,
true);
codeTrue.Should().BeTrue();
codeFalse.Should().BeFalse();
coworkTrue.Should().BeTrue();
failedFalse.Should().BeFalse();
}
[Fact]
public void EvaluateDecisionHelpers_ReturnExpectedTransitionShape()
{
var devSkip = InvokePrivateStatic<(bool ShouldContinue, string? TerminalResponse, string? ToolResultMessage)>(
"EvaluateDevStepDecision",
"건너뛰기");
var devStop = InvokePrivateStatic<(bool ShouldContinue, string? TerminalResponse, string? ToolResultMessage)>(
"EvaluateDevStepDecision",
"중단");
var scopeSkip = InvokePrivateStatic<(bool ShouldContinue, string? TerminalResponse, string? ToolResultMessage)>(
"EvaluateScopeDecision",
"건너뛰기");
var scopeCancel = InvokePrivateStatic<(bool ShouldContinue, string? TerminalResponse, string? ToolResultMessage)>(
"EvaluateScopeDecision",
"취소");
devSkip.ShouldContinue.Should().BeTrue();
devSkip.ToolResultMessage.Should().Contain("SKIPPED");
devStop.TerminalResponse.Should().NotBeNull();
scopeSkip.ShouldContinue.Should().BeTrue();
scopeCancel.TerminalResponse.Should().NotBeNull();
}
[Fact]
public void IsTransientLlmError_DetectsRateLimitAndTimeoutPatterns()
{
var rateLimit = InvokePrivateStatic<bool>(
"IsTransientLlmError",
new Exception("429 Too Many Requests: rate limit exceeded"));
var timeout = InvokePrivateStatic<bool>(
"IsTransientLlmError",
new TimeoutException("request timed out"));
var fatal = InvokePrivateStatic<bool>(
"IsTransientLlmError",
new Exception("invalid api key"));
rateLimit.Should().BeTrue();
timeout.Should().BeTrue();
fatal.Should().BeFalse();
}
[Fact]
public void ComputeTransientLlmBackoffDelayMs_UsesRetryAfterWhenPresent()
{
var delay = InvokePrivateStatic<int>(
"ComputeTransientLlmBackoffDelayMs",
1,
new Exception("Rate limited. retry-after: 7"));
delay.Should().Be(7000);
}
[Fact]
public void ComputeTransientLlmBackoffDelayMs_GrowsWithRetryCountWithoutRetryAfter()
{
var delay1 = InvokePrivateStatic<int>(
"ComputeTransientLlmBackoffDelayMs",
1,
new Exception("service unavailable"));
var delay2 = InvokePrivateStatic<int>(
"ComputeTransientLlmBackoffDelayMs",
2,
new Exception("service unavailable"));
delay1.Should().BeGreaterThanOrEqualTo(800);
delay1.Should().BeLessThan(1050);
delay2.Should().BeGreaterThanOrEqualTo(1600);
delay2.Should().BeLessThan(1850);
}
[Fact]
public void IsContextOverflowError_DetectsMaxOutputTokenPatterns()
{
var maxOutput = InvokePrivateStatic<bool>(
"IsContextOverflowError",
"Request failed: maximum output tokens exceeded");
var truncated = InvokePrivateStatic<bool>(
"IsContextOverflowError",
"response was truncated due to token limit");
var unrelated = InvokePrivateStatic<bool>(
"IsContextOverflowError",
"invalid api key");
maxOutput.Should().BeTrue();
truncated.Should().BeTrue();
unrelated.Should().BeFalse();
}
[Fact]
public void IsLikelyWithheldOrOverflowResponse_DetectsWithheldAndLimitHints()
{
var withheld = InvokePrivateStatic<bool>(
"IsLikelyWithheldOrOverflowResponse",
"output withheld because prompt too long");
var maxOutput = InvokePrivateStatic<bool>(
"IsLikelyWithheldOrOverflowResponse",
"max_output_tokens reached");
var normal = InvokePrivateStatic<bool>(
"IsLikelyWithheldOrOverflowResponse",
"completed successfully");
withheld.Should().BeTrue();
maxOutput.Should().BeTrue();
normal.Should().BeFalse();
}
[Fact]
public void GetToolExecutionTimeoutMs_UsesDefaultWhenEnvMissing()
{
var original = Environment.GetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS");
try
{
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", null);
var timeout = InvokePrivateStatic<int>("GetToolExecutionTimeoutMs");
timeout.Should().Be(90000);
}
finally
{
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", original);
}
}
[Fact]
public void GetToolExecutionTimeoutMs_ClampsOutOfRangeValues()
{
var original = Environment.GetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS");
try
{
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", "1000");
InvokePrivateStatic<int>("GetToolExecutionTimeoutMs").Should().Be(5000);
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", "9999999");
InvokePrivateStatic<int>("GetToolExecutionTimeoutMs").Should().Be(600000);
}
finally
{
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", original);
}
}
[Fact]
public void InjectTaskTypeGuidance_AddsBugfixGuidanceOnlyOnce()
{
var messages = new List<ChatMessage>
{
new() { Role = "user", Content = "fix this error" }
};
InvokePrivateStaticVoid("InjectTaskTypeGuidance", messages, "bugfix");
InvokePrivateStaticVoid("InjectTaskTypeGuidance", messages, "bugfix");
messages.Count(m => m.Role == "user" && m.Content.StartsWith("[System:TaskType]", StringComparison.Ordinal))
.Should().Be(1);
messages[0].Content.Should().Contain("bug-fix");
}
[Fact]
public void ShouldEnforceForkExecution_RequiresNonCompliantToolWithinAttemptLimit()
{
var shouldEnforce = InvokePrivateStatic<bool>(
"ShouldEnforceForkExecution",
true,
false,
"file_read",
0,
2);
shouldEnforce.Should().BeTrue();
}
[Fact]
public void ShouldEnforceForkExecution_DoesNotEnforceForSpawnAgentOrAfterLimit()
{
var compliantTool = InvokePrivateStatic<bool>(
"ShouldEnforceForkExecution",
true,
false,
"spawn_agent",
0,
2);
var overLimit = InvokePrivateStatic<bool>(
"ShouldEnforceForkExecution",
true,
false,
"file_edit",
2,
2);
var afterDelegation = InvokePrivateStatic<bool>(
"ShouldEnforceForkExecution",
true,
true,
"file_edit",
0,
2);
compliantTool.Should().BeFalse();
overLimit.Should().BeFalse();
afterDelegation.Should().BeFalse();
}
private static object CreateRuntimeOverridesForHookFilter(string filter)
{
var loopType = typeof(AgentLoopService);
var filterType = loopType.GetNestedType("HookFilterRule", BindingFlags.NonPublic);
var runtimeType = loopType.GetNestedType("SkillRuntimeOverrides", BindingFlags.NonPublic);
filterType.Should().NotBeNull();
runtimeType.Should().NotBeNull();
var filterCtor = filterType!.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
.First(c => c.GetParameters().Length == 6);
var filterListType = typeof(List<>).MakeGenericType(filterType);
var filterList = (System.Collections.IList)Activator.CreateInstance(filterListType)!;
var order = 0;
foreach (var raw in filter.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
{
var parts = raw.Trim().Split('@', StringSplitOptions.TrimEntries);
var hookNameRaw = parts.Length > 0 ? parts[0] : "*";
var isExclude = hookNameRaw.StartsWith("!");
var hookName = isExclude ? hookNameRaw[1..] : hookNameRaw;
if (string.IsNullOrWhiteSpace(hookName))
hookName = "*";
var timing = parts.Length > 1 ? parts[1] : "*";
var toolName = parts.Length > 2 ? parts[2] : "*";
var specificity = (hookName == "*" ? 0 : 1) + (timing == "*" ? 0 : 1) + (toolName == "*" ? 0 : 1);
var filterObj = filterCtor.Invoke([hookName, timing, toolName, isExclude, specificity, order++]);
filterList.Add(filterObj);
}
var runtimeCtor = runtimeType!.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
.First(c => c.GetParameters().Length == 8);
return runtimeCtor.Invoke(
[
null, // service
null, // model
null, // temperature
null, // effort
false, // requireForkExecution
new HashSet<string>(StringComparer.OrdinalIgnoreCase), // allowedTools
new HashSet<string>(StringComparer.OrdinalIgnoreCase), // hookNames
filterList // hookFilters
])!;
}
private static ChatMessage CreateToolResult(string toolName, string content = "ok")
=> new()
{
Role = "user",
Content = JsonSerializer.Serialize(new
{
type = "tool_result",
tool_name = toolName,
content
})
};
private static T InvokePrivateStatic<T>(string methodName, params object?[] arguments)
{
var method = ResolvePrivateStatic(methodName, arguments);
method.Should().NotBeNull($"{methodName} should exist on AgentLoopService");
var result = method!.Invoke(null, arguments);
result.Should().NotBeNull();
return (T)result!;
}
private static void InvokePrivateStaticVoid(string methodName, params object?[] arguments)
{
var method = ResolvePrivateStatic(methodName, arguments);
method.Should().NotBeNull($"{methodName} should exist on AgentLoopService");
method!.Invoke(null, arguments);
}
private static MethodInfo? ResolvePrivateStatic(string methodName, object?[] arguments)
{
return typeof(AgentLoopService)
.GetMethods(BindingFlags.NonPublic | BindingFlags.Static)
.Where(m => m.Name == methodName)
.FirstOrDefault(m =>
{
var parameters = m.GetParameters();
if (parameters.Length != arguments.Length)
return false;
for (var i = 0; i < parameters.Length; i++)
{
var arg = arguments[i];
if (arg == null)
continue;
var targetType = parameters[i].ParameterType;
if (!targetType.IsInstanceOfType(arg))
return false;
}
return true;
});
}
}