변경 목적: - AX Agent의 도구 이름, 내부 설정, 스킬 정책, 실행 루프 사이의 불일치를 줄이고 전체 동작 품질을 높인다. - claw-code 수준의 일관된 동작 품질을 참고하되 AX 구조에 맞는 고유한 카탈로그·정규화 레이어로 재구성한다. 핵심 수정사항: - 도구 canonical id, legacy alias, 탭 노출, 설정 카테고리, read-only 분류를 중앙 카탈로그로 통합했다. - ToolRegistry, AgentLoopService, 병렬 실행 분류, 권한 처리, 훅 처리, 스킬 allowed-tools 해석이 같은 이름 체계를 사용하도록 정리했다. - Agent 설정/일반 설정/도움말의 도구 카드와 훅 편집기, 스킬 설명을 현재 런타임 구조에 맞게 갱신했다. - 컨텍스트 압축, intent gate, spawn agents, session learning, model prompt adapter, workspace context 관련 변경과 테스트 추가를 함께 반영했다. - 문서 이력과 비교/로드맵 문서를 최신 상태로 갱신했다. 검증 결과: - dotnet build src/AxCopilot/AxCopilot.csproj -c Release -v minimal -p:OutputPath=bin\verify_toolcat\ -p:IntermediateOutputPath=obj\verify_toolcat\ : 경고 0 / 오류 0 - dotnet test src/AxCopilot.Tests/AxCopilot.Tests.csproj -c Release -v minimal --filter AgentToolCatalogTests -p:OutputPath=bin\verify_toolcat_tests\ -p:IntermediateOutputPath=obj\verify_toolcat_tests\ : 통과 8
2154 lines
71 KiB
C#
2154 lines
71 KiB
C#
using System.Reflection;
|
|
using System.IO;
|
|
using System.Text.Json;
|
|
using AxCopilot.Models;
|
|
using AxCopilot.Services;
|
|
using AxCopilot.Services.Agent;
|
|
using FluentAssertions;
|
|
using Xunit;
|
|
|
|
namespace AxCopilot.Tests.Services;
|
|
|
|
public class AgentLoopCodeQualityTests
|
|
{
|
|
[Fact]
|
|
public void IsHighImpactCodeModification_ReturnsTrueForServiceFile()
|
|
{
|
|
var result = ToolResult.Ok("updated", @"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs");
|
|
|
|
var highImpact = InvokePrivateStatic<bool>(
|
|
"IsHighImpactCodeModification",
|
|
"Code",
|
|
"file_edit",
|
|
result);
|
|
|
|
highImpact.Should().BeTrue();
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("fix this error", "Code", "bugfix")]
|
|
[InlineData("add a new feature", "Code", "feature")]
|
|
[InlineData("refactor this flow", "Code", "refactor")]
|
|
[InlineData("review this code", "Code", "review")]
|
|
[InlineData("write a report", "Cowork", "docs")]
|
|
public void ClassifyTaskType_ReturnsExpectedType(string query, string tab, string expected)
|
|
{
|
|
var taskType = AgentLoopService.ClassifyTaskType(query, tab);
|
|
|
|
taskType.Should().Be(expected);
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildTaskTypeGuidanceMessage_IncludesToolOrderHints()
|
|
{
|
|
var bugfix = AgentLoopService.BuildTaskTypeGuidanceMessage("bugfix");
|
|
var refactor = AgentLoopService.BuildTaskTypeGuidanceMessage("refactor");
|
|
|
|
bugfix.Should().Contain("Preferred tool order");
|
|
bugfix.Should().Contain("build_run/test_loop");
|
|
refactor.Should().Contain("git_tool(diff)");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildCodeQualityFollowUpPrompt_IncludesBaselineAndTestReviewForProductionCode()
|
|
{
|
|
var result = ToolResult.Ok("updated", @"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs");
|
|
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildCodeQualityFollowUpPrompt",
|
|
"file_edit",
|
|
result,
|
|
false,
|
|
false,
|
|
"bugfix");
|
|
|
|
prompt.Should().Contain("baseline build/test");
|
|
prompt.Should().Contain("grep 또는 glob");
|
|
prompt.Should().Contain("build_run");
|
|
prompt.Should().Contain("테스트 부재 사실");
|
|
prompt.Should().Contain("Task type: bugfix");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildCodeQualityFollowUpPrompt_StrengthensHighImpactVerification()
|
|
{
|
|
var result = ToolResult.Ok("updated", @"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs");
|
|
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildCodeQualityFollowUpPrompt",
|
|
"file_edit",
|
|
result,
|
|
true,
|
|
true,
|
|
"refactor");
|
|
|
|
prompt.Should().Contain("spawn_agent");
|
|
prompt.Should().Contain("build_run");
|
|
prompt.Should().Contain("grep 또는 glob");
|
|
prompt.Should().Contain("테스트 부재 사실");
|
|
prompt.Should().Contain("영향 범위가 넓을 가능성");
|
|
prompt.Should().Contain("Task type: refactor");
|
|
}
|
|
|
|
[Fact]
|
|
public void HasCodeVerificationEvidenceAfterLastModification_RequiresReferenceSearchForHighImpact()
|
|
{
|
|
var messages =
|
|
new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("file_read"),
|
|
CreateToolResult("git_tool"),
|
|
CreateToolResult("build_run")
|
|
};
|
|
|
|
var hasEvidence = InvokePrivateStatic<bool>(
|
|
"HasCodeVerificationEvidenceAfterLastModification",
|
|
messages,
|
|
true);
|
|
|
|
hasEvidence.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasCodeVerificationEvidenceAfterLastModification_AcceptsReferenceSearchAndExecutionForHighImpact()
|
|
{
|
|
var messages =
|
|
new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("file_read"),
|
|
CreateToolResult("grep"),
|
|
CreateToolResult("build_run")
|
|
};
|
|
|
|
var hasEvidence = InvokePrivateStatic<bool>(
|
|
"HasCodeVerificationEvidenceAfterLastModification",
|
|
messages,
|
|
true);
|
|
|
|
hasEvidence.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasCodeVerificationEvidenceAfterLastModification_AcceptsDelegatedInvestigationForHighImpact()
|
|
{
|
|
var messages =
|
|
new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("file_read"),
|
|
CreateToolResult("wait_agents"),
|
|
CreateToolResult("build_run")
|
|
};
|
|
|
|
var hasEvidence = InvokePrivateStatic<bool>(
|
|
"HasCodeVerificationEvidenceAfterLastModification",
|
|
messages,
|
|
true);
|
|
|
|
hasEvidence.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasCodeVerificationEvidenceAfterLastModification_AcceptsInspectionOnlyForNormalChange()
|
|
{
|
|
var messages =
|
|
new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("file_read")
|
|
};
|
|
|
|
var hasEvidence = InvokePrivateStatic<bool>(
|
|
"HasCodeVerificationEvidenceAfterLastModification",
|
|
messages,
|
|
false);
|
|
|
|
hasEvidence.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildFailureInvestigationPrompt_IncludesHighImpactChecks()
|
|
{
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildFailureInvestigationPrompt",
|
|
"build_run",
|
|
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
|
|
true,
|
|
"bugfix");
|
|
|
|
prompt.Should().Contain("OrderService.cs");
|
|
prompt.Should().Contain("spawn_agent");
|
|
prompt.Should().Contain("build/test");
|
|
prompt.Should().Contain("테스트 부재 사실");
|
|
prompt.Should().Contain("symptom is no longer reproducible");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildFailureInvestigationPrompt_ChangesByTaskType()
|
|
{
|
|
var featurePrompt = InvokePrivateStatic<string>(
|
|
"BuildFailureInvestigationPrompt",
|
|
"build_run",
|
|
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
|
|
false,
|
|
"feature");
|
|
|
|
var refactorPrompt = InvokePrivateStatic<string>(
|
|
"BuildFailureInvestigationPrompt",
|
|
"build_run",
|
|
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
|
|
false,
|
|
"refactor");
|
|
|
|
featurePrompt.Should().Contain("feature path and caller linkage");
|
|
refactorPrompt.Should().Contain("behavior-compatible");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildFailureReflectionMessage_IncludesFallbackSequenceForBuildFailures()
|
|
{
|
|
var result = ToolResult.Fail("build failed");
|
|
var message = InvokePrivateStatic<string>(
|
|
"BuildFailureReflectionMessage",
|
|
"build_run",
|
|
result,
|
|
2,
|
|
3,
|
|
"bugfix");
|
|
|
|
message.Should().Contain("Fallback sequence");
|
|
message.Should().Contain("file_read -> grep/glob -> git_tool(diff)");
|
|
message.Should().Contain("repro/root-cause");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildFailureNextToolPriorityPrompt_IncludesOrderedPriority()
|
|
{
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildFailureNextToolPriorityPrompt",
|
|
"build_run",
|
|
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
|
|
true,
|
|
"feature");
|
|
|
|
prompt.Should().Contain("[System:NextToolPriority]");
|
|
prompt.Should().Contain("file_read -> grep/glob -> git_tool(diff) -> file_edit -> build_run/test_loop");
|
|
prompt.Should().Contain("작업유형: feature");
|
|
prompt.Should().Contain("고영향 변경");
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeAdaptiveMaxRetry_AdjustsByTaskType()
|
|
{
|
|
InvokePrivateStatic<int>(
|
|
"ComputeAdaptiveMaxRetry",
|
|
3,
|
|
"bugfix").Should().Be(4);
|
|
|
|
InvokePrivateStatic<int>(
|
|
"ComputeAdaptiveMaxRetry",
|
|
3,
|
|
"feature").Should().Be(3);
|
|
|
|
InvokePrivateStatic<int>(
|
|
"ComputeAdaptiveMaxRetry",
|
|
3,
|
|
"refactor").Should().Be(2);
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeQualityAwareMaxRetry_AdjustsByRecentQuality()
|
|
{
|
|
InvokePrivateStatic<int>(
|
|
"ComputeQualityAwareMaxRetry",
|
|
3,
|
|
0.30,
|
|
"feature").Should().Be(4);
|
|
|
|
InvokePrivateStatic<int>(
|
|
"ComputeQualityAwareMaxRetry",
|
|
3,
|
|
0.90,
|
|
"feature").Should().Be(2);
|
|
|
|
InvokePrivateStatic<int>(
|
|
"ComputeQualityAwareMaxRetry",
|
|
3,
|
|
0.90,
|
|
"bugfix").Should().Be(3);
|
|
}
|
|
|
|
[Fact]
|
|
public void HasSufficientFinalReportEvidence_RequiresRiskForHighImpact()
|
|
{
|
|
var weakReport = "변경: 서비스 파일 수정. 검증: build/test 통과.";
|
|
var strongReport = "Changed service file. Verified by build and test. Feature flow path reviewed. Caller reference impact scope checked. Remaining risk: integration follow-up needed.";
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
weakReport,
|
|
"feature",
|
|
true).Should().BeFalse();
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
strongReport,
|
|
"feature",
|
|
true).Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasSufficientFinalReportEvidence_BugfixRequiresTaskSpecificSignals()
|
|
{
|
|
var generic = "Changed file and build/test verified.";
|
|
var specific = "Changed file and build/test verified. Root cause was null path; fix applied; repro steps now pass.";
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
generic,
|
|
"bugfix",
|
|
false).Should().BeFalse();
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
specific,
|
|
"bugfix",
|
|
false).Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void TryParseApprovedPlanDecision_ParsesEditedSteps()
|
|
{
|
|
var decision = $"{AgentLoopService.ApprovedPlanDecisionPrefix}\n1. 파일 구조 점검\n2. 테스트 실행";
|
|
|
|
var ok = AgentLoopService.TryParseApprovedPlanDecision(decision, out var planText, out var steps);
|
|
|
|
ok.Should().BeTrue();
|
|
steps.Should().Equal("파일 구조 점검", "테스트 실행");
|
|
planText.Should().Contain("1. 파일 구조 점검");
|
|
planText.Should().Contain("2. 테스트 실행");
|
|
}
|
|
|
|
[Fact]
|
|
public void TryParseApprovedPlanDecision_ReturnsFalseForNormalDecision()
|
|
{
|
|
var ok = AgentLoopService.TryParseApprovedPlanDecision("승인", out var planText, out var steps);
|
|
|
|
ok.Should().BeFalse();
|
|
planText.Should().BeEmpty();
|
|
steps.Should().BeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasSufficientFinalReportEvidence_FeatureRequiresBehaviorAndImpactSignals()
|
|
{
|
|
var missingBehavior = "Changed files and build/test verified. caller references checked.";
|
|
var strong = "Changed files and build/test verified. New input/output flow added and caller impact scope checked via references.";
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
missingBehavior,
|
|
"feature",
|
|
false).Should().BeFalse();
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
strong,
|
|
"feature",
|
|
false).Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildFinalReportQualityPrompt_IncludesTaskSpecificGuidance()
|
|
{
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildFinalReportQualityPrompt",
|
|
"bugfix",
|
|
true);
|
|
|
|
prompt.Should().Contain("무엇을 변경했는지");
|
|
prompt.Should().Contain("build/test/검증 근거");
|
|
prompt.Should().Contain("bug fix");
|
|
prompt.Should().Contain("남은 리스크");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildFailurePatternGuidance_ReturnsEmptyWhenNoPatterns()
|
|
{
|
|
var guidance = AgentLoopService.BuildFailurePatternGuidance(Array.Empty<string>(), "bugfix");
|
|
|
|
guidance.Should().BeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildFailurePatternGuidance_IncludesTaskSpecificFocusAndPatterns()
|
|
{
|
|
var guidance = AgentLoopService.BuildFailurePatternGuidance(
|
|
new[]
|
|
{
|
|
"code-failure | task:bugfix | tool:build_run | retry:1/3 | error:CS1002",
|
|
"code-failure | task:bugfix | tool:test_loop | retry:2/3 | error:NRE"
|
|
},
|
|
"bugfix");
|
|
|
|
guidance.Should().Contain("[System:FailurePatterns]");
|
|
guidance.Should().Contain("reproduction");
|
|
guidance.Should().Contain("CS1002");
|
|
guidance.Should().Contain("NRE");
|
|
}
|
|
|
|
[Fact]
|
|
public void UnknownAndDisallowedRecoveryPrompts_ShouldGuideToolSearch()
|
|
{
|
|
var unknownPrompt = InvokePrivateStatic<string>(
|
|
"BuildUnknownToolRecoveryPrompt",
|
|
"read_file_typo",
|
|
new[] { "file_read", "file_edit", "tool_search" });
|
|
|
|
var disallowedPrompt = InvokePrivateStatic<string>(
|
|
"BuildDisallowedToolRecoveryPrompt",
|
|
"shell_exec",
|
|
new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "file_read", "tool_search" },
|
|
new[] { "file_read", "tool_search" });
|
|
|
|
unknownPrompt.Should().Contain("tool_search");
|
|
disallowedPrompt.Should().Contain("tool_search");
|
|
}
|
|
|
|
[Fact]
|
|
public void IsFailurePatternForTaskType_MatchesCaseInsensitiveTaskToken()
|
|
{
|
|
var matched = InvokePrivateStatic<bool>(
|
|
"IsFailurePatternForTaskType",
|
|
"code-failure | task:BugFix | tool:build_run | error:CS1002",
|
|
"bugfix");
|
|
var notMatched = InvokePrivateStatic<bool>(
|
|
"IsFailurePatternForTaskType",
|
|
"code-failure | task:feature | tool:build_run | error:CS1002",
|
|
"bugfix");
|
|
|
|
matched.Should().BeTrue();
|
|
notMatched.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void IsFailurePatternForTaskType_HandlesSpacedAndShuffledTokens()
|
|
{
|
|
var matched = InvokePrivateStatic<bool>(
|
|
"IsFailurePatternForTaskType",
|
|
"code-failure | retry: 2 / 3 | impact:high | tool:build_run | task: bugfix | error:CS1002",
|
|
"bugfix");
|
|
|
|
matched.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void ExtractRetryCurrent_ReturnsZeroForMalformedPattern()
|
|
{
|
|
var retry = InvokePrivateStatic<int>(
|
|
"ExtractRetryCurrent",
|
|
"code-failure | task:bugfix | retry:abc | error:CS1002");
|
|
|
|
retry.Should().Be(0);
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildFailureMemoryContent_SanitizesPipeAndNewlineInErrorAndPath()
|
|
{
|
|
var result = ToolResult.Ok("line1|line2\nline3");
|
|
|
|
var content = InvokePrivateStatic<string>(
|
|
"BuildFailureMemoryContent",
|
|
"build_run",
|
|
result,
|
|
"bugfix",
|
|
@"E:\AX Copilot - Codex\src\A|B\Foo.cs",
|
|
2,
|
|
3);
|
|
|
|
content.Should().Contain("task:bugfix");
|
|
content.Should().Contain("retry:2/3");
|
|
content.Should().NotContain("|line2\n");
|
|
content.Should().Contain("line1/line2 line3");
|
|
content.Should().Contain(@"file:E:\AX Copilot - Codex\src\A/B\Foo.cs");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildFailureMemoryContent_NormalizesToolNameDelimiters()
|
|
{
|
|
var result = ToolResult.Ok("err");
|
|
|
|
var content = InvokePrivateStatic<string>(
|
|
"BuildFailureMemoryContent",
|
|
"build|run",
|
|
result,
|
|
"feature",
|
|
null,
|
|
1,
|
|
3);
|
|
|
|
content.Should().Contain("tool:build/run");
|
|
}
|
|
|
|
[Fact]
|
|
public void SelectTopFailurePatterns_PrioritizesExactTaskAndHighRetry()
|
|
{
|
|
var now = DateTime.Now;
|
|
var entries = new List<MemoryEntry>
|
|
{
|
|
new()
|
|
{
|
|
Type = "correction",
|
|
Content = "code-failure | task:feature | tool:build_run | retry:3/3 | impact:high | error:CS1002",
|
|
LastUsedAt = now,
|
|
UseCount = 4,
|
|
Relevance = 0.2
|
|
},
|
|
new()
|
|
{
|
|
Type = "correction",
|
|
Content = "code-failure | task:bugfix | tool:build_run | retry:1/3 | error:NRE",
|
|
LastUsedAt = now.AddDays(-2),
|
|
UseCount = 1,
|
|
Relevance = 0.8
|
|
},
|
|
new()
|
|
{
|
|
Type = "correction",
|
|
Content = "code-failure | task:general | tool:test_loop | retry:3/3 | error:timeout",
|
|
LastUsedAt = now,
|
|
UseCount = 5,
|
|
Relevance = 0.9
|
|
}
|
|
};
|
|
|
|
var selected = AgentLoopService.SelectTopFailurePatterns(entries, "feature", 2);
|
|
|
|
selected.Should().HaveCount(2);
|
|
selected[0].Should().Contain("task:feature");
|
|
}
|
|
|
|
[Fact]
|
|
public void SelectTopFailurePatterns_PrioritizesRecentPatternsWhenTaskMatches()
|
|
{
|
|
var now = DateTime.Now;
|
|
var entries = new List<MemoryEntry>
|
|
{
|
|
new()
|
|
{
|
|
Type = "correction",
|
|
Content = "code-failure | task:bugfix | tool:build_run | retry:1/3 | error:old",
|
|
LastUsedAt = now.AddDays(-40),
|
|
UseCount = 10,
|
|
Relevance = 0.7
|
|
},
|
|
new()
|
|
{
|
|
Type = "correction",
|
|
Content = "code-failure | task:bugfix | tool:build_run | retry:1/3 | error:new",
|
|
LastUsedAt = now.AddHours(-5),
|
|
UseCount = 1,
|
|
Relevance = 0.4
|
|
}
|
|
};
|
|
|
|
var selected = AgentLoopService.SelectTopFailurePatterns(entries, "bugfix", 1);
|
|
|
|
selected.Should().HaveCount(1);
|
|
selected[0].Should().Contain("error:new");
|
|
}
|
|
|
|
[Fact]
|
|
public void SelectTopFailurePatterns_DeduplicatesSameRootCauseSignature()
|
|
{
|
|
var now = DateTime.Now;
|
|
var entries = new List<MemoryEntry>
|
|
{
|
|
new()
|
|
{
|
|
Type = "correction",
|
|
Content = "code-failure | task:bugfix | tool:build_run | retry:1/3 | error:CS1002 token missing",
|
|
LastUsedAt = now.AddHours(-1),
|
|
UseCount = 1,
|
|
Relevance = 0.6
|
|
},
|
|
new()
|
|
{
|
|
Type = "correction",
|
|
Content = "code-failure | task:bugfix | tool:build_run | retry:3/3 | error:CS1002 token missing",
|
|
LastUsedAt = now,
|
|
UseCount = 2,
|
|
Relevance = 0.4
|
|
},
|
|
new()
|
|
{
|
|
Type = "correction",
|
|
Content = "code-failure | task:bugfix | tool:test_loop | retry:1/3 | error:NRE",
|
|
LastUsedAt = now,
|
|
UseCount = 1,
|
|
Relevance = 0.4
|
|
}
|
|
};
|
|
|
|
var selected = AgentLoopService.SelectTopFailurePatterns(entries, "bugfix", 3);
|
|
|
|
selected.Should().HaveCount(2);
|
|
selected.Count(x => x.Contains("CS1002", StringComparison.OrdinalIgnoreCase)).Should().Be(1);
|
|
selected.Should().Contain(x => x.Contains("retry:3/3", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildToolCallSignature_IncludesToolAndCanonicalInput()
|
|
{
|
|
var input = JsonDocument.Parse("""{"path":"src/A.cs","line":10}""").RootElement.Clone();
|
|
var call = new ContentBlock
|
|
{
|
|
Type = "tool_use",
|
|
ToolName = "file_edit",
|
|
ToolId = "t1",
|
|
ToolInput = input
|
|
};
|
|
|
|
var signature = InvokePrivateStatic<string>("BuildToolCallSignature", call);
|
|
|
|
signature.Should().StartWith("file_edit|");
|
|
signature.Should().Contain("\"path\":\"src/A.cs\"");
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldBlockRepeatedFailedCall_ReturnsTrueWhenThresholdReached()
|
|
{
|
|
var shouldBlock = InvokePrivateStatic<bool>(
|
|
"ShouldBlockRepeatedFailedCall",
|
|
"sig",
|
|
"sig",
|
|
3,
|
|
3);
|
|
|
|
var shouldNotBlock = InvokePrivateStatic<bool>(
|
|
"ShouldBlockRepeatedFailedCall",
|
|
"sig-a",
|
|
"sig-b",
|
|
5,
|
|
3);
|
|
|
|
shouldBlock.Should().BeTrue();
|
|
shouldNotBlock.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void CreateParallelExecutionPlan_DisabledFlagKeepsSequentialOnly()
|
|
{
|
|
var calls = new List<ContentBlock>
|
|
{
|
|
new() { Type = "tool_use", ToolName = "file_read", ToolId = "t1", ToolInput = JsonDocument.Parse("""{"path":"a.txt"}""").RootElement.Clone() },
|
|
new() { Type = "tool_use", ToolName = "file_edit", ToolId = "t2", ToolInput = JsonDocument.Parse("""{"path":"a.txt","old":"a","new":"b"}""").RootElement.Clone() }
|
|
};
|
|
|
|
var plan = InvokePrivateStatic<(bool ShouldRun, List<ContentBlock> ParallelBatch, List<ContentBlock> SequentialBatch)>(
|
|
"CreateParallelExecutionPlan",
|
|
false,
|
|
calls,
|
|
0);
|
|
|
|
plan.ShouldRun.Should().BeFalse();
|
|
plan.ParallelBatch.Should().BeEmpty();
|
|
plan.SequentialBatch.Should().HaveCount(2);
|
|
}
|
|
|
|
[Fact]
|
|
public void CreateParallelExecutionPlan_UsesOnlyReadOnlyPrefixForParallelBatch()
|
|
{
|
|
var calls = new List<ContentBlock>
|
|
{
|
|
new() { Type = "tool_use", ToolName = "file_read", ToolId = "t1", ToolInput = JsonDocument.Parse("""{"path":"a.txt"}""").RootElement.Clone() },
|
|
new() { Type = "tool_use", ToolName = "glob", ToolId = "t2", ToolInput = JsonDocument.Parse("""{"pattern":"*.cs"}""").RootElement.Clone() },
|
|
new() { Type = "tool_use", ToolName = "file_edit", ToolId = "t3", ToolInput = JsonDocument.Parse("""{"path":"a.txt","old":"a","new":"b"}""").RootElement.Clone() },
|
|
new() { Type = "tool_use", ToolName = "file_read", ToolId = "t4", ToolInput = JsonDocument.Parse("""{"path":"b.txt"}""").RootElement.Clone() }
|
|
};
|
|
|
|
var plan = InvokePrivateStatic<(bool ShouldRun, List<ContentBlock> ParallelBatch, List<ContentBlock> SequentialBatch)>(
|
|
"CreateParallelExecutionPlan",
|
|
true,
|
|
calls,
|
|
0);
|
|
|
|
plan.ShouldRun.Should().BeTrue();
|
|
plan.ParallelBatch.Select(x => x.ToolId).Should().Equal("t1", "t2");
|
|
plan.SequentialBatch.Select(x => x.ToolId).Should().Equal("t3", "t4");
|
|
}
|
|
|
|
[Fact]
|
|
public void CreateParallelExecutionPlan_RecognizesAliasReadOnlyToolInPrefix()
|
|
{
|
|
var calls = new List<ContentBlock>
|
|
{
|
|
new() { Type = "tool_use", ToolName = "Read", ToolId = "t1", ToolInput = JsonDocument.Parse("""{"path":"a.txt"}""").RootElement.Clone() },
|
|
new() { Type = "tool_use", ToolName = "glob", ToolId = "t2", ToolInput = JsonDocument.Parse("""{"pattern":"*.cs"}""").RootElement.Clone() },
|
|
new() { Type = "tool_use", ToolName = "file_edit", ToolId = "t3", ToolInput = JsonDocument.Parse("""{"path":"a.txt","old":"a","new":"b"}""").RootElement.Clone() }
|
|
};
|
|
|
|
var plan = InvokePrivateStatic<(bool ShouldRun, List<ContentBlock> ParallelBatch, List<ContentBlock> SequentialBatch)>(
|
|
"CreateParallelExecutionPlan",
|
|
true,
|
|
calls,
|
|
0);
|
|
|
|
plan.ShouldRun.Should().BeTrue();
|
|
plan.ParallelBatch.Select(x => x.ToolId).Should().Equal("t1", "t2");
|
|
plan.SequentialBatch.Select(x => x.ToolId).Should().Equal("t3");
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeFailureTransitionState_IncrementsRepeatAndConsecutiveOnSameSignature()
|
|
{
|
|
var state = InvokePrivateStatic<(string? LastFailedToolSignature, int RepeatedFailedToolSignatureCount, int ConsecutiveErrors, bool CanRetry)>(
|
|
"ComputeFailureTransitionState",
|
|
"sig-a",
|
|
"sig-a",
|
|
2,
|
|
1,
|
|
3);
|
|
|
|
state.LastFailedToolSignature.Should().Be("sig-a");
|
|
state.RepeatedFailedToolSignatureCount.Should().Be(3);
|
|
state.ConsecutiveErrors.Should().Be(2);
|
|
state.CanRetry.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeFailureTransitionState_ResetsRepeatOnDifferentSignatureAndDetectsRetryLimit()
|
|
{
|
|
var state = InvokePrivateStatic<(string? LastFailedToolSignature, int RepeatedFailedToolSignatureCount, int ConsecutiveErrors, bool CanRetry)>(
|
|
"ComputeFailureTransitionState",
|
|
"sig-new",
|
|
"sig-old",
|
|
4,
|
|
2,
|
|
2);
|
|
|
|
state.LastFailedToolSignature.Should().Be("sig-new");
|
|
state.RepeatedFailedToolSignatureCount.Should().Be(1);
|
|
state.ConsecutiveErrors.Should().Be(3);
|
|
state.CanRetry.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void IsNonRetriableToolFailure_DetectsPermissionAndSchemaFailures()
|
|
{
|
|
var permissionDenied = ToolResult.Fail("Permission denied while writing file");
|
|
var invalidArgs = ToolResult.Fail("schema validation failed: missing path");
|
|
var transient = ToolResult.Fail("timeout while executing command");
|
|
|
|
InvokePrivateStatic<bool>("IsNonRetriableToolFailure", permissionDenied).Should().BeTrue();
|
|
InvokePrivateStatic<bool>("IsNonRetriableToolFailure", invalidArgs).Should().BeTrue();
|
|
InvokePrivateStatic<bool>("IsNonRetriableToolFailure", transient).Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void IsNonRetriableToolFailure_DetectsUnknownToolFailures()
|
|
{
|
|
var unknown = ToolResult.Fail("알 수 없는 도구: x_tool");
|
|
InvokePrivateStatic<bool>("IsNonRetriableToolFailure", unknown).Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldBlockNoProgressReadOnlyLoop_BlocksAfterThresholdForReadOnlyTools()
|
|
{
|
|
var blocked = InvokePrivateStatic<bool>(
|
|
"ShouldBlockNoProgressReadOnlyLoop",
|
|
"file_read",
|
|
4);
|
|
var notYet = InvokePrivateStatic<bool>(
|
|
"ShouldBlockNoProgressReadOnlyLoop",
|
|
"file_read",
|
|
3);
|
|
|
|
blocked.Should().BeTrue();
|
|
notYet.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldBlockNoProgressReadOnlyLoop_DoesNotBlockWriteTools()
|
|
{
|
|
var blocked = InvokePrivateStatic<bool>(
|
|
"ShouldBlockNoProgressReadOnlyLoop",
|
|
"file_edit",
|
|
8);
|
|
|
|
blocked.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldRequestDocumentArtifact_RequiresDocsTaskAndMissingArtifact()
|
|
{
|
|
var tempFile = Path.GetTempFileName();
|
|
var shouldRequest = InvokePrivateStatic<bool>(
|
|
"ShouldRequestDocumentArtifact",
|
|
"docs",
|
|
null,
|
|
0);
|
|
var noRequestWithArtifact = InvokePrivateStatic<bool>(
|
|
"ShouldRequestDocumentArtifact",
|
|
"docs",
|
|
tempFile,
|
|
0);
|
|
var noRequestAfterRetries = InvokePrivateStatic<bool>(
|
|
"ShouldRequestDocumentArtifact",
|
|
"docs",
|
|
null,
|
|
2);
|
|
var noRequestForCode = InvokePrivateStatic<bool>(
|
|
"ShouldRequestDocumentArtifact",
|
|
"bugfix",
|
|
null,
|
|
0);
|
|
|
|
shouldRequest.Should().BeTrue();
|
|
noRequestWithArtifact.Should().BeFalse();
|
|
noRequestAfterRetries.Should().BeFalse();
|
|
noRequestForCode.Should().BeFalse();
|
|
File.Delete(tempFile);
|
|
}
|
|
|
|
[Fact]
|
|
public void HasMaterializedArtifact_ReturnsTrueOnlyForExistingFiles()
|
|
{
|
|
var tempFile = Path.GetTempFileName();
|
|
try
|
|
{
|
|
InvokePrivateStatic<bool>("HasMaterializedArtifact", tempFile).Should().BeTrue();
|
|
InvokePrivateStatic<bool>("HasMaterializedArtifact", tempFile + ".missing").Should().BeFalse();
|
|
InvokePrivateStatic<bool>("HasMaterializedArtifact", (object?)null).Should().BeFalse();
|
|
}
|
|
finally
|
|
{
|
|
if (File.Exists(tempFile))
|
|
File.Delete(tempFile);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void HasDocumentVerificationEvidenceAfterLastArtifact_ReturnsFalseWithoutReadAfterCreate()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("html_create"),
|
|
CreateToolResult("grep_tool")
|
|
};
|
|
|
|
var hasVerification = InvokePrivateStatic<bool>(
|
|
"HasDocumentVerificationEvidenceAfterLastArtifact",
|
|
messages);
|
|
|
|
hasVerification.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasDocumentVerificationEvidenceAfterLastArtifact_ReturnsTrueWithReadAfterCreate()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("html_create"),
|
|
CreateToolResult("file_read")
|
|
};
|
|
|
|
var hasVerification = InvokePrivateStatic<bool>(
|
|
"HasDocumentVerificationEvidenceAfterLastArtifact",
|
|
messages);
|
|
|
|
hasVerification.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasSufficientFinalReportEvidence_DocsRequiresVerificationEvidenceAfterArtifact()
|
|
{
|
|
var response = "Changed report file and verified structure with file_read. File: report.html";
|
|
var withoutVerification = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("html_create")
|
|
};
|
|
var withVerification = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("html_create"),
|
|
CreateToolResult("file_read")
|
|
};
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
response,
|
|
"docs",
|
|
false,
|
|
withoutVerification).Should().BeTrue();
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
response,
|
|
"docs",
|
|
false,
|
|
withVerification).Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasExplicitDocumentVerificationToolMention_DetectsReadVerificationKeywords()
|
|
{
|
|
var withTool = InvokePrivateStatic<bool>(
|
|
"HasExplicitDocumentVerificationToolMention",
|
|
"검증은 file_read로 수행했습니다.");
|
|
var withKorean = InvokePrivateStatic<bool>(
|
|
"HasExplicitDocumentVerificationToolMention",
|
|
"문서 내용을 다시 확인 후 문제 없음으로 판단했습니다.");
|
|
var withoutHint = InvokePrivateStatic<bool>(
|
|
"HasExplicitDocumentVerificationToolMention",
|
|
"문서 생성 완료. report.html 저장.");
|
|
|
|
withTool.Should().BeTrue();
|
|
withKorean.Should().BeTrue();
|
|
withoutHint.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasExplicitDiffToolMention_DetectsDiffSignals()
|
|
{
|
|
var withGitTool = InvokePrivateStatic<bool>(
|
|
"HasExplicitDiffToolMention",
|
|
"검증은 git_tool과 git diff 기준으로 확인했습니다.");
|
|
var withGenericDiff = InvokePrivateStatic<bool>(
|
|
"HasExplicitDiffToolMention",
|
|
"Changed files diff를 검토했습니다.");
|
|
var withoutDiff = InvokePrivateStatic<bool>(
|
|
"HasExplicitDiffToolMention",
|
|
"파일 확인 및 테스트 검증 완료.");
|
|
|
|
withGitTool.Should().BeTrue();
|
|
withGenericDiff.Should().BeTrue();
|
|
withoutDiff.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasExplicitVerificationToolMention_DetectsCommonExecutionCommands()
|
|
{
|
|
var withTool = InvokePrivateStatic<bool>(
|
|
"HasExplicitVerificationToolMention",
|
|
"Verified with build_run and test_loop. dotnet test passed.");
|
|
var withCommand = InvokePrivateStatic<bool>(
|
|
"HasExplicitVerificationToolMention",
|
|
"Validation: dotnet build succeeded.");
|
|
var withoutTool = InvokePrivateStatic<bool>(
|
|
"HasExplicitVerificationToolMention",
|
|
"Verified behavior and checked logs.");
|
|
|
|
withTool.Should().BeTrue();
|
|
withCommand.Should().BeTrue();
|
|
withoutTool.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasExplicitFileLikeMention_DetectsCommonSourcePaths()
|
|
{
|
|
var withFile = InvokePrivateStatic<bool>(
|
|
"HasExplicitFileLikeMention",
|
|
"Changed src/AxCopilot/Services/AgentLoopService.cs and verified build.");
|
|
var withWindowsPath = InvokePrivateStatic<bool>(
|
|
"HasExplicitFileLikeMention",
|
|
@"Updated E:\AX Copilot - Codex\src\AxCopilot\Views\ChatWindow.xaml.");
|
|
var withoutFile = InvokePrivateStatic<bool>(
|
|
"HasExplicitFileLikeMention",
|
|
"Changed the flow and verified tests.");
|
|
|
|
withFile.Should().BeTrue();
|
|
withWindowsPath.Should().BeTrue();
|
|
withoutFile.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void UpdateConsecutiveReadOnlySuccessTools_IncrementsForReadOnlyAndResetsForWrite()
|
|
{
|
|
var next = InvokePrivateStatic<int>(
|
|
"UpdateConsecutiveReadOnlySuccessTools",
|
|
2,
|
|
"file_read",
|
|
true);
|
|
var resetByWrite = InvokePrivateStatic<int>(
|
|
"UpdateConsecutiveReadOnlySuccessTools",
|
|
5,
|
|
"file_edit",
|
|
true);
|
|
|
|
next.Should().Be(3);
|
|
resetByWrite.Should().Be(0);
|
|
}
|
|
|
|
[Fact]
|
|
public void UpdateConsecutiveReadOnlySuccessTools_ResetsOnFailure()
|
|
{
|
|
var reset = InvokePrivateStatic<int>(
|
|
"UpdateConsecutiveReadOnlySuccessTools",
|
|
4,
|
|
"file_read",
|
|
false);
|
|
|
|
reset.Should().Be(0);
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldTriggerNoProgressExecutionRecovery_MatchesThresholdAndRetryLimit()
|
|
{
|
|
var shouldTrigger = InvokePrivateStatic<bool>(
|
|
"ShouldTriggerNoProgressExecutionRecovery",
|
|
8,
|
|
0);
|
|
var belowThreshold = InvokePrivateStatic<bool>(
|
|
"ShouldTriggerNoProgressExecutionRecovery",
|
|
7,
|
|
0);
|
|
var overRetry = InvokePrivateStatic<bool>(
|
|
"ShouldTriggerNoProgressExecutionRecovery",
|
|
10,
|
|
2);
|
|
|
|
shouldTrigger.Should().BeTrue();
|
|
belowThreshold.Should().BeFalse();
|
|
overRetry.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldAbortNoProgressExecution_RequiresHighCountAfterRecoveryRetries()
|
|
{
|
|
var shouldAbort = InvokePrivateStatic<bool>(
|
|
"ShouldAbortNoProgressExecution",
|
|
12,
|
|
2);
|
|
var notEnoughCount = InvokePrivateStatic<bool>(
|
|
"ShouldAbortNoProgressExecution",
|
|
11,
|
|
3);
|
|
var notEnoughRetry = InvokePrivateStatic<bool>(
|
|
"ShouldAbortNoProgressExecution",
|
|
20,
|
|
1);
|
|
|
|
shouldAbort.Should().BeTrue();
|
|
notEnoughCount.Should().BeFalse();
|
|
notEnoughRetry.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void GetReadOnlySignatureLoopThreshold_UsesEnvAndClamp()
|
|
{
|
|
var original = Environment.GetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD", "1");
|
|
InvokePrivateStatic<int>("GetReadOnlySignatureLoopThreshold").Should().Be(2);
|
|
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD", "9");
|
|
InvokePrivateStatic<int>("GetReadOnlySignatureLoopThreshold").Should().Be(9);
|
|
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD", "999");
|
|
InvokePrivateStatic<int>("GetReadOnlySignatureLoopThreshold").Should().Be(12);
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_READONLY_SIGNATURE_LOOP_THRESHOLD", original);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void GetNoProgressRecoveryMaxRetries_UsesDefaultWhenEnvMissing()
|
|
{
|
|
var original = Environment.GetEnvironmentVariable("AXCOPILOT_NOPROGRESS_RECOVERY_MAX_RETRIES");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_NOPROGRESS_RECOVERY_MAX_RETRIES", null);
|
|
InvokePrivateStatic<int>("GetNoProgressRecoveryMaxRetries").Should().Be(2);
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_NOPROGRESS_RECOVERY_MAX_RETRIES", original);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveThresholdValue_ReturnsDefaultWhenInputInvalid()
|
|
{
|
|
InvokePrivateStatic<int>(
|
|
"ResolveThresholdValue",
|
|
"abc",
|
|
7,
|
|
2,
|
|
10).Should().Be(7);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveThresholdValue_AppliesClamp()
|
|
{
|
|
InvokePrivateStatic<int>(
|
|
"ResolveThresholdValue",
|
|
"1",
|
|
7,
|
|
2,
|
|
10).Should().Be(2);
|
|
InvokePrivateStatic<int>(
|
|
"ResolveThresholdValue",
|
|
"99",
|
|
7,
|
|
2,
|
|
10).Should().Be(10);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveConfiguredOrEnvThresholdValue_PrefersConfiguredValue()
|
|
{
|
|
var resolved = InvokePrivateStatic<int>(
|
|
"ResolveConfiguredOrEnvThresholdValue",
|
|
9,
|
|
"4",
|
|
7,
|
|
2,
|
|
10);
|
|
|
|
resolved.Should().Be(9);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveConfiguredOrEnvThresholdValue_UsesEnvThenDefault()
|
|
{
|
|
var fromEnv = InvokePrivateStatic<int>(
|
|
"ResolveConfiguredOrEnvThresholdValue",
|
|
null,
|
|
"15",
|
|
7,
|
|
2,
|
|
10);
|
|
var fromDefault = InvokePrivateStatic<int>(
|
|
"ResolveConfiguredOrEnvThresholdValue",
|
|
null,
|
|
"invalid",
|
|
7,
|
|
2,
|
|
10);
|
|
|
|
fromEnv.Should().Be(10);
|
|
fromDefault.Should().Be(7);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveToolExecutionTimeoutMs_PrioritizesConfiguredValue()
|
|
{
|
|
var timeout = InvokePrivateStatic<int>(
|
|
"ResolveToolExecutionTimeoutMs",
|
|
45000,
|
|
"120000");
|
|
|
|
timeout.Should().Be(45000);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveToolExecutionTimeoutMs_UsesEnvWhenConfiguredIsZero()
|
|
{
|
|
var timeout = InvokePrivateStatic<int>(
|
|
"ResolveToolExecutionTimeoutMs",
|
|
0,
|
|
"120000");
|
|
|
|
timeout.Should().Be(120000);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveToolExecutionTimeoutMs_UsesDefaultWhenBothMissing()
|
|
{
|
|
var timeout = InvokePrivateStatic<int>(
|
|
"ResolveToolExecutionTimeoutMs",
|
|
0,
|
|
"not-a-number");
|
|
|
|
timeout.Should().Be(90000);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveNoToolCallResponseThreshold_UsesDefaultAndClamps()
|
|
{
|
|
AgentLoopRuntimeThresholds.ResolveNoToolCallResponseThreshold(null).Should().Be(2);
|
|
AgentLoopRuntimeThresholds.ResolveNoToolCallResponseThreshold("0").Should().Be(1);
|
|
AgentLoopRuntimeThresholds.ResolveNoToolCallResponseThreshold("99").Should().Be(6);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveNoToolCallRecoveryMaxRetries_UsesDefaultAndClamps()
|
|
{
|
|
AgentLoopRuntimeThresholds.ResolveNoToolCallRecoveryMaxRetries(null).Should().Be(3);
|
|
AgentLoopRuntimeThresholds.ResolveNoToolCallRecoveryMaxRetries("-1").Should().Be(0);
|
|
AgentLoopRuntimeThresholds.ResolveNoToolCallRecoveryMaxRetries("99").Should().Be(6);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolvePlanExecutionRetryMax_UsesDefaultAndClamps()
|
|
{
|
|
AgentLoopRuntimeThresholds.ResolvePlanExecutionRetryMax(null).Should().Be(2);
|
|
AgentLoopRuntimeThresholds.ResolvePlanExecutionRetryMax("-5").Should().Be(0);
|
|
AgentLoopRuntimeThresholds.ResolvePlanExecutionRetryMax("10").Should().Be(6);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveTerminalEvidenceGateMaxRetries_UsesDefaultAndClamps()
|
|
{
|
|
AgentLoopRuntimeThresholds.ResolveTerminalEvidenceGateMaxRetries(null).Should().Be(1);
|
|
AgentLoopRuntimeThresholds.ResolveTerminalEvidenceGateMaxRetries("-2").Should().Be(0);
|
|
AgentLoopRuntimeThresholds.ResolveTerminalEvidenceGateMaxRetries("9").Should().Be(3);
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldSkipTerminalEvidenceGateForAnalysisQuery_MatchesAnalysisKeywords()
|
|
{
|
|
var policy = TaskTypePolicy.FromTaskType("feature");
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"ShouldSkipTerminalEvidenceGateForAnalysisQuery",
|
|
"코드 분석 결과 설명해줘",
|
|
policy).Should().BeTrue();
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"ShouldSkipTerminalEvidenceGateForAnalysisQuery",
|
|
"OrderService 파일을 수정해줘",
|
|
policy).Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasAnySuccessfulProgressToolResult_DetectsMutatingOrExecutionSuccess()
|
|
{
|
|
var readOnlyMessages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_read", "ok")
|
|
};
|
|
var progressMessages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit", "updated"),
|
|
CreateToolResult("build_run", "Build succeeded")
|
|
};
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasAnySuccessfulProgressToolResult",
|
|
readOnlyMessages).Should().BeFalse();
|
|
InvokePrivateStatic<bool>(
|
|
"HasAnySuccessfulProgressToolResult",
|
|
progressMessages).Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void UpdateConsecutiveNonMutatingSuccessTools_ResetsOnMutatingTools()
|
|
{
|
|
var incremented = InvokePrivateStatic<int>(
|
|
"UpdateConsecutiveNonMutatingSuccessTools",
|
|
3,
|
|
"file_read",
|
|
true);
|
|
var resetByMutation = InvokePrivateStatic<int>(
|
|
"UpdateConsecutiveNonMutatingSuccessTools",
|
|
6,
|
|
"file_edit",
|
|
true);
|
|
|
|
incremented.Should().Be(4);
|
|
resetByMutation.Should().Be(0);
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildIterationLimitFallbackResponse_IncludesArtifactAndMetrics()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("build_run"),
|
|
new() { Role = "assistant", Content = "updated src/AxCopilot/Services/AgentLoopService.cs" }
|
|
};
|
|
var policy = TaskTypePolicy.FromTaskType("docs");
|
|
|
|
var response = InvokePrivateStatic<string>(
|
|
"BuildIterationLimitFallbackResponse",
|
|
25,
|
|
policy,
|
|
12,
|
|
8,
|
|
4,
|
|
@"E:\AX Copilot - Codex\out\report.html",
|
|
new List<string> { "file_read", "html_create" },
|
|
new Dictionary<string, int> { ["file_read"] = 2, ["html_create"] = 1 },
|
|
messages);
|
|
|
|
response.Should().Contain("최대 반복 횟수");
|
|
response.Should().Contain("도구 호출: 총 12회");
|
|
response.Should().Contain("E:\\AX Copilot - Codex\\out\\report.html");
|
|
response.Should().Contain("html_create");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildVerificationReport_UsesStructuredFormatAndPassMetadata()
|
|
{
|
|
var report = InvokePrivateStatic<string>(
|
|
"BuildVerificationReport",
|
|
"draft.md",
|
|
@"E:\AX Copilot - Codex\out\draft.md",
|
|
false,
|
|
"검증 통과\n핵심 근거: markdown 구조 정상",
|
|
false,
|
|
2);
|
|
|
|
report.Should().Contain("[VerificationReport]");
|
|
report.Should().Contain("- Status: PASS");
|
|
report.Should().Contain("- Type: document");
|
|
report.Should().Contain(@"- Target: E:\AX Copilot - Codex\out\draft.md");
|
|
report.Should().Contain("- ReadTools: 2");
|
|
report.Should().Contain("- Highlights:");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildVerificationReport_UsesFallbackTargetAndFailMetadata()
|
|
{
|
|
var report = InvokePrivateStatic<string>(
|
|
"BuildVerificationReport",
|
|
"OrderService.cs",
|
|
null,
|
|
true,
|
|
"오류 발견: null 검사 누락",
|
|
true,
|
|
4);
|
|
|
|
report.Should().Contain("- Status: FAIL");
|
|
report.Should().Contain("- Type: code");
|
|
report.Should().Contain("- Target: OrderService.cs");
|
|
report.Should().Contain("- ReadTools: 4");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildTopFailureSummary_OrdersByCountAndLimitsToTopThree()
|
|
{
|
|
var histogram = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
["build_run"] = 4,
|
|
["file_edit"] = 7,
|
|
["grep_tool"] = 3,
|
|
["test_loop"] = 7,
|
|
["file_read"] = 1,
|
|
};
|
|
|
|
var summary = InvokePrivateStatic<string>(
|
|
"BuildTopFailureSummary",
|
|
histogram);
|
|
|
|
summary.Should().Be("file_edit(7), test_loop(7), build_run(4)");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildNoProgressAbortResponse_IncludesRetrySummaryAndNextAction()
|
|
{
|
|
var policy = TaskTypePolicy.FromTaskType("bugfix");
|
|
var response = InvokePrivateStatic<string>(
|
|
"BuildNoProgressAbortResponse",
|
|
policy,
|
|
14,
|
|
2,
|
|
@"E:\AX Copilot - Codex\src\AxCopilot\Services\OrderService.cs",
|
|
new List<string> { "file_read", "grep_tool", "build_run" });
|
|
|
|
response.Should().Contain("비진행 상태");
|
|
response.Should().Contain("연속 비진행 호출: 14");
|
|
response.Should().Contain("복구 시도 횟수: 2");
|
|
response.Should().Contain("OrderService.cs");
|
|
}
|
|
|
|
[Fact]
|
|
public void HasDiffEvidenceAfterLastModification_ReturnsFalseWithoutGitDiffAfterEdit()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("file_read")
|
|
};
|
|
|
|
var hasDiff = InvokePrivateStatic<bool>(
|
|
"HasDiffEvidenceAfterLastModification",
|
|
messages);
|
|
|
|
hasDiff.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasDiffEvidenceAfterLastModification_ReturnsTrueWithGitDiffAfterEdit()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("git_tool", "diff --git a/src/A.cs b/src/A.cs\n@@ -1,2 +1,2 @@")
|
|
};
|
|
|
|
var hasDiff = InvokePrivateStatic<bool>(
|
|
"HasDiffEvidenceAfterLastModification",
|
|
messages);
|
|
|
|
hasDiff.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasDiffEvidenceAfterLastModification_ReturnsFalseForGitStatusOnly()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("git_tool", "On branch main\nnothing to commit, working tree clean")
|
|
};
|
|
|
|
var hasDiff = InvokePrivateStatic<bool>(
|
|
"HasDiffEvidenceAfterLastModification",
|
|
messages);
|
|
|
|
hasDiff.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasBuildOrTestEvidenceAfterLastModification_ReturnsFalseWhenExecutionIsBeforeEdit()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("build_run"),
|
|
CreateToolResult("file_edit")
|
|
};
|
|
|
|
var hasRecentExecution = InvokePrivateStatic<bool>(
|
|
"HasBuildOrTestEvidenceAfterLastModification",
|
|
messages);
|
|
|
|
hasRecentExecution.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasBuildOrTestEvidenceAfterLastModification_ReturnsTrueWhenExecutionIsAfterEdit()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("test_loop")
|
|
};
|
|
|
|
var hasRecentExecution = InvokePrivateStatic<bool>(
|
|
"HasBuildOrTestEvidenceAfterLastModification",
|
|
messages);
|
|
|
|
hasRecentExecution.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasBuildOrTestEvidenceAfterLastModification_IgnoresFailedExecution()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("build_run", "Build failed with error CS1002")
|
|
};
|
|
|
|
var hasRecentExecution = InvokePrivateStatic<bool>(
|
|
"HasBuildOrTestEvidenceAfterLastModification",
|
|
messages);
|
|
|
|
hasRecentExecution.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void IsSuccessfulBuildOrTestResult_DetectsSuccessAndFailureSignals()
|
|
{
|
|
var success = InvokePrivateStatic<bool>(
|
|
"IsSuccessfulBuildOrTestResult",
|
|
"Build succeeded. Tests passed.");
|
|
var failure = InvokePrivateStatic<bool>(
|
|
"IsSuccessfulBuildOrTestResult",
|
|
"빌드 실패: 오류가 발생했습니다.");
|
|
|
|
success.Should().BeTrue();
|
|
failure.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasAnyBuildOrTestAttempt_DetectsAttemptsRegardlessOfOutcome()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("build_run", "Build failed with error")
|
|
};
|
|
|
|
var hasAttempt = InvokePrivateStatic<bool>(
|
|
"HasAnyBuildOrTestAttempt",
|
|
messages);
|
|
|
|
hasAttempt.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasSuccessfulBuildAndTestAfterLastModification_RequiresBothSuccessfulSignals()
|
|
{
|
|
var onlyBuild = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("build_run", "Build succeeded")
|
|
};
|
|
var buildAndTest = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("build_run", "Build succeeded"),
|
|
CreateToolResult("test_loop", "Tests passed")
|
|
};
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSuccessfulBuildAndTestAfterLastModification",
|
|
onlyBuild).Should().BeFalse();
|
|
InvokePrivateStatic<bool>(
|
|
"HasSuccessfulBuildAndTestAfterLastModification",
|
|
buildAndTest).Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void HasSufficientFinalReportEvidence_HighImpactRequiresBothBuildAndTestSignals()
|
|
{
|
|
var response = "Changed src/AxCopilot/Services/OrderService.cs. build_run success with output logs. " +
|
|
"Feature flow path validated and caller impact scope reviewed. Remaining risk documented.";
|
|
var onlyBuild = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("build_run", "Build succeeded")
|
|
};
|
|
var buildAndTest = new List<ChatMessage>
|
|
{
|
|
CreateToolResult("file_edit"),
|
|
CreateToolResult("build_run", "Build succeeded"),
|
|
CreateToolResult("test_loop", "Tests passed")
|
|
};
|
|
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
response,
|
|
"feature",
|
|
true,
|
|
onlyBuild).Should().BeFalse();
|
|
InvokePrivateStatic<bool>(
|
|
"HasSufficientFinalReportEvidence",
|
|
response + " test_loop success.",
|
|
"feature",
|
|
true,
|
|
buildAndTest).Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildRecentExecutionEvidencePrompt_IncludesExecutionGuidance()
|
|
{
|
|
var policy = TaskTypePolicy.FromTaskType("feature");
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildRecentExecutionEvidencePrompt",
|
|
policy);
|
|
|
|
prompt.Should().Contain("RecentExecutionGate");
|
|
prompt.Should().Contain("build_run");
|
|
prompt.Should().Contain("test_loop");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildExecutionSuccessGatePrompt_IncludesSuccessGuidance()
|
|
{
|
|
var policy = TaskTypePolicy.FromTaskType("bugfix");
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildExecutionSuccessGatePrompt",
|
|
policy);
|
|
|
|
prompt.Should().Contain("ExecutionSuccessGate");
|
|
prompt.Should().Contain("build_run");
|
|
prompt.Should().Contain("test_loop");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildUnknownToolRecoveryPrompt_ContainsFallbackToolList()
|
|
{
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildUnknownToolRecoveryPrompt",
|
|
"write_file",
|
|
new List<string> { "file_write", "file_read", "glob", "grep_tool" });
|
|
|
|
prompt.Should().Contain("UnknownToolRecovery");
|
|
prompt.Should().Contain("write_file");
|
|
prompt.Should().Contain("file_write");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildUnknownToolRecoveryPrompt_IncludesAliasHintWhenAvailable()
|
|
{
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildUnknownToolRecoveryPrompt",
|
|
"Read",
|
|
new List<string> { "file_read", "file_write", "glob" });
|
|
|
|
prompt.Should().Contain("자동 매핑 후보");
|
|
prompt.Should().Contain("Read");
|
|
prompt.Should().Contain("file_read");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildUnknownToolLoopAbortResponse_ContainsAbortSummary()
|
|
{
|
|
var response = InvokePrivateStatic<string>(
|
|
"BuildUnknownToolLoopAbortResponse",
|
|
"bad_tool",
|
|
3,
|
|
new List<string> { "file_read", "file_write", "glob" });
|
|
|
|
response.Should().Contain("중단");
|
|
response.Should().Contain("bad_tool");
|
|
response.Should().Contain("반복 횟수: 3");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildDisallowedToolRecoveryPrompt_ContainsPolicyAndActiveToolHints()
|
|
{
|
|
var prompt = InvokePrivateStatic<string>(
|
|
"BuildDisallowedToolRecoveryPrompt",
|
|
"write_file",
|
|
new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "file_read", "file_write" },
|
|
new List<string> { "file_read", "file_write", "glob" });
|
|
|
|
prompt.Should().Contain("DisallowedToolRecovery");
|
|
prompt.Should().Contain("write_file");
|
|
prompt.Should().Contain("file_read");
|
|
prompt.Should().Contain("file_write");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildDisallowedToolLoopAbortResponse_ContainsAbortSummary()
|
|
{
|
|
var response = InvokePrivateStatic<string>(
|
|
"BuildDisallowedToolLoopAbortResponse",
|
|
"delete_all",
|
|
3,
|
|
new List<string> { "file_read", "file_write", "glob" });
|
|
|
|
response.Should().Contain("중단");
|
|
response.Should().Contain("delete_all");
|
|
response.Should().Contain("반복 횟수: 3");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildUnknownToolLoopAbortResponse_ShouldGuideToolSearch()
|
|
{
|
|
var response = InvokePrivateStatic<string>(
|
|
"BuildUnknownToolLoopAbortResponse",
|
|
"unknown_exec",
|
|
4,
|
|
new List<string> { "tool_search", "file_read", "process" });
|
|
|
|
response.Should().Contain("tool_search");
|
|
response.Should().Contain("unknown_exec");
|
|
}
|
|
|
|
[Fact]
|
|
public void BuildDisallowedToolLoopAbortResponse_ShouldGuideToolSearch()
|
|
{
|
|
var response = InvokePrivateStatic<string>(
|
|
"BuildDisallowedToolLoopAbortResponse",
|
|
"dangerous_exec",
|
|
5,
|
|
new List<string> { "tool_search", "file_read", "file_edit" });
|
|
|
|
response.Should().Contain("tool_search");
|
|
response.Should().Contain("dangerous_exec");
|
|
}
|
|
|
|
[Fact]
|
|
public void MixedRecoverySignals_ShouldRemainConsistentAcrossUnknownDisallowedAndNoProgress()
|
|
{
|
|
var unknownPrompt = InvokePrivateStatic<string>(
|
|
"BuildUnknownToolRecoveryPrompt",
|
|
"bad_tool",
|
|
new List<string> { "tool_search", "file_read", "glob" });
|
|
|
|
var disallowedPrompt = InvokePrivateStatic<string>(
|
|
"BuildDisallowedToolRecoveryPrompt",
|
|
"unsafe_tool",
|
|
new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "tool_search", "file_read" },
|
|
new List<string> { "tool_search", "file_read" });
|
|
|
|
unknownPrompt.Should().Contain("tool_search");
|
|
disallowedPrompt.Should().Contain("tool_search");
|
|
unknownPrompt.Should().Contain("bad_tool");
|
|
disallowedPrompt.Should().Contain("unsafe_tool");
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveRequestedToolName_MapsCommonAliasWhenTargetIsActive()
|
|
{
|
|
var resolved = InvokePrivateStatic<string>(
|
|
"ResolveRequestedToolName",
|
|
"Read",
|
|
new List<string> { "file_read", "file_write", "process" });
|
|
|
|
resolved.Should().Be("file_read");
|
|
}
|
|
|
|
[Fact]
|
|
public void GetRuntimeHooksForCall_AppliesTimingAndToolFilter()
|
|
{
|
|
var hooks = new List<AgentHookEntry>
|
|
{
|
|
new() { Name = "lint-pre", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
|
|
new() { Name = "verify-post", ToolName = "*", Timing = "post", ScriptPath = "x.cmd", Enabled = true },
|
|
};
|
|
|
|
var runtimeOverrides = CreateRuntimeOverridesForHookFilter("lint-pre@pre@file_edit");
|
|
|
|
var filtered = InvokePrivateStatic<IReadOnlyList<AgentHookEntry>>(
|
|
"GetRuntimeHooksForCall",
|
|
hooks.AsReadOnly(),
|
|
runtimeOverrides,
|
|
"file_edit",
|
|
"pre");
|
|
|
|
filtered.Select(h => h.Name).Should().Equal("lint-pre");
|
|
}
|
|
|
|
[Fact]
|
|
public void GetRuntimeHooksForCall_ReturnsEmptyWhenFilterDoesNotMatch()
|
|
{
|
|
var hooks = new List<AgentHookEntry>
|
|
{
|
|
new() { Name = "lint-pre", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
|
|
};
|
|
var runtimeOverrides = CreateRuntimeOverridesForHookFilter("lint-pre@pre@file_write");
|
|
|
|
var filtered = InvokePrivateStatic<IReadOnlyList<AgentHookEntry>>(
|
|
"GetRuntimeHooksForCall",
|
|
hooks.AsReadOnly(),
|
|
runtimeOverrides,
|
|
"file_edit",
|
|
"pre");
|
|
|
|
filtered.Should().BeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void GetRuntimeHooksForCall_ExcludeRuleOverridesWildcardAllow()
|
|
{
|
|
var hooks = new List<AgentHookEntry>
|
|
{
|
|
new() { Name = "hook-a", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
|
|
new() { Name = "hook-b", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
|
|
};
|
|
|
|
var runtimeOverrides = CreateRuntimeOverridesForHookFilter("hook-a@*@*, !hook-b@pre@math_eval");
|
|
var filtered = InvokePrivateStatic<IReadOnlyList<AgentHookEntry>>(
|
|
"GetRuntimeHooksForCall",
|
|
hooks.AsReadOnly(),
|
|
runtimeOverrides,
|
|
"math_eval",
|
|
"pre");
|
|
|
|
filtered.Select(h => h.Name).Should().Equal("hook-a");
|
|
}
|
|
|
|
[Fact]
|
|
public void GetRuntimeHooksForCall_MoreSpecificRuleWins()
|
|
{
|
|
var hooks = new List<AgentHookEntry>
|
|
{
|
|
new() { Name = "hook-a", ToolName = "*", Timing = "pre", ScriptPath = "x.cmd", Enabled = true },
|
|
};
|
|
|
|
var runtimeOverrides = CreateRuntimeOverridesForHookFilter("hook-a@*@*, !hook-a@pre@math_eval");
|
|
var filtered = InvokePrivateStatic<IReadOnlyList<AgentHookEntry>>(
|
|
"GetRuntimeHooksForCall",
|
|
hooks.AsReadOnly(),
|
|
runtimeOverrides,
|
|
"math_eval",
|
|
"pre");
|
|
|
|
filtered.Should().BeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveRequestedToolName_UsesNormalizedNameMatchBeforeUnknown()
|
|
{
|
|
var resolved = InvokePrivateStatic<string>(
|
|
"ResolveRequestedToolName",
|
|
"grep-tool",
|
|
new List<string> { "file_read", "grep_tool", "glob" });
|
|
|
|
resolved.Should().Be("grep_tool");
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveRequestedToolName_ReturnsOriginalWhenAliasTargetIsUnavailable()
|
|
{
|
|
var resolved = InvokePrivateStatic<string>(
|
|
"ResolveRequestedToolName",
|
|
"Write",
|
|
new List<string> { "file_read", "glob" });
|
|
|
|
resolved.Should().Be("Write");
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveRequestedToolName_MapsRgAliasToGrep()
|
|
{
|
|
var resolved = InvokePrivateStatic<string>(
|
|
"ResolveRequestedToolName",
|
|
"rg",
|
|
new List<string> { "file_read", "grep", "glob" });
|
|
|
|
resolved.Should().Be("grep");
|
|
}
|
|
|
|
[Fact]
|
|
public void ResolveRequestedToolName_MapsCodeSearchAliasToSearchCodebase()
|
|
{
|
|
var resolved = InvokePrivateStatic<string>(
|
|
"ResolveRequestedToolName",
|
|
"code_search",
|
|
new List<string> { "search_codebase", "file_read" });
|
|
|
|
resolved.Should().Be("search_codebase");
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldRunPostToolVerification_MatchesTabAndToolTypeRules()
|
|
{
|
|
var codeTrue = InvokePrivateStatic<bool>(
|
|
"ShouldRunPostToolVerification",
|
|
"Code",
|
|
"file_edit",
|
|
true,
|
|
true,
|
|
false);
|
|
var codeFalse = InvokePrivateStatic<bool>(
|
|
"ShouldRunPostToolVerification",
|
|
"Code",
|
|
"document_plan",
|
|
true,
|
|
true,
|
|
true);
|
|
var coworkTrue = InvokePrivateStatic<bool>(
|
|
"ShouldRunPostToolVerification",
|
|
"Cowork",
|
|
"html_create",
|
|
true,
|
|
false,
|
|
true);
|
|
var failedFalse = InvokePrivateStatic<bool>(
|
|
"ShouldRunPostToolVerification",
|
|
"Code",
|
|
"file_edit",
|
|
false,
|
|
true,
|
|
true);
|
|
|
|
codeTrue.Should().BeTrue();
|
|
codeFalse.Should().BeFalse();
|
|
coworkTrue.Should().BeTrue();
|
|
failedFalse.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void EvaluateDecisionHelpers_ReturnExpectedTransitionShape()
|
|
{
|
|
var devSkip = InvokePrivateStatic<(bool ShouldContinue, string? TerminalResponse, string? ToolResultMessage)>(
|
|
"EvaluateDevStepDecision",
|
|
"건너뛰기");
|
|
var devStop = InvokePrivateStatic<(bool ShouldContinue, string? TerminalResponse, string? ToolResultMessage)>(
|
|
"EvaluateDevStepDecision",
|
|
"중단");
|
|
var scopeSkip = InvokePrivateStatic<(bool ShouldContinue, string? TerminalResponse, string? ToolResultMessage)>(
|
|
"EvaluateScopeDecision",
|
|
"건너뛰기");
|
|
var scopeCancel = InvokePrivateStatic<(bool ShouldContinue, string? TerminalResponse, string? ToolResultMessage)>(
|
|
"EvaluateScopeDecision",
|
|
"취소");
|
|
|
|
devSkip.ShouldContinue.Should().BeTrue();
|
|
devSkip.ToolResultMessage.Should().Contain("SKIPPED");
|
|
devStop.TerminalResponse.Should().NotBeNull();
|
|
scopeSkip.ShouldContinue.Should().BeTrue();
|
|
scopeCancel.TerminalResponse.Should().NotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void IsTransientLlmError_DetectsRateLimitAndTimeoutPatterns()
|
|
{
|
|
var rateLimit = InvokePrivateStatic<bool>(
|
|
"IsTransientLlmError",
|
|
new Exception("429 Too Many Requests: rate limit exceeded"));
|
|
var timeout = InvokePrivateStatic<bool>(
|
|
"IsTransientLlmError",
|
|
new TimeoutException("request timed out"));
|
|
var fatal = InvokePrivateStatic<bool>(
|
|
"IsTransientLlmError",
|
|
new Exception("invalid api key"));
|
|
|
|
rateLimit.Should().BeTrue();
|
|
timeout.Should().BeTrue();
|
|
fatal.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeTransientLlmBackoffDelayMs_UsesRetryAfterWhenPresent()
|
|
{
|
|
var delay = InvokePrivateStatic<int>(
|
|
"ComputeTransientLlmBackoffDelayMs",
|
|
1,
|
|
new Exception("Rate limited. retry-after: 7"));
|
|
|
|
delay.Should().Be(7000);
|
|
}
|
|
|
|
[Fact]
|
|
public void ComputeTransientLlmBackoffDelayMs_GrowsWithRetryCountWithoutRetryAfter()
|
|
{
|
|
var delay1 = InvokePrivateStatic<int>(
|
|
"ComputeTransientLlmBackoffDelayMs",
|
|
1,
|
|
new Exception("service unavailable"));
|
|
var delay2 = InvokePrivateStatic<int>(
|
|
"ComputeTransientLlmBackoffDelayMs",
|
|
2,
|
|
new Exception("service unavailable"));
|
|
|
|
delay1.Should().BeGreaterThanOrEqualTo(800);
|
|
delay1.Should().BeLessThan(1050);
|
|
delay2.Should().BeGreaterThanOrEqualTo(1600);
|
|
delay2.Should().BeLessThan(1850);
|
|
}
|
|
|
|
[Fact]
|
|
public void IsContextOverflowError_DetectsMaxOutputTokenPatterns()
|
|
{
|
|
var maxOutput = InvokePrivateStatic<bool>(
|
|
"IsContextOverflowError",
|
|
"Request failed: maximum output tokens exceeded");
|
|
var truncated = InvokePrivateStatic<bool>(
|
|
"IsContextOverflowError",
|
|
"response was truncated due to token limit");
|
|
var unrelated = InvokePrivateStatic<bool>(
|
|
"IsContextOverflowError",
|
|
"invalid api key");
|
|
|
|
maxOutput.Should().BeTrue();
|
|
truncated.Should().BeTrue();
|
|
unrelated.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void IsLikelyWithheldOrOverflowResponse_DetectsWithheldAndLimitHints()
|
|
{
|
|
var withheld = InvokePrivateStatic<bool>(
|
|
"IsLikelyWithheldOrOverflowResponse",
|
|
"output withheld because prompt too long");
|
|
var maxOutput = InvokePrivateStatic<bool>(
|
|
"IsLikelyWithheldOrOverflowResponse",
|
|
"max_output_tokens reached");
|
|
var normal = InvokePrivateStatic<bool>(
|
|
"IsLikelyWithheldOrOverflowResponse",
|
|
"completed successfully");
|
|
|
|
withheld.Should().BeTrue();
|
|
maxOutput.Should().BeTrue();
|
|
normal.Should().BeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void GetToolExecutionTimeoutMs_UsesDefaultWhenEnvMissing()
|
|
{
|
|
var original = Environment.GetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", null);
|
|
|
|
var timeout = InvokePrivateStatic<int>("GetToolExecutionTimeoutMs");
|
|
timeout.Should().Be(90000);
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", original);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void GetToolExecutionTimeoutMs_ClampsOutOfRangeValues()
|
|
{
|
|
var original = Environment.GetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS");
|
|
try
|
|
{
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", "1000");
|
|
InvokePrivateStatic<int>("GetToolExecutionTimeoutMs").Should().Be(5000);
|
|
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", "9999999");
|
|
InvokePrivateStatic<int>("GetToolExecutionTimeoutMs").Should().Be(600000);
|
|
}
|
|
finally
|
|
{
|
|
Environment.SetEnvironmentVariable("AXCOPILOT_TOOL_TIMEOUT_MS", original);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void InjectTaskTypeGuidance_AddsBugfixGuidanceOnlyOnce()
|
|
{
|
|
var messages = new List<ChatMessage>
|
|
{
|
|
new() { Role = "user", Content = "fix this error" }
|
|
};
|
|
|
|
InvokePrivateStaticVoid("InjectTaskTypeGuidance", messages, "bugfix");
|
|
InvokePrivateStaticVoid("InjectTaskTypeGuidance", messages, "bugfix");
|
|
|
|
messages.Count(m => m.Role == "user" && m.Content.StartsWith("[System:TaskType]", StringComparison.Ordinal))
|
|
.Should().Be(1);
|
|
messages[0].Content.Should().Contain("bug-fix");
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldEnforceForkExecution_RequiresNonCompliantToolWithinAttemptLimit()
|
|
{
|
|
var shouldEnforce = InvokePrivateStatic<bool>(
|
|
"ShouldEnforceForkExecution",
|
|
true,
|
|
false,
|
|
"file_read",
|
|
0,
|
|
2);
|
|
|
|
shouldEnforce.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void ShouldEnforceForkExecution_DoesNotEnforceForSpawnAgentOrAfterLimit()
|
|
{
|
|
var compliantTool = InvokePrivateStatic<bool>(
|
|
"ShouldEnforceForkExecution",
|
|
true,
|
|
false,
|
|
"spawn_agent",
|
|
0,
|
|
2);
|
|
var overLimit = InvokePrivateStatic<bool>(
|
|
"ShouldEnforceForkExecution",
|
|
true,
|
|
false,
|
|
"file_edit",
|
|
2,
|
|
2);
|
|
var afterDelegation = InvokePrivateStatic<bool>(
|
|
"ShouldEnforceForkExecution",
|
|
true,
|
|
true,
|
|
"file_edit",
|
|
0,
|
|
2);
|
|
|
|
compliantTool.Should().BeFalse();
|
|
overLimit.Should().BeFalse();
|
|
afterDelegation.Should().BeFalse();
|
|
}
|
|
|
|
private static object CreateRuntimeOverridesForHookFilter(string filter)
|
|
{
|
|
var loopType = typeof(AgentLoopService);
|
|
var filterType = loopType.GetNestedType("HookFilterRule", BindingFlags.NonPublic);
|
|
var runtimeType = loopType.GetNestedType("SkillRuntimeOverrides", BindingFlags.NonPublic);
|
|
filterType.Should().NotBeNull();
|
|
runtimeType.Should().NotBeNull();
|
|
|
|
var filterCtor = filterType!.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
|
|
.First(c => c.GetParameters().Length == 6);
|
|
|
|
var filterListType = typeof(List<>).MakeGenericType(filterType);
|
|
var filterList = (System.Collections.IList)Activator.CreateInstance(filterListType)!;
|
|
var order = 0;
|
|
foreach (var raw in filter.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
|
|
{
|
|
var parts = raw.Trim().Split('@', StringSplitOptions.TrimEntries);
|
|
var hookNameRaw = parts.Length > 0 ? parts[0] : "*";
|
|
var isExclude = hookNameRaw.StartsWith("!");
|
|
var hookName = isExclude ? hookNameRaw[1..] : hookNameRaw;
|
|
if (string.IsNullOrWhiteSpace(hookName))
|
|
hookName = "*";
|
|
var timing = parts.Length > 1 ? parts[1] : "*";
|
|
var toolName = parts.Length > 2 ? parts[2] : "*";
|
|
var specificity = (hookName == "*" ? 0 : 1) + (timing == "*" ? 0 : 1) + (toolName == "*" ? 0 : 1);
|
|
var filterObj = filterCtor.Invoke([hookName, timing, toolName, isExclude, specificity, order++]);
|
|
filterList.Add(filterObj);
|
|
}
|
|
|
|
var runtimeCtor = runtimeType!.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
|
|
.First(c => c.GetParameters().Length == 8);
|
|
return runtimeCtor.Invoke(
|
|
[
|
|
null, // service
|
|
null, // model
|
|
null, // temperature
|
|
null, // effort
|
|
false, // requireForkExecution
|
|
new HashSet<string>(StringComparer.OrdinalIgnoreCase), // allowedTools
|
|
new HashSet<string>(StringComparer.OrdinalIgnoreCase), // hookNames
|
|
filterList // hookFilters
|
|
])!;
|
|
}
|
|
|
|
private static ChatMessage CreateToolResult(string toolName, string content = "ok")
|
|
=> new()
|
|
{
|
|
Role = "user",
|
|
Content = JsonSerializer.Serialize(new
|
|
{
|
|
type = "tool_result",
|
|
tool_name = toolName,
|
|
content
|
|
})
|
|
};
|
|
|
|
private static T InvokePrivateStatic<T>(string methodName, params object?[] arguments)
|
|
{
|
|
var method = ResolvePrivateStatic(methodName, arguments);
|
|
method.Should().NotBeNull($"{methodName} should exist on AgentLoopService");
|
|
|
|
var result = method!.Invoke(null, arguments);
|
|
result.Should().NotBeNull();
|
|
return (T)result!;
|
|
}
|
|
|
|
private static void InvokePrivateStaticVoid(string methodName, params object?[] arguments)
|
|
{
|
|
var method = ResolvePrivateStatic(methodName, arguments);
|
|
method.Should().NotBeNull($"{methodName} should exist on AgentLoopService");
|
|
method!.Invoke(null, arguments);
|
|
}
|
|
|
|
private static MethodInfo? ResolvePrivateStatic(string methodName, object?[] arguments)
|
|
{
|
|
return typeof(AgentLoopService)
|
|
.GetMethods(BindingFlags.NonPublic | BindingFlags.Static)
|
|
.Where(m => m.Name == methodName)
|
|
.FirstOrDefault(m =>
|
|
{
|
|
var parameters = m.GetParameters();
|
|
if (parameters.Length != arguments.Length)
|
|
return false;
|
|
|
|
for (var i = 0; i < parameters.Length; i++)
|
|
{
|
|
var arg = arguments[i];
|
|
if (arg == null)
|
|
continue;
|
|
|
|
var targetType = parameters[i].ParameterType;
|
|
if (!targetType.IsInstanceOfType(arg))
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
});
|
|
}
|
|
}
|