Skip to content

Commit d009196

Browse files
scbeddscbedd
authored
Proxy Integration Test Framework + Initial Functionality Tests + Session/Individual Test Matcher Support (#1117)
* we now support global and per-test override of matchers * add proxy integration test harness + initial set of tests * fix crossplat proxy startup
1 parent 8011417 commit d009196

File tree

8 files changed

+412
-14
lines changed

8 files changed

+412
-14
lines changed

‎core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/Azure.Mcp.Core.LiveTests.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22
<PropertyGroup>
33
<IsTestProject>true</IsTestProject>
44
<OutputType>Exe</OutputType>

‎core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/RecordingFramework/RecordedCommandTestHarness.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System.Collections.Generic;
5+
using System.IO;
6+
using System.Text;
7+
using Azure.Mcp.Tests.Client;
8+
using Azure.Mcp.Tests.Client.Attributes;
9+
using Azure.Mcp.Tests.Client.Helpers;
10+
using Azure.Mcp.Tests.Generated.Models;
11+
using Azure.Mcp.Tests.Helpers;
12+
using Xunit;
13+
14+
namespace Azure.Mcp.Core.LiveTests.RecordingFramework;
15+
16+
/// <summary>
17+
/// Harness for testing RecordedCommandTestsBase functionality. Intended for proper abstraction of livetest settings etc to allow both record and playback modes in the same test for full roundtrip testing.
18+
/// </summary>
19+
/// <param name="output"></param>
20+
/// <param name="fixture"></param>
21+
internal sealed class RecordedCommandTestHarness(ITestOutputHelper output, TestProxyFixture fixture) : RecordedCommandTestsBase(output, fixture)
22+
{
23+
public TestMode DesiredMode { get; set; } = TestMode.Record;
24+
25+
public IReadOnlyDictionary<string, string> Variables => TestVariables;
26+
27+
public string GetRecordingAbsolutePath(string displayName)
28+
{
29+
var sanitized = RecordingPathResolver.Sanitize(displayName);
30+
var relativeDirectory = PathResolver.GetSessionDirectory(GetType(), variantSuffix: null)
31+
.Replace('/', Path.DirectorySeparatorChar);
32+
var fileName = RecordingPathResolver.BuildFileName(sanitized, IsAsync, VersionQualifier);
33+
var absoluteDirectory = Path.Combine(PathResolver.RepositoryRoot, relativeDirectory);
34+
Directory.CreateDirectory(absoluteDirectory);
35+
return Path.Combine(absoluteDirectory, fileName);
36+
}
37+
38+
protected override ValueTask LoadSettingsAsync()
39+
{
40+
Settings = new LiveTestSettings
41+
{
42+
SubscriptionId = "00000000-0000-0000-0000-000000000000",
43+
TenantId = "00000000-0000-0000-0000-000000000000",
44+
ResourceBaseName = "Sanitized",
45+
SubscriptionName = "Sanitized",
46+
TenantName = "Sanitized",
47+
TestMode = TestMode.Playback
48+
};
49+
50+
Settings.TestMode = DesiredMode;
51+
TestMode = DesiredMode;
52+
53+
return ValueTask.CompletedTask;
54+
}
55+
56+
public void ResetVariables()
57+
{
58+
TestVariables.Clear();
59+
}
60+
61+
public string GetRecordingId()
62+
{
63+
return RecordingId;
64+
}
65+
}

‎core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/RecordingFramework/RecordedCommandTestsBaseTests.cs‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Reflection;
6+
using System.Text.Json;
7+
using Azure.Mcp.Tests.Client;
8+
using Azure.Mcp.Tests.Client.Attributes;
9+
using Azure.Mcp.Tests.Client.Helpers;
10+
using Azure.Mcp.Tests.Generated.Models;
11+
using Azure.Mcp.Tests.Helpers;
12+
using Microsoft.Extensions.FileSystemGlobbing;
13+
using NSubstitute;
14+
using Xunit;
15+
using Xunit.v3;
16+
17+
namespace Azure.Mcp.Core.LiveTests.RecordingFramework;
18+
19+
public sealed class RecordedCommandTestsBaseTest : IAsyncLifetime
20+
{
21+
private string RecordingFileLocation = string.Empty;
22+
private string TestDisplayName = string.Empty;
23+
private TestProxyFixture Fixture = new TestProxyFixture();
24+
private ITestOutputHelper CollectedOutput = Substitute.For<ITestOutputHelper>();
25+
private RecordedCommandTestHarness? DefaultHarness;
26+
27+
[Fact]
28+
public async Task ProxyRecordProducesRecording()
29+
{
30+
await DefaultHarness!.InitializeAsync();
31+
32+
Assert.NotNull(Fixture.Proxy);
33+
Assert.False(string.IsNullOrWhiteSpace(Fixture.Proxy!.BaseUri));
34+
35+
DefaultHarness!.RegisterVariable("sampleKey", "sampleValue");
36+
await DefaultHarness!.DisposeAsync();
37+
38+
Assert.True(File.Exists(RecordingFileLocation));
39+
40+
using var document = JsonDocument.Parse(await File.ReadAllTextAsync(RecordingFileLocation, CancellationToken.None));
41+
Assert.True(document.RootElement.TryGetProperty("Variables", out var variablesElement));
42+
Assert.Equal("sampleValue", variablesElement.GetProperty("sampleKey").GetString());
43+
}
44+
45+
[CustomMatcher(IgnoreQueryOrdering = true, CompareBodies = true)]
46+
[Fact]
47+
public async Task PerTestMatcherAttributeAppliesWhenPresent()
48+
{
49+
var activeMatcher = GetActiveMatcher();
50+
Assert.NotNull(activeMatcher);
51+
Assert.True(activeMatcher!.CompareBodies);
52+
Assert.True(activeMatcher.IgnoreQueryOrdering);
53+
54+
DefaultHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
55+
{
56+
DesiredMode = TestMode.Record,
57+
EnableDefaultSanitizerAdditions = false,
58+
};
59+
var recordingId = string.Empty;
60+
61+
await DefaultHarness.InitializeAsync();
62+
DefaultHarness.RegisterVariable("attrKey", "attrValue");
63+
await DefaultHarness.DisposeAsync();
64+
65+
var playbackHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
66+
{
67+
DesiredMode = TestMode.Playback,
68+
EnableDefaultSanitizerAdditions = false,
69+
};
70+
71+
await playbackHarness.InitializeAsync();
72+
recordingId = playbackHarness.GetRecordingId();
73+
await playbackHarness.DisposeAsync();
74+
75+
CollectedOutput.Received().WriteLine(Arg.Is<string>(s => s.Contains($"Applying custom matcher to recordingId \"{recordingId}\"")));
76+
}
77+
78+
[Fact]
79+
public void CustomMatcherAttributeClearsAfterExecution()
80+
{
81+
var attribute = new CustomMatcherAttribute(compareBody: true, ignoreQueryordering: true);
82+
var xunitTest = Substitute.For<IXunitTest>();
83+
var methodInfo = typeof(RecordedCommandTestsBaseTest).GetMethod(nameof(CustomMatcherAttributeClearsAfterExecution))
84+
?? throw new InvalidOperationException("Unable to locate test method for CustomMatcherAttribute verification.");
85+
86+
attribute.Before(methodInfo, xunitTest);
87+
try
88+
{
89+
var active = GetActiveMatcher();
90+
Assert.Same(attribute, active);
91+
Assert.True(active!.CompareBodies);
92+
Assert.True(active.IgnoreQueryOrdering);
93+
}
94+
finally
95+
{
96+
attribute.After(methodInfo, xunitTest);
97+
}
98+
99+
Assert.Null(GetActiveMatcher());
100+
}
101+
102+
private static CustomMatcherAttribute? GetActiveMatcher()
103+
{
104+
var method = typeof(CustomMatcherAttribute).GetMethod("GetActive", BindingFlags.NonPublic | BindingFlags.Static);
105+
return (CustomMatcherAttribute?)method?.Invoke(null, null);
106+
}
107+
108+
[Fact]
109+
public async Task GlobalMatcherAndSanitizerAppliesWhenPresent()
110+
{
111+
DefaultHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
112+
{
113+
DesiredMode = TestMode.Record,
114+
TestMatcher = new CustomDefaultMatcher
115+
{
116+
CompareBodies = true,
117+
IgnoreQueryOrdering = true,
118+
}
119+
};
120+
121+
DefaultHarness.GeneralRegexSanitizers.Add(new GeneralRegexSanitizer(new GeneralRegexSanitizerBody
122+
{
123+
Regex = "sample",
124+
Value = "sanitized",
125+
}));
126+
DefaultHarness.DisabledDefaultSanitizers.Add("UriSubscriptionIdSanitizer");
127+
128+
await DefaultHarness.InitializeAsync();
129+
await DefaultHarness.DisposeAsync();
130+
131+
CollectedOutput.Received().WriteLine(Arg.Is<string>(s => s.Contains("Applying custom matcher to global settings")));
132+
}
133+
134+
[Fact]
135+
public async Task VariableSurvivesRecordPlaybackRoundtrip()
136+
{
137+
await DefaultHarness!.InitializeAsync();
138+
DefaultHarness.RegisterVariable("roundtrip", "value");
139+
await DefaultHarness.DisposeAsync();
140+
141+
var playbackHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
142+
{
143+
DesiredMode = TestMode.Playback,
144+
};
145+
await playbackHarness.InitializeAsync();
146+
Assert.True(playbackHarness.Variables.TryGetValue("roundtrip", out var variableValue));
147+
Assert.Equal("value", variableValue);
148+
await playbackHarness.DisposeAsync();
149+
}
150+
151+
public ValueTask InitializeAsync()
152+
{
153+
TestDisplayName = TestContext.Current?.Test?.TestCase?.TestCaseDisplayName ?? throw new InvalidDataException("Test case display name is not available.");
154+
155+
var harness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
156+
{
157+
DesiredMode = TestMode.Record
158+
};
159+
160+
RecordingFileLocation = harness.GetRecordingAbsolutePath(TestDisplayName);
161+
162+
if (File.Exists(RecordingFileLocation))
163+
{
164+
File.Delete(RecordingFileLocation);
165+
}
166+
167+
DefaultHarness = harness;
168+
return ValueTask.CompletedTask;
169+
}
170+
171+
public async ValueTask DisposeAsync()
172+
{
173+
// always clean up this recording file on our way out of the test if it exists
174+
if (File.Exists(RecordingFileLocation))
175+
{
176+
File.Delete(RecordingFileLocation);
177+
}
178+
179+
// automatically collect the proxy fixture so that writers of tests don't need to remember to do so and the proxy process doesn't run forever
180+
await Fixture.DisposeAsync();
181+
}
182+
}

‎core/Azure.Mcp.Core/tests/Azure.Mcp.Tests/Attributes/CustomMatcherAttribute.cs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System.Reflection;
5+
using System.Threading;
6+
using Xunit.v3;
7+
8+
namespace Azure.Mcp.Tests.Client.Attributes;
9+
10+
/// <summary>
11+
/// Attribute to customize the test-proxy matcher for a specific test method.
12+
/// Apply this to individual test methods to override default matching behavior for that test only.
13+
///
14+
/// Tests other than what this is applied to will use the default matcher behavior as defined in default test configuration.
15+
/// </summary>
16+
public sealed class CustomMatcherAttribute : BeforeAfterTestAttribute
17+
{
18+
private static readonly AsyncLocal<CustomMatcherAttribute?> Current = new();
19+
20+
/// <summary>
21+
/// When true, the request/response body will be compared during playback matching. Otherwise, body comparison is skipped. Defaults to true.
22+
/// </summary>
23+
public bool CompareBodies { get; set; }
24+
25+
/// <summary>
26+
/// When true, query parameter ordering will be ignored during playback matching. Defaults to false.
27+
/// </summary>
28+
public bool IgnoreQueryOrdering { get; set; }
29+
30+
public CustomMatcherAttribute(
31+
bool compareBody = false,
32+
bool ignoreQueryordering = false)
33+
{
34+
CompareBodies = compareBody;
35+
IgnoreQueryOrdering = ignoreQueryordering;
36+
}
37+
38+
public override void Before(MethodInfo methodUnderTest, IXunitTest xunitTest)
39+
{
40+
base.Before(methodUnderTest, xunitTest);
41+
Current.Value = this;
42+
}
43+
44+
public override void After(MethodInfo methodUnderTest, IXunitTest xunitTest)
45+
{
46+
base.After(methodUnderTest, xunitTest);
47+
Current.Value = null;
48+
}
49+
50+
internal static CustomMatcherAttribute? GetActive() => Current.Value;
51+
}

‎core/Azure.Mcp.Core/tests/Azure.Mcp.Tests/Client/CommandTestsBase.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,13 @@ public abstract class CommandTestsBase(ITestOutputHelper output) : IAsyncLifetim
1818
protected const string TenantNameReason = "Service principals cannot use TenantName for lookup";
1919

2020
protected McpClient Client { get; private set; } = default!;
21-
protected LiveTestSettings Settings { get; private set; } = default!;
21+
protected LiveTestSettings Settings { get; set; } = default!;
2222
protected StringBuilder FailureOutput { get; } = new();
2323
protected ITestOutputHelper Output { get; } = output;
2424

2525
public string[]? CustomArguments;
2626
public TestMode TestMode = TestMode.Live;
2727

28-
2928
/// <summary>
3029
/// Sets custom arguments for the MCP server. Call this before InitializeAsync().
3130
/// </summary>

‎core/Azure.Mcp.Core/tests/Azure.Mcp.Tests/Client/Helpers/RecordingPathResolver.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ namespace Azure.Mcp.Tests.Client.Helpers;
88
/// <summary>
99
/// Provides path resolution for session records and related assets.
1010
/// </summary>
11-
internal sealed class RecordingPathResolver
11+
public sealed class RecordingPathResolver
1212
{
1313
private static readonly char[] _invalidChars = ['\\', '/', ':', '*', '?', '"', '<', '>', '|'];
1414

0 commit comments

Comments
 (0)