Skip to content

Commit

Permalink
rysweet-4150-xlang-ci-test (microsoft#4596)
Browse files Browse the repository at this point in the history
adds aspire-based integration test that validates:
* registration
* subscriptions
* event delivery
* python -> .NET server -> python subscriber
* .NET -> .NET server  -> python subscriber
* python -> .NET server -> .NET subscriber
  • Loading branch information
rysweet authored Dec 9, 2024
1 parent 38cb532 commit d969972
Show file tree
Hide file tree
Showing 8 changed files with 625 additions and 7 deletions.
20 changes: 20 additions & 0 deletions .github/workflows/codeql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,26 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.11
- name: Install jupyter and ipykernel
run: |
python -m pip install --upgrade pip
python -m pip install jupyter
python -m pip install ipykernel
- name: list available kernels
run: |
python -m jupyter kernelspec list
- uses: astral-sh/setup-uv@v3
with:
enable-cache: true
- run: uv sync --locked --all-extras
working-directory: ./python
- name: Prepare python venv
run: |
source ${{ github.workspace }}/python/.venv/bin/activate
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
Expand Down
76 changes: 70 additions & 6 deletions .github/workflows/dotnet-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,18 @@ jobs:
- name: list available kernels
run: |
python -m jupyter kernelspec list
- name: Setup .NET
- uses: astral-sh/setup-uv@v3
with:
enable-cache: true
- run: uv sync --locked --all-extras
working-directory: ./python
- name: Prepare python venv
run: |
source ${{ github.workspace }}/python/.venv/bin/activate
- name: Setup .NET 8.0
uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
- name: Install .NET Aspire workload
run: dotnet workload install aspire
- name: Restore dependencies
run: |
# dotnet nuget add source --name dotnet-tool https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json --configfile NuGet.config
Expand All @@ -96,7 +102,57 @@ jobs:
echo "Build AutoGen"
dotnet build --no-restore --configuration Release -bl /p:SignAssembly=true
- name: Unit Test
run: dotnet test --no-build -bl --configuration Release
run: dotnet test --no-build -bl --configuration Release --filter type=!integration

integration-test:
strategy:
fail-fast: true
matrix:
os: [ ubuntu-latest]
version: [ net8.0 ]
needs: build
defaults:
run:
working-directory: dotnet
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
lfs: true
- uses: astral-sh/setup-uv@v3
with:
enable-cache: true
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- run: uv sync --locked --all-extras
working-directory: ./python
- name: Prepare python venv
run: |
source ${{ github.workspace }}/python/.venv/bin/activate
- name: Setup .NET 9.0
uses: actions/setup-dotnet@v4
with:
dotnet-version: '9.0.x'
- name: Install Temp Global.JSON
run: |
echo "{\"sdk\": {\"version\": \"9.0.101\"}}" > global.json
- name: Install .NET Aspire workload
run: dotnet workload install aspire
- name: Install dev certs
run: dotnet --version && dotnet dev-certs https --trust
- name: Restore dependencies
run: |
dotnet restore -bl
- name: Build
run: |
echo "Build AutoGen"
dotnet build --no-restore --configuration Release -bl /p:SignAssembly=true
- name: Integration Test
run: dotnet --version && dotnet test --no-build -bl --configuration Release --filter type=integration
- name: Restore the global.json
run: rm global.json && git checkout -- global.json

aot-test: # this make sure the AutoGen.Core is aot compatible
strategy:
fail-fast: false # ensures the entire test matrix is run, even if one permutation fails
Expand Down Expand Up @@ -147,9 +203,17 @@ jobs:
- name: list available kernels
run: |
python -m jupyter kernelspec list
- name: Setup .NET
- uses: astral-sh/setup-uv@v3
with:
enable-cache: true
- uses: actions/setup-python@v5
with:
python-version: "3.11"

- name: Setup .NET 8.0
uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
global-json-file: dotnet/global.json
- name: Restore dependencies
run: |
Expand All @@ -159,7 +223,7 @@ jobs:
echo "Build AutoGen"
dotnet build --no-restore --configuration Release -bl /p:SignAssembly=true
- name: OpenAI Test
run: dotnet test --no-build -bl --configuration Release
run: dotnet test --no-build -bl --configuration Release --filter type!=integration
env:
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
Expand Down
7 changes: 7 additions & 0 deletions dotnet/AutoGen.sln
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AutoGen.Extension
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AutoGen.Agents.Tests", "test\Microsoft.AutoGen.Agents.Tests\Microsoft.AutoGen.Agents.Tests.csproj", "{394FDAF8-74F9-4977-94A5-3371737EB774}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.AutoGen.Integration.Tests", "test\Microsoft.AutoGen.Integration.Tests\Microsoft.AutoGen.Integration.Tests.csproj", "{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -342,6 +344,10 @@ Global
{394FDAF8-74F9-4977-94A5-3371737EB774}.Debug|Any CPU.Build.0 = Debug|Any CPU
{394FDAF8-74F9-4977-94A5-3371737EB774}.Release|Any CPU.ActiveCfg = Release|Any CPU
{394FDAF8-74F9-4977-94A5-3371737EB774}.Release|Any CPU.Build.0 = Release|Any CPU
{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D04C6153-8EAF-4E54-9852-52CEC1BE8D31}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -402,6 +408,7 @@ Global
{64EF61E7-00A6-4E5E-9808-62E10993A0E5} = {7EB336C2-7C0A-4BC8-80C6-A3173AB8DC45}
{65059914-5527-4A00-9308-9FAF23D5E85A} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
{394FDAF8-74F9-4977-94A5-3371737EB774} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
{D04C6153-8EAF-4E54-9852-52CEC1BE8D31} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {93384647-528D-46C8-922C-8DB36A382F0B}
Expand Down
10 changes: 9 additions & 1 deletion dotnet/Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
<ItemGroup>
<PackageVersion Include="Aspire.Hosting" Version="9.0.0" />
<PackageVersion Include="Aspire.Hosting.Python" Version="9.0.0" />
<PackageVersion Include="Aspire.Hosting.Testing" Version="9.0.0" />
<PackageVersion Include="AspNetCore.Authentication.ApiKey" Version="8.0.1" />
<PackageVersion Include="Aspire.Azure.AI.OpenAI" Version="8.0.1-preview.8.24267.1" />
<PackageVersion Include="Aspire.Hosting.AppHost" Version="9.0.0" />
Expand All @@ -31,12 +32,17 @@
<PackageVersion Include="Azure.ResourceManager.ContainerInstance" Version="1.2.1" />
<PackageVersion Include="Azure.Storage.Files.Shares" Version="12.21.0" />
<PackageVersion Include="CloudNative.CloudEvents.SystemTextJson" Version="2.7.1" />
<PackageVersion Include="GitHubActionsTestLogger" Version="2.4.1">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageVersion>
<PackageVersion Include="Grpc.AspNetCore" Version="2.67.0" />
<PackageVersion Include="Grpc.Core" Version="2.46.6" />
<PackageVersion Include="Grpc.Net.ClientFactory" Version="2.67.0" />
<PackageVersion Include="Grpc.Tools" Version="2.67.0" />
<PackageVersion Include="Grpc.Net.Client" Version="2.65.0" />
<PackageVersion Include="Google.Protobuf" Version="3.28.3" />
<PackageVersion Include="MartinCostello.Logging.XUnit" Version="0.4.0" />
<PackageVersion Include="Microsoft.AspNetCore.App" Version="8.0.4" />
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.8" />
<PackageVersion Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
Expand All @@ -54,6 +60,7 @@
<PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="$(MicrosoftExtensionConfiguration)" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="$(MicrosoftExtensionDependencyInjection)" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(MicrosoftExtensionDependencyInjection)" />
<PackageVersion Include="Microsoft.Extensions.Diagnostics.Testing" Version="9.0.0" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="9.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionLogging)" />
Expand Down Expand Up @@ -118,5 +125,6 @@
<PackageVersion Include="Moq" Version="4.20.72" />
<PackageVersion Include="Microsoft.PowerShell.SDK" Version="7.4.5" />
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="8.0.11" />
<PackageVersion Include="coverlet.collector" Version="6.0.2" />
</ItemGroup>
</Project>
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// HelloAppHostIntegrationTests.cs

using System.Text.Json;
using Xunit.Abstractions;

namespace Microsoft.AutoGen.Integration.Tests;

public class HelloAppHostIntegrationTests(ITestOutputHelper testOutput)
{
[Theory, Trait("type", "integration")]
[MemberData(nameof(AppHostAssemblies))]
public async Task AppHostRunsCleanly(string appHostPath)
{
var appHost = await DistributedApplicationTestFactory.CreateAsync(appHostPath, testOutput);
await using var app = await appHost.BuildAsync().WaitAsync(TimeSpan.FromSeconds(15));

await app.StartAsync().WaitAsync(TimeSpan.FromSeconds(120));
await app.WaitForResourcesAsync().WaitAsync(TimeSpan.FromSeconds(120));

app.EnsureNoErrorsLogged();
await app.StopAsync().WaitAsync(TimeSpan.FromSeconds(15));
}

[Theory, Trait("type", "integration")]
[MemberData(nameof(TestEndpoints))]
public async Task AppHostLogsHelloAgentE2E(TestEndpoints testEndpoints)
{
var appHostName = testEndpoints.AppHost!;
var appHostPath = $"{appHostName}.dll";
var appHost = await DistributedApplicationTestFactory.CreateAsync(appHostPath, testOutput);
await using var app = await appHost.BuildAsync().WaitAsync(TimeSpan.FromSeconds(15));

await app.StartAsync().WaitAsync(TimeSpan.FromSeconds(120));
await app.WaitForResourcesAsync().WaitAsync(TimeSpan.FromSeconds(120));
if (testEndpoints.WaitForResources?.Count > 0)
{
// Wait until each resource transitions to the required state
var timeout = TimeSpan.FromMinutes(5);
foreach (var (ResourceName, TargetState) in testEndpoints.WaitForResources)
{
await app.WaitForResource(ResourceName, TargetState).WaitAsync(timeout);
}
}
//sleep 5 seconds to make sure the app is running
await Task.Delay(5000);
app.EnsureNoErrorsLogged();
app.EnsureLogContains("HelloAgents said Goodbye");
app.EnsureLogContains("Wild Hello from Python!");

await app.StopAsync().WaitAsync(TimeSpan.FromSeconds(15));
}
public static TheoryData<string> AppHostAssemblies()
{
var appHostAssemblies = GetSamplesAppHostAssemblyPaths();
var theoryData = new TheoryData<string, bool>();
return new(appHostAssemblies.Select(p => Path.GetRelativePath(AppContext.BaseDirectory, p)));
}

public static TheoryData<TestEndpoints> TestEndpoints() =>
new([
new TestEndpoints("Hello.AppHost", new() {
{ "backend", ["/"] }
}),
]);

private static IEnumerable<string> GetSamplesAppHostAssemblyPaths()
{
// All the AppHost projects are referenced by this project so we can find them by looking for all their assemblies in the base directory
return Directory.GetFiles(AppContext.BaseDirectory, "*.AppHost.dll")
.Where(fileName => !fileName.EndsWith("Aspire.Hosting.AppHost.dll", StringComparison.OrdinalIgnoreCase));
}
}

public class TestEndpoints : IXunitSerializable
{
// Required for deserialization
public TestEndpoints() { }

public TestEndpoints(string appHost, Dictionary<string, List<string>> resourceEndpoints)
{
AppHost = appHost;
ResourceEndpoints = resourceEndpoints;
}

public string? AppHost { get; set; }

public List<ResourceWait>? WaitForResources { get; set; }

public Dictionary<string, List<string>>? ResourceEndpoints { get; set; }

public void Deserialize(IXunitSerializationInfo info)
{
AppHost = info.GetValue<string>(nameof(AppHost));
WaitForResources = JsonSerializer.Deserialize<List<ResourceWait>>(info.GetValue<string>(nameof(WaitForResources)));
ResourceEndpoints = JsonSerializer.Deserialize<Dictionary<string, List<string>>>(info.GetValue<string>(nameof(ResourceEndpoints)));
}

public void Serialize(IXunitSerializationInfo info)
{
info.AddValue(nameof(AppHost), AppHost);
info.AddValue(nameof(WaitForResources), JsonSerializer.Serialize(WaitForResources));
info.AddValue(nameof(ResourceEndpoints), JsonSerializer.Serialize(ResourceEndpoints));
}

public override string? ToString() => $"{AppHost} ({ResourceEndpoints?.Count ?? 0} resources)";

public class ResourceWait(string resourceName, string targetState)
{
public string ResourceName { get; } = resourceName;

public string TargetState { get; } = targetState;

public void Deconstruct(out string resourceName, out string targetState)
{
resourceName = ResourceName;
targetState = TargetState;
}
}
}
Loading

0 comments on commit d969972

Please sign in to comment.