Combine test attributes (Azure#461)

Replace all E2E attribute usage by Integration attribute; and remove E2E attribute.
pwissneth · Oct 22, 2018 · 3bf4b9c · 3bf4b9c
1 parent e7c4122
commit 3bf4b9c
Show file tree

Hide file tree

Showing 14 changed files with 24 additions and 53 deletions.
diff --git a/doc/build-test.md b/doc/build-test.md
@@ -27,9 +27,10 @@ Test
 
 Our C# tests use [xUnit](https://xunit.github.io/docs/getting-started-dotnet-core). Each test is tagged with one of the following attributes:
 
-| Attribute   | Goal                                                                                | Notes                                                                                         | Runs in build(s)  |
-|-------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------|-------------------|
-| Unit        | Fast, reliable, repeatable tests against low-level components of the codebase.      | Should not depend on external resources (e.g., network, disk) or configuration.               | Checkin           |
-| Integration | Pretty fast tests that verify our interaction with dependencies _in isolation_.     | Require little/no setup an no external configuration.                                         | Checkin           |
-| E2E         | High-level tests that verify core end-to-end scenarios.                             | Not suitable for scenarios which could be covered by unit or integration tests.               | CI                |
-| Stress      | Specialized end-to-end tests for scenarios under heavy load.                        |                                                                                               | N/A               |
+| Attribute   | Goal                                                                                            | Notes                                                                                 | Runs in build(s)  |
+|-------------|-------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------|-------------------|
+| Unit        | Fast, reliable, repeatable tests against low-level components of the codebase.                  | Should not depend on external resources (e.g., network, disk) or configuration.       | Checkin           |
+| Integration | Tests that verify interaction with dependencies _in isolation_ and core end-to-end scenarios.    | Require external configuration.                                                       | CI                |
+| Stress      | Specialized end-to-end tests for scenarios under heavy load.                                    |                                                                                       | N/A               |
+
+Besides these tests, E2E scenario testing is implemented in IoTEdgeQuickStart and LeafDevice project in smoke folder.  These tests will be run in Linux, Raspberry Pi and Windows to ensure all core IoT Edge features function correctly.
diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test/AgentTests.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test/AgentTests.cs
@@ -29,7 +29,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Docker.E2E.Test
 
     public class AgentTests
     {
-        [E2E]
+        [Integration]
         [Theory]
         [MemberData(nameof(GenerateStartTestData))]
         public async Task AgentStartsUpModules(TestConfig testConfig)

diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/EdgeAgentConnectionTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.IoTHub.Test/EdgeAgentConnectionTest.cs
@@ -27,7 +27,7 @@ public class EdgeAgentConnectionTest
     {
         const string DockerType = "docker";
 
-        [E2E]
+        [Integration]
         [Fact]
         public async Task EdgeAgentConnectionBasicTest()
         {
@@ -289,7 +289,7 @@ public static async Task SetAgentDesiredProperties(RegistryManager rm, string de
             await rm.ApplyConfigurationContentOnDeviceAsync(deviceId, cc);
         }
 
-        [E2E]
+        [Integration]
         [Fact]
         public async Task EdgeAgentConnectionConfigurationTest()
         {
@@ -1124,7 +1124,7 @@ public async Task GetDeploymentConfigInfoAsyncReturnsConfigWhenThereAreNoErrorsW
             Assert.Equal(deploymentConfigInfo.OrDefault().DeploymentConfig, deploymentConfig);
         }
 
-        [E2E]
+        [Integration]
         [Fact]
         public async Task EdgeAgentConnectionStatusTest()
         {
@@ -1213,7 +1213,7 @@ public async Task EdgeAgentConnectionStatusTest()
             }
         }
 
-        [E2E]
+        [Integration]
         [Fact]
         public async Task EdgeAgentPingMethodTest()
         {

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/CloudProxyTest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/CloudProxyTest.cs
@@ -19,7 +19,7 @@ namespace Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test
     using Newtonsoft.Json.Linq;
     using Xunit;
 
-    [E2E]
+    [Integration]
     [Collection("Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test")]
     [TestCaseOrderer("Microsoft.Azure.Devices.Edge.Util.Test.PriorityOrderer", "Microsoft.Azure.Devices.Edge.Util.Test")]
     public class CloudProxyTest

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Core.Test/DeviceScopeIdentitiesCacheTest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.Core.Test/DeviceScopeIdentitiesCacheTest.cs
@@ -241,7 +241,7 @@ public async Task RefreshCacheWithRefreshRequestTest()
             deviceScopeIdentitiesCache.InitiateCacheRefresh();
 
             // Wait for the 2 refresh cycles to complete, this time because of the refresh request
-            await Task.Delay(TimeSpan.FromSeconds(2));
+            await Task.Delay(TimeSpan.FromSeconds(5));
             receivedServiceIdentity1 = await deviceScopeIdentitiesCache.GetServiceIdentity("d1");
             receivedServiceIdentity2 = await deviceScopeIdentitiesCache.GetServiceIdentity("d2/m1");
             receivedServiceIdentity3 = await deviceScopeIdentitiesCache.GetServiceIdentity("d3");

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/Cloud2DeviceTest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/Cloud2DeviceTest.cs
@@ -14,7 +14,7 @@ namespace Microsoft.Azure.Devices.Edge.Hub.E2E.Test
     using Xunit;
     using Message = Microsoft.Azure.Devices.Message;
 
-    [E2E]
+    [Integration]
     [Collection("Microsoft.Azure.Devices.Edge.Hub.E2E.Test")]
     [TestCaseOrderer("Microsoft.Azure.Devices.Edge.Util.Test.PriorityOrderer", "Microsoft.Azure.Devices.Edge.Util.Test")]
     public class Cloud2DeviceTest : IClassFixture<ProtocolHeadFixture>

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/EdgeHubConnectionTest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/EdgeHubConnectionTest.cs
@@ -23,7 +23,7 @@ namespace Microsoft.Azure.Devices.Edge.Hub.E2E.Test
     using Newtonsoft.Json;
     using Xunit;
 
-    [E2E]
+    [Integration]
     [Collection("Microsoft.Azure.Devices.Edge.Hub.E2E.Test")]
     public class EdgeHubConnectionTest : IClassFixture<ProtocolHeadFixture>
     {

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/EdgeToDeviceMethodTest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/EdgeToDeviceMethodTest.cs
@@ -11,7 +11,7 @@ namespace Microsoft.Azure.Devices.Edge.Hub.E2E.Test
     using Newtonsoft.Json;
     using Xunit;
 
-    [E2E]
+    [Integration]
     [Collection("Microsoft.Azure.Devices.Edge.Hub.E2E.Test")]
     public class EdgeToDeviceMethodTest : IClassFixture<ProtocolHeadFixture>
     {

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/StressTest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/StressTest.cs
@@ -10,7 +10,7 @@ namespace Microsoft.Azure.Devices.Edge.Hub.E2E.Test
     using Microsoft.Azure.Devices.Edge.Util.Test.Common;
     using Xunit;
 
-    [E2E, Stress]
+    [Integration, Stress]
     [Collection("Microsoft.Azure.Devices.Edge.Hub.E2E.Test")]
     [TestCaseOrderer("Microsoft.Azure.Devices.Edge.Util.Test.PriorityOrderer", "Microsoft.Azure.Devices.Edge.Util.Test")]
     public class StressTest : IClassFixture<ProtocolHeadFixture>

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/TelemetryTest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/TelemetryTest.cs
@@ -10,7 +10,7 @@ namespace Microsoft.Azure.Devices.Edge.Hub.E2E.Test
     using Microsoft.Azure.Devices.Edge.Util.Test.Common;
     using Xunit;
 
-    [E2E]
+    [Integration]
     [Collection("Microsoft.Azure.Devices.Edge.Hub.E2E.Test")]
     public class TelemetryTest : IClassFixture<ProtocolHeadFixture>
     {

diff --git a/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/TwinDiffE2ETest.cs b/edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/TwinDiffE2ETest.cs
@@ -10,7 +10,7 @@ namespace Microsoft.Azure.Devices.Edge.Hub.E2E.Test
     using Newtonsoft.Json.Linq;
     using Xunit;
 
-    [E2E]
+    [Integration]
     [Collection("Microsoft.Azure.Devices.Edge.Hub.E2E.Test")]
     public class TwinDiffE2ETest : IClassFixture<ProtocolHeadFixture>
     {

diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Util.Test.Common/E2EAttribute.cs b/edge-util/test/Microsoft.Azure.Devices.Edge.Util.Test.Common/E2EAttribute.cs
diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Util.Test.Common/E2EDiscoverer.cs b/edge-util/test/Microsoft.Azure.Devices.Edge.Util.Test.Common/E2EDiscoverer.cs
diff --git a/scripts/windows/test/Test-Branch.ps1 b/scripts/windows/test/Test-Branch.ps1
@@ -78,6 +78,7 @@ $Success = $True
 foreach ($Project in (Get-ChildItem $BuildRepositoryLocalPath -Include $TEST_PROJ_PATTERN -Recurse)) {
     Write-Host "Running tests for $Project."
     if (Test-Path $OPENCOVER -PathType "Leaf") {
+		Write-Host "Run command: $OPENCOVER -register:user -target:$DOTNET_PATH -targetargs:'$BaseTestCommand $Project' -skipautoprops -hideskipped:All -oldstyle -output:$CODE_COVERAGE -mergeoutput:$CODE_COVERAGE -returntargetcode ..."
         &$OPENCOVER `
             -register:user `
             -target:$DOTNET_PATH `
@@ -87,9 +88,11 @@ foreach ($Project in (Get-ChildItem $BuildRepositoryLocalPath -Include $TEST_PRO
             -oldstyle `
             -output:$CODE_COVERAGE `
             -mergeoutput:$CODE_COVERAGE `
-            -returntargetcode
+            -returntargetcode `
+			-filter:"+[*]* -[Moq*]* -[App.Metrics.Reporting*]*"
     }
     else {
+		Write-Host "Run command: '" + $DOTNET_PATH + "' " + $BaseTestCommand " -o " + $BuildBinariesDirectory + " " + $Project
         Invoke-Expression "&`"$DOTNET_PATH`" $BaseTestCommand -o $BuildBinariesDirectory $Project"
     }