E2E/CI: Port disables for flaky tests (Azure#5468)

Cherry pick: Azure#5444 Azure#5449 Azure#5469
snandan18 · Sep 7, 2021 · cc268df · cc268df
1 parent b4b2d7d
commit cc268df
Show file tree

Hide file tree

Showing 15 changed files with 36 additions and 19 deletions.
diff --git a/builds/e2e/templates/e2e-run.yaml b/builds/e2e/templates/e2e-run.yaml
@@ -8,16 +8,16 @@ steps:
     $testFile = '$(binDir)/Microsoft.Azure.Devices.Edge.Test.dll'
     $test_type = '${{ parameters.test_type }}'
 
-    # Filter out unstable tests.
-    $filter = 'Category!=Unstable'
+    # Filter out flaky tests.
+    $filter = 'Category!=Flaky'
 
     if ('$(Agent.Name)'.Contains('centos'))
     {
       $filter += '&Category=CentOsSafe'
     }
     elseif ('$(arch)' -eq 'arm32v7' -Or '$(arch)' -eq 'arm64v8')
     {
-      $filter += '&Category!=UnstableOnArm'
+      $filter += '&Category!=FlakyOnArm'
     }
     
     if ($test_type -eq 'nestededge_mqtt')
@@ -26,13 +26,21 @@ steps:
       $filter += '&Category!=NestedEdgeAmqpOnly'
       $filter += '&Category!=LegacyMqttRequired'
     
+      $filter += '&Category!=FlakyOnNested'
+
+      # Below tests were disabled and marked for re-enable when a blocking item was resolved.
+      # When it was resolved the tests were never enabled. We need to re-enable these.
       $filter += '&FullyQualifiedName!~Provisioning&FullyQualifiedName!~SasOutOfScope&FullyQualifiedName!~X509ManualProvision&FullyQualifiedName!~AuthorizationPolicyUpdateTest&FullyQualifiedName!~AuthorizationPolicyExplicitPolicyTest'
     }
     elseif ($test_type -eq 'nestededge_amqp')
     {
       $filter += '&Category!=SingleNodeOnly'
       $filter += '&Category!=BrokerRequired'
 
+      $filter += '&Category!=FlakyOnNested'
+
+      # Below tests were disabled and marked for re-enable when a blocking item was resolved.
+      # When it was resolved the tests were never enabled. We need to re-enable these.
       $filter += '&FullyQualifiedName!~Provisioning&FullyQualifiedName!~SasOutOfScope&FullyQualifiedName!~X509ManualProvision&FullyQualifiedName!~AuthorizationPolicyUpdateTest&FullyQualifiedName!~AuthorizationPolicyExplicitPolicyTest'
     }
     elseif ($test_type -eq 'nestededge_isa95')

diff --git a/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/CloudProxyTest.cs b/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/CloudProxyTest.cs
@@ -31,7 +31,7 @@ public class CloudProxyTest
         static readonly int EventHubMessageReceivedRetry = 10;
         static readonly ILogger logger = Logger.Factory.CreateLogger(nameof(CloudProxyTest));
 
-        [Fact]
+        [Fact(Skip = "Flaky")]
         [TestPriority(401)]
         public async Task SendMessageTest()
         {
@@ -53,7 +53,7 @@ public async Task SendMessageTest()
             await CheckMessageInEventHub(sentMessagesByDevice, startTime);
         }
 
-        [Fact]
+        [Fact(Skip = "Flaky")]
         [TestPriority(402)]
         public async Task SendMessageMultipleDevicesTest()
         {

diff --git a/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/Cloud2DeviceTest.cs b/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/Cloud2DeviceTest.cs
@@ -24,7 +24,7 @@ public class Cloud2DeviceTest
         const string DeviceNamePrefix = "E2E_c2d_";
         static readonly TimeSpan ClockSkewAdjustment = TimeSpan.FromSeconds(35);
 
-        [Theory]
+        [Theory(Skip = "Flaky")]
         [TestPriority(101)]
         [InlineData(TransportType.Mqtt_Tcp_Only)]
         // [InlineData(TransportType.Mqtt_WebSocket_Only)] // Disabled: need a valid server cert for WebSocket to work

diff --git a/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/EdgeToDeviceMethodTest.cs b/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/EdgeToDeviceMethodTest.cs
@@ -29,7 +29,7 @@ public void Dispose()
             this.logger.Dispose();
         }
 
-        [Theory]
+        [Theory(Skip = "Flaky")]
         [MemberData(nameof(TestSettings.TransportSettings), MemberType = typeof(TestSettings))]
         public async Task InvokeMethodOnModuleTest(ITransportSettings[] transportSettings)
         {

diff --git a/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/InMemoryDbStoreLimitsTest.cs b/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/InMemoryDbStoreLimitsTest.cs
@@ -14,7 +14,7 @@ public InMemoryDbStoreLimitsTest()
         {
         }
 
-        [Fact]
+        [Fact(Skip = "Flaky")]
         async Task InMemoryDbStoreLimitValidationTest() => await this.StoreLimitValidationTestAsync();
     }
 }
diff --git a/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/TelemetryTest.cs b/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/TelemetryTest.cs
@@ -63,7 +63,7 @@ async Task SendTelemetryTest(ITransportSettings[] transportSettings)
             await Task.Delay(TimeSpan.FromSeconds(10));
         }
 
-        [Theory]
+        [Theory(Skip = "Flaky")]
         [MemberData(nameof(TestSettings.TransportSettings), MemberType = typeof(TestSettings))]
         async Task SendOneTelemetryMessageTest(ITransportSettings[] transportSettings)
         {
@@ -114,7 +114,7 @@ async Task SendOneTelemetryMessageTest(ITransportSettings[] transportSettings)
             await Task.Delay(TimeSpan.FromSeconds(10));
         }
 
-        [Theory]
+        [Theory(Skip = "Flaky")]
         [MemberData(nameof(TestSettings.TransportSettings), MemberType = typeof(TestSettings))]
         async Task SendTelemetryMultipleInputsTest(ITransportSettings[] transportSettings)
         {
@@ -210,7 +210,7 @@ async Task SendLargeMessageHandleExceptionTest(ITransportSettings[] transportSet
             await Task.Delay(TimeSpan.FromSeconds(10));
         }
 
-        [Theory]
+        [Theory(Skip = "Flaky")]
         [MemberData(nameof(TestSettings.TransportSettings), MemberType = typeof(TestSettings))]
         async Task SendTelemetryWithDelayedReceiverTest(ITransportSettings[] transportSettings)
         {

diff --git a/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/TwinDiffE2ETest.cs b/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/TwinDiffE2ETest.cs
@@ -16,7 +16,7 @@ public class TwinDiffE2ETest
     {
         const string DeviceNamePrefix = "E2E_twin_";
 
-        [Theory]
+        [Theory(Skip = "Flaky")]
         [MemberData(nameof(TestSettings.TransportSettings), MemberType = typeof(TestSettings))]
         public async Task AddPropertySuccess(ITransportSettings[] transportSettings)
         {
@@ -57,7 +57,7 @@ await this.RunTestCase(
                 });
         }
 
-        [Theory]
+        [Theory(Skip = "Flaky")]
         [MemberData(nameof(TestSettings.TransportSettings), MemberType = typeof(TestSettings))]
         public async Task OverwritePropertySuccess(ITransportSettings[] transportSettings)
         {
@@ -94,7 +94,7 @@ await this.RunTestCase(
                 });
         }
 
-        [Theory]
+        [Theory(Skip = "Flaky")]
         [MemberData(nameof(TestSettings.TransportSettings), MemberType = typeof(TestSettings))]
         public async Task UnchangedPropertySuccess(ITransportSettings[] transportSettings)
         {
@@ -260,7 +260,7 @@ await this.RunTestCase(
                 });
         }
 
-        [Theory]
+        [Theory(Skip = "Flaky")]
         [MemberData(nameof(TestSettings.TransportSettings), MemberType = typeof(TestSettings))]
         public async Task OverwriteValueWithObjectSuccess(ITransportSettings[] transportSettings)
         {

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/Device.cs b/test/Microsoft.Azure.Devices.Edge.Test/Device.cs
@@ -57,6 +57,7 @@ await TryFinally.DoAsync(
         [Test]
         [Category("CentOsSafe")]
         [Category("NestedEdgeOnly")]
+        [Category("FlakyOnNested")]
         public async Task QuickstartChangeSasKey()
         {
             CancellationToken token = this.TestToken;
@@ -192,6 +193,7 @@ await Profiler.Run(
 
         [Test]
         [Category("CentOsSafe")]
+        [Category("FlakyOnNested")]
         public async Task DisableReenableParentEdge()
         {
             CancellationToken token = this.TestToken;

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs b/test/Microsoft.Azure.Devices.Edge.Test/DeviceWithCustomCertificates.cs
@@ -17,6 +17,7 @@ namespace Microsoft.Azure.Devices.Edge.Test
     class DeviceWithCustomCertificates : CustomCertificatesFixture
     {
         [Test]
+        [Category("Flaky")]
         public async Task TransparentGateway(
             [Values] TestAuthenticationType testAuth,
             [Values(Protocol.Mqtt, Protocol.Amqp)] Protocol protocol)
@@ -75,6 +76,7 @@ await TryFinally.DoAsync(
 
         [Test]
         [Category("NestedEdgeOnly")]
+        [Category("FlakyOnNested")]
         [Description("A test to verify a leaf device can be registered under grandparent device scope.")]
         public async Task GrandparentScopeDevice(
             [Values(

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs b/test/Microsoft.Azure.Devices.Edge.Test/EdgeAgentDirectMethods.cs
@@ -39,6 +39,7 @@ public async Task TestPing()
         }
 
         [Test]
+        [Category("Flaky")]
         public async Task TestGetModuleLogs()
         {
             string moduleName = "NumberLogger";

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/Metrics.cs b/test/Microsoft.Azure.Devices.Edge.Test/Metrics.cs
@@ -22,6 +22,7 @@ public class Metrics : SasManualProvisioningFixture
         public const string ModuleName = "metricsValidator";
 
         [Test]
+        [Category("FlakyOnArm")]
         public async Task ValidateMetrics()
         {
             CancellationToken token = this.TestToken;

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/Module.cs b/test/Microsoft.Azure.Devices.Edge.Test/Module.cs
@@ -116,6 +116,7 @@ public async Task TempFilter()
         }
 
         [Test]
+        [Category("FlakyOnArm")]
         // Test Temperature Filter Function: https://docs.microsoft.com/en-us/azure/iot-edge/tutorial-deploy-function
         public async Task TempFilterFunc()
         {

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/PriorityQueues.cs b/test/Microsoft.Azure.Devices.Edge.Test/PriorityQueues.cs
@@ -33,7 +33,7 @@ public class PriorityQueues : SasManualProvisioningFixture
         const string DefaultLoadGenTestStartDelay = "00:00:20";
 
         [Test]
-        [Category("UnstableOnArm")]
+        [Category("FlakyOnArm")]
         public async Task PriorityQueueModuleToModuleMessages()
         {
             CancellationToken token = this.TestToken;
@@ -55,7 +55,7 @@ public async Task PriorityQueueModuleToModuleMessages()
         }
 
         [Test]
-        [Category("UnstableOnArm")]
+        [Category("Flaky")]
         public async Task PriorityQueueModuleToHubMessages()
         {
             CancellationToken token = this.TestToken;
@@ -89,7 +89,7 @@ public async Task PriorityQueueModuleToHubMessages()
         }
 
         [Test]
-        [Category("UnstableOnArm")]
+        [Category("FlakyOnArm")]
         public async Task PriorityQueueTimeToLive()
         {
             CancellationToken token = this.TestToken;

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/Provisioning.cs b/test/Microsoft.Azure.Devices.Edge.Test/Provisioning.cs
@@ -37,6 +37,7 @@ string DeriveDeviceKey(byte[] groupKey, string registrationId)
 
         [Test]
         [Category("CentOsSafe")]
+        [Category("FlakyOnArm")]
         public async Task DpsSymmetricKey()
         {
             string idScope = Context.Current.DpsIdScope.Expect(() => new InvalidOperationException("Missing DPS ID scope (check dpsIdScope in context.json)"));
@@ -80,6 +81,7 @@ await this.daemon.ConfigureAsync(
         }
 
         [Test]
+        [Category("FlakyOnArm")]
         public async Task DpsX509()
         {
             (string, string, string) rootCa =

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs b/test/Microsoft.Azure.Devices.Edge.Test/X509Device.cs
@@ -6,7 +6,6 @@ namespace Microsoft.Azure.Devices.Edge.Test
     using System.Threading;
     using System.Threading.Tasks;
     using Microsoft.Azure.Devices.Edge.Test.Common;
-    using Microsoft.Azure.Devices.Edge.Test.Common.Certs;
     using Microsoft.Azure.Devices.Edge.Test.Common.Config;
     using Microsoft.Azure.Devices.Edge.Test.Helpers;
     using Microsoft.Azure.Devices.Edge.Util;
@@ -17,6 +16,7 @@ namespace Microsoft.Azure.Devices.Edge.Test
     class X509Device : X509ManualProvisioningFixture
     {
         [Test]
+        [Category("Flaky")]
         public async Task X509ManualProvision()
         {
             CancellationToken token = this.TestToken;
-Original file line number
+Diff line change
@@ Expand Up / @@ -14,7 +14,7 @@ public InMemoryDbStoreLimitsTest() @@
             {
             }
-            [Fact]
+            [Fact(Skip = "Flaky")]
             async Task InMemoryDbStoreLimitValidationTest() => await this.StoreLimitValidationTestAsync();
         }
     }