Skip to content

Commit

Permalink
Connectivity / Nested Connectivity: Add tolerances for failure cases (#…
Browse files Browse the repository at this point in the history 
…5512)
  • Loading branch information
@and-rewsmith
and-rewsmith authored Sep 21, 2021
1 parent 31c4afa commit 673eb87
Show file tree
Hide file tree
Showing 21 changed files with 244 additions and 32 deletions.
21 changes: 14 additions & 7 deletions e2e_deployment_files/nestededge_bottomLayerBaseDeployment_connectivity_mqtt.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -726,28 +726,32 @@
"TwinTestPropertyType": "Desired",
"ExpectedSource": "twinTester1.desiredUpdated",
"ActualSource": "twinTester2.desiredReceived",
"TestDescription": "twin | desired property | amqp"
"TestDescription": "twin | desired property | amqp",
"Topology": "Nested"
},
"reportMetadata6": {
"TestReportType": "TwinCountingReport",
"TwinTestPropertyType": "Reported",
"ExpectedSource": "twinTester2.reportedReceived",
"ActualSource": "twinTester2.reportedUpdated",
"TestDescription": "twin | reported property | amqp"
"TestDescription": "twin | reported property | amqp",
"Topology": "Nested"
},
"reportMetadata7": {
"TestReportType": "TwinCountingReport",
"TwinTestPropertyType": "Desired",
"ExpectedSource": "twinTester3.desiredUpdated",
"ActualSource": "twinTester4.desiredReceived",
"TestDescription": "twin | desired property | mqtt"
"TestDescription": "twin | desired property | mqtt",
"Topology": "Nested"
},
"reportMetadata8": {
"TestReportType": "TwinCountingReport",
"TwinTestPropertyType": "Reported",
"ExpectedSource": "twinTester4.reportedReceived",
"ActualSource": "twinTester4.reportedUpdated",
"TestDescription": "twin | reported property | mqtt"
"TestDescription": "twin | reported property | mqtt",
"Topology": "Nested"
},
"reportMetadata9": {
"TestReportType": "DeploymentTestReport",
Expand All @@ -760,20 +764,23 @@
"SenderSource": "directMethodSender1.send",
"ReceiverSource": "directMethodReceiver1.receive",
"TolerancePeriod": "00:01:00",
"TestDescription": "direct method | cloud | amqp"
"TestDescription": "direct method | cloud | amqp",
"Topology": "Nested"
},
"reportMetadata11": {
"TestReportType": "DirectMethodConnectivityReport",
"SenderSource": "directMethodSender2.send",
"ReceiverSource": "directMethodReceiver2.receive",
"TolerancePeriod": "00:01:00",
"TestDescription": "direct method | cloud | mqtt"
"TestDescription": "direct method | cloud | mqtt",
"Topology": "Nested"
},
"reportMetadata12": {
"TestReportType": "DirectMethodConnectivityReport",
"SenderSource": "directMethodSender3.send",
"TolerancePeriod": "00:01:00",
"TestDescription": "edge agent ping"
"TestDescription": "edge agent ping",
"Topology": "Nested"
},
"reportMetadata15": {
"TestReportType": "NetworkControllerReport",
Expand Down
13 changes: 13 additions & 0 deletions test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ namespace Modules.Test.TestResultCoordinator.Reports
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using global::TestResultCoordinator;
using global::TestResultCoordinator.Reports;
using Microsoft.Azure.Devices.Edge.ModuleUtil;
using Microsoft.Azure.Devices.Edge.Storage;
Expand Down Expand Up @@ -60,6 +61,7 @@ public void TestConstructorSuccess()

var reportGenerator = new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
expectedSource,
expectedResults.GetAsyncEnumerator(),
Expand Down Expand Up @@ -90,6 +92,7 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReportGenerator(
testDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
"expectedSource",
mockExpectedResults.Object,
Expand All @@ -114,6 +117,7 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId)
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
trackingId,
"expectedSource",
mockExpectedResults.Object,
Expand All @@ -138,6 +142,7 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
expectedSource,
mockExpectedResults.Object,
Expand All @@ -159,6 +164,7 @@ public void TestConstructorThrowsWhenExpectedStoreIsNotProvided()
ArgumentNullException ex = Assert.Throws<ArgumentNullException>(
() => new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
"expectedSource",
null,
Expand All @@ -183,6 +189,7 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
"expectedSource",
mockExpectedResults.Object,
Expand All @@ -204,6 +211,7 @@ public void TestConstructorThrowsWhenActualStoreIsNotProvided()
ArgumentNullException ex = Assert.Throws<ArgumentNullException>(
() => new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
"expectedSource",
mockExpectedResults.Object,
Expand All @@ -228,6 +236,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType)
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
"expectedSource",
mockExpectedResults.Object,
Expand All @@ -250,6 +259,7 @@ public void TestConstructorThrowsWhenTestResultComparerIsNotProvided()
ArgumentNullException ex = Assert.Throws<ArgumentNullException>(
() => new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
"expectedSource",
mockExpectedResults.Object,
Expand All @@ -273,6 +283,7 @@ public void TestConstructorThrowsWhenUnmatchedResultsMaxSizeIsNonPositive(ushort
ArgumentOutOfRangeException ex = Assert.Throws<ArgumentOutOfRangeException>(
() => new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
"expectedSource",
mockExpectedResults.Object,
Expand All @@ -298,6 +309,7 @@ public async Task TestCreateReportAsyncWithEmptyResults()

var reportGenerator = new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
expectedSource,
expectedResults.GetAsyncEnumerator(),
Expand Down Expand Up @@ -351,6 +363,7 @@ public async Task TestCreateReportAsync(

var reportGenerator = new CountingReportGenerator(
TestDescription,
TestMode.Connectivity,
Guid.NewGuid().ToString(),
expectedSource,
expectedResults.GetAsyncEnumerator(),
Expand Down
7 changes: 7 additions & 0 deletions test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ namespace Modules.Test.TestResultCoordinator.Reports
{
using System;
using System.Collections.Generic;
using global::TestResultCoordinator;
using global::TestResultCoordinator.Reports;
using Microsoft.Azure.Devices.Edge.ModuleUtil;
using Microsoft.Azure.Devices.Edge.Util;
Expand All @@ -19,6 +20,7 @@ public void TestConstructorSuccess()
{
var report = new CountingReport(
TestDescription,
TestMode.Connectivity,
"trackingId123",
"expectedSource",
"actualSource",
Expand Down Expand Up @@ -74,6 +76,7 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReport(
testDescription,
TestMode.Connectivity,
"trackingId123",
"expectedSource",
"actualSource",
Expand Down Expand Up @@ -105,6 +108,7 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId)
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReport(
TestDescription,
TestMode.Connectivity,
trackingId,
"expectedSource",
"actualSource",
Expand Down Expand Up @@ -136,6 +140,7 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReport(
TestDescription,
TestMode.Connectivity,
"trackingId-23434",
expectedSource,
"actualSource",
Expand Down Expand Up @@ -167,6 +172,7 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReport(
TestDescription,
TestMode.Connectivity,
"trackingId-23434",
"expectedSource",
actualSource,
Expand Down Expand Up @@ -198,6 +204,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType)
ArgumentException ex = Assert.Throws<ArgumentException>(
() => new CountingReport(
TestDescription,
TestMode.Connectivity,
"trackingId-23434",
"expectedSource",
"actualSource",
Expand Down
47 changes: 42 additions & 5 deletions ...tor/Reports/DirectMethod/DirectMethodConnectivityReportDataWithSenderAndReceiverSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class DirectMethodConnectivityReportDataWithSenderAndReceiverSource
},
new object[]
{
// NetworkOffSuccess test
// NetworkOffSuccess test with NotFound exception
Enumerable.Range(1, 7).Select(v => (ulong)v),
new[] { 1UL, 2UL, 4UL, 5UL, 6UL, 7UL },
new List<HttpStatusCode> { HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.NotFound, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK },
Expand All @@ -50,6 +50,24 @@ class DirectMethodConnectivityReportDataWithSenderAndReceiverSource
10, 6, 1, 0, 0, 0, 0, 0, 0, true
},
new object[]
{
// NetworkOffSuccess test with FailedDependency exception
Enumerable.Range(1, 7).Select(v => (ulong)v),
new[] { 1UL, 2UL, 4UL, 5UL, 6UL, 7UL },
new List<HttpStatusCode> { HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.FailedDependency, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK },
new DateTime[]
{
new DateTime(2020, 1, 1, 9, 10, 12, 10),
new DateTime(2020, 1, 1, 9, 10, 13, 10),
new DateTime(2020, 1, 1, 9, 10, 16, 10),
new DateTime(2020, 1, 1, 9, 10, 22, 10),
new DateTime(2020, 1, 1, 9, 10, 23, 10),
new DateTime(2020, 1, 1, 9, 10, 24, 10),
new DateTime(2020, 1, 1, 9, 10, 24, 15)
},
10, 6, 1, 0, 0, 0, 0, 0, 0, true
},
new object[]
{
// NetworkOnToleratedSuccess test
Enumerable.Range(1, 7).Select(v => (ulong)v),
Expand Down Expand Up @@ -90,7 +108,7 @@ class DirectMethodConnectivityReportDataWithSenderAndReceiverSource
// NetworkOnFailure test
Enumerable.Range(1, 7).Select(v => (ulong)v),
new[] { 1UL, 2UL, 3UL, 5UL, 6UL, 7UL },
new List<HttpStatusCode> { HttpStatusCode.InternalServerError, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.NotFound, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK },
new List<HttpStatusCode> { HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.InternalServerError, HttpStatusCode.NotFound, HttpStatusCode.InternalServerError, HttpStatusCode.InternalServerError, HttpStatusCode.InternalServerError },
new DateTime[]
{
new DateTime(2020, 1, 1, 9, 10, 12, 10),
Expand All @@ -101,7 +119,7 @@ class DirectMethodConnectivityReportDataWithSenderAndReceiverSource
new DateTime(2020, 1, 1, 9, 10, 24, 10),
new DateTime(2020, 1, 1, 9, 10, 24, 15)
},
10, 5, 0, 0, 0, 2, 0, 0, 0, false
10, 2, 0, 0, 0, 5, 0, 0, 0, false
},
new object[]
{
Expand Down Expand Up @@ -185,7 +203,7 @@ class DirectMethodConnectivityReportDataWithSenderAndReceiverSource
// Non-Offline test
Enumerable.Range(1, 7).Select(v => (ulong)v),
Enumerable.Range(1, 7).Select(v => (ulong)v),
new List<HttpStatusCode> { HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.NotFound, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK },
new List<HttpStatusCode> { HttpStatusCode.NotFound, HttpStatusCode.NotFound, HttpStatusCode.NotFound, HttpStatusCode.NotFound, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK },
new DateTime[]
{
new DateTime(2020, 1, 1, 9, 10, 12, 10),
Expand All @@ -196,7 +214,7 @@ class DirectMethodConnectivityReportDataWithSenderAndReceiverSource
new DateTime(2020, 1, 1, 9, 10, 24, 10),
new DateTime(2020, 1, 1, 9, 10, 24, 15)
},
10, 6, 0, 0, 0, 1, 0, 0, 0, false,
10, 3, 0, 0, 0, 4, 0, 0, 0, false,
NetworkControllerType.Satellite
},
new object[]
Expand All @@ -218,6 +236,25 @@ class DirectMethodConnectivityReportDataWithSenderAndReceiverSource
10, 6, 0, 0, 0, 0, 0, 1, 0, true,
NetworkControllerType.Satellite
},
new object[]
{
// Duplicate receiver result test
Enumerable.Range(1, 7).Select(v => (ulong)v),
new[] { 1UL, 2UL, 3UL, 3UL, 4UL, 5UL, 6UL, 7UL },
new List<HttpStatusCode> { HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK },
new DateTime[]
{
new DateTime(2020, 1, 1, 9, 10, 12, 10),
new DateTime(2020, 1, 1, 9, 10, 13, 10),
new DateTime(2020, 1, 1, 9, 10, 21, 10),
new DateTime(2020, 1, 1, 9, 10, 21, 10),
new DateTime(2020, 1, 1, 9, 10, 22, 10),
new DateTime(2020, 1, 1, 9, 10, 23, 10),
new DateTime(2020, 1, 1, 9, 10, 24, 10),
new DateTime(2020, 1, 1, 9, 10, 24, 15)
},
10, 7, 0, 0, 0, 0, 0, 0, 0, true
},
};
}
}
4 changes: 2 additions & 2 deletions ...oordinator/Reports/DirectMethod/DirectMethodConnectivityReportDataWithSenderSourceOnly.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public class DirectMethodConnectivityReportDataWithSenderSourceOnly
{
// NetworkOnFailure test
Enumerable.Range(1, 7).Select(v => (ulong)v),
new List<HttpStatusCode> { HttpStatusCode.InternalServerError, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.NotFound, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK },
new List<HttpStatusCode> { HttpStatusCode.InternalServerError, HttpStatusCode.InternalServerError, HttpStatusCode.InternalServerError, HttpStatusCode.NotFound, HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.OK },
new DateTime[]
{
new DateTime(2020, 1, 1, 9, 10, 12, 10),
Expand All @@ -95,7 +95,7 @@ public class DirectMethodConnectivityReportDataWithSenderSourceOnly
new DateTime(2020, 1, 1, 9, 10, 24, 10),
new DateTime(2020, 1, 1, 9, 10, 24, 15)
},
10, 5, 0, 0, 0, 2, 0, 0, 0, false
10, 3, 0, 0, 0, 4, 0, 0, 0, false
},
new object[]
{
Expand Down
Loading

0 comments on commit 673eb87

Please sign in to comment.