Merge branch 'master' into iiot

kiranpradeep · Oct 8, 2020 · 1ed82eb · 1ed82eb
2 parents 17c313c + 215dd42
commit 1ed82eb
Show file tree

Hide file tree

Showing 32 changed files with 5,460 additions and 204 deletions.
diff --git a/builds/e2e/nested-connectivity.yaml b/builds/e2e/nested-connectivity.yaml
@@ -0,0 +1,172 @@
+trigger: none
+pr: none
+
+variables:
+  images.artifact.name.linux: 'core-linux'
+  vsts.project: $(System.TeamProjectId)
+
+jobs:
+################################################################################
+  - job: linux_amd64_moby
+################################################################################
+    displayName: Linux AMD64 Moby
+    condition: ne(variables['agent.group'], '')
+    timeoutInMinutes: 180
+    strategy:
+      matrix:
+        Online:
+          testrun.config: 1
+          testrun.network.description: "Online"
+          testrun.network.frequencies: "00:00:00 00:00:00 0"
+          testrun.network.runProfile: "Online"
+          testrun.duration: "01:00:00"
+        Offline(5/5/5):
+          testrun.config: 2
+          testrun.network.description: "Offline(5/5/5)"
+          testrun.network.frequencies: "00:05:00 00:05:00 5"
+          testrun.network.runProfile: "Offline"
+          testrun.duration: "00:50:00"
+        Offline(10/10/3):
+          testrun.config: 3
+          testrun.network.description: "Offline(10/10/3)"
+          testrun.network.frequencies: "00:10:00 00:10:00 3"
+          testrun.network.runProfile: "Offline"
+          testrun.duration: "01:00:00"
+        Offline(15/15/3):
+          testrun.config: 4
+          testrun.network.description: "Offline(15/15/3)"
+          testrun.network.frequencies: "00:15:00 00:15:00 3"
+          testrun.network.runProfile: "Offline"
+          testrun.duration: "01:30:00"
+        Offline(30/30/2):
+          testrun.config: 5
+          testrun.network.description: "Offline(30/30/2)"
+          testrun.network.frequencies: "00:30:00 00:30:00 2"
+          testrun.network.runProfile: "Offline"
+          testrun.duration: "02:00:00"
+        Offline(5/20/3):
+          testrun.config: 6
+          testrun.network.description: "Offline(5/20/3)"
+          testrun.network.frequencies: "00:05:00 00:20:00 3"
+          testrun.network.runProfile: "Offline"
+          testrun.duration: "01:15:00"
+        SatelliteGood(60/0/1):
+          testrun.config: 7
+          testrun.network.description: "SatelliteGood(60/0/1)"
+          testrun.network.frequencies: "01:00:00 00:00:00 1"
+          testrun.network.runProfile: "SatelliteGood"
+          testrun.duration: "01:00:00"
+        Cellular3G(60/0/1):
+          testrun.config: 8
+          testrun.network.description: "Cellular3G(60/0/1)"
+          testrun.network.frequencies: "01:00:00 00:00:00 1"
+          testrun.network.runProfile: "Cellular3G"
+          testrun.duration: "01:00:00"
+    pool:
+      name: $(pool.name)
+      demands:
+        - agent-group -equals $(agent.group)
+        - Agent.OS -equals Linux
+        - Agent.OSArchitecture -equals X64
+        - run-connectivity -equals true
+        - nested-edge -equals L3
+    variables:
+      edgelet.artifact.name: 'iotedged-ubuntu16.04-amd64'
+    steps:
+      - task: Bash@3
+        name: Print_test_parameters
+        displayName: 'Print test parameters'
+        inputs:
+          targetType: inline
+          script: |
+            # print all matrix variables here
+            echo "testrun.config=$(testrun.config)"
+            echo "testrun.network.frequencies=$(testrun.network.frequencies)"
+            echo "testrun.network.runProfile=$(testrun.network.runProfile)"
+            echo "testrun.duration=$(testrun.duration)"
+            echo "deploymentFileName=$(deploymentFileName)"
+            echo "test.runtimeLogLevel=$(test.runtimeLogLevel)"
+      - template: templates/set-run-flag.yaml
+      - checkout: none
+        condition: and(succeeded(), eq(variables['run.flag'], 1))
+      - task: AzureKeyVault@1
+        condition: eq(variables['run.flag'], 1)
+        displayName: 'EdgeBuild Azure Key Vault'
+        inputs:
+          azureSubscription: $(azure.subscription)
+          KeyVaultName: edgebuildkv
+          SecretsFilter: 'edgebuilds-azurecr-io-username,edgebuilds-azurecr-io-pwd,kvLogAnalyticWorkspaceId,kvLogAnalyticSharedKey,EdgeConnectivityStorageAccountConnString'
+      - task: AzureKeyVault@1
+        condition: eq(variables['run.flag'], 1)
+        displayName: 'Nested Edge Azure Key Vault'
+        inputs:
+          azureSubscription: $(azure.subscription)
+          KeyVaultName: edgebuildkv
+          SecretsFilter: 'ConnectivityNestedEdge-ConnectionString,ConnectivityNestedEdge-EventHubConnectionString'
+      - task: DownloadBuildArtifacts@0
+        condition: and(succeeded(), eq(variables['run.flag'], 1))
+        displayName: 'Download Edgelet Artifacts'
+        inputs:
+          buildType: specific
+          project: $(vsts.project)
+          pipeline: $(edgelet.package.build)
+          branchName: $(edgelet.package.branchName)
+          buildVersionToDownload: latestFromBranch
+          downloadPath: '$(Build.StagingDirectory)'
+          artifactName: $(edgelet.artifact.name)
+      - task: DownloadBuildArtifacts@0
+        condition: and(succeeded(), eq(variables['run.flag'], 1))
+        displayName: 'Download Images Artifacts'
+        inputs:
+          buildType: specific
+          project: $(vsts.project)
+          pipeline: $(images.build)
+          branchName: $(images.branchName)
+          buildVersionToDownload: latestFromBranch
+          downloadPath: '$(Build.StagingDirectory)'
+          artifactName: $(images.artifact.name.linux)
+          itemPattern: |
+            $(images.artifact.name.linux)/IotEdgeQuickstart.linux-x64.tar.gz
+            $(images.artifact.name.linux)/e2e_deployment_files/$(deploymentFileName)
+            $(images.artifact.name.linux)/scripts/connectivityTest.sh
+            $(images.artifact.name.linux)/scripts/testHelper.sh
+            $(images.artifact.name.linux)/artifactInfo.txt
+      # Deploy connectivity test
+      - template: templates/connectivity-deploy.yaml
+        parameters:
+          release.label: 'ct$(agent.group)'
+          test.buildNumber: '$(Build.BuildNumber)'
+          test.buildId: '$(Build.BuildId)'
+          build.source.branch: '$(Build.SourceBranchName)'
+          edgelet.source.branch: '$(edgelet.package.branchName)'
+          images.source.branch: '$(images.branchName)'
+          edgelet.artifact.name: '$(edgelet.artifact.name)'
+          images.artifact.name: '$(images.artifact.name.linux)'
+          container.registry: '$(container.registry)'
+          container.registry.username: '$(edgebuilds-azurecr-io-username)'
+          container.registry.password: '$(edgebuilds-azurecr-io-pwd)'
+          iotHub.connectionString: '$(ConnectivityNestedEdge-ConnectionString)'
+          eventHub.connectionString: '$(ConnectivityNestedEdge-EventHubConnectionString)'
+          deploymentFileName: '$(deploymentFileName)'
+          upstream.protocol: '$(upstream.protocol)'
+          loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          edgeHubRestartTest.restartPeriod: '$(edgeHubRestartTest.restartPeriod)'
+          edgeHubRestartTest.sdkOperationTimeout: '$(edgeHubRestartTest.sdkOperationTimeout)'
+          testDuration: '$(testrun.duration)'
+          testStartDelay: '$(testStartDelay)'
+          networkController.frequencies: '$(testrun.network.frequencies)'
+          networkController.runProfile: '$(testrun.network.runProfile)'
+          networkController.description: '$(testrun.network.description)'
+          logAnalyticsWorkspaceId: '$(kvLogAnalyticWorkspaceId)'
+          logAnalyticsSharedKey: '$(kvLogAnalyticSharedKey)'
+          testResultCoordinator.logAnalyticsLogType: '$(testResultCoordinator.logAnalyticsLogType)'
+          testResultCoordinator.verificationDelay: '$(testResultCoordinator.verificationDelay)'
+          testResultCoordinator.storageAccountConnectionString: '$(EdgeConnectivityStorageAccountConnString)'
+          metricsCollector.metricsEndpointsCSV: '$(metricsCollector.metricsEndpointsCSV)'
+          metricsCollector.scrapeFrequencyInSecs: '$(metricsCollector.scrapeFrequencyInSecs)'
+          metricsCollector.uploadTarget: '$(metricsCollector.uploadTarget)'
+          metricsCollector.hostPlatform: 'linux_amd64_moby'
+          edgeRuntime.buildNumber: '$(edgeRuntime.buildNumber)'
+          customEdgeAgent.image: '$(customEdgeAgent.image)'
+          customEdgeHub.image: '$(customEdgeHub.image)'
+          test.runtimeLogLevel: '$(test.runtimeLogLevel)'
diff --git a/builds/e2e/nested-longhaul.yaml b/builds/e2e/nested-longhaul.yaml
@@ -0,0 +1,126 @@
+trigger: none
+pr: none
+
+variables:
+  images.artifact.name.linux: 'core-linux'
+  vsts.project: $(System.TeamProjectId)
+  # Variable defined in VSTS
+  #azure.keyVault: ''
+  #azure.subscription: ''
+  #edgelet.package.build: <edglet package build id>
+  #images.build: <images build id>
+  #pool.name: ''
+  #container.registry: ''
+  #snitch.storage.account: ''
+  # Variables settable at queue time
+  #test.startDelay.arm
+  #test.startDelay.amd
+  #edgelet.package.branchName
+  #images.branchName
+  #loadGen.message.frequency.amd64: 'xx:xx:xx.xx'
+  #loadGen.message.frequency.arm32v7: 'xx:xx:xx.xx'
+  #loadGen.message.frequency.arm64v8: 'xx:xx:xx.xx'
+  #run.linux.amd64.moby: 'true/false'
+  #run.linux.arm32v7.moby: 'true/false'
+  #run.linux.arm64v8.docker: 'true/false'
+  #longhaul.desiredModulesToRestartCSV: ''
+  #longhaul.restartIntervalInMins: ''
+  #twinTester.twinUpdateSize: ''
+  #twinTester.twinUpdateFrequency: 'xx:xx:xx.xx' 
+  #twinTester.twinUpdateFailureThreshold: 'xx:xx:xx.xx'
+  #twinTester.edgeHubRestartFailureTolerance: 'xx:xx:xx.xx'
+  #metricsCollector.metricsEndpointsCSV
+  #metricsCollector.scrapeFrequencyInSecs: 'xx:xx:xx.xx'
+  #metricsCollector.uploadTarget
+
+jobs:
+################################################################################
+  - job: linux_amd64_moby
+################################################################################
+    displayName: Linux AMD64 Moby
+    condition: ne(variables['agent.group'], '')
+    pool:
+      name: $(pool.name)
+      demands:
+        - agent-group -equals $(agent.group)
+        - Agent.OS -equals Linux
+        - Agent.OSArchitecture -equals X64
+        - run-long-haul -equals true
+        - nested-edge -equals L3
+    variables:
+      edgelet.artifact.name: 'iotedged-ubuntu16.04-amd64'
+    steps:
+      - checkout: none
+      - task: AzureKeyVault@1
+        displayName: 'Azure Key Vault'
+        inputs:
+          azureSubscription: $(azure.subscription)
+          KeyVaultName: edgebuildkv
+          SecretsFilter: 'edgebuilds-azurecr-io-username,edgebuilds-azurecr-io-pwd,StorageAccountMasterKeyStress,SnitchLongHaulAlertUrl,kvLogAnalyticWorkspaceId,kvLogAnalyticSharedKey'
+      - task: AzureKeyVault@1
+        displayName: 'Azure Key Vault'
+        inputs:
+          azureSubscription: $(azure.subscription)
+          KeyVaultName: edgebuildkv
+          SecretsFilter: 'LongHaulNestedEdge-ConnectionString,LongHaulNestedEdge-EventHubConnectionString'
+      - task: DownloadBuildArtifacts@0
+        displayName: 'Download Edgelet Artifacts'
+        inputs:
+          buildType: specific
+          project: $(vsts.project)
+          pipeline: $(edgelet.package.build)
+          branchName: $(edgelet.package.branchName)
+          buildVersionToDownload: latestFromBranch
+          downloadPath: '$(Build.StagingDirectory)'
+          artifactName: $(edgelet.artifact.name)
+      - task: DownloadBuildArtifacts@0
+        displayName: 'Download Images Artifacts'
+        inputs:
+          buildType: specific
+          project: $(vsts.project)
+          pipeline: $(images.build)
+          branchName: $(images.branchName)
+          buildVersionToDownload: latestFromBranch
+          downloadPath: '$(Build.StagingDirectory)'
+          artifactName: $(images.artifact.name.linux)
+          itemPattern: |
+            $(images.artifact.name.linux)/IotEdgeQuickstart.linux-x64.tar.gz
+            $(images.artifact.name.linux)/e2e_deployment_files/nested_long_haul_deployment.template.json
+            $(images.artifact.name.linux)/scripts/linux/runE2ETest.sh
+            $(images.artifact.name.linux)/artifactInfo.txt
+      # Deploy long haul
+      - template: templates/longhaul-deploy.yaml
+        parameters:
+          release.label: 'lh$(agent.group)'
+          test.buildNumber: '$(Build.BuildNumber)'
+          test.buildId: '$(Build.BuildId)'
+          test.startDelay: '$(test.startDelay.amd)'
+          build.source.branch: '$(Build.SourceBranchName)'
+          edgelet.source.branch: '$(edgelet.package.branchName)'
+          images.source.branch: '$(images.branchName)'
+          edgelet.artifact.name: '$(edgelet.artifact.name)'
+          images.artifact.name: '$(images.artifact.name.linux)'
+          container.registry: '$(container.registry)'
+          container.registry.username: '$(edgebuilds-azurecr-io-username)'
+          container.registry.password: '$(edgebuilds-azurecr-io-pwd)'
+          iotHub.connectionString: '$(LongHaulNestedEdge-ConnectionString)'
+          eventHub.connectionString: '$(LongHaulNestedEdge-EventHubConnectionString)'
+          snitch.build.number: '$(snitch.build.number)'
+          snitch.alert.url: '$(SnitchLongHaulAlertUrl)'
+          snitch.storage.account: '$(snitch.storage.account)'
+          snitch.storage.masterKey: '$(StorageAccountMasterKeyStress)'
+          loadGen.message.frequency: '$(loadGen.message.frequency.amd64)'
+          longHaul.desiredModulesToRestartCSV: '$(longHaul.desiredModulesToRestartCSV)'
+          longHaul.restartIntervalInMins: '$(longHaul.restartIntervalInMins)'
+          test.runtimeLogLevel: '$(test.runtimeLogLevel)'
+          analyzer.consumerGroupId: 'longhaul_linux_amd64'
+          logAnalytics.workspaceId: '$(kvLogAnalyticWorkspaceId)'
+          logAnalytics.sharedKey: '$(kvLogAnalyticSharedKey)'
+          twinTester.twinUpdateSize: '$(twinTester.twinUpdateSize)'
+          twinTester.twinUpdateFrequency: '$(twinTester.twinUpdateFrequency)'
+          twinTester.twinUpdateFailureThreshold: '$(twinTester.twinUpdateFailureThreshold)'
+          twinTester.edgeHubRestartFailureTolerance: '$(twinTester.edgeHubRestartFailureTolerance)'
+          metricsCollector.metricsEndpointsCSV: '$(metricsCollector.metricsEndpointsCSV)'
+          metricsCollector.scrapeFrequencyInSecs: '$(metricsCollector.scrapeFrequencyInSecs)'
+          metricsCollector.uploadTarget: '$(metricsCollector.uploadTarget)'
+          metricsCollector.hostPlatform: 'linux_amd64_moby'
diff --git a/builds/misc/addons-release.yaml b/builds/misc/addons-release.yaml
@@ -0,0 +1,38 @@
+name: $(version)
+jobs: 
+################################################################################
+  - job: linux_API_proxy_module
+################################################################################
+    displayName: linuxAPIProxy
+    pool:
+      vmImage: 'ubuntu-18.04'
+    steps:
+      - bash: |
+            # Need to login to EdgeBuilds Container Registry for base images.
+            docker login '$(build.registry.address)' --username '$(build.registry.user)' --password '$(build.registry.password)'
+            docker login '$(registry.address)' --username '$(registry.user)' --password '$(registry.password)'
+        displayName: 'Docker Login'
+      # Build API Proxy executable
+      - template: templates/build-api-proxy.yaml
+      # Build API Proxy Image
+      - template: templates/image-linux.yaml
+        parameters:
+          name: API Proxy
+          imageName: api-proxy-module
+          project: api-proxy-module      
+      # Check API proxy
+      - task: ComponentGovernanceComponentDetection@0
+        inputs:
+          dockerImagesToScan: '$(registry.address)/microsoft/api-proxy-module:$(Build.BuildNumber)-linux-amd64,$(registry.address)/microsoft/api-proxy-module:$(Build.BuildNumber)-linux-arm32v7,$(registry.address)/microsoft/api-proxy-module:$(Build.BuildNumber)-linux-arm64v8' 
+
+################################################################################
+  - job: manifest
+################################################################################
+    displayName: Publish Manifest Images
+    pool:
+      vmImage: 'ubuntu-18.04'
+    dependsOn:
+      - linux_API_proxy_module
+    steps:
+    - script: scripts/linux/buildManifest.sh -r $(registry.address) -u $(registry.user) -p $(registry.password) -v $(Build.BuildNumber) -t $(System.DefaultWorkingDirectory)/edge-modules/api-proxy-module/docker/manifest.yaml.template -n microsoft --tags "$(tags)"
+      displayName: 'Publish api-proxy-module Manifest'
diff --git a/builds/misc/images-release.yaml b/builds/misc/images-release.yaml
@@ -326,30 +326,6 @@ jobs:
         inputs:
           PathtoPublish: '$(Build.BinariesDirectory)/publish'
           ArtifactName: 'publish-linux'
-################################################################################
-  - job: linux_API_proxy_module
-################################################################################
-    timeoutInMinutes: 80
-    displayName: linuxAPIProxy
-    pool:
-      vmImage: 'ubuntu-16.04'
-    steps:
-      - bash: |
-            docker login '$(build.registry.address)' --username '$(build.registry.user)' --password '$(build.registry.password)'
-            docker login '$(registry.address)' --username '$(registry.user)' --password '$(registry.password)'
-        displayName: 'Docker Login'
-      # Build API Proxy executable
-      - template: templates/build-api-proxy.yaml
-      # Build API Proxy Image
-      - template: templates/image-linux.yaml
-        parameters:
-          name: API Proxy
-          imageName: api-proxy-module
-          project: api-proxy-module      
-      # Check API proxy
-      - task: ComponentGovernanceComponentDetection@0
-        inputs:
-          dockerImagesToScan: '$(registry.address)/microsoft/api-proxy-module:$(Build.BuildNumber)-linux-amd64,$(registry.address)/microsoft/api-proxy-module:$(Build.BuildNumber)-linux-arm32v7,$(registry.address)/microsoft/api-proxy-module:$(Build.BuildNumber)-linux-arm64v8' 
 
 ################################################################################
   - job: manifest
@@ -359,7 +335,6 @@ jobs:
       vmImage: 'ubuntu-16.04'
     dependsOn:
       - linux_dotnet_projects
-      - linux_API_proxy_module
     steps:
     - script: scripts/linux/buildManifest.sh -r '$(registry.address)' -u '$(registry.user)' -p '$(registry.password)' -v '$(version)' -t '$(System.DefaultWorkingDirectory)/edge-modules/iotedge-diagnostics-dotnet/docker/manifest.yaml.template' -n '$(namespace)' --tags '$(tags)'
       displayName: 'Publish azureiotedge-diagnostics Manifest'
@@ -368,6 +343,4 @@ jobs:
     - script: scripts/linux/buildManifest.sh -r '$(registry.address)' -u '$(registry.user)' -p '$(registry.password)' -v '$(version)' -t '$(System.DefaultWorkingDirectory)/edge-hub/docker/manifest.yaml.template' -n '$(namespace)' --tags '$(tags)'
       displayName: 'Publish Edge Hub Manifest'
     - script: scripts/linux/buildManifest.sh -r '$(registry.address)' -u '$(registry.user)' -p '$(registry.password)' -v '$(version)' -t '$(System.DefaultWorkingDirectory)/edge-modules/SimulatedTemperatureSensor/docker/manifest.yaml.template' -n '$(namespace)' --tags '$(tags)'
-      displayName: 'Publish Temperature Sensor Manifest'
-    - script: scripts/linux/buildManifest.sh -r $(registry.address) -u $(registry.user) -p $(registry.password) -v $(Build.BuildNumber) -t $(System.DefaultWorkingDirectory)/edge-modules/api-proxy-module/docker/manifest.yaml.template -n microsoft --tags "$(tags)"
-      displayName: 'Publish api-proxy-module Manifest'
+      displayName: 'Publish Temperature Sensor Manifest'