Fix aks pipeline to test e2e

openshift-cloud-team · Jul 29, 2022 · 901819c · 901819c
1 parent c7f6eb1
commit 901819c
Show file tree

Hide file tree

Showing 7 changed files with 32 additions and 26 deletions.
diff --git a/.pipelines/run-autoscaling-e2e.yml b/.pipelines/run-autoscaling-e2e.yml
@@ -1,14 +1,11 @@
-pr:
+schedules:
+- cron: "0 1 * * *"
+ displayName: Daily AKS Autoscaling E2E Test
  branches:
  include:
  - master
- - release-*
- paths:
- exclude:
- - docs
- - site
- - helm
- - .github
+
+pr: none
 
 pool:
  vmImage: ubuntu-18.04
@@ -60,7 +57,7 @@ stages:
  REGISTRY_PASSWORD: $(registry.password)
  - job: create_aks_cluster_and_run_tests_job
  dependsOn: [build_push_ccm_image_job, build_push_cnm_image_job]
- timeoutInMinutes: 100
+ timeoutInMinutes: 300
  steps:
  - bash: |
  CLUSTER_TYPE="autoscaling"

diff --git a/.pipelines/run-autoscaling-multipool-e2e.yml b/.pipelines/run-autoscaling-multipool-e2e.yml
@@ -1,14 +1,11 @@
-pr:
+schedules:
+- cron: "0 5 * * *"
+ displayName: Daily AKS Autoscaling Multipool E2E Test
  branches:
  include:
  - master
- - release-*
- paths:
- exclude:
- - docs
- - site
- - helm
- - .github
+
+pr: none
 
 pool:
  vmImage: ubuntu-18.04
@@ -60,7 +57,7 @@ stages:
  REGISTRY_PASSWORD: $(registry.password)
  - job: create_aks_cluster_and_run_tests_job
  dependsOn: [build_push_ccm_image_job, build_push_cnm_image_job]
- timeoutInMinutes: 100
+ timeoutInMinutes: 300
  steps:
  - bash: |
  CLUSTER_TYPE="autoscaling-multipool"

diff --git a/.pipelines/run-basic-lb-e2e.yml b/.pipelines/run-basic-lb-e2e.yml
@@ -60,10 +60,10 @@ stages:
  REGISTRY_PASSWORD: $(registry.password)
  - job: create_aks_cluster_and_run_tests_job
  dependsOn: [build_push_ccm_image_job, build_push_cnm_image_job]
- timeoutInMinutes: 100
+ timeoutInMinutes: 300
  steps:
  - bash: |
- CCLUSTER_TYPE="basic-lb"
+ CLUSTER_TYPE="basic-lb"
  echo "##vso[task.setvariable variable=CLUSTER_TYPE]${CLUSTER_TYPE}"
  displayName: set cluster type
  - bash: |

diff --git a/.pipelines/scripts/run-e2e.sh b/.pipelines/scripts/run-e2e.sh
@@ -18,7 +18,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
+REPO_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/../..)
 export GOPATH="/home/vsts/go"
 export PATH="${PATH}:${GOPATH}/bin"
 export AKS_CLUSTER_ID="/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourcegroups/${RESOURCE_GROUP}/providers/Microsoft.ContainerService/managedClusters/${CLUSTER_NAME}"
@@ -27,7 +27,7 @@ az extension add -n aks-preview
 az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}"
 
 get_random_location() {
- local LOCATIONS=("eastus" "eastus2" "southcentralus" "westus3" "australiaeast")
+ local LOCATIONS=("eastus" "eastus2" "southcentralus" "westus3")
  echo "${LOCATIONS[${RANDOM} % ${#LOCATIONS[@]}]}"
 }
 
@@ -39,6 +39,9 @@ cleanup() {
 trap cleanup EXIT
 
 export AZURE_LOCATION="$(get_random_location)"
+if [[ "${CLUSTER_TYPE}" =~ "autoscaling" ]]; then
+ export AZURE_LOCATION="australiaeast"
+fi
 
 echo "Setting up customconfiguration.json"
 IMAGE_TAG="$(git rev-parse --short=7 HEAD)"
@@ -83,5 +86,14 @@ kubectl wait --for=condition=Ready node --all --timeout=5m
 kubectl get node -owide
 
 echo "Running e2e"
+
+# TODO: We should do it in autoscaling-multipool.json
+if [[ "${CLUSTER_TYPE}" == "autoscaling-multipool" ]]; then
+ az aks update --resource-group "${RESOURCE_GROUP}" --name "${CLUSTER_NAME}" --cluster-autoscaler-profile balance-similar-node-groups=true
+fi
+
 export E2E_ON_AKS_CLUSTER=true
+if [[ "${CLUSTER_TYPE}" =~ "autoscaling" ]]; then
+ export LABEL_FILTER="Feature:Autoscaling"
+fi
 make test-ccm-e2e
diff --git a/.pipelines/templates/autoscaling-multipool.json b/.pipelines/templates/autoscaling-multipool.json
@@ -14,7 +14,7 @@
  "agentPoolProfiles": [
  {
  "name": "agentpool1",
- "count": 2,
+ "count": 1,
  "mode" : "System",
  "vmSize": "Standard_DS2_v2",
  "osType": "Linux",
@@ -26,7 +26,7 @@
  },
  {
  "name": "agentpool2",
- "count": 2,
+ "count": 1,
  "mode" : "System",
  "vmSize": "Standard_DS2_v2",
  "osType": "Linux",

diff --git a/.pipelines/templates/autoscaling.json b/.pipelines/templates/autoscaling.json
@@ -14,7 +14,7 @@
  "agentPoolProfiles": [
  {
  "name": "agentpool1",
- "count": 2,
+ "count": 1,
  "mode" : "System",
  "vmSize": "Standard_DS2_v2",
  "osType": "Linux",

diff --git a/hack/test-ccm-e2e.sh b/hack/test-ccm-e2e.sh
@@ -23,7 +23,7 @@ LABEL_FILTER="${LABEL_FILTER:-${DEFAULT_LABEL_FILTER}}"
 
 if [[ -n "${E2E_ON_AKS_CLUSTER}" ]]; then
  AKS_CLUSTER_E2E_LABEL_FILTER="!Autoscaling && !Multi-Nodepool && !Single-Nodepool && !VMSS && !Spot-VM && !Kubenet && !Multi-Group && !AvailabilitySet && !PLS && !SLBOutbound && !ServiceAnnotation && !Credential && !Node && !LB && !Multi-Ports && !NSG"
- LABEL_FILTER="${LABEL_FILTER} && ${AKS_CLUSTER_E2E_LABEL_FILTER}"
+ LABEL_FILTER="${AKS_CLUSTER_E2E_LABEL_FILTER}"
 fi
 
 source "${REPO_ROOT}/hack/ensure-ginkgo-v2.sh"