Add functional tests (minio#666)

Signed-off-by: Bala.FA <[email protected]> Signed-off-by: Bala.FA <[email protected]>
varac · Oct 27, 2022 · 0d2dfca · 0d2dfca
1 parent 4dc2cba
commit 0d2dfca
Show file tree

Hide file tree

Showing 9 changed files with 395 additions and 79 deletions.
diff --git a/cmd/kubectl-directpv/release.go b/cmd/kubectl-directpv/release.go
@@ -121,6 +121,7 @@ func validateReleaseCmd() error {
 
 func releaseMain(ctx context.Context) {
 	var processed bool
+	var failed bool
 
 	ctx, cancelFunc := context.WithCancel(ctx)
 	defer cancelFunc()
@@ -147,6 +148,7 @@ func releaseMain(ctx context.Context) {
 		default:
 			volumeCount := result.Drive.GetVolumeCount()
 			if volumeCount > 0 {
+				failed = true
 				eprintf(quietFlag, true, "%v/%v: %v volumes still exist\n", result.Drive.GetNodeID(), result.Drive.GetDriveName(), volumeCount)
 			} else {
 				result.Drive.Status.Status = directpvtypes.DriveStatusReleased
@@ -155,6 +157,7 @@ func releaseMain(ctx context.Context) {
 					_, err = client.DriveClient().Update(ctx, &result.Drive, metav1.UpdateOptions{})
 				}
 				if err != nil {
+					failed = true
 					eprintf(quietFlag, true, "%v/%v: %v\n", result.Drive.GetNodeID(), result.Drive.GetDriveName(), err)
 				} else {
 					fmt.Printf("Releasing %v/%v\n", result.Drive.GetNodeID(), result.Drive.GetDriveName())
@@ -172,4 +175,8 @@ func releaseMain(ctx context.Context) {
 
 		os.Exit(1)
 	}
+
+	if failed {
+		os.Exit(1)
+	}
 }
diff --git a/functests/common.sh b/functests/common.sh
@@ -0,0 +1,187 @@
+#!/usr/bin/env bash
+#
+# This file is part of MinIO DirectPV
+# Copyright (c) 2021, 2022 MinIO, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -ex
+
+export LV_LOOP_DEVICE=
+export VG_NAME="testvg${RANDOM}"
+export LV_DEVICE=
+export LUKS_LOOP_DEVICE=
+export LUKS_DEVICE=
+export DIRECTPV_CLIENT=
+
+# usage: create_loop <newfile> <size>
+function create_loop() {
+    truncate --size="$2" "$1"
+    sudo losetup --find "$1"
+    if [ -n "${RHEL7_TEST}" ]; then
+        sudo losetup --output NAME --associated "$1" | tail -n +2
+    else
+        sudo losetup --noheadings --output NAME --associated "$1"
+    fi
+}
+
+function setup_lvm() {
+    LV_LOOP_DEVICE=$(create_loop testpv.img 1G)
+    sudo pvcreate "${LV_LOOP_DEVICE}"
+    sudo vgcreate "${VG_NAME}" "${LV_LOOP_DEVICE}"
+    sudo lvcreate --name=testlv --extents=100%FREE "${VG_NAME}"
+    LV_DEVICE=$(basename "$(readlink -f "/dev/${VG_NAME}/testlv")")
+}
+
+function remove_lvm() {
+    sudo lvchange --quiet --activate n "${VG_NAME}/testlv"
+    sudo lvremove --quiet --yes "${VG_NAME}/testlv"
+    sudo vgremove --quiet "${VG_NAME}"
+    sudo pvremove --quiet "${LV_LOOP_DEVICE}"
+    sudo losetup --detach "${LV_LOOP_DEVICE}"
+    rm -f testpv.img
+}
+
+function setup_luks() {
+    LUKS_LOOP_DEVICE=$(create_loop testluks.img 1G)
+    echo "mylukspassword" > lukspassfile
+    yes YES | sudo cryptsetup luksFormat "${LUKS_LOOP_DEVICE}" lukspassfile
+    sudo cryptsetup luksOpen "${LUKS_LOOP_DEVICE}" myluks --key-file=lukspassfile
+    LUKS_DEVICE=$(basename "$(readlink -f /dev/mapper/myluks)")
+}
+
+function remove_luks() {
+    sudo cryptsetup luksClose myluks
+    sudo losetup --detach "${LUKS_LOOP_DEVICE}"
+    rm -f testluks.img
+}
+
+function install_directpv() {
+    "${DIRECTPV_CLIENT}" install --quiet
+
+    required_count=5
+    running_count=0
+    while [[ $running_count -lt $required_count ]]; do
+        echo "$ME: waiting for $(( required_count - running_count )) DirectPV pods to come up"
+        sleep $(( required_count - running_count ))
+        running_count=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=directpv-min-io | wc -l)
+    done
+
+    while ! "${DIRECTPV_CLIENT}" info --quiet; do
+        echo "$ME: waiting for DirectPV to come up"
+        sleep 5
+    done
+}
+
+function uninstall_directpv() {
+    "${DIRECTPV_CLIENT}" uninstall --quiet
+
+    pending=5
+    while [[ $pending -gt 0 ]]; do
+        echo "$ME: waiting for ${pending} DirectPV pods to go down"
+        sleep ${pending}
+        pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=directpv-min-io | wc -l)
+    done
+
+    while kubectl get namespace directpv-min-io --no-headers | grep -q .; do
+        echo "$ME: waiting for directpv-min-io namespace to be removed"
+        sleep 5
+    done
+
+    return 0
+}
+
+# usage: check_drives_status <status>
+function check_drives_status() {
+    status="$1"
+
+    if ! "${DIRECTPV_CLIENT}" get drives -d "${LV_DEVICE}" --no-headers | grep -q -e "${LV_DEVICE}.*${status}"; then
+        echo "$ME: error: LVM device ${LV_DEVICE} not found in ${status} state"
+        return 1
+    fi
+
+    if ! "${DIRECTPV_CLIENT}" get drives -d "${LUKS_DEVICE}" --no-headers | grep -q -e "${LUKS_DEVICE}.*${status}"; then
+        echo "$ME: error: LUKS device ${LUKS_DEVICE} not found in ${status} state"
+        return 1
+    fi
+}
+
+function export_admin_server() {
+    admin_server=$(kubectl get pods -n directpv-min-io --no-headers | awk '/^admin-server/ { print $1 }')
+    (kubectl port-forward pod/"${admin_server}" 40443:40443 -n directpv-min-io &)
+    sleep 3
+}
+
+function unexport_admin_server() {
+    pid=$(ps ax | awk '/[k]ubectl port-forward/ { print $1 }')
+    kill "${pid}"
+}
+
+function add_drives() {
+    echo -e 'ALL\nALL\nYes\n' | ./kubectl-directpv format --api-server localhost:40443 --allowed --force
+
+    # Show output for manual debugging.
+    "${DIRECTPV_CLIENT}" get drives --all
+
+    check_drives_status Ready
+}
+
+function remove_drives() {
+    ./kubectl-directpv release --all --quiet
+}
+
+function deploy_minio() {
+    kubectl apply -f functests/minio.yaml
+
+    required_count=4
+    running_count=0
+    while [[ $running_count -lt $required_count ]]; do
+        echo "$ME: waiting for $(( required_count - running_count )) minio pods to come up"
+        sleep $(( required_count - running_count ))
+        running_count=$(kubectl get pods --field-selector=status.phase=Running --no-headers | grep -c '^minio-' || true)
+    done
+}
+
+function uninstall_minio() {
+    kubectl delete -f functests/minio.yaml
+    pending=4
+    retry_count=0
+    while [[ $pending -gt 0 ]]; do
+        if [[ $retry_count -gt 50 ]]; then
+            kubectl delete pods --all --force --grace-period 0
+        fi
+        retry_count=$((retry_count + 1))
+        echo "$ME: waiting for ${pending} minio pods to go down"
+        sleep ${pending}
+        pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers | grep -c '^minio-' || true)
+    done
+
+    kubectl delete pvc --all --force
+    sleep 3
+
+    # purge all the volumes
+    "${DIRECTPV_CLIENT}" purge --all || true
+
+    while true; do
+        count=$("${DIRECTPV_CLIENT}" get volumes --all --no-headers | tee /dev/stderr | wc -l)
+        if [[ $count -eq 0 ]]; then
+            break
+        fi
+        echo "$ME: waiting for ${count} volumes to be removed"
+        sleep 3
+    done
+
+    # Show output for manual debugging.
+    "${DIRECTPV_CLIENT}" get drives --all
+}
diff --git a/functests/execute.sh b/functests/execute.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# This file is part of MinIO DirectPV
+# Copyright (c) 2021, 2022 MinIO, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+function execute() {
+    # Open /var/tmp/check.sh.xtrace for fd 3
+    exec 3>/var/tmp/check.sh.xtrace
+
+    # Set PS4 to [filename:lineno]: for tracing
+    # Save all traces to fd 3
+    # Run passed command with enabled errexit/xtrace
+    PS4='+ [${BASH_SOURCE}:${FUNCNAME[0]:+${FUNCNAME[0]}():}${LINENO}]: ' BASH_XTRACEFD=3 bash -ex "$@"
+    exit_status=$?
+
+    # Close fd 3
+    exec 3>&-
+
+    if [ "$exit_status" -ne 0 ]; then
+        echo
+        echo "xtrace:"
+        tail /var/tmp/check.sh.xtrace | tac
+    fi
+
+    return "$exit_status"
+}
+
+execute "$@"
diff --git a/functests/minio.yaml b/functests/minio.yaml
@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: minio
+  labels:
+    app: minio
+spec:
+  serviceName: "minio"
+  replicas: 4
+  selector:
+    matchLabels:
+      app: minio
+  template:
+    metadata:
+      labels:
+        app: minio
+        directpv.min.io/organization: minio
+        directpv.min.io/app: minio-example
+        directpv.min.io/tenant: tenant-1
+    spec:
+      containers:
+      - name: minio
+        image: minio/minio
+        env:
+        - name: MINIO_ACCESS_KEY
+          value: minio
+        - name: MINIO_SECRET_KEY
+          value: minio123
+        volumeMounts:
+        - name: minio-data-1
+          mountPath: /data1
+        - name: minio-data-2
+          mountPath: /data2
+        - name: minio-data-3
+          mountPath: /data3
+        - name: minio-data-4
+          mountPath: /data4
+        args:
+        - "server"
+        - "http://minio-{0...3}.minio.default.svc:9000/data{1...4}"
+  volumeClaimTemplates: # This is the specification in which you reference the StorageClass
+  - metadata:
+      name: minio-data-1
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 16Mi
+      storageClassName: directpv-min-io # This field references the existing StorageClass
+  - metadata:
+      name: minio-data-2
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 16Mi
+      storageClassName: directpv-min-io # This field references the existing StorageClass
+  - metadata:
+      name: minio-data-3
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 16Mi
+      storageClassName: directpv-min-io # This field references the existing StorageClass
+  - metadata:
+      name: minio-data-4
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 16Mi
+      storageClassName: directpv-min-io # This field references the existing StorageClass
diff --git a/functests/run.sh b/functests/run.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# This file is part of MinIO DirectPV
+# Copyright (c) 2021, 2022 MinIO, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+ME=$(basename "$0")
+export ME
+
+SCRIPT_DIR=$(dirname "$0")
+export SCRIPT_DIR
+
+if [[ $# -ne 1 ]]; then
+    echo "error: build version must be provided"
+    echo "usage: $ME <BUILD-VERSION>"
+    exit 255
+fi
+
+BUILD_VERSION="$1"
+export BUILD_VERSION
+
+"${SCRIPT_DIR}/execute.sh" "${SCRIPT_DIR}/tests.sh"