Implement MicroShift RPM upgrade tests on a non-ostree system

test/bin/build_images.sh

@@ -259,7 +259,7 @@ do_group() {
 
     if ${BUILD_INSTALLER}; then
         for image_installer in "${groupdir}"/*.image-installer; do
-            blueprint=$(cat "${image_installer}")
+            blueprint=$(envsubst < "${image_installer}")
             echo "Building image-installer from ${blueprint}"
             buildid=$(sudo composer-cli compose start \
                            "${blueprint}" \

test/bin/create_local_repo.sh

@@ -29,7 +29,7 @@ make_repo() {
     createrepo "${repodir}"
 
     echo "Fixing permissions of RPM repo contents"
-    find "${repodir}" -type f -print -exec chmod a+r  {} \;
+    find "${repodir}" -type f -exec chmod a+r  {} \;
     find "${repodir}" -type d -exec chmod a+rx {} \;
 }
 

test/bin/scenario.sh

@@ -106,21 +106,42 @@ function get_vm_ip {
 
 # Try to login to the host via ssh until the connection is accepted
 wait_for_ssh() {
-    local ip="${1}"
+    local -r ip="${1}"
 
     echo "Waiting ${VM_BOOT_TIMEOUT} for ssh access to ${ip}"
-    timeout "${VM_BOOT_TIMEOUT}" bash -c "until ssh -oBatchMode=yes -oStrictHostKeyChecking=accept-new redhat@${ip} 'echo host is up'; do date; sleep 5; done"
+
+    local -r start_time=$(date +%s)
+    while [ $(( $(date +%s) - start_time )) -lt "${VM_BOOT_TIMEOUT}" ] ; do
+        if ssh -oConnectTimeout=10 -oBatchMode=yes -oStrictHostKeyChecking=accept-new "redhat@${ip}" "echo host is up" ; then
+            return 0
+        fi
+        date
+        sleep 5
+    done
+    # Return an error if non of the ssh attempts succeeded
+    return 1
 }
 
 # Wait for greenboot health check to complete, without checking the results
 wait_for_greenboot() {
-    local vmname="${1}"
-    local ip="${2}"
+    local -r vmname="${1}"
+    local -r ip="${2}"
 
     echo "Waiting ${VM_BOOT_TIMEOUT} for greenboot on ${vmname} to complete"
-    timeout "${VM_BOOT_TIMEOUT}" bash -c "until ssh redhat@${ip} \"sudo journalctl -n 5 -u greenboot-healthcheck; sudo systemctl status greenboot-healthcheck | grep 'active (exited)'\"; do date; sleep 10; done"
-}
 
+    local -r start_time=$(date +%s)
+    while [ $(( $(date +%s) - start_time )) -lt "${VM_BOOT_TIMEOUT}" ] ; do
+        if ssh -oConnectTimeout=10 -oBatchMode=yes -oStrictHostKeyChecking=accept-new "redhat@${ip}" \
+                "sudo journalctl -n 5 -u greenboot-healthcheck; \
+                 systemctl show --property=SubState --value greenboot-healthcheck | grep -w exited" ; then
+            return 0
+        fi
+        date
+        sleep 10
+    done
+    # Return an error if non of the ssh attempts succeeded
+    return 1
+}
 
 start_junit() {
     local outputfile="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/junit.xml"

test/image-blueprints/group2/rhel92-microshift-previous-minor.image-installer

@@ -0,0 +1 @@
+rhel-9.2-microshift-4.${PREVIOUS_MINOR_VERSION}

test/kickstart-templates/kickstart-liveimg.ks.template

@@ -0,0 +1,106 @@
+lang en_US.UTF-8
+keyboard us
+timezone UTC
+text
+reboot
+
+# Configure network to use DHCP and activate on boot
+network --bootproto=dhcp --device=link --activate --onboot=on --hostname=REPLACE_HOST_NAME
+
+# Partition disk with a 1GB boot XFS partition and an LVM volume containing a 10GB+ system root
+# The remainder of the volume will be used by the CSI driver for storing data
+#
+# For example, a 20GB disk would be partitioned in the following way:
+#
+# NAME          MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
+# sda             8:0    0  20G  0 disk
+# ├─sda1          8:1    0 200M  0 part /boot/efi
+# ├─sda1          8:1    0 800M  0 part /boot
+# └─sda2          8:2    0  19G  0 part
+#  └─rhel-root  253:0    0  10G  0 lvm  /sysroot
+#
+zerombr
+clearpart --all --initlabel
+part /boot/efi --fstype=efi --size=200
+part /boot --fstype=xfs --asprimary --size=800
+# Uncomment this line to add a SWAP partition of the recommended size
+#part swap --fstype=swap --recommended
+part pv.01 --grow
+volgroup rhel pv.01
+logvol / --vgname=rhel --fstype=xfs --size=REPLACE_LVM_SYSROOT_SIZE --name=root
+
+# Lock root user account
+rootpw --lock
+
+# Configure a non-ostree system
+liveimg --url file:///run/install/repo/liveimg.tar.gz
+
+%post --log=/var/log/anaconda/post-install.log --erroronfail
+
+# Set the configuration of MicroShift to include subjectAltNames
+mkdir -p /etc/microshift
+cat - >/etc/microshift/config.yaml <<EOF
+apiServer:
+  subjectAltNames:
+EOF
+
+DEFAULT_ROUTE_IP="$(ip route show default 2>/dev/null | head -1 | awk '{print $9}')"
+if [ -n "${DEFAULT_ROUTE_IP}" ] ; then
+    echo "    - ${DEFAULT_ROUTE_IP}" >>/etc/microshift/config.yaml
+else
+    MAIN_HOST_IP="$(hostname -I | awk '{print $1}')"
+    echo "    - ${MAIN_HOST_IP}" >>/etc/microshift/config.yaml
+fi
+if [ -n "REPLACE_PUBLIC_IP" ]; then
+    echo "    - REPLACE_PUBLIC_IP" >>/etc/microshift/config.yaml
+fi
+
+# The pull secret is mandatory for MicroShift builds on top of OpenShift, but not OKD
+# The /etc/crio/crio.conf.d/microshift.conf references the /etc/crio/openshift-pull-secret file
+mkdir -p /etc/crio
+cat > /etc/crio/openshift-pull-secret <<EOF
+REPLACE_PULL_SECRET
+EOF
+chmod 600 /etc/crio/openshift-pull-secret
+
+# Create a default redhat user, allowing it to run sudo commands without password
+useradd -m -d /home/redhat -p \$5\$XDVQ6DxT8S5YWLV7\$8f2om5JfjK56v9ofUkUAwZXTxJl3Sqnc9yPnza4xoJ0 redhat
+echo -e 'redhat\tALL=(ALL)\tNOPASSWD: ALL' > /etc/sudoers.d/microshift
+
+# # Add authorized ssh keys
+mkdir -m 700 /home/redhat/.ssh
+cat >> /home/redhat/.ssh/authorized_keys <<EOF
+REPLACE_REDHAT_AUTHORIZED_KEYS
+EOF
+chmod 600 /home/redhat/.ssh/authorized_keys
+
+# Make sure redhat user directory contents ownership is correct
+chown -R redhat:redhat /home/redhat/
+
+# Make the KUBECONFIG from MicroShift directly available for the root user
+echo -e 'export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig' >> /root/.profile
+
+# Configure systemd journal service to persist logs between boots and limit their size to 1G
+sudo mkdir -p /etc/systemd/journald.conf.d
+cat > /etc/systemd/journald.conf.d/microshift.conf <<EOF
+[Journal]
+Storage=persistent
+SystemMaxUse=1G
+RuntimeMaxUse=1G
+EOF
+
+# Make sure all the Ethernet network interfaces are connected automatically
+for uuid in $(nmcli -f uuid,type,autoconnect connection | awk '$2 == "ethernet" && $3 == "no" {print $1}') ; do
+    # Remove autoconnect option from the configuration file to keep it enabled after reboot
+    file=$(nmcli -f uuid,filename connection | awk -v uuid=${uuid} '$1 == uuid' | sed "s/${uuid}//g" | xargs)
+    sed -i '/autoconnect=.*/d' "${file}"
+done
+
+# Disable a timer that sets boot_success to 1 after two minutes from a user login.
+# It impacts greenboot checks because grub script decrementing boot_counter works
+# only if boot_success is 0 (see /etc/grub.d/08_fallback_counting). In case of a
+# user login, this results in more than requested amount of red-boot-induced reboots
+# and system needing much more time to roll back.
+ln -sf /dev/null /etc/systemd/user/grub-boot-success.timer
+
+%end

test/resources/microshift-rpm.resource

@@ -0,0 +1,54 @@
+*** Settings ***
+Documentation       Keywords for working with the MicroShift on a non-ostree system
+
+Resource            common.resource
+Library             SSHLibrary
+
+
+*** Variables ***
+${REPO_FILE_NAME}       "/etc/yum.repos.d/microshift-local.repo"
+
+
+*** Keywords ***
+Install MicroShift RPM Packages
+    [Documentation]    Installs MicroShift RPM packages from the specified URL
+    [Arguments]    ${repo_url}
+
+    ${repo_file_contents}=    Catenate    SEPARATOR=\n
+    ...    [microshift-local]
+    ...    name=Microshift Local Repository
+    ...    baseurl=${repo_url}
+    ...    enabled=1
+    ...    gpgcheck=0
+
+    Upload String To File    ${repo_file_contents}    ${REPO_FILE_NAME}
+    Update Dnf Cache
+
+    ${stdout}    ${stderr}    ${rc}=    SSHLibrary.Execute Command
+    ...    dnf install -y microshift
+    ...    sudo=True    return_rc=True    return_stderr=True    return_stdout=True
+    Log    ${stderr}
+    Should Be Equal As Integers    0    ${rc}
+
+    ${rc}=    SSHLibrary.Execute Command    rm -f "${REPO_FILE_NAME}"
+    ...    sudo=True    return_rc=True    return_stderr=False    return_stdout=False
+    Should Be Equal As Integers    0    ${rc}
+
+    Update Dnf Cache
+
+Uninstall MicroShift RPM Packages
+    [Documentation]    Uninstalls MicroShift RPM packages
+
+    ${stdout}    ${stderr}    ${rc}=    SSHLibrary.Execute Command
+    ...    dnf remove -y --exclude=microshift-test-agent microshift\*
+    ...    sudo=True    return_rc=True    return_stderr=True    return_stdout=True
+    Log    ${stderr}
+    Should Be Equal As Integers    0    ${rc}
+
+Update Dnf Cache
+    [Documentation]    Updates dnf cache
+
+    ${stderr}    ${rc}=    SSHLibrary.Execute Command    dnf makecache
+    ...    sudo=True    return_rc=True    return_stderr=True    return_stdout=False
+    Log    ${stderr}
+    Should Be Equal As Integers    0    ${rc}

test/scenarios/el92-prel@upgrade-rpm-ok.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Sourced from scenario.sh and uses functions defined there.
+
+scenario_create_vms() {
+    prepare_kickstart host1 kickstart-liveimg.ks.template ""
+    launch_vm host1 "rhel-9.2-microshift-4.$(previous_minor_version)"
+}
+
+scenario_remove_vms() {
+    remove_vm host1
+}
+
+scenario_run_tests() {
+    local -r reponame=$(basename "${LOCAL_REPO}")
+    run_tests host1 \
+        --variable "TARGET_REPO_URL:${WEB_SERVER_URL}/rpm-repos/${reponame}" \
+        suites/upgrade/upgrade-rpm-successful.robot
+}

test/suites/upgrade/upgrade-rpm-successful.robot

@@ -0,0 +1,52 @@
+*** Settings ***
+Documentation       Tests related to upgrading MicroShift on a non-ostree system
+
+Resource            ../../resources/common.resource
+Resource            ../../resources/microshift-rpm.resource
+Resource            ../../resources/microshift-host.resource
+Resource            ../../resources/microshift-process.resource
+Resource            ../../resources/ostree-data.resource
+Resource            ../../resources/ostree-health.resource
+Library             Collections
+
+Suite Setup         Setup
+Suite Teardown      Teardown
+
+Test Tags           rpm-based-system
+
+
+*** Variables ***
+${TARGET_REPO_URL}      ${EMPTY}
+
+
+*** Test Cases ***
+Upgrade
+    [Documentation]    Performs an upgrade from a given RPM repository URL
+    ...    and checks if it was successful
+
+    # Health of the system is implicitly checked by greenboot successful exit
+    MicroShift 413 Should Not Have Upgrade Artifacts
+    Wait Until Greenboot Health Check Exited
+
+    # Upgrade MicroShift and reboot
+    Install MicroShift RPM Packages    ${TARGET_REPO_URL}
+    Reboot MicroShift Host
+
+    # Health of the system is implicitly checked by greenboot successful exit
+    Wait Until Greenboot Health Check Exited
+
+
+*** Keywords ***
+Setup
+    [Documentation]    Test suite setup
+    Check Required Env Variables
+    Should Not Be Empty    ${TARGET_REPO_URL}    TARGET_REPO_URL variable is required
+    Login MicroShift Host
+
+    # Make sure we run on a non-ostree system
+    ${is_ostree}=    Is System OSTree
+    Should Not Be True    ${is_ostree}
+
+Teardown
+    [Documentation]    Test suite teardown
+    Logout MicroShift Host