Update the wheel-tester with the following enhancements:

- Add Ubuntu 24.04 (Noble) to test matrix - Convert to using virtualenvs to install latest version of pip to containers to account for latest distros (AL2023, Ubuntu 24.04) not allowing system-wide pip installs - Use --prefer-binary on pip install to let the pkg manager search for the latest known binary wheel for a package if the latest is only a source release. Add a badge to the report stating what version for the tested distro and python interpretter was installed. - Run a second --dry-run pip install to detect if the installed wheel version is different from the latest release. Add a badge if the installed version and the latest package release version that supports the current interpretter differ.
geoffreyblake · Aug 13, 2024 · 33f83b7 · 33f83b7
1 parent 4b50191
commit 33f83b7
Show file tree

Hide file tree

Showing 10 changed files with 64 additions and 8 deletions.
diff --git a/test/container-script.sh b/test/container-script.sh
@@ -3,5 +3,12 @@
 set -e
 
 cd /io
-pip3 install $PIP_EXTRA_ARGS $PACKAGE_LIST
+python3 -m venv .test
+source .test/bin/activate
+pip3 install --progress-bar off --upgrade pip
+
+# Check if we will have a mismatch between latest release and wheel, by doing a dry-run
+pip3 install --dry-run --progress-bar off --report pip_latest.json $PIP_EXTRA_ARGS $PACKAGE_LIST &> dryrun_output.log
+
+pip3 install --prefer-binary --progress-bar off --report pip_binary.json $PIP_EXTRA_ARGS $PACKAGE_LIST
 python3 test-script.py
diff --git a/test/docker/Dockerfile.amazon-linux2 b/test/docker/Dockerfile.amazon-linux2
@@ -8,4 +8,4 @@ RUN yum install -y \
         mesa-libGL
 ADD https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-aarch64.sh /root/anaconda.sh
 RUN bash ~/anaconda.sh -b -p $HOME/anaconda
-RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install virtualenv
diff --git a/test/docker/Dockerfile.amazon-linux2-py38 b/test/docker/Dockerfile.amazon-linux2-py38
@@ -8,4 +8,4 @@ RUN yum install -y \
         python38-pip \
         libgomp \
         mesa-libGL
-RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install virtualenv
diff --git a/test/docker/Dockerfile.amazon-linux2023 b/test/docker/Dockerfile.amazon-linux2023
@@ -4,5 +4,6 @@ RUN yum install -y \
         "@Development tools" \
         python3-devel \
         python3-pip \
+        python3-virtualenv \
         libgomp \
         mesa-libGL
diff --git a/test/docker/Dockerfile.focal b/test/docker/Dockerfile.focal
@@ -4,8 +4,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get upgrade -y && apt-get install -y \
     python3-dev \
     python3-pip \
+    python3-venv \
     libgl1 \
     libglib2.0-0
-RUN python3 -m pip install --upgrade pip
 ADD https://repo.anaconda.com/archive/Anaconda3-2021.04-Linux-aarch64.sh /root/anaconda.sh
 RUN bash ~/anaconda.sh -b -p $HOME/anaconda
diff --git a/test/docker/Dockerfile.jammy b/test/docker/Dockerfile.jammy
@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get upgrade -y && apt-get install -y \
     python3-dev \
     python3-pip \
+    python3-venv \
     libgl1 \
     libglib2.0-0 \
     tzdata
-RUN python3 -m pip install --upgrade pip
diff --git a/test/docker/Dockerfile.noble b/test/docker/Dockerfile.noble
@@ -0,0 +1,10 @@
+FROM ubuntu:noble
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get upgrade -y && apt-get install -y \
+    python3-dev \
+    python3-pip \
+    python3-venv \
+    libgl1 \
+    libglib2.0-0 \
+    tzdata
diff --git a/test/process-results.py b/test/process-results.py
@@ -178,7 +178,7 @@ def get_package_name_class(test_name):
         return 'package-pip'
 
 def get_distribution_name(test_name):
-    distros = ["amazon-linux2", "centos8", "focal", "jammy"]
+    distros = ["amazon-linux2", "centos8", "focal", "jammy", "noble"]
     for distro in distros:
         if distro in test_name:
             return distro
@@ -383,6 +383,10 @@ def date_of_last_passing_html(wheel, test_name):
                 else:
                     html.append(make_badge(classes=['failed'], text='failed'))
                     show_output = True
+                if result["installed-version"]:
+                    html.append(make_badge(classes=['passed'], text=f"installed version {result['installed-version']}"))
+                if result["installed-version"] and (result["installed-version"] != result["latest-version"]):
+                    html.append(make_badge(classes=['warning'], text=f"latest version {result['latest-version']}"))
                 if result['build-required']:
                     html.append(make_badge(classes=['warning'], text='build required'))
                 if result['slow-install']:

diff --git a/test/setup-containers.sh b/test/setup-containers.sh
@@ -7,11 +7,11 @@ cd "$( dirname "${BASH_SOURCE[0]}" )"
 
 # fetch the latest version of each base image
 # without this step the build phase will use cached (old) versions of the base containers
-for image in 'ubuntu:focal' 'ubuntu:jammy' 'amazonlinux:2' 'amazonlinux:2023'; do
+for image in 'ubuntu:focal' 'ubuntu:jammy' 'ubuntu:noble' 'amazonlinux:2' 'amazonlinux:2023'; do
     docker pull ${image}
 done
 
-for image in 'focal' 'jammy' 'amazon-linux2' 'amazon-linux2-py38' 'amazon-linux2023'; do
+for image in 'focal' 'jammy' 'noble' 'amazon-linux2' 'amazon-linux2-py38' 'amazon-linux2023'; do
     docker build -t wheel-tester/${image} -f docker/Dockerfile.${image} .
 done
 

diff --git a/test/test-packages.py b/test/test-packages.py
@@ -19,6 +19,7 @@
 SLOW_INSTALL_TIME = 60
 TIMEOUT = 180
 
+
 def main():
     parser = argparse.ArgumentParser(description="Run wheel tests")
     parser.add_argument('--token', type=str, help="Github API token")
@@ -44,6 +45,7 @@ def get_test_set():
             'amazon-linux2': ['PIP', 'CONDA'],
             'focal': ['PIP', 'APT', 'CONDA'],
             'jammy': ['PIP', 'APT'],
+            'noble': ['PIP'],
             'amazon-linux2-py38': ['PIP'],
             'amazon-linux2023': ['PIP', 'YUM'],
         }
@@ -116,20 +118,45 @@ def get_test_set():
 
 
 process_work_dir = ''
+
+
 def do_test_initializer():
     global process_work_dir
     process_work_dir = "work_pid_%d" % os.getpid()
     os.mkdir(process_work_dir)
     subprocess.run(f'cp container-* {process_work_dir}/', shell=True)
 
+
 def do_test_lambda(x):
     return do_test(*x)
+
+
+def process_pip_report(report, package):
+    result = None
+    try:
+        f = open(report)
+        install_data = json.load(f)
+        # assume the primary package is the first one listed
+        for entry in install_data["install"]:
+            metadata = entry["metadata"]
+            if metadata["name"].lower() == package.lower():
+                result = metadata["version"]
+                break
+        f.close()
+    except Exception:
+        pass
+
+    return result
+
+
 def do_test(package_main_name, package_list, container, test_sh_script, test_py_script, test_name, package_manager):
     result = {
         'test-passed': False,
         'build-required': False,
         'binary-wheel': False,
         'slow-install': False,
+        'latest-version': None,
+        'installed-version': None,
         'timeout': False,
         'runtime': 0,
         'wheel': package_main_name,
@@ -182,6 +209,13 @@ def do_test(package_main_name, package_list, container, test_sh_script, test_py_
     if re.search(f'Downloading {package_main_name}[^\n]*aarch64[^\n]*whl', output) is not None:
         result['binary-wheel'] = True
 
+    # Primary package assumed the first listed
+    primary_package = package_list.split()[0] 
+    if latest_version := process_pip_report(f"{wd}/{process_work_dir}/pip_latest.json", primary_package):
+        result["latest-version"] = latest_version
+    if binary_version := process_pip_report(f"{wd}/{process_work_dir}/pip_binary.json", primary_package):
+        result['installed-version'] = binary_version
+
     result['output'] = output
 
     outcome = "passed" if result['test-passed'] else "failed"