Merge tag 'v1.2.1' into uffd-superbranch-3

Changed

* Upgraded Rust toolchain from 1.64.0 to 1.66.1.

Fixed

* A race condition that has been identified between the API thread and the VMM
  thread due to a misconfiguration of the `api_event_fd`.

.buildkite/common.py

@@ -0,0 +1,50 @@
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Common helpers to create Buildkite pipelines
+"""
+
+import json
+
+DEFAULT_INSTANCES = [
+    "m5d.metal",
+    "m6i.metal",
+    "m6a.metal",
+    "m6g.metal",
+]
+
+DEFAULT_PLATFORMS = [("al2", "linux_4.14"), ("al2", "linux_5.10")]
+
+DEFAULT_QUEUE = "public-prod-us-east-1"
+
+
+def group(label, command, instances, platforms, agent_tags=None, **kwargs):
+    """
+    Generate a group step with specified parameters, for each instance+kernel
+    combination
+
+    https://buildkite.com/docs/pipelines/group-step
+    """
+    if agent_tags is None:
+        agent_tags = []
+    # Use the 1st character of the group name (should be an emoji)
+    label1 = label[0]
+    steps = []
+    for instance in instances:
+        for (os, kv) in platforms:
+            agents = [f"instance={instance}", f"kv={kv}", f"os={os}"] + agent_tags
+            step = {
+                "command": command,
+                "label": f"{label1} {instance} {os} {kv}",
+                "agents": agents,
+                **kwargs,
+            }
+            steps.append(step)
+
+    return {"group": label, "steps": steps}
+
+
+def pipeline_to_json(pipeline):
+    """Serialize a pipeline dictionary to JSON"""
+    return json.dumps(pipeline, indent=4, sort_keys=True, ensure_ascii=False)

.buildkite/pipeline_pr.py

@@ -4,103 +4,90 @@
 
 """Generate Buildkite pipelines dynamically"""
 
-import json
+import subprocess
+from pathlib import Path
 
-INSTANCES = [
-    "m5d.metal",
-    "m6i.metal",
-    "m6a.metal",
-    "m6gd.metal",
-]
+from common import DEFAULT_INSTANCES, DEFAULT_PLATFORMS, DEFAULT_QUEUE, group, pipeline_to_json
 
-KERNELS = ["4.14", "5.10"]
+# Buildkite default job priority is 0. Setting this to 1 prioritizes PRs over
+# scheduled jobs and other batch jobs.
+DEFAULT_PRIORITY = 1
 
 
-def group(group_name, command, agent_tags=None, priority=0, timeout=30):
+def get_changed_files(branch):
     """
-    Generate a group step with specified parameters, for each instance+kernel
-    combination
-
-    https://buildkite.com/docs/pipelines/group-step
+    Get all files changed since `branch`
     """
-    if agent_tags is None:
-        agent_tags = []
-    # Use the 1st character of the group name (should be an emoji)
-    label1 = group_name[0]
-    steps = []
-    for instance in INSTANCES:
-        for kv in KERNELS:
-            agents = [
-                f"type={instance}",
-                f"kv={kv}",
-            ]
-            agents.extend(agent_tags)
-            step = {
-                "command": command,
-                "label": f"{label1} {instance} kv={kv}",
-                "priority": priority,
-                "timeout": timeout,
-                "agents": agents,
-            }
-            steps.append(step)
-
-    return {"group": group_name, "steps": steps}
-
-
-step_block_unless_maintainer = {
-    "block": "Waiting for approval to run",
-    "if": '(build.creator.teams includes "prod-compute-capsule") == false',
-}
+    stdout = subprocess.check_output(["git", "diff", "--name-only", branch])
+    return [Path(line) for line in stdout.decode().splitlines()]
+
 
 step_style = {
     "command": "./tools/devtool -y test -- ../tests/integration_tests/style/",
     "label": "🪶 Style",
-    # we only install the required dependencies in x86_64
-    "agents": ["platform=x86_64.metal"]
+    "priority": DEFAULT_PRIORITY,
+}
+
+defaults = {
+    "instances": DEFAULT_INSTANCES,
+    "platforms": DEFAULT_PLATFORMS,
+    # buildkite step parameters
+    "priority": DEFAULT_PRIORITY,
+    "timeout_in_minutes": 30,
 }
 
 build_grp = group(
     "📦 Build",
     "./tools/devtool -y test -- ../tests/integration_tests/build/",
-    priority=1,
+    **defaults
 )
 
 functional_1_grp = group(
     "⚙ Functional [a-n]",
     "./tools/devtool -y test -- `cd tests; ls integration_tests/functional/test_[a-n]*.py`",
-    priority=1,
+    **defaults
 )
 
 functional_2_grp = group(
     "⚙ Functional [o-z]",
     "./tools/devtool -y test -- `cd tests; ls integration_tests/functional/test_[o-z]*.py`",
-    priority=1,
+    **defaults
 )
 
 security_grp = group(
     "🔒 Security",
     "./tools/devtool -y test -- ../tests/integration_tests/security/",
-    priority=1,
+    **defaults
+)
+
+defaults_for_performance = defaults.copy()
+defaults_for_performance.update(
+    # We specify higher priority so the ag=1 jobs get picked up before the ag=n
+    # jobs in ag=1 agents
+    priority=DEFAULT_PRIORITY + 1,
+    agent_tags=["ag=1"],
 )
 
 performance_grp = group(
     "⏱ Performance",
     "./tools/devtool -y test -- ../tests/integration_tests/performance/",
-    priority=1,
-    agent_tags=["ag=1"],
+    **defaults_for_performance,
 )
 
-pipeline = {
-    "agents": {"queue": "default"},
-    "steps": [
-        step_block_unless_maintainer,
-        step_style,
+steps = [step_style]
+changed_files = get_changed_files("main")
+if any(x.suffix != ".md" for x in changed_files):
+    steps += [
         build_grp,
         functional_1_grp,
         functional_2_grp,
         security_grp,
         performance_grp,
-    ],
-}
+    ]
 
-print(json.dumps(pipeline, indent=4, sort_keys=True, ensure_ascii=False))
+pipeline = {
+    "env": {},
+    "agents": {"queue": DEFAULT_QUEUE},
+    "steps": steps,
+}
+print(pipeline_to_json(pipeline))

.gitignore

@@ -9,3 +9,4 @@ __pycache__
 .vscode
 test_results/*
 *.core
+*.profraw

.gitlint

@@ -1,5 +1,5 @@
 [title-max-length]
-line-length=50
+line-length=72
 
 [body-max-line-length]
 line-length=72

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## [1.2.1]
+
+### Changed
+
+- Upgraded Rust toolchain from 1.64.0 to 1.66.1.
+
+### Fixed
+
+- A race condition that has been identified between the API thread and the VMM
+  thread due to a misconfiguration of the `api_event_fd`.
+
 ## [1.2.0]
 
 ### Added

CONTRIBUTING.md

@@ -79,7 +79,7 @@ Your contribution needs to meet the following standards:
 - A good commit message may look like
 
   ```
-  A descriptive title of 50 characters or fewer
+  A descriptive title of 72 characters or fewer
 
   A concise description where each line is 72 characters or fewer.