CI: set timeout for install check

#job_Install_packages_amd64 #job_style_check
ohAitch · Mar 25, 2024 · 727aeb7 · 727aeb7
1 parent f78a66c
commit 727aeb7
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 8 deletions.
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
@@ -220,7 +220,7 @@ class JobConfig:
     digest: DigestConfig = field(default_factory=DigestConfig)
     # will be triggered for the job if omited in CI workflow yml
     run_command: str = ""
-    # job timeout
+    # job timeout, seconds
     timeout: Optional[int] = None
     # sets number of batches for multi-batch job
     num_batches: int = 1
@@ -517,10 +517,11 @@ class TestConfig:
     ),
     "run_command": 'clickbench.py "$CHECK_NAME"',
 }
-install_test_params = {
-    "digest": install_check_digest,
-    "run_command": 'install_check.py "$CHECK_NAME"',
-}
+install_test_params = JobConfig(
+    digest=install_check_digest,
+    run_command='install_check.py "$CHECK_NAME"',
+    timeout=900,
+)
 
 
 @dataclass
@@ -1105,10 +1106,10 @@ def validate(self) -> None:
     },
     test_configs={
         JobNames.INSTALL_TEST_AMD: TestConfig(
-            Build.PACKAGE_RELEASE, job_config=JobConfig(**install_test_params)  # type: ignore
+            Build.PACKAGE_RELEASE, job_config=install_test_params
         ),
         JobNames.INSTALL_TEST_ARM: TestConfig(
-            Build.PACKAGE_AARCH64, job_config=JobConfig(**install_test_params)  # type: ignore
+            Build.PACKAGE_AARCH64, job_config=install_test_params
         ),
         JobNames.STATEFUL_TEST_ASAN: TestConfig(
             Build.PACKAGE_ASAN, job_config=JobConfig(**stateful_test_common_params)  # type: ignore

diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
@@ -1,5 +1,6 @@
 from contextlib import contextmanager
 import os
+import signal
 from typing import Any, List, Union, Iterator
 from pathlib import Path
 
@@ -48,3 +49,14 @@ def print_in_group(group_name: str, lines: Union[Any, List[Any]]) -> None:
         for line in lines:
             print(line)
         print("::endgroup::")
+
+
+def set_job_timeout():
+    def timeout_handler(_signum, _frame):
+        print("Timeout expired")
+        raise TimeoutError("Job's KILL_TIMEOUT expired")
+
+    kill_timeout = int(os.getenv("KILL_TIMEOUT", "0"))
+    assert kill_timeout > 0, "kill timeout must be provided in KILL_TIMEOUT env"
+    signal.signal(signal.SIGALRM, timeout_handler)
+    signal.alarm(kill_timeout)
diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py
@@ -14,10 +14,11 @@
 
 from compress_files import compress_fast
 from docker_images_helper import DockerImage, pull_image, get_docker_image
-from env_helper import REPORT_PATH, TEMP_PATH as TEMP
+from env_helper import CI, REPORT_PATH, TEMP_PATH as TEMP
 from report import JobReport, TestResults, TestResult, FAILURE, FAIL, OK, SUCCESS
 from stopwatch import Stopwatch
 from tee_popen import TeePopen
+from ci_utils import set_job_timeout
 
 
 RPM_IMAGE = "clickhouse/install-rpm-test"
@@ -255,6 +256,9 @@ def main():
 
     args = parse_args()
 
+    if CI:
+        set_job_timeout()
+
     TEMP_PATH.mkdir(parents=True, exist_ok=True)
     LOGS_PATH.mkdir(parents=True, exist_ok=True)