Adding uberenv support for Gitlab CI (#477)

* Added .uberenv_config.json

* Added spack CachedCMakePackage in radiuss-spack-configs for Caliper

* Added .gitmodules

* Began build-and-test.sh

* Trying to fix submodules uberenv and radiuss-spack-configs

* Successfully builds caliper in a weird directory (to be fixed)

* Initial testing works with the exception of MPI testing

* Fixed 'gcc not found' error

* Minor cleanup and option checking

* Minor updates and radiuss-spack-configs commit

* Fixed CI tests failing on Quartz

* Exclude MPI tests from Lassen

* Updated prefix generation

* Updated gitlab-ci

* lassen-build-and-test-extra

* update lassen build and test extra

* update

* update

* update

* moved build-and-test.sh to gitlab folder

* update package in radiuss-spack-configs

* update testing CI

* fix

* update

* update

* update

* added build and test for other machines

* Remove extra test requirement

* Updated CI

* Fix radiuss-shared-ci include

* Update time limits

* update time allocation

* Update reading of SPEC

* Update package and change to spack v0.19.0

* Updates to radiuss-spack-configs caliper package

* Updated ROCm include

* Adding quartz to machines

* Move include of quartz build and test

* Added extra tioga spec

* Removed quartz include

* Added build-and-test-extra for ruby, corona, and tioga

* Fix compiler availability and update to caliper package

* Fix .gitlab-ci include

* Update radiuss-spack-configs and shared-ci

* Deactivate quartz (not removing .gitlab/quartz-build-and-test.yml yet).

* Update radiuss-spack-configs to define elfutils as an external

* Deactivate job mixing clang and rocm in favor of rocmcc

* Set papi as external

* Update radiuss-spack-configs to fix mpi management

* Update radiuss spack configs to Fix formatting

* Comment job using that was mixing clang and rocm

* Fix caliper package PAPI prefix

* Update radiuss spack configs with fix for intel+gcc compiler

* on Ruby intel either needs its module, or [email protected] to be set

* Update radiuss spack configs with added [email protected]

* Fix gcc 8.5.0 path

* Merge radiuss-spack-configs and radiuss-shared-ci

* Enable adiak by default

* Remove MPI from adiak on lassen

* Fix GLIB_not_found on Tioga master spec

* Update compilers in radiuss-spack-configs

* Update radiuss-spack-configs with new caliper package now requiring +tests variant in CI

* Add missing +tests in extra jobs

* Update radiuss-spack-configs

* Update radiuss-spack-configs

* Delete quartz-build-and-test.yml

* Apply suggestions from code review

* Add mpi externals for 3 main compilers, update [email protected] externals

* restrict mpi providers list

* Fix cray_mpich -> cray-mpich

* Adjust allocation time, remove quartz, run ruby jobs concurrently

* Fix prefix creation to avoid collitions

* Fix the fix

* Add an allocation delay to improve robustness

* Update radiuss-spack-configs to main ref

---------

Co-authored-by: Adrien M. BERNEDE <[email protected]>

.gitlab-ci.yml

@@ -1,53 +1,74 @@
-.build-quartz:
-  stage: build
-  tags: 
-    - quartz
-    - shell
-  script:
-    - echo "==== ${build_config} ===="
-    - mkdir build-${build_config} && cd build-${build_config}
-    - cmake -C ../cmake/hostconfig/gitlab-${build_config}.cmake ..
-    - make
-    - ctest --output-on-failure
+###############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
 
-build-quartz-gcc:
-  variables:
-    build_config: "quartz-gcc"
-  extends: .build-quartz
-  artifacts:
-    paths:
-      - build-quartz-gcc/
+# DESCRIPTION:
+###############################################################################
+# General GitLab pipelines configurations for supercomputers and Linux clusters
+# at Lawrence Livermore National Laboratory (LLNL).
+# This entire pipeline is LLNL-specific
+#
+# Important note: This file is a template provided by
+# llnl/radiuss-shared-ci. It should not require any change from the project to
+# get started but could feature project-specific stages.
+#
+# Instead, each project should provide:
+# - .gitlab/custom-jobs-and-variables.yml
+# - .gitlab/subscribed-pipelines.yml
+# - .gitlab/${MACHINE}-build-and-test-extra.yml
+###############################################################################
 
-build-quartz-intel:
-  variables:
-    build_config: "quartz-intel"
-  extends: .build-quartz
+# We define the following GitLab pipeline variables:
+variables:
+# Required information about GitHub repository
+  GITHUB_PROJECT_NAME: "Caliper"
+  GITHUB_PROJECT_ORG: "LLNL"
+# Use an LLNL service user to run CI. This prevents from running pipelines as
+# an actual user.
+  LLNL_SERVICE_USER: ""
+# Use the service user workspace. Solves permission issues, stores everything
+# at the same location whoever triggers a pipeline.
+#  CUSTOM_CI_BUILDS_DIR: ""
+# Tells Gitlab to recursively update the submodules when cloning the project.
+  GIT_SUBMODULE_STRATEGY: recursive
+# We build the projects in the CI clone directory.
+# TODO: add a clean-up mechanism
+  BUILD_ROOT: ${CI_PROJECT_DIR}
+# Set the build-and-test command.
+  BUILD_AND_TEST_CMD: "./scripts/gitlab/build-and-test.sh"
+# Override the list of branch that will skip the "draft PR test".
+# Add protected branches here. Defaults to "develop main master".
+#  ALWAYS_RUN_LIST: ""
 
-build-quartz-pgi:
-  variables:
-    build_config: "quartz-pgi"
-  extends: .build-quartz
+# We organize the build-and-test stage in sub-pipelines. Each sub-pipeline
+# corresponds to a test batch on a given machine.
 
-perftest-quartz:
-  stage: test
-  tags:
-    - quartz
-    - shell
-  script:
-    - cd build-quartz-gcc
-    - ../scripts/run-annotation-perftest.sh
-  dependencies:
-    - build-quartz-gcc
+# High level stages
+stages:
+  - machine-checks
+  - build-and-test
 
-build-butte-xlc:
-  stage: build
-  tags:
-    - butte
-    - shell
-  script:
-    - echo "==== Build butte/xlc ===="
-    - module load cmake
-    - mkdir build-butte-xlc && cd build-butte-xlc
-    - cmake -C ../cmake/hostconfig/gitlab-butte-xlc.cmake ..
-    - make
-    - ctest --output-on-failure
+# Template for jobs triggering a build-and-test sub-pipelines:
+.build-and-test:
+  stage: build-and-test
+  trigger:
+    include:
+      - local: '.gitlab/custom-jobs-and-variables.yml'
+      - project: 'radiuss/radiuss-shared-ci'
+        ref: v2023.03.1
+        file: '${CI_MACHINE}-build-and-test.yml'
+      - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml'
+    strategy: depend
+    forward:
+      pipeline_variables: true
+
+include:
+  # [Optional] checks preliminary to running the actual CI test
+  - project: 'radiuss/radiuss-shared-ci'
+    ref: v2023.03.1
+    file: 'preliminary-ignore-draft-pr.yml'
+  # pipelines subscribed by the project
+  - local: .gitlab/subscribed-pipelines.yml

.gitlab/corona-build-and-test-extra.yml

@@ -0,0 +1,22 @@
+##############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# With GitLab CI, included files cannot be empty.
+variables:
+  INCLUDED_FILE_CANNOT_BE_EMPTY: "True"
+
+# The correct way is now to use rocmcc compiler directly as done in shared spec
+### Template for extra jobs defined by the project.
+#clang14.0.6-rocm5.4.1:
+#  variables:
+#    SPEC: "%[email protected] +rocm amdgpu_target=gfx906 ^[email protected] "
+#  extends: .build_and_test_on_corona
+
+master:
+  variables:
+    SPEC: "@master +tests"
+  extends: .build_and_test_on_corona

.gitlab/custom-jobs-and-variables.yml

@@ -0,0 +1,61 @@
+###############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+# We define the following GitLab pipeline variables:
+variables:
+# On LLNL's ruby, this pipeline creates only one allocation shared among jobs
+# in order to save time and resources. This allocation has to be uniquely named
+# so that we are sure to retrieve it and avoid collisions.
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+
+# Ruby
+# Arguments for top level allocation
+  RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --partition=pdebug --time=30 --nodes=1"
+# Arguments for job level allocation
+  RUBY_BUILD_AND_TEST_JOB_ALLOC: "--overlap --time=20 --nodes=1"
+# Project specific variants for ruby
+  PROJECT_RUBY_VARIANTS: " +tests"
+# Project specific deps for ruby
+  PROJECT_RUBY_DEPS: ""
+
+# Corona
+# Arguments for top level allocation
+  CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=15m --nodes=1"
+# Arguments for job level allocation
+  CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s"
+# Project specific variants for corona
+  PROJECT_CORONA_VARIANTS: " +tests"
+# Project specific deps for corona
+  PROJECT_CORONA_DEPS: ""
+
+# Tioga
+# Arguments for top level allocation
+  TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=20m --nodes=1"
+# Arguments for job level allocation
+  TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s"
+# Project specific variants for tioga
+  PROJECT_TIOGA_VARIANTS: " +tests"
+# Project specific deps for tioga
+  PROJECT_TIOGA_DEPS: ""
+
+# Lassen and Butte use a different job scheduler (spectrum lsf) that does not
+# allow pre-allocation the same way slurm does.
+# Arguments for job level allocation
+  LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 10"
+# Project specific variants for lassen
+  PROJECT_LASSEN_VARIANTS: " +tests"
+# Project specific deps for lassen
+  PROJECT_LASSEN_DEPS: "^adiak ~mpi"
+
+
+# Configuration shared by build and test jobs specific to this project.
+# Not all configuration can be shared. Here projects can fine tune the
+# CI behavior.
+# See Umpire for an example (export junit test reports).
+.custom_build_and_test:
+  variables:
+    JOB_TEMPLATE_CANNOT_BE_EMPTY: "True"

.gitlab/lassen-build-and-test-extra.yml

@@ -0,0 +1,16 @@
+##############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# With GitLab CI, included files cannot be empty.
+variables:
+  INCLUDED_FILE_CANNOT_BE_EMPTY: "True"
+
+## Template for extra jobs defined by the project.
+master:
+  variables:
+    SPEC: "@master +tests"
+  extends: .build_and_test_on_lassen

.gitlab/ruby-build-and-test-extra.yml

@@ -0,0 +1,16 @@
+##############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# With GitLab CI, included files cannot be empty.
+variables:
+  INCLUDED_FILE_CANNOT_BE_EMPTY: "True"
+
+## Template for extra jobs defined by the project.
+master:
+  variables:
+    SPEC: "@master +tests"
+  extends: .build_and_test_on_ruby

.gitlab/subscribed-pipelines.yml

@@ -0,0 +1,92 @@
+###############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+# The template job to test whether a machine is up.
+# Expects CI_MACHINE defined to machine name.
+.machine-check:
+  stage: machine-checks
+  tags: [shell, oslic]
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - |
+      if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]]
+      then
+        echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m"
+        curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \
+             --header 'Content-Type: application/json' \
+             --header "authorization: Bearer ${GITHUB_TOKEN}" \
+             --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }"
+        exit 1
+      fi
+
+###
+# Trigger a build-and-test pipeline for a machine.
+# Comment the jobs for machines you don’t need.
+###
+
+# RUBY
+ruby-up-check:
+  variables:
+    CI_MACHINE: "ruby"
+  extends: [.machine-check]
+
+ruby-build-and-test:
+  variables:
+    CI_MACHINE: "ruby"
+  needs: [ruby-up-check]
+  extends: [.build-and-test]
+
+# Quartz it very similar to ruby, let’s deactivate it for now
+## QUARTZ
+#quartz-up-check:
+#  variables:
+#    CI_MACHINE: "QUARTZ"
+#  extends: [.machine-check]
+#
+#quartz-build-and-test:
+#  variables:
+#    CI_MACHINE: "quartz"
+#  needs: [quartz-up-check]
+#  extends: [.build-and-test]
+
+# CORONA
+corona-up-check:
+  variables:
+    CI_MACHINE: "corona"
+  extends: [.machine-check]
+
+corona-build-and-test:
+  variables:
+    CI_MACHINE: "corona"
+  needs: [corona-up-check]
+  extends: [.build-and-test]
+
+# TIOGA
+tioga-up-check:
+  variables:
+    CI_MACHINE: "tioga"
+  extends: [.machine-check]
+
+tioga-build-and-test:
+  variables:
+    CI_MACHINE: "tioga"
+  needs: [tioga-up-check]
+  extends: [.build-and-test]
+
+# LASSEN
+lassen-up-check:
+  variables:
+    CI_MACHINE: "lassen"
+  extends: [.machine-check]
+
+lassen-build-and-test:
+  variables:
+    CI_MACHINE: "lassen"
+  needs: [lassen-up-check]
+  extends: [.build-and-test]
+

.gitlab/tioga-build-and-test-extra.yml

@@ -0,0 +1,22 @@
+##############################################################################
+# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# With GitLab CI, included files cannot be empty.
+variables:
+  INCLUDED_FILE_CANNOT_BE_EMPTY: "True"
+
+# The correct way is now to use rocmcc compiler directly as done in shared spec
+### Template for extra jobs defined by the project.
+#clang13.0.1-rocm5.4.1:
+#  variables:
+#    SPEC: "%[email protected] +rocm amdgpu_target=gfx90a ^[email protected]"
+#  extends: .build_and_test_on_tioga
+
+master:
+  variables:
+    SPEC: "@master +tests %[email protected]"
+  extends: .build_and_test_on_tioga

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "scripts/uberenv"]
+	path = scripts/uberenv
+	url = https://github.com/LLNL/uberenv.git
+[submodule "scripts/radiuss-spack-configs"]
+	path = scripts/radiuss-spack-configs
+	url = https://github.com/LLNL/radiuss-spack-configs.git

.uberenv_config.json

@@ -0,0 +1,14 @@
+{
+    "package_name" : "caliper",
+    "package_version" : "develop",
+    "package_source_dir" : "../..",
+    "package_final_phase" : "initconfig",
+    "spack_url": "https://github.com/spack/spack.git",
+    "spack_branch": "e4s-23.02",
+    "spack_activate" : {},
+    "spack_configs_path": "scripts/radiuss-spack-configs",
+    "spack_packages_path": "scripts/radiuss-spack-configs/packages",
+    "spack_concretizer": "clingo",
+    "spack_setup_clingo": false,
+    "debug": true
+    }