Merge pull request #5 from chen-mao/xqdev

Support for running xdxct-container-runtime on Kylin server systems
chen-mao · Oct 24, 2023 · 605f554 · 605f554
2 parents e41d323 + b3ba803
commit 605f554
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 22 deletions.
diff --git a/mk/Dockerfile.centos b/mk/Dockerfile.centos
@@ -3,14 +3,21 @@ FROM ${BASEIMAGE}
 
 SHELL ["/bin/bash", "-c"]
 
+RUN sed -e 's|^mirrorlist=|#mirrorlist=|g' \
+        -e 's|^#baseurl=http://mirror.centos.org/centos|baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos|g' \
+        -i.bak \
+        /etc/yum.repos.d/CentOS-*.repo
+RUN yum makecache
+
 ARG OS_VERSION
 RUN if [ "${OS_VERSION}" = "8" ]; then \
         yum --enablerepo=powertools install -y \
             rpcgen \
             libseccomp-devel; \
     else \
         yum install -y \
-            libseccomp-devel; \
+            libseccomp-devel \
+            applydeltarpm; \
     fi
 
 RUN yum install -y \
@@ -48,15 +55,6 @@ ENV WITH_LIBELF=${WITH_LIBELF}
 ENV WITH_TIRPC=${WITH_TIRPC}
 ENV WITH_SECCOMP=${WITH_SECCOMP}
 
-RUN if [ "$WITH_LIBELF" = "no" ]; then \
-        arch=${OS_ARCH} && \
-        cd $(mktemp -d) && \
-        curl -fsSL -O https://mirrors.kernel.org/mageia/distrib/7.1/${arch}/media/core/release/pmake-1.45-17.mga7.${arch}.rpm && \
-        curl -fsSL -O https://mirrors.kernel.org/mageia/distrib/7.1/${arch}/media/core/release/bmake-20181221-1.mga7.${arch}.rpm && \
-        rpm -i *.rpm && \
-        rm -rf $PWD \
-    ; fi
-
 WORKDIR /tmp/libxdxct-container
 COPY . .
 

diff --git a/mk/docker.mk b/mk/docker.mk
@@ -126,6 +126,7 @@ docker-amd64-verify: $(patsubst %, %-verify, $(AMD64_TARGETS)) \
 # private centos target with overrides
 --centos%: OS := centos
 --centos%: WITH_TIRPC = yes
+--centos%: WITH_LIBELF = yes
 --centos8%: BASEIMAGE = quay.io/centos/centos:stream8
 
 # private opensuse-leap target with overrides

diff --git a/src/common.h b/src/common.h
@@ -49,7 +49,7 @@
 #elif defined(__aarch64__)
 # define LIB_ARCH                 LD_AARCH64_LIB64
 # define LIB32_ARCH               LD_ARM_LIBHF
-# define USR_LIB_MULTIARCH_DIR    "/usr/lib/aarch64-linux-gnu/"
+# define XDX_LIB_MULTIARCH_DIR    "/usr/lib/aarch64-linux-gnu/"
 # define USR_LIB32_MULTIARCH_DIR  "/var/empty"
 # if !defined(__NR_execveat)
 #  define __NR_execveat 387

diff --git a/src/nvc.h b/src/nvc.h
@@ -115,9 +115,10 @@ struct nvc_device_info {
 struct nvc_container_config {
         pid_t pid;
         char *rootfs;
-        char *bins_dir;
-        char *libs_dir;
-        char *libs32_dir;
+        char *bins_dir;       // The bins mount point in container
+        char *libs_dir;       // The libs mount point in container
+        char *libs32_dir;     // The libs32 mount point in container
+        char *dris_dir;       // The dri's path in Host 
         char *cudart_dir;
         char *ldconfig;
 };

diff --git a/src/nvc_container.c b/src/nvc_container.c
@@ -3,6 +3,8 @@
  */
 
 #include <sys/types.h>
+#include <sys/utsname.h>
+#include <sys/stat.h>
 
 #include <inttypes.h>
 #include <limits.h>
@@ -124,8 +126,11 @@ copy_config(struct error *err, struct nvc_container *cnt, const struct nvc_conta
         const char *bins_dir = cfg->bins_dir;
         const char *libs_dir = cfg->libs_dir;
         const char *libs32_dir = cfg->libs32_dir;
+        const char *dris_dir = cfg->dris_dir;
         const char *cudart_dir = cfg->cudart_dir;
         const char *ldconfig = cfg->ldconfig;
+        struct utsname systemInfo;
+        struct stat st;
         char *rootfs;
         int multiarch, ret;
         int rv = -1;
@@ -186,6 +191,29 @@ copy_config(struct error *err, struct nvc_container *cnt, const struct nvc_conta
                         }
                 }
         }
+        if (dris_dir == NULL) {
+                // process the path of dris according to the type of the host.
+                if (uname(&systemInfo) != 0) {
+                        error_set(err, "uname");
+                        goto fail;
+                }
+                if (str_has_suffix(systemInfo.machine, "x86_64") && \
+                    stat("/etc/apt", &st) == 0 && S_ISDIR(st.st_mode)) {
+                        dris_dir = XDX_LIB_DRI_X86_DEB;
+                }
+                else if (str_has_suffix(systemInfo.machine, "aarch64") && \
+                    stat("/etc/apt", &st) == 0 && S_ISDIR(st.st_mode)) {
+                        dris_dir = XDX_LIB_DRI_ARM_DEB;
+                }    
+                else if (str_has_suffix(systemInfo.machine, "x86_64") && \
+                    stat("/etc/rpm", &st) == 0 && S_ISDIR(st.st_mode)) {
+                        dris_dir = XDX_LIB_DRI_X86_RPM;
+                }           
+                else {
+                        log_errf("Unsupported architecture %s and system version.\n", systemInfo.machine);
+                        goto fail;
+                }
+        }
         if (cudart_dir == NULL)
                 cudart_dir = CUDA_RUNTIME_DIR;
         if (ldconfig == NULL) {
@@ -206,6 +234,8 @@ copy_config(struct error *err, struct nvc_container *cnt, const struct nvc_conta
                 goto fail;
         if ((cnt->cfg.libs32_dir = xstrdup(err, libs32_dir)) == NULL)
                 goto fail;
+        if ((cnt->cfg.dris_dir = xstrdup(err, dris_dir)) == NULL)
+                goto fail;
         if ((cnt->cfg.cudart_dir = xstrdup(err, cudart_dir)) == NULL)
                 goto fail;
         if ((cnt->cfg.ldconfig = xstrdup(err, ldconfig)) == NULL)
@@ -225,7 +255,7 @@ nvc_container_new(struct nvc_context *ctx, const struct nvc_container_config *cf
 
         if (validate_context(ctx) < 0)
                 return (NULL);
-        if (validate_args(ctx, cfg != NULL && cfg->pid > 0 && cfg->rootfs != NULL && !str_empty(cfg->rootfs) && cfg->rootfs[0] == '/' &&
+        if (validate_args(ctx, cfg != NULL && cfg->pid > 0 && cfg->rootfs != NULL && !str_empty(cfg->rootfs) && cfg->rootfs[0] == '/' && !str_empty(cfg->dris_dir) &&
             !str_empty(cfg->bins_dir) && !str_empty(cfg->libs_dir) && !str_empty(cfg->libs32_dir) && !str_empty(cfg->cudart_dir) && !str_empty(cfg->ldconfig)) < 0)
                 return (NULL);
         if (opts == NULL)
@@ -265,6 +295,7 @@ nvc_container_new(struct nvc_context *ctx, const struct nvc_container_config *cf
         log_infof("setting bins directory to %s", cnt->cfg.bins_dir);
         log_infof("setting libs directory to %s", cnt->cfg.libs_dir);
         log_infof("setting libs32 directory to %s", cnt->cfg.libs32_dir);
+        log_infof("setting dris directory to %s", cnt->cfg.dris_dir);
         log_infof("setting cudart directory to %s", cnt->cfg.cudart_dir);
         log_infof("setting ldconfig to %s%s", cnt->cfg.ldconfig, (cnt->cfg.ldconfig[0] == '@') ? " (host relative)" : "");
         log_infof("setting mount namespace to %s", cnt->mnt_ns);
@@ -288,6 +319,7 @@ nvc_container_free(struct nvc_container *cnt)
         free(cnt->cfg.bins_dir);
         free(cnt->cfg.libs_dir);
         free(cnt->cfg.libs32_dir);
+        free(cnt->cfg.dris_dir);
         free(cnt->cfg.cudart_dir);
         free(cnt->cfg.ldconfig);
         free(cnt->mnt_ns);

diff --git a/src/nvc_info.c b/src/nvc_info.c
@@ -314,8 +314,6 @@ lookup_libraries(struct error *err, struct dxcore_context *dxcore, struct nvc_dr
         ptr = array_append(ptr, compute_libs, nitems(compute_libs));
         ptr = array_append(ptr, video_libs, nitems(video_libs));
         ptr = array_append(ptr, graphics_libs, nitems(graphics_libs));
-        // TODO
-        // log_infof("****239:The root path is %s", root);
 
         if (dxcore->initialized)
                 ptr = array_append(ptr, dxcore_libs, nitems(dxcore_libs));

diff --git a/src/nvc_internal.h b/src/nvc_internal.h
@@ -47,8 +47,10 @@
 #define NV_APP_PROFILE_DIR       "/etc/nvidia/nvidia-application-profiles-rc.d"
 #define NV_CAPS_MIG_MINORS_PATH  NV_CAPS_PROC_DRIVER "/mig-minors"
 // #define XDX_SYS_CLASS_DRM        "/sys/class/drm"
-#define XDX_LIB_DRI              "/usr/lib/x86_64-linux-gnu/dri"
-#define XDX_VIDEO_LIB_NUM        10
+#define XDX_LIB_DRI_X86_DEB      "/usr/lib/x86_64-linux-gnu/dri"
+#define XDX_LIB_DRI_ARM_DEB      "/usr/lib/aarch64-linux-gnu/dri"
+#define XDX_LIB_DRI_X86_RPM      "/usr/lib64/dri"
+#define XDX_VIDEO_LIB_NUM        4
 #define LD_CONFIG_DIR            "/etc/ld.so.conf.d"
 #define SMI_FRONT_DIR            "/usr/lib/python3/dist-packages/xdxsmi"
 #define NV_PROC_DRIVER_CAPS    NV_PROC_DRIVER "/capabilities"

diff --git a/src/nvc_mount.c b/src/nvc_mount.c
@@ -866,11 +866,11 @@ device_mount_native(struct nvc_context *ctx, const struct nvc_container *cnt, co
         char *dri_libs[XDX_VIDEO_LIB_NUM];
         size_t nlibs;
 
-        if (library_search_in_dir(&ctx->err, dri_libs, XDX_LIB_DRI, pattern, &nlibs) < 0) {
+        if (library_search_in_dir(&ctx->err, dri_libs, cnt->cfg.dris_dir, pattern, &nlibs) < 0) {
                 log_errf("%s", ctx->err.msg);
                 return -1;
         }
-        
+
         int rv = -1;
         if (!(cnt->flags & OPT_NO_DEVBIND)) {
                 if ((dev_mnt = mount_device(&ctx->err, ctx->cfg.root, cnt, &dev->node)) == NULL)
@@ -890,9 +890,11 @@ device_mount_native(struct nvc_context *ctx, const struct nvc_container *cnt, co
                 free(tmp);
         }
         if (cnt->flags & OPT_VIDEO_LIBS || cnt->flags & OPT_GRAPHICS_LIBS) {
-                if ((tmp = (const char **)mount_files(&ctx->err, ctx->cfg.root, cnt, XDX_LIB_DRI, dri_libs, nlibs)) == NULL){
+                if ((tmp = (const char **)mount_files(&ctx->err, ctx->cfg.root, cnt, cnt->cfg.dris_dir, dri_libs, nlibs)) == NULL){
                         goto fail;
                 }
+                for (size_t i = 0; i < nlibs; ++i)
+                        free(dri_libs[i]);
                 free(tmp);
         }
         if (!(cnt->flags & OPT_NO_CGROUPS)) {