check ocp proxy TrustedCA before rendering configMap mounts

tian-men · Sep 18, 2023 · 03b4ac4 · 03b4ac4
1 parent 898efac
commit 03b4ac4
Show file tree

Hide file tree

Showing 8 changed files with 58 additions and 74 deletions.
diff --git a/controllers/clusterinfo/clusterinfo.go b/controllers/clusterinfo/clusterinfo.go
@@ -20,10 +20,10 @@ import (
 	"context"
 	"fmt"
 	"maps"
-	"sort"
 	"strings"
 
-	configv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
+	configv1 "github.com/openshift/api/config/v1"
+	ocpconfigv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
 	imagesv1 "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,7 +34,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
-	gpuv1 "github.com/NVIDIA/gpu-operator/api/v1"
 	"github.com/NVIDIA/gpu-operator/internal/consts"
 )
 
@@ -44,7 +43,7 @@ type Interface interface {
 	GetOpenshiftVersion() (string, error)
 	GetRHCOSVersions(map[string]string) ([]string, error)
 	GetOpenshiftDriverToolkitImages() map[string]string
-	GetOpenshiftProxyEnvars() ([]gpuv1.EnvVar, error)
+	GetOpenshiftProxySpec() (*configv1.ProxySpec, error)
 	GetKernelVersions(map[string]string) ([]string, error)
 }
 
@@ -63,7 +62,7 @@ type clusterInfo struct {
 	rhcosVersions                []string
 	openshiftDriverToolkitImages map[string]string
 	kernelVersions               []string
-	proxyEnvars                  []gpuv1.EnvVar
+	proxySpec                    *configv1.ProxySpec
 }
 
 // New creates a new instance of clusterinfo API
@@ -219,12 +218,12 @@ func (l *clusterInfo) GetKernelVersions(labelSelector map[string]string) ([]stri
 	return getKernelVersions(l.ctx, l.config, labelSelector)
 }
 
-func (l *clusterInfo) GetOpenshiftProxyEnvars() ([]gpuv1.EnvVar, error) {
+func (l *clusterInfo) GetOpenshiftProxySpec() (*configv1.ProxySpec, error) {
 	if l.oneshot {
-		return l.proxyEnvars, nil
+		return l.proxySpec, nil
 	}
 
-	return getOpenshiftProxyEnvars(l.ctx, l.config)
+	return getOpenshiftProxySpec(l.ctx, l.config)
 }
 
 func getKubernetesVersion(config *rest.Config) (string, error) {
@@ -242,7 +241,7 @@ func getKubernetesVersion(config *rest.Config) (string, error) {
 }
 
 func getOpenshiftVersion(ctx context.Context, config *rest.Config) (string, error) {
-	client, err := configv1.NewForConfig(config)
+	client, err := ocpconfigv1.NewForConfig(config)
 	if err != nil {
 		return "", err
 	}
@@ -355,51 +354,18 @@ func getKernelVersions(ctx context.Context, config *rest.Config, selector map[st
 	return kernelVersions, nil
 }
 
-func getOpenshiftProxyEnvars(ctx context.Context, cfg *rest.Config) ([]gpuv1.EnvVar, error) {
-	var proxyEnvars []gpuv1.EnvVar
+func getOpenshiftProxySpec(ctx context.Context, cfg *rest.Config) (*configv1.ProxySpec, error) {
 	logger := log.FromContext(ctx)
 
-	client, err := configv1.NewForConfig(cfg)
+	client, err := ocpconfigv1.NewForConfig(cfg)
 	if err != nil {
 		logger.Error(err, "error instantiating openshift config client")
 	}
 
 	proxy, err := client.Proxies().Get(ctx, "cluster", metav1.GetOptions{})
 	if err != nil {
 		logger.Error(err, "error retrieving proxies for openshift cluster")
+		return nil, err
 	}
-
-	if proxy == nil {
-		return nil, nil
-	}
-
-	proxies := map[string]string{
-		"HTTPS_PROXY": proxy.Spec.HTTPSProxy,
-		"HTTP_PROXY":  proxy.Spec.HTTPProxy,
-		"NO_PROXY":    proxy.Spec.NoProxy,
-	}
-	var envs []string
-	for k := range proxies {
-		envs = append(envs, k)
-	}
-	// ensure ordering is preserved when we add these env to pod spec
-	sort.Strings(envs)
-
-	for _, e := range envs {
-		v := proxies[e]
-		if len(v) == 0 {
-			continue
-		}
-		upperCaseEnvvar := gpuv1.EnvVar{
-			Name:  strings.ToUpper(e),
-			Value: v,
-		}
-		lowerCaseEnvvar := gpuv1.EnvVar{
-			Name:  strings.ToLower(e),
-			Value: v,
-		}
-		proxyEnvars = append(proxyEnvars, upperCaseEnvvar, lowerCaseEnvvar)
-	}
-
-	return proxyEnvars, nil
+	return &proxy.Spec, nil
 }
diff --git a/internal/state/driver.go b/internal/state/driver.go
@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strings"
 
+	configv1 "github.com/openshift/api/config/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -59,7 +60,7 @@ type driverRuntimeSpec struct {
 	OpenshiftDriverToolkitEnabled bool
 	OpenshiftRHCOSVersions        []string
 	OpenshiftDriverToolkitImages  map[string]string
-	OpenshiftProxyEnvars          []gpuv1.EnvVar
+	OpenshiftProxySpec            *configv1.ProxySpec
 	KernelVersions                []string
 }
 
@@ -385,11 +386,11 @@ func getRuntimeSpec(info clusterinfo.Interface, spec *nvidiav1alpha1.NVIDIADrive
 	// running on an Openshift cluster and precompiled drivers are disabled.
 	if openshiftVersion != "" && !spec.UsePrecompiledDrivers() {
 
-		openshiftProxyEnvars, err := info.GetOpenshiftProxyEnvars()
+		openshiftProxySpec, err := info.GetOpenshiftProxySpec()
 		if err != nil {
 			return nil, fmt.Errorf("failed to retrieve proxy settings for openshift cluster: %w", err)
 		}
-		rs.OpenshiftProxyEnvars = openshiftProxyEnvars
+		rs.OpenshiftProxySpec = openshiftProxySpec
 
 		rhcosVersions, err := info.GetRHCOSVersions(spec.NodeSelector)
 		if err != nil {

diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go
@@ -24,6 +24,7 @@ import (
 	"strings"
 	"testing"
 
+	configv1 "github.com/openshift/api/config/v1"
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -392,18 +393,12 @@ func TestDriverOpenshiftDriverToolkit(t *testing.T) {
 	}
 	renderData.Runtime.OpenshiftDriverToolkitEnabled = true
 	renderData.Runtime.OpenshiftVersion = "4.13"
-	renderData.Runtime.OpenshiftProxyEnvars = []gpuv1.EnvVar{
-		{
-			Name:  "HTTP_PROXY",
-			Value: "http://user:pass@example:8080",
-		},
-		{
-			Name:  "HTTPS_PROXY",
-			Value: "https://user:pass@example:8085",
-		},
-		{
-			Name:  "NO_PROXY",
-			Value: "internal.example.com",
+	renderData.Runtime.OpenshiftProxySpec = &configv1.ProxySpec{
+		HTTPProxy:  "http://user:pass@example:8080",
+		HTTPSProxy: "https://user:pass@example:8085",
+		NoProxy:    "internal.example.com",
+		TrustedCA: configv1.ConfigMapNameReference{
+			Name: "gpu-operator-trusted-ca",
 		},
 	}
 

diff --git a/internal/state/golden/driver-openshift-drivertoolkit.yaml b/internal/state/golden/driver-openshift-drivertoolkit.yaml
@@ -163,6 +163,8 @@ data:
   ca-bundle.crt: ""
 kind: ConfigMap
 metadata:
+  labels:
+    config.openshift.io/inject-trusted-cabundle: "true"
   name: gpu-operator-trusted-ca
   namespace: test-operator
 ---

diff --git a/internal/state/testdata/0420_proxycfg.openshift.yaml b/internal/state/testdata/0420_proxycfg.openshift.yaml
@@ -1,9 +1,11 @@
-{{ if and (.Openshift) (.Runtime.OpenshiftProxyEnvars)  }}
+{{ if and (.Openshift) (.Runtime.OpenshiftProxySpec.TrustedCA) (.Runtime.OpenshiftProxySpec.TrustedCA.Name) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: gpu-operator-trusted-ca
   namespace: {{ .Runtime.Namespace }}
+  labels:
+    config.openshift.io/inject-trusted-cabundle: "true"
 data:
   ca-bundle.crt: ""
 {{ end }}
diff --git a/internal/state/testdata/0500_daemonset.yaml b/internal/state/testdata/0500_daemonset.yaml
@@ -223,10 +223,18 @@ spec:
         - name: RHCOS_VERSION
           value: {{ .Openshift.RHCOSVersion }}
       {{- end }}
-      {{- if and (.Openshift) (.Runtime.OpenshiftProxyEnvars) }}
-      {{- range .Runtime.OpenshiftProxyEnvars }}
-        - name: {{ .Name }}
-          value : {{ .Value }}
+      {{- if and (.Openshift) (.Runtime.OpenshiftProxySpec) }}
+        {{- if .Runtime.OpenshiftProxySpec.HTTPProxy }}
+        - name: "HTTP_PROXY"
+          value : {{ .Runtime.OpenshiftProxySpec.HTTPProxy | quote }}
+        {{- end }}
+        {{- if .Runtime.OpenshiftProxySpec.HTTPSProxy }}
+        - name: "HTTPS_PROXY"
+          value : {{ .Runtime.OpenshiftProxySpec.HTTPSProxy | quote }}
+        {{- end }}
+        {{- if .Runtime.OpenshiftProxySpec.NoProxy }}
+        - name: "NO_PROXY"
+          value : {{ .Runtime.OpenshiftProxySpec.NoProxy | quote }}
         {{- end }}
       {{- end }}
         volumeMounts:
@@ -260,7 +268,7 @@ spec:
           - name: shared-nvidia-driver-toolkit
             mountPath: /mnt/shared-nvidia-driver-toolkit
           {{- end}}
-          {{- if and (.Openshift) (.Runtime.OpenshiftProxyEnvars) }}
+          {{- if and (.Openshift) (.Runtime.OpenshiftProxySpec.TrustedCA) (.Runtime.OpenshiftProxySpec.TrustedCA.Name) }}
           - name: gpu-operator-trusted-ca
             mountPath: /etc/pki/ca-trust/extracted/pem
             readOnly: true
@@ -499,7 +507,7 @@ spec:
         - name: shared-nvidia-driver-toolkit
           emptyDir: {}
         {{- end }}
-        {{- if and (.Openshift) (.Runtime.OpenshiftProxyEnvars) }}
+        {{- if and (.Openshift) (.Runtime.OpenshiftProxySpec.TrustedCA) (.Runtime.OpenshiftProxySpec.TrustedCA.Name) }}
         - name: gpu-operator-trusted-ca
           configMap: 
             name: gpu-operator-trusted-ca

diff --git a/manifests/state-driver/0420_proxycfg.openshift.yaml b/manifests/state-driver/0420_proxycfg.openshift.yaml
@@ -4,6 +4,8 @@ kind: ConfigMap
 metadata:
   name: gpu-operator-trusted-ca
   namespace: {{ .Runtime.Namespace }}
+  labels:
+    config.openshift.io/inject-trusted-cabundle: "true"
 data:
   ca-bundle.crt: ""
 {{ end }}
diff --git a/manifests/state-driver/0500_daemonset.yaml b/manifests/state-driver/0500_daemonset.yaml
@@ -223,10 +223,18 @@ spec:
         - name: RHCOS_VERSION
           value: {{ .Openshift.RHCOSVersion }}
       {{- end }}
-      {{- if and (.Openshift) (.Runtime.OpenshiftProxyEnvars) }}
-      {{- range .Runtime.OpenshiftProxyEnvars }}
-        - name: {{ .Name }}
-          value : {{ .Value }}
+      {{- if and (.Openshift) (.Runtime.OpenshiftProxySpec) }}
+        {{- if .Runtime.OpenshiftProxySpec.HTTPProxy }}
+        - name: "HTTP_PROXY"
+          value : {{ .Runtime.OpenshiftProxySpec.HTTPProxy | quote }}
+        {{- end }}
+        {{- if .Runtime.OpenshiftProxySpec.HTTPSProxy }}
+        - name: "HTTPS_PROXY"
+          value : {{ .Runtime.OpenshiftProxySpec.HTTPSProxy | quote }}
+        {{- end }}
+        {{- if .Runtime.OpenshiftProxySpec.NoProxy }}
+        - name: "NO_PROXY"
+          value : {{ .Runtime.OpenshiftProxySpec.NoProxy | quote }}
         {{- end }}
       {{- end }}
         volumeMounts:
@@ -260,7 +268,7 @@ spec:
           - name: shared-nvidia-driver-toolkit
             mountPath: /mnt/shared-nvidia-driver-toolkit
           {{- end}}
-          {{- if and (.Openshift) (.Runtime.OpenshiftProxyEnvars) }}
+          {{- if and (.Openshift) (.Runtime.OpenshiftProxySpec.TrustedCA) (.Runtime.OpenshiftProxySpec.TrustedCA.Name) }}
           - name: gpu-operator-trusted-ca
             mountPath: /etc/pki/ca-trust/extracted/pem
             readOnly: true
@@ -499,7 +507,7 @@ spec:
         - name: shared-nvidia-driver-toolkit
           emptyDir: {}
         {{- end }}
-        {{- if and (.Openshift) (.Runtime.OpenshiftProxyEnvars) }}
+        {{- if and (.Openshift) (.Runtime.OpenshiftProxySpec.TrustedCA) (.Runtime.OpenshiftProxySpec.TrustedCA.Name) }}
         - name: gpu-operator-trusted-ca
           configMap: 
             name: gpu-operator-trusted-ca