Skip to content

Commit

Permalink
Avoid adding duplicate secrets in the daemon set spec
Browse files Browse the repository at this point in the history
  • Loading branch information
@shivamerla
shivamerla committed Nov 7, 2023
1 parent 911c9e4 commit e67adb4
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 56 deletions.
92 changes: 38 additions & 54 deletions controllers/object_controls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -774,9 +774,7 @@ func TransformGPUDiscoveryPlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPol

// set image pull secrets
if len(config.GPUFeatureDiscovery.ImagePullSecrets) > 0 {
for _, secret := range config.GPUFeatureDiscovery.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.GPUFeatureDiscovery.ImagePullSecrets)
}

// set resource limits
Expand Down Expand Up @@ -1076,10 +1074,9 @@ func TransformToolkit(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n

// set image pull secrets
if len(config.Toolkit.ImagePullSecrets) > 0 {
for _, secret := range config.Toolkit.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.Toolkit.ImagePullSecrets)
}

// set resource limits
if config.Toolkit.Resources != nil {
// apply resource limits to all containers
Expand Down Expand Up @@ -1209,12 +1206,12 @@ func TransformDevicePlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe

// update image pull policy
obj.Spec.Template.Spec.Containers[0].ImagePullPolicy = gpuv1.ImagePullPolicy(config.DevicePlugin.ImagePullPolicy)

// set image pull secrets
if len(config.DevicePlugin.ImagePullSecrets) > 0 {
for _, secret := range config.DevicePlugin.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.DevicePlugin.ImagePullSecrets)
}

// set resource limits
if config.DevicePlugin.Resources != nil {
// apply resource limits to all containers
Expand Down Expand Up @@ -1282,9 +1279,7 @@ func TransformSandboxDevicePlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPo
obj.Spec.Template.Spec.Containers[0].ImagePullPolicy = gpuv1.ImagePullPolicy(config.SandboxDevicePlugin.ImagePullPolicy)
// set image pull secrets
if len(config.SandboxDevicePlugin.ImagePullSecrets) > 0 {
for _, secret := range config.SandboxDevicePlugin.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.SandboxDevicePlugin.ImagePullSecrets)
}
// set resource limits
if config.SandboxDevicePlugin.Resources != nil {
Expand Down Expand Up @@ -1326,9 +1321,7 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
obj.Spec.Template.Spec.Containers[0].ImagePullPolicy = gpuv1.ImagePullPolicy(config.DCGMExporter.ImagePullPolicy)
// set image pull secrets
if len(config.DCGMExporter.ImagePullSecrets) > 0 {
for _, secret := range config.DCGMExporter.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.DCGMExporter.ImagePullSecrets)
}
// set resource limits
if config.DCGMExporter.Resources != nil {
Expand Down Expand Up @@ -1463,9 +1456,7 @@ func TransformDCGM(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n Clu
obj.Spec.Template.Spec.Containers[0].ImagePullPolicy = gpuv1.ImagePullPolicy(config.DCGM.ImagePullPolicy)
// set image pull secrets
if len(config.DCGM.ImagePullSecrets) > 0 {
for _, secret := range config.DCGM.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.DCGM.ImagePullSecrets)
}
// set resource limits
if config.DCGM.Resources != nil {
Expand Down Expand Up @@ -1522,9 +1513,7 @@ func TransformMIGManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec,

// set image pull secrets
if len(config.MIGManager.ImagePullSecrets) > 0 {
for _, secret := range config.MIGManager.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.MIGManager.ImagePullSecrets)
}

// set resource limits
Expand Down Expand Up @@ -1601,9 +1590,7 @@ func TransformKataManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec

// set image pull secrets
if len(config.KataManager.ImagePullSecrets) > 0 {
for _, secret := range config.KataManager.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.KataManager.ImagePullSecrets)
}

// set resource limits
Expand Down Expand Up @@ -1711,9 +1698,7 @@ func TransformVFIOManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec

// set image pull secrets
if len(config.VFIOManager.ImagePullSecrets) > 0 {
for _, secret := range config.VFIOManager.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.VFIOManager.ImagePullSecrets)
}

// set resource limits
Expand Down Expand Up @@ -1754,9 +1739,7 @@ func TransformCCManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec,

// set image pull secrets
if len(config.CCManager.ImagePullSecrets) > 0 {
for _, secret := range config.CCManager.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.CCManager.ImagePullSecrets)
}

// set resource limits
Expand Down Expand Up @@ -1807,9 +1790,7 @@ func TransformVGPUDeviceManager(obj *appsv1.DaemonSet, config *gpuv1.ClusterPoli

// set image pull secrets
if len(config.VGPUDeviceManager.ImagePullSecrets) > 0 {
for _, secret := range config.VGPUDeviceManager.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.VGPUDeviceManager.ImagePullSecrets)
}

// set resource limits
Expand Down Expand Up @@ -1932,9 +1913,7 @@ func TransformValidatorShared(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
obj.Spec.Template.Spec.Containers[0].ImagePullPolicy = gpuv1.ImagePullPolicy(config.Validator.ImagePullPolicy)
// set image pull secrets
if len(config.Validator.ImagePullSecrets) > 0 {
for _, secret := range config.Validator.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.Validator.ImagePullSecrets)
}
// set resource limits
if config.Validator.Resources != nil {
Expand Down Expand Up @@ -2096,9 +2075,7 @@ func TransformNodeStatusExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPol

// set image pull secrets
if len(config.NodeStatusExporter.ImagePullSecrets) > 0 {
for _, secret := range config.NodeStatusExporter.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.NodeStatusExporter.ImagePullSecrets)
}

// set resource limits
Expand Down Expand Up @@ -2413,9 +2390,7 @@ func transformDriverManagerInitContainer(obj *appsv1.DaemonSet, driverManagerSpe

// add any pull secrets needed for driver-manager image
if len(driverManagerSpec.ImagePullSecrets) > 0 {
for _, secret := range driverManagerSpec.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, driverManagerSpec.ImagePullSecrets)
}

return nil
Expand Down Expand Up @@ -2494,9 +2469,7 @@ func transformGDSContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe

// set image pull secrets
if len(config.GPUDirectStorage.ImagePullSecrets) > 0 {
for _, secret := range config.GPUDirectStorage.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.GPUDirectStorage.ImagePullSecrets)
}

// set/append environment variables for GDS container
Expand Down Expand Up @@ -2848,9 +2821,7 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy

// set image pull secrets
if len(config.Driver.ImagePullSecrets) > 0 {
for _, secret := range config.Driver.ImagePullSecrets {
podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.Driver.ImagePullSecrets)
}
// set resource limits
if config.Driver.Resources != nil {
Expand Down Expand Up @@ -3077,9 +3048,7 @@ func transformVGPUManagerContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterP

// set image pull secrets
if len(config.VGPUManager.ImagePullSecrets) > 0 {
for _, secret := range config.VGPUManager.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.VGPUManager.ImagePullSecrets)
}
// set resource limits
if config.VGPUManager.Resources != nil {
Expand Down Expand Up @@ -3194,13 +3163,28 @@ func transformValidationInitContainer(obj *appsv1.DaemonSet, config *gpuv1.Clust
}
// add any pull secrets needed for validation image
if len(config.Validator.ImagePullSecrets) > 0 {
for _, secret := range config.Validator.ImagePullSecrets {
obj.Spec.Template.Spec.ImagePullSecrets = append(obj.Spec.Template.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
addPullSecrets(&obj.Spec.Template.Spec, config.Validator.ImagePullSecrets)
}
return nil
}

func addPullSecrets(podSpec *v1.PodSpec, secrets []string) {
for _, secret := range secrets {
if !containsSecret(podSpec.ImagePullSecrets, secret) {
podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, v1.LocalObjectReference{Name: secret})
}
}
}

func containsSecret(secrets []v1.LocalObjectReference, secretName string) bool {
for _, s := range secrets {
if s.Name == secretName {
return true
}
}
return false
}

func isDeploymentReady(name string, n ClusterPolicyController) gpuv1.State {
opts := []client.ListOption{
client.MatchingLabels{"app": name},
Expand Down
18 changes: 16 additions & 2 deletions controllers/object_controls_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,10 +476,12 @@ func getDriverTestInput(testCase string) *gpuv1.ClusterPolicy {
cp.Spec.Driver.Repository = "nvcr.io/nvidia"
cp.Spec.Driver.Image = "driver"
cp.Spec.Driver.Version = "470.57.02"
cp.Spec.Driver.ImagePullSecrets = []string{"ngc-secret"}

cp.Spec.Driver.Manager.Repository = "nvcr.io/nvidia/cloud-native"
cp.Spec.Driver.Manager.Image = "k8s-driver-manager"
cp.Spec.Driver.Manager.Version = "test"
cp.Spec.Driver.Manager.ImagePullSecrets = []string{"ngc-secret"}

cp.Spec.Driver.StartupProbe = &gpuv1.ContainerProbeSpec{InitialDelaySeconds: 20, PeriodSeconds: 5, FailureThreshold: 1, TimeoutSeconds: 60}

Expand All @@ -505,6 +507,7 @@ func getDriverTestOutput(testCase string) map[string]interface{} {
"mofedValidationPresent": false,
"nvPeerMemPresent": false,
"driverManagerImage": "nvcr.io/nvidia/cloud-native/k8s-driver-manager:test",
"imagePullSecret": "ngc-secret",
}

switch testCase {
Expand Down Expand Up @@ -575,6 +578,8 @@ func TestDriver(t *testing.T) {
require.Equal(t, tc.output["nvPeerMemPresent"], nvPeerMemPresent, "Unexpected configuration for nv-peermem container")
require.Equal(t, tc.output["driverImage"], driverImage, "Unexpected configuration for nvidia-driver-ctr image")
require.Equal(t, tc.output["driverManagerImage"], driverManagerImage, "Unexpected configuration for k8s-driver-manager image")
require.Equal(t, len(ds.Spec.Template.Spec.ImagePullSecrets), 1, "Incorrect number of imagePullSecrets in the daemon set spec")
require.Equal(t, tc.output["imagePullSecret"], ds.Spec.Template.Spec.ImagePullSecrets[0].Name, "Incorrect imagePullSecret in the daemon set spec")

// cleanup by deleting all kubernetes objects
err = removeState(&clusterPolicyController, clusterPolicyController.idx-1)
Expand All @@ -596,10 +601,12 @@ func getDevicePluginTestInput(testCase string) *gpuv1.ClusterPolicy {
cp.Spec.DevicePlugin.Repository = "nvcr.io/nvidia"
cp.Spec.DevicePlugin.Image = "k8s-device-plugin"
cp.Spec.DevicePlugin.Version = "v0.12.0-ubi8"
cp.Spec.DevicePlugin.ImagePullSecrets = []string{"ngc-secret"}

cp.Spec.Validator.Repository = "nvcr.io/nvidia/cloud-native"
cp.Spec.Validator.Image = "gpu-operator-validator"
cp.Spec.Validator.Version = "v1.11.0"
cp.Spec.Validator.ImagePullSecrets = []string{"ngc-secret"}

switch testCase {
case "default":
Expand All @@ -622,6 +629,7 @@ func getDevicePluginTestOutput(testCase string) map[string]interface{} {
"configManagerInitPresent": false,
"configManagerSidecarPresent": false,
"devicePluginImage": "nvcr.io/nvidia/k8s-device-plugin:v0.12.0-ubi8",
"imagePullSecret": "ngc-secret",
}

switch testCase {
Expand Down Expand Up @@ -730,6 +738,8 @@ func getVGPUManagerTestInput(testCase string) *gpuv1.ClusterPolicy {
cp.Spec.VGPUManager.DriverManager.Repository = "nvcr.io/nvidia/cloud-native"
cp.Spec.VGPUManager.DriverManager.Image = "k8s-driver-manager"
cp.Spec.VGPUManager.DriverManager.Version = "v0.3.0"
cp.Spec.VGPUManager.ImagePullSecrets = []string{"ngc-secret"}
cp.Spec.VGPUManager.DriverManager.ImagePullSecrets = []string{"ngc-secret"}
clusterPolicyController.sandboxEnabled = true

switch testCase {
Expand All @@ -750,6 +760,7 @@ func getVGPUManagerTestOutput(testCase string) map[string]interface{} {
"numDaemonsets": 1,
"driverImage": "nvcr.io/nvidia/vgpu-manager:470.57.02-ubuntu22.04",
"driverManagerImage": "nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.3.0",
"imagePullSecret": "ngc-secret",
}

switch testCase {
Expand Down Expand Up @@ -837,10 +848,12 @@ func getSandboxDevicePluginTestInput(testCase string) *gpuv1.ClusterPolicy {
cp.Spec.SandboxDevicePlugin.Image = "kubevirt-device-plugin"
cp.Spec.SandboxDevicePlugin.Version = "v1.1.0"
clusterPolicyController.sandboxEnabled = true
cp.Spec.SandboxDevicePlugin.ImagePullSecrets = []string{"ngc-secret"}

cp.Spec.Validator.Repository = "nvcr.io/nvidia/cloud-native"
cp.Spec.Validator.Image = "gpu-operator-validator"
cp.Spec.Validator.Version = "v1.11.0"
cp.Spec.Validator.ImagePullSecrets = []string{"ngc-secret"}

switch testCase {
case "default":
Expand All @@ -857,8 +870,9 @@ func getSandboxDevicePluginTestInput(testCase string) *gpuv1.ClusterPolicy {
func getSandboxDevicePluginTestOutput(testCase string) map[string]interface{} {
// default output
output := map[string]interface{}{
"numDaemonsets": 1,
"image": "nvcr.io/nvidia/kubevirt-device-plugin:v1.1.0",
"numDaemonsets": 1,
"image": "nvcr.io/nvidia/kubevirt-device-plugin:v1.1.0",
"imagePullSecret": "ngc-secret",
}

switch testCase {
Expand Down

0 comments on commit e67adb4

Please sign in to comment.