Skip to content

Commit

Permalink
Fix debug workflow not work when pod in external cluster (koderover#2541
Browse files Browse the repository at this point in the history 
)

* wip

Signed-off-by: M-Cosmosss <[email protected]>

* wip

Signed-off-by: M-Cosmosss <[email protected]>

* wip

Signed-off-by: M-Cosmosss <[email protected]>

---------

Signed-off-by: M-Cosmosss <[email protected]>
Co-authored-by: M-Cosmosss <[email protected]>
  • Loading branch information
@M-Cosmosss
M-Cosmosss and M-Cosmosss authored Apr 20, 2023
1 parent 098113c commit 697e5bd
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 13 deletions.
55 changes: 45 additions & 10 deletions pkg/microservice/aslan/core/workflow/service/workflow/workflow_task_v4.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ import (
"github.com/koderover/zadig/pkg/microservice/user/core"
"github.com/koderover/zadig/pkg/microservice/user/core/repository/orm"
"github.com/koderover/zadig/pkg/setting"
kubeclient "github.com/koderover/zadig/pkg/shared/kube/client"
e "github.com/koderover/zadig/pkg/tool/errors"
krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client"
"github.com/koderover/zadig/pkg/tool/kube/getter"
"github.com/koderover/zadig/pkg/tool/kube/podexec"
larktool "github.com/koderover/zadig/pkg/tool/lark"
Expand Down Expand Up @@ -557,16 +557,34 @@ FOR:
logger.Infof("set workflowTaskV4 breakpoint success: %s-%s %v", jobName, position, set)
return nil
}

jobTaskSpec := &commonmodels.JobTaskFreestyleSpec{}
if err := commonmodels.IToi(task.Spec, jobTaskSpec); err != nil {
logger.Errorf("set workflowTaskV4 breakpoint failed: IToi %v", err)
return e.ErrSetBreakpoint.AddDesc("修改断点意外失败: convert job task spec")
}

kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), jobTaskSpec.Properties.ClusterID)
if err != nil {
log.Errorf("set workflowTaskV4 breakpoint failed: get kube client error: %s", err)
return e.ErrSetBreakpoint.AddDesc("修改断点意外失败: get kube client")
}
clientSet, err := kubeclient.GetClientset(config.HubServerAddress(), jobTaskSpec.Properties.ClusterID)
if err != nil {
log.Errorf("set workflowTaskV4 breakpoint failed: get kube client set error: %s", err)
return e.ErrSetBreakpoint.AddDesc("修改断点意外失败: get kube client set")
}
restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), jobTaskSpec.Properties.ClusterID)
if err != nil {
log.Errorf("set workflowTaskV4 breakpoint failed: get kube rest config error: %s", err)
return e.ErrSetBreakpoint.AddDesc("修改断点意外失败: get kube rest config")
}

// job task is running, check whether shell step has run, and touch breakpoint file
// if job task status is debug_after, only breakpoint operation can do is unset breakpoint_after, which should be done by StopDebugWorkflowTaskJobV4
// if job task status is prepare, setting breakpoints has a low probability of not taking effect, and the current design allows for this flaw
if task.Status == config.StatusRunning || task.Status == config.StatusDebugBefore || task.Status == config.StatusPrepare {
jobTaskSpec := &commonmodels.JobTaskFreestyleSpec{}
if err := commonmodels.IToi(task.Spec, jobTaskSpec); err != nil {
logger.Errorf("set workflowTaskV4 breakpoint failed: IToi %v", err)
return e.ErrSetBreakpoint.AddDesc("修改断点意外失败")
}
pods, err := getter.ListPods(jobTaskSpec.Properties.Namespace, labels.Set{"job-name": task.K8sJobName}.AsSelector(), krkubeclient.Client())
pods, err := getter.ListPods(jobTaskSpec.Properties.Namespace, labels.Set{"job-name": task.K8sJobName}.AsSelector(), kubeClient)
if err != nil {
logger.Errorf("set workflowTaskV4 breakpoint failed: list pods %v", err)
return e.ErrSetBreakpoint.AddDesc("修改断点意外失败: ListPods")
Expand All @@ -589,7 +607,7 @@ FOR:
ContainerName: pod.Spec.Containers[0].Name,
Command: []string{"sh", "-c", cmd},
}
_, stderr, success, _ := podexec.KubeExec(krkubeclient.Clientset(), krkubeclient.RESTConfig(), opt)
_, stderr, success, _ := podexec.KubeExec(clientSet, restConfig, opt)
logger.Errorf("set workflowTaskV4 breakpoint exec %s error: %s", cmd, stderr)
return success
}
Expand Down Expand Up @@ -687,7 +705,24 @@ FOR:
logger.Errorf("stop workflowTaskV4 debug shell failed: IToi %v", err)
return e.ErrStopDebugShell.AddDesc("结束调试意外失败")
}
pods, err := getter.ListPods(jobTaskSpec.Properties.Namespace, labels.Set{"job-name": task.K8sJobName}.AsSelector(), krkubeclient.Client())

kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), jobTaskSpec.Properties.ClusterID)
if err != nil {
log.Errorf("stop workflowTaskV4 debug shell failed: get kube client error: %s", err)
return e.ErrSetBreakpoint.AddDesc("结束调试意外失败: get kube client")
}
clientSet, err := kubeclient.GetClientset(config.HubServerAddress(), jobTaskSpec.Properties.ClusterID)
if err != nil {
log.Errorf("stop workflowTaskV4 debug shell failed: get kube client set error: %s", err)
return e.ErrSetBreakpoint.AddDesc("结束调试意外失败: get kube client set")
}
restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), jobTaskSpec.Properties.ClusterID)
if err != nil {
log.Errorf("stop workflowTaskV4 debug shell failed: get kube rest config error: %s", err)
return e.ErrSetBreakpoint.AddDesc("结束调试意外失败: get kube rest config")
}

pods, err := getter.ListPods(jobTaskSpec.Properties.Namespace, labels.Set{"job-name": task.K8sJobName}.AsSelector(), kubeClient)
if err != nil {
logger.Errorf("stop workflowTaskV4 debug shell failed: list pods %v", err)
return e.ErrStopDebugShell.AddDesc("结束调试意外失败: ListPods")
Expand All @@ -710,7 +745,7 @@ FOR:
ContainerName: pod.Spec.Containers[0].Name,
Command: []string{"sh", "-c", cmd},
}
_, stderr, success, _ := podexec.KubeExec(krkubeclient.Clientset(), krkubeclient.RESTConfig(), opt)
_, stderr, success, _ := podexec.KubeExec(clientSet, restConfig, opt)
logger.Errorf("stop workflowTaskV4 debug shell exec %s error: %s", cmd, stderr)
return success
}
Expand Down
23 changes: 20 additions & 3 deletions pkg/microservice/podexec/core/service/pod_server_ws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"

"github.com/koderover/zadig/pkg/microservice/aslan/config"
commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models"
"github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/workflowcontroller"
internalhandler "github.com/koderover/zadig/pkg/shared/handler"
kubeclient "github.com/koderover/zadig/pkg/shared/kube/client"
e "github.com/koderover/zadig/pkg/tool/errors"
krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client"
"github.com/koderover/zadig/pkg/tool/kube/getter"
"github.com/koderover/zadig/pkg/tool/log"
)
Expand Down Expand Up @@ -157,7 +158,23 @@ FOR:
_ = pty.Close()
}()

pods, err := getter.ListPods(jobTaskSpec.Properties.Namespace, labels.Set{"job-name": task.K8sJobName}.AsSelector(), krkubeclient.Client())
kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), jobTaskSpec.Properties.ClusterID)
if err != nil {
log.Errorf("debug workflow failed: get kube client error: %s", err)
return e.ErrGetDebugShell.AddDesc("启动调试终端意外失败: get kube client")
}
clientSet, err := kubeclient.GetClientset(config.HubServerAddress(), jobTaskSpec.Properties.ClusterID)
if err != nil {
log.Errorf("debug workflow failed: get kube client set error: %s", err)
return e.ErrGetDebugShell.AddDesc("启动调试终端意外失败: get kube client set")
}
restConfig, err := kubeclient.GetRESTConfig(config.HubServerAddress(), jobTaskSpec.Properties.ClusterID)
if err != nil {
log.Errorf("debug workflow failed: get kube rest config error: %s", err)
return e.ErrGetDebugShell.AddDesc("启动调试终端意外失败: get kube rest config")
}

pods, err := getter.ListPods(jobTaskSpec.Properties.Namespace, labels.Set{"job-name": task.K8sJobName}.AsSelector(), kubeClient)
if err != nil {
logger.Errorf("debug workflow failed: list pods %v", err)
return e.ErrGetDebugShell.AddDesc("启动调试终端意外失败: ListPods")
Expand All @@ -184,7 +201,7 @@ FOR:
}
script += "bash\n"

err = ExecPod(krkubeclient.Clientset(), krkubeclient.RESTConfig(), []string{"/bin/sh", "-c", script}, pty, jobTaskSpec.Properties.Namespace, pod.Name, pod.Spec.Containers[0].Name)
err = ExecPod(clientSet, restConfig, []string{"/bin/sh", "-c", script}, pty, jobTaskSpec.Properties.Namespace, pod.Name, pod.Spec.Containers[0].Name)
if err != nil {
msg := fmt.Sprintf("Exec to pod error! err: %v", err)
log.Errorf(msg)
Expand Down

0 comments on commit 697e5bd

Please sign in to comment.