Skip to content

Commit

Permalink
enhancement: support target-container procfs correct (aylei#83)
Browse files Browse the repository at this point in the history
* enhancement: support target-container procfs correct

* update Makefile 

remove some wrong adjustments for my own dev environment

* Update cmd.go

fix my typo error I made for travis CI

* fix: agent_daemonset yaml support lxcfs

* add:使用alpine linux,减小debug-agent镜像大小

* fix: 修复nsenter在alpine发行版中的一些error

* fix: reinforce the compatibility of debug-agent for kubectl-debug && fix typo && go fmt
  • Loading branch information
markzhang0928 authored and aylei committed Nov 14, 2019
1 parent 63e1ecc commit ae58ddf
Show file tree
Hide file tree
Showing 11 changed files with 308 additions and 37 deletions.
2 changes: 1 addition & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
*
!debug-agent

!./scripts/start.sh
19 changes: 17 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,21 @@
FROM gcr.io/distroless/static
FROM ubuntu:xenial as build

RUN apt-get update && apt-get install libcgmanager-dev libnih-dbus-dev libnih-dev libfuse-dev automake libtool libpam-dev wget gcc automake -y && apt-get clean && rm -rf /var/lib/apt/lists/*

ENV LXCFS_VERSION 3.1.2
RUN wget https://linuxcontainers.org/downloads/lxcfs/lxcfs-$LXCFS_VERSION.tar.gz && \
mkdir /lxcfs && tar xzvf lxcfs-$LXCFS_VERSION.tar.gz -C /lxcfs --strip-components=1 && \
cd /lxcfs && ./configure && make

FROM alpine:3.10

COPY --from=build /lxcfs/lxcfs /usr/local/bin/lxcfs
COPY --from=build /lxcfs/.libs/liblxcfs.so /usr/local/lib/lxcfs/liblxcfs.so
COPY --from=build /lxcfs/lxcfs /lxcfs/lxcfs
COPY --from=build /lxcfs/.libs/liblxcfs.so /lxcfs/liblxcfs.so
COPY ./scripts/start.sh /
COPY ./debug-agent /bin/debug-agent

EXPOSE 10027

ENTRYPOINT ["/bin/debug-agent"]
CMD ["/start.sh"]
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ plugin:
GO111MODULE=on CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o kubectl-debug cmd/plugin/main.go

agent-docker: agent
docker build . -t aylei/debug-agent:latest
docker build . -t aylei/debug-agent:0.0.2

agent:
$(GO) build -ldflags '$(LDFLAGS)' -o debug-agent cmd/agent/main.go
Expand Down
62 changes: 62 additions & 0 deletions pkg/agent/lxcfs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package agent

import (
"bufio"
"bytes"
"fmt"
"io"
"os"
)

// List of LXC filesystem files
const (
MemFile string = "/proc/meminfo"
CpuFile string = "/proc/cpuinfo"
UpTimeFile string = "/proc/uptime"
SwapsFile string = "/proc/swaps"
StatFile string = "/proc/stat"
DiskStatsFile string = "/proc/diskstats"
LoadavgFile string = "/proc/loadavg"
)

var (
// IsLxcfsEnabled means whether to enable lxcfs
LxcfsEnabled bool

// LxcfsRootDir
LxcfsRootDir = "/var/lib/lxc"

// LxcfsHomeDir means /var/lib/lxc/lxcfs
LxcfsHomeDir = "/var/lib/lxc/lxcfs"

// LxcfsFiles is a list of LXC files
LxcfsProcFiles = []string{MemFile, CpuFile, UpTimeFile, SwapsFile, StatFile, DiskStatsFile, LoadavgFile}
)

// CheckLxcfsMount check if the the mount point of lxcfs exists
func CheckLxcfsMount() error {
isMount := false
f, err := os.Open("/proc/1/mountinfo")
if err != nil {
return fmt.Errorf("Check lxcfs mounts failed: %v", err)
}
fr := bufio.NewReader(f)
for {
line, err := fr.ReadBytes('\n')
if err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("Check lxcfs mounts failed: %v", err)
}

if bytes.Contains(line, []byte(LxcfsHomeDir)) {
isMount = true
break
}
}
if !isMount {
return fmt.Errorf("%s is not a mount point, please run \" lxcfs %s \" before debug", LxcfsHomeDir, LxcfsHomeDir)
}
return nil
}
55 changes: 48 additions & 7 deletions pkg/agent/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ import (
"log"
"time"

term "github.com/aylei/kubectl-debug/pkg/util"
"github.com/aylei/kubectl-debug/pkg/nsenter"
"github.com/aylei/kubectl-debug/pkg/util"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/strslice"
Expand Down Expand Up @@ -41,11 +42,12 @@ func NewRuntimeManager(host string, timeout time.Duration) (*RuntimeManager, err
// DebugAttacher implements Attacher
// we use this struct in order to inject debug info (image, command) in the debug procedure
type DebugAttacher struct {
runtime *RuntimeManager
image string
authStr string
command []string
client *dockerclient.Client
runtime *RuntimeManager
image string
authStr string
lxcfsEnabled bool
command []string
client *dockerclient.Client

// control the preparing of debug container
stopListenEOF chan struct{}
Expand All @@ -58,11 +60,12 @@ func (a *DebugAttacher) AttachContainer(name string, uid kubetype.UID, container
}

// GetAttacher returns an implementation of Attacher
func (m *RuntimeManager) GetAttacher(image string, authStr string, command []string, context context.Context, cancel context.CancelFunc) kubeletremote.Attacher {
func (m *RuntimeManager) GetAttacher(image, authStr string, lxcfsEnabled bool, command []string, context context.Context, cancel context.CancelFunc) kubeletremote.Attacher {
return &DebugAttacher{
runtime: m,
image: image,
authStr: authStr,
lxcfsEnabled: lxcfsEnabled,
command: command,
context: context,
client: m.client,
Expand Down Expand Up @@ -101,6 +104,17 @@ func (m *DebugAttacher) DebugContainer(container, image string, authStr string,
// }
// }
//} ()
// step 0: set container procfs correct by lxcfs
stdout.Write([]byte(fmt.Sprintf("set container procfs correct %t .. \n\r", m.lxcfsEnabled)))
if m.lxcfsEnabled {
if err := CheckLxcfsMount(); err != nil {
return err
}

if err := m.SetContainerLxcfs(container); err != nil {
return err
}
}

// step 1: pull image
stdout.Write([]byte(fmt.Sprintf("pulling image %s... \n\r", image)))
Expand Down Expand Up @@ -129,6 +143,33 @@ func (m *DebugAttacher) DebugContainer(container, image string, authStr string,
return nil
}

func (m *DebugAttacher) SetContainerLxcfs(container string) error {
ctx, cancel := m.getContextWithTimeout()
defer cancel()
containerInstance, err := m.client.ContainerInspect(ctx, container)
if err != nil {
return err
}
for _, mount := range containerInstance.Mounts {
if mount.Destination == LxcfsRootDir {
log.Printf("remount lxcfs when the rootdir of lxcfs of target container has been mounted. \n\t ")
for _, procfile := range LxcfsProcFiles {
nsenter := &nsenter.MountNSEnter{
Target: containerInstance.State.Pid,
MountLxcfs: true,
}
_, stderr, err := nsenter.Execute("--", "mount", "-B", LxcfsHomeDir+procfile, procfile)
if err != nil {
log.Printf("bind mount lxcfs files failed. \n\t reason: %s", stderr)
return err
}
}
}
}

return nil
}

// Run a new container, this container will join the network,
// mount, and pid namespace of the given container
func (m *DebugAttacher) RunDebugContainer(targetId string, image string, command []string) (string, error) {
Expand Down
8 changes: 7 additions & 1 deletion pkg/agent/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ func (s *Server) ServeDebug(w http.ResponseWriter, req *http.Request) {
Stderr: false,
TTY: true,
}
lxcfsEnabled := req.FormValue("lxcfsEnabled")
if lxcfsEnabled == "" || lxcfsEnabled == "false" {
LxcfsEnabled = false
} else if lxcfsEnabled == "true" {
LxcfsEnabled = true
}

context, cancel := context.WithCancel(req.Context())
defer cancel()
Expand All @@ -105,7 +111,7 @@ func (s *Server) ServeDebug(w http.ResponseWriter, req *http.Request) {
kubeletremote.ServeAttach(
w,
req,
s.runtimeApi.GetAttacher(image, authStr, commandSlice, context, cancel),
s.runtimeApi.GetAttacher(image, authStr, LxcfsEnabled, commandSlice, context, cancel),
"",
"",
dockerContainerId,
Expand Down
61 changes: 61 additions & 0 deletions pkg/nsenter/nsenter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package nsenter

import (
"bytes"
"context"
"fmt"
"os/exec"
"strconv"
)

// MountNSEnter is the client used to enter the mount namespace
type MountNSEnter struct {
Target int // target PID (required)
MountLxcfs bool // enter mount namespace or not
MountFile string // Mount namespace location, default to /proc/PID/ns/mnt
}

// Execute runs the given command with a default background context
func (cli *MountNSEnter) Execute(command string, args ...string) (stdout, stderr string, err error) {
return cli.ExecuteContext(context.Background(), command, args...)
}

// ExecuteContext the given command using the specific nsenter config
func (cli *MountNSEnter) ExecuteContext(ctx context.Context, command string, args ...string) (string, string, error) {
cmd, err := cli.setCommand(ctx)
if err != nil {
return "", "", fmt.Errorf("Error when set command: %v", err)
}

var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
cmd.Args = append(cmd.Args, command)
cmd.Args = append(cmd.Args, args...)

err = cmd.Run()
if err != nil {
return stdout.String(), stderr.String(), fmt.Errorf("Error while executing command: %v", err)
}

return stdout.String(), stderr.String(), nil
}

func (cli *MountNSEnter) setCommand(ctx context.Context) (*exec.Cmd, error) {
if cli.Target == 0 {
return nil, fmt.Errorf("Target must be specified")
}
var args []string
args = append(args, "--target", strconv.Itoa(cli.Target))

if cli.MountLxcfs {
if cli.MountFile != "" {
args = append(args, fmt.Sprintf("--mount=%s", cli.MountFile))
} else {
args = append(args, "--mount")
}
}

cmd := exec.CommandContext(ctx, "/usr/bin/nsenter", args...)
return cmd, nil
}
49 changes: 49 additions & 0 deletions pkg/plugin/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ You may set default configuration such as image and command in the config file,

defaultPortForward = true
defaultAgentless = true
defaultLxcfsEnable = true
)

// DebugOptions specify how to run debug container in a running pod
Expand Down Expand Up @@ -114,6 +115,8 @@ type DebugOptions struct {
AgentPodNamespace string
AgentPodNode string
AgentPodResource agentPodResources
// enable lxcfs
IsLxcfsEnabled bool

Flags *genericclioptions.ConfigFlags
CoreClient coreclient.CoreV1Interface
Expand Down Expand Up @@ -214,6 +217,8 @@ func NewDebugCmd(streams genericclioptions.IOStreams) *cobra.Command {
fmt.Sprintf("Agentless mode, agent pod cpu limits, default is not set"))
cmd.Flags().StringVar(&opts.AgentPodResource.MemoryLimits, "agent-pod-memory-limits", "",
fmt.Sprintf("Agentless mode, agent pod memory limits, default is not set"))
cmd.Flags().BoolVarP(&opts.IsLxcfsEnabled, "enable-lxcfs", "", true,
fmt.Sprintf("Enable Lxcfs, the target container can use its proc files, default to %t", defaultLxcfsEnable))
opts.Flags.AddFlags(cmd.Flags())

return cmd
Expand Down Expand Up @@ -372,6 +377,12 @@ func (o *DebugOptions) Complete(cmd *cobra.Command, args []string, argsLenAtDash
}
}

if o.IsLxcfsEnabled {
o.IsLxcfsEnabled = config.IsLxcfsEnabled
} else {
o.IsLxcfsEnabled = defaultLxcfsEnable
}

if config.PortForward {
o.PortForward = true
}
Expand Down Expand Up @@ -534,6 +545,11 @@ func (o *DebugOptions) Run() error {
params := url.Values{}
params.Add("image", o.Image)
params.Add("container", containerID)
if o.IsLxcfsEnabled {
params.Add("lxcfsEnabled", "true")
} else {
params.Add("lxcfsEnabled", "false")
}
var authStr string
registrySecret, err := o.CoreClient.Secrets(o.RegistrySecretNamespace).Get(o.RegistrySecretName, v1.GetOptions{})
if err != nil {
Expand Down Expand Up @@ -725,6 +741,9 @@ func (o *DebugOptions) launchPod(pod *corev1.Pod) (*corev1.Pod, error) {

// getAgentPod construnct agentPod from agent pod template
func (o *DebugOptions) getAgentPod() *corev1.Pod {
prop := corev1.MountPropagationBidirectional
directoryCreate := corev1.HostPathDirectoryOrCreate
priveleged := true
agentPod := &corev1.Pod{
TypeMeta: v1.TypeMeta{
Kind: "Pod",
Expand All @@ -735,6 +754,7 @@ func (o *DebugOptions) getAgentPod() *corev1.Pod {
Namespace: o.AgentPodNamespace,
},
Spec: corev1.PodSpec{
HostPID: true,
NodeName: o.AgentPodNode,
Containers: []corev1.Container{
{
Expand All @@ -754,12 +774,24 @@ func (o *DebugOptions) getAgentPod() *corev1.Pod {
TimeoutSeconds: 1,
FailureThreshold: 3,
},
SecurityContext: &corev1.SecurityContext{
Privileged: &priveleged,
},
Resources: o.buildAgentResourceRequirements(),
VolumeMounts: []corev1.VolumeMount{
{
Name: "docker",
MountPath: "/var/run/docker.sock",
},
{
Name: "cgroup",
MountPath: "/sys/fs/cgroup",
},
{
Name: "lxcfs",
MountPath: "/var/lib/lxc/lxcfs",
MountPropagation: &prop,
},
},
Ports: []corev1.ContainerPort{
{
Expand All @@ -779,6 +811,23 @@ func (o *DebugOptions) getAgentPod() *corev1.Pod {
},
},
},
{
Name: "cgroup",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/sys/fs/cgroup",
},
},
},
{
Name: "lxcfs",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/var/lib/lxc/lxcfs",
Type: &directoryCreate,
},
},
},
},
RestartPolicy: corev1.RestartPolicyNever,
},
Expand Down
Loading

0 comments on commit ae58ddf

Please sign in to comment.