*: Initial eBPF tracing support (go-delve#2625)

.gitignore

@@ -2,11 +2,12 @@
 .tags*
 tags
 .dbg_history
-cmd/dlv/dlv
+**/**/dlv
 .vagrant
 **/*.swp
 localtests
 .idea
 *.iml
 .teamcity/target
 .vscode
+**/*.o

Documentation/cli/starlark.md

@@ -26,6 +26,7 @@ clear_breakpoint(Id, Name) | Equivalent to API call [ClearBreakpoint](https://go
 clear_checkpoint(ID) | Equivalent to API call [ClearCheckpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ClearCheckpoint)
 raw_command(Name, ThreadID, GoroutineID, ReturnInfoLoadConfig, Expr, UnsafeCall) | Equivalent to API call [Command](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Command)
 create_breakpoint(Breakpoint) | Equivalent to API call [CreateBreakpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.CreateBreakpoint)
+create_ebpf_tracepoint(FunctionName) | Equivalent to API call [CreateEBPFTracepoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.CreateEBPFTracepoint)
 create_watchpoint(Scope, Expr, Type) | Equivalent to API call [CreateWatchpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.CreateWatchpoint)
 detach(Kill) | Equivalent to API call [Detach](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Detach)
 disassemble(Scope, StartPC, EndPC, Flavour) | Equivalent to API call [Disassemble](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Disassemble)
@@ -37,6 +38,7 @@ examine_memory(Address, Length) | Equivalent to API call [ExamineMemory](https:/
 find_location(Scope, Loc, IncludeNonExecutableLines, SubstitutePathRules) | Equivalent to API call [FindLocation](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.FindLocation)
 function_return_locations(FnName) | Equivalent to API call [FunctionReturnLocations](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.FunctionReturnLocations)
 get_breakpoint(Id, Name) | Equivalent to API call [GetBreakpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.GetBreakpoint)
+get_buffered_tracepoints() | Equivalent to API call [GetBufferedTracepoints](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.GetBufferedTracepoints)
 get_thread(Id) | Equivalent to API call [GetThread](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.GetThread)
 is_multiclient() | Equivalent to API call [IsMulticlient](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.IsMulticlient)
 last_modified() | Equivalent to API call [LastModified](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.LastModified)

Documentation/usage/dlv_trace.md

@@ -21,11 +21,12 @@ dlv trace [package] regexp [flags]
 ### Options
 
 ```
+      --ebpf            Trace using eBPF (experimental).
   -e, --exec string     Binary file to exec and trace.
   -h, --help            help for trace
       --output string   Output path for the binary. (default "debug")
   -p, --pid int         Pid to attach to.
-  -s, --stack int       Show stack trace with given depth.
+  -s, --stack int       Show stack trace with given depth. (Ignored with -ebpf)
   -t, --test            Trace a test binary.
 ```
 

Makefile

@@ -1,12 +1,30 @@
 .DEFAULT_GOAL=test
 
+BPF_OBJ := pkg/proc/internal/ebpf/trace_probe/trace.o
+BPF_SRC := $(shell find . -type f -name '*.bpf.*')
+GO_SRC := $(shell find . -type f -not -path './_fixtures/*' -not -path './vendor/*' -not -path './_scripts/*' -not -path './localtests/*' -name '*.go')
+
 check-cert:
 	@go run _scripts/make.go check-cert
 
-build:
+build: $(GO_SRC)
 	@go run _scripts/make.go build
 
-install:
+$(BPF_OBJ): $(BPF_SRC)
+	clang \
+		-I /usr/include \
+		-I /usr/src/kernels/$(uname -r)/tools/lib \
+		-I /usr/src/kernels/$(uname -r)/tools/bpf/resolve_btfids/libbpf \
+		-g -O2 \
+		-c \
+		-target bpf \
+		-o $(BPF_OBJ) \
+		pkg/proc/internal/ebpf/trace_probe/trace.bpf.c
+
+build-bpf: $(BPF_OBJ) $(GO_SRC)
+	@env CGO_LDFLAGS="/usr/lib64/libbpf.a" go run _scripts/make.go build --tags=ebpf
+
+install: $(GO_SRC)
 	@go run _scripts/make.go install
 
 uninstall:
@@ -27,4 +45,4 @@ test-integration-run:
 vendor:
 	@go run _scripts/make.go vendor
 
-.PHONY: vendor test-integration-run test-proc-run test check-cert install build vet
+.PHONY: vendor test-integration-run test-proc-run test check-cert install build vet build-bpf uninstall

_scripts/gen-starlark-bindings.go

@@ -111,6 +111,8 @@ func processServerMethods(serverMethods []*types.Func) []binding {
 			name = "set_expr"
 		case "command":
 			name = "raw_command"
+		case "create_e_b_p_f_tracepoint":
+			name = "create_ebpf_tracepoint"
 		default:
 			// remove list_ prefix, it looks better
 			const listPrefix = "list_"

_scripts/make.go

@@ -21,6 +21,7 @@ var Verbose bool
 var NOTimeout bool
 var TestIncludePIE bool
 var TestSet, TestRegex, TestBackend, TestBuildMode string
+var Tags *[]string
 
 func NewMakeCommands() *cobra.Command {
 	RootCommand := &cobra.Command{
@@ -34,17 +35,27 @@ func NewMakeCommands() *cobra.Command {
 		Run:   checkCertCmd,
 	})
 
-	RootCommand.AddCommand(&cobra.Command{
+	buildCmd := &cobra.Command{
 		Use:   "build",
 		Short: "Build delve",
 		Run: func(cmd *cobra.Command, args []string) {
 			tagFlag := prepareMacnative()
-			execute("go", "build", tagFlag, buildFlags(), DelveMainPackagePath)
+			if len(*Tags) > 0 {
+				if len(tagFlag) == 0 {
+					tagFlag = "-tags="
+				} else {
+					tagFlag += ","
+				}
+				tagFlag += strings.Join(*Tags, ",")
+			}
+			execute("go", "build", "-ldflags", "-extldflags -static", tagFlag, buildFlags(), DelveMainPackagePath)
 			if runtime.GOOS == "darwin" && os.Getenv("CERT") != "" && canMacnative() {
 				codesign("./dlv")
 			}
 		},
-	})
+	}
+	Tags = buildCmd.PersistentFlags().StringArray("tags", []string{}, "Build tags")
+	RootCommand.AddCommand(buildCmd)
 
 	RootCommand.AddCommand(&cobra.Command{
 		Use:   "install",

cmd/dlv/cmds/commands.go

@@ -9,6 +9,7 @@ import (
 	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"strconv"
 	"strings"
@@ -76,6 +77,7 @@ var (
 	traceExecFile   string
 	traceTestBinary bool
 	traceStackDepth int
+	traceUseEBPF    bool
 
 	// redirect specifications for target process
 	redirects []string
@@ -283,7 +285,8 @@ only see the output of the trace operations you can redirect stdout.`,
 	traceCommand.Flags().IntVarP(&traceAttachPid, "pid", "p", 0, "Pid to attach to.")
 	traceCommand.Flags().StringVarP(&traceExecFile, "exec", "e", "", "Binary file to exec and trace.")
 	traceCommand.Flags().BoolVarP(&traceTestBinary, "test", "t", false, "Trace a test binary.")
-	traceCommand.Flags().IntVarP(&traceStackDepth, "stack", "s", 0, "Show stack trace with given depth.")
+	traceCommand.Flags().BoolVarP(&traceUseEBPF, "ebpf", "", false, "Trace using eBPF (experimental).")
+	traceCommand.Flags().IntVarP(&traceStackDepth, "stack", "s", 0, "Show stack trace with given depth. (Ignored with -ebpf)")
 	traceCommand.Flags().String("output", "debug", "Output path for the binary.")
 	rootCommand.AddCommand(traceCommand)
 
@@ -522,13 +525,14 @@ func traceCmd(cmd *cobra.Command, args []string) {
 			dlvArgs = dlvArgs[:dlvArgsLen-1]
 		}
 
+		var debugname string
 		if traceAttachPid == 0 {
 			if dlvArgsLen >= 2 && traceExecFile != "" {
 				fmt.Fprintln(os.Stderr, "Cannot specify package when using exec.")
 				return 1
 			}
 
-			debugname := traceExecFile
+			debugname = traceExecFile
 			if traceExecFile == "" {
 				debugname, err = filepath.Abs(cmd.Flag("output").Value.String())
 				if err != nil {
@@ -583,39 +587,79 @@ func traceCmd(cmd *cobra.Command, args []string) {
 			return 1
 		}
 		for i := range funcs {
-			_, err = client.CreateBreakpoint(&api.Breakpoint{
-				FunctionName: funcs[i],
-				Tracepoint:   true,
-				Line:         -1,
-				Stacktrace:   traceStackDepth,
-				LoadArgs:     &terminal.ShortLoadConfig,
-			})
-			if err != nil && !isBreakpointExistsErr(err) {
-				fmt.Fprintln(os.Stderr, err)
-				return 1
-			}
-			addrs, err := client.FunctionReturnLocations(funcs[i])
-			if err != nil {
-				fmt.Fprintln(os.Stderr, err)
-				return 1
-			}
-			for i := range addrs {
+			if traceUseEBPF {
+				err := client.CreateEBPFTracepoint(funcs[i])
+				if err != nil {
+					fmt.Fprintln(os.Stderr, err)
+					return 1
+				}
+			} else {
+				// Fall back to breakpoint based tracing if we get an error.
 				_, err = client.CreateBreakpoint(&api.Breakpoint{
-					Addr:        addrs[i],
-					TraceReturn: true,
-					Stacktrace:  traceStackDepth,
-					Line:        -1,
-					LoadArgs:    &terminal.ShortLoadConfig,
+					FunctionName: funcs[i],
+					Tracepoint:   true,
+					Line:         -1,
+					Stacktrace:   traceStackDepth,
+					LoadArgs:     &terminal.ShortLoadConfig,
 				})
 				if err != nil && !isBreakpointExistsErr(err) {
 					fmt.Fprintln(os.Stderr, err)
 					return 1
 				}
+				addrs, err := client.FunctionReturnLocations(funcs[i])
+				if err != nil {
+					fmt.Fprintln(os.Stderr, err)
+					return 1
+				}
+				for i := range addrs {
+					_, err = client.CreateBreakpoint(&api.Breakpoint{
+						Addr:        addrs[i],
+						TraceReturn: true,
+						Stacktrace:  traceStackDepth,
+						Line:        -1,
+						LoadArgs:    &terminal.ShortLoadConfig,
+					})
+					if err != nil && !isBreakpointExistsErr(err) {
+						fmt.Fprintln(os.Stderr, err)
+						return 1
+					}
+				}
 			}
 		}
 		cmds := terminal.DebugCommands(client)
 		t := terminal.New(client, nil)
 		defer t.Close()
+		if traceUseEBPF {
+			done := make(chan struct{})
+			defer close(done)
+			go func() {
+				for {
+					select {
+					case <-done:
+						return
+					default:
+						tracepoints, err := client.GetBufferedTracepoints()
+						if err != nil {
+							panic(err)
+						}
+						for _, t := range tracepoints {
+							var params strings.Builder
+							for _, p := range t.InputParams {
+								if params.Len() > 0 {
+									params.WriteString(", ")
+								}
+								if p.Kind == reflect.String {
+									params.WriteString(fmt.Sprintf("%q", p.Value))
+								} else {
+									params.WriteString(p.Value)
+								}
+							}
+							fmt.Printf("%s:%d %s(%s)\n", t.File, t.Line, t.FunctionName, params.String())
+						}
+					}
+				}
+			}()
+		}
 		cmds.Call("continue", t)
 		return 0
 	}()

go.mod

@@ -1,8 +1,9 @@
 module github.com/go-delve/delve
 
-go 1.11
+go 1.16
 
 require (
+	github.com/aquasecurity/libbpfgo v0.1.2-0.20210708203834-4928d36fafac
 	github.com/cosiner/argv v0.1.0
 	github.com/creack/pty v1.1.9
 	github.com/google/go-dap v0.5.0
@@ -14,7 +15,7 @@ require (
 	github.com/spf13/cobra v1.1.3
 	go.starlark.net v0.0.0-20200821142938-949cc6f4b097
 	golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4
-	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
+	golang.org/x/sys v0.0.0-20210514084401-e8d321eab015
 	golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -16,6 +16,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/aquasecurity/libbpfgo v0.1.2-0.20210708203834-4928d36fafac h1:oehMMAySC3p8eSwcwQ8lTXxeCkkPll+AwNesUNowbJ8=
+github.com/aquasecurity/libbpfgo v0.1.2-0.20210708203834-4928d36fafac/go.mod h1:/+clceXE103FaXvVTIY2HAkQjxNtkra4DRWvZYr2SKw=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -181,10 +183,13 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -257,8 +262,9 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E=
+golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -317,6 +323,8 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/dwarf/godwarf/type.go

@@ -787,6 +787,7 @@ func readType(d *dwarf.Data, name string, r *dwarf.Reader, off dwarf.Offset, typ
 		case reflect.String:
 			str := new(StringType)
 			t = &str.StructType
+			str.ReflectKind = reflect.String
 			typ = str
 		default:
 			typ = t

pkg/dwarf/op/op.go

@@ -145,7 +145,7 @@ func PrettyPrint(out io.Writer, instructions []byte) {
 
 func callframecfa(opcode Opcode, ctxt *context) error {
 	if ctxt.CFA == 0 {
-		return fmt.Errorf("Could not retrieve CFA for current PC")
+		return errors.New("could not retrieve CFA for current PC")
 	}
 	ctxt.stack = append(ctxt.stack, int64(ctxt.CFA))
 	return nil