API Server: Enhanced profiling with godeltaprof (grafana#88939)

Author: marefr

devenv/docker/blocks/self-instrumentation/agent.flow

@@ -50,4 +50,38 @@ pyroscope.scrape "default" {
     {"__address__" = "host.docker.internal:6060", "service_name"="grafana"},
   ]
   forward_to = [pyroscope.write.default.receiver]
+
+  profiling_config {
+    profile.process_cpu {
+      enabled = true
+    }
+
+    profile.godeltaprof_memory {
+      enabled = true
+    }
+
+    profile.memory { // disable memory, use godeltaprof_memory instead
+      enabled = false
+    }
+
+    profile.godeltaprof_mutex {
+      enabled = true
+    }
+
+    profile.mutex { // disable mutex, use godeltaprof_mutex instead
+      enabled = false
+    }
+
+    profile.godeltaprof_block {
+      enabled = true
+    }
+
+    profile.block { // disable block, use godeltaprof_block instead
+      enabled = false
+    }
+
+    profile.goroutine {
+      enabled = true
+    }
+  }
 }

pkg/cmd/grafana/apiserver/server.go

@@ -7,10 +7,12 @@ import (
 	"net"
 	"path"
 
+	"github.com/grafana/pyroscope-go/godeltaprof/http/pprof"
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/mux"
 	"k8s.io/client-go/tools/clientcmd"
 	netutils "k8s.io/utils/net"
 
@@ -178,5 +180,19 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf
 		}
 	}
 
+	if config.EnableProfiling {
+		deltaProfiling{}.Install(server.Handler.NonGoRestfulMux)
+	}
+
 	return server.PrepareRun().Run(stopCh)
 }
+
+// deltaProfiling adds godeltapprof handlers for pprof under /debug/pprof.
+type deltaProfiling struct{}
+
+// Install register godeltapprof handlers to the given mux.
+func (d deltaProfiling) Install(c *mux.PathRecorderMux) {
+	c.UnlistedHandleFunc("/debug/pprof/delta_heap", pprof.Heap)
+	c.UnlistedHandleFunc("/debug/pprof/delta_block", pprof.Block)
+	c.UnlistedHandleFunc("/debug/pprof/delta_mutex", pprof.Mutex)
+}