Tracing: Enable traces to profiles (grafana#88896)

Author: marefr

devenv/datasources.yaml

@@ -311,3 +311,13 @@ datasources:
           spanEndTimeShift: '-5m'
           customQuery: true
           query: '{filename="/var/log/grafana/grafana.log"} |="$${__span.traceId}"'
+      tracesToProfiles:
+        datasourceUid: gdev-pyroscope
+        profileTypeId: "process_cpu:cpu:nanoseconds:cpu:nanoseconds"
+
+  - name: gdev-pyroscope
+    type: grafana-pyroscope-datasource
+    uid: gdev-pyroscope
+    access: proxy
+    url: http://localhost:4040
+    editable: false

devenv/docker/blocks/self-instrumentation/docker-compose.yaml

@@ -36,6 +36,8 @@
     image: "grafana/pyroscope:latest"
     ports:
       - "4040:4040"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
   agent:
     image: grafana/agent:v0.37.0
     environment:

go.mod

@@ -106,6 +106,7 @@ require (
 	// For local development grafana/grafana will always use the local files
 	// Check go.work file for details
 	github.com/grafana/grafana/pkg/promlib v0.0.6 // @grafana/observability-metrics
+	github.com/grafana/otel-profiling-go v0.5.1 // @grafana/grafana-backend-group
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // @grafana/observability-traces-and-profiling
 	github.com/grafana/pyroscope/api v0.3.0 // @grafana/observability-traces-and-profiling
 	github.com/grafana/tempo v1.5.1-0.20230524121406-1dc1bfe7085b // @grafana/observability-traces-and-profiling
@@ -242,6 +243,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
 	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
@@ -282,6 +284,7 @@ require (
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/emicklei/proto v1.10.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
@@ -343,6 +346,7 @@ require (
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
+	github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/klauspost/compress v1.17.8 // indirect
@@ -457,12 +461,6 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect; @grafana-app-platform-squad
 )
 
-require (
-	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
-	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
-	github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 // indirect
-)
-
 // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream
 replace github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240523142256-cc370b98af7c
 

go.sum

@@ -2344,6 +2344,8 @@ github.com/grafana/grafana/pkg/promlib v0.0.6 h1:FuRyHMIgVVXkLuJnCflNfk3gqJflmyi
 github.com/grafana/grafana/pkg/promlib v0.0.6/go.mod h1:shFkrG1fQ/PPNRGhxAPNMLp0SAeG/jhqaLoG6n2191M=
 github.com/grafana/grafana/pkg/util/xorm v0.0.1 h1:72QZjxWIWpSeOF8ob4aMV058kfgZyeetkAB8dmeti2o=
 github.com/grafana/grafana/pkg/util/xorm v0.0.1/go.mod h1:eNfbB9f2jM8o9RfwqwjY8SYm5tvowJ8Ly+iE4P9rXII=
+github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8=
+github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls=
 github.com/grafana/prometheus-alertmanager v0.25.1-0.20240422145632-c33c6b5b6e6b h1:HCbWyVL6vi7gxyO76gQksSPH203oBJ1MJ3JcG1OQlsg=
 github.com/grafana/prometheus-alertmanager v0.25.1-0.20240422145632-c33c6b5b6e6b/go.mod h1:01sXtHoRwI8W324IPAzuxDFOmALqYLCOhvSC2fUHWXc=
 github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo=

pkg/infra/tracing/tracing.go

@@ -272,6 +272,8 @@ func (ots *TracingService) initOpentelemetryTracer() error {
 		}
 	}
 
+	tp = NewProfilingTracerProvider(tp)
+
 	// Register our TracerProvider as the global so any imported
 	// instrumentation in the future will default to using it
 	// only if tracing is enabled

pkg/infra/tracing/tracing_profiling.go

@@ -0,0 +1,27 @@
+package tracing
+
+import (
+	"context"
+
+	otelpyroscope "github.com/grafana/otel-profiling-go"
+	trace "go.opentelemetry.io/otel/trace"
+)
+
+type profilingTracerProvider struct {
+	trace.TracerProvider
+	wrappedTp tracerProvider
+}
+
+// NewProfilingTracerProvider creates a new tracer provider that annotates pprof
+// samples with span_id label. This allows to establish a relationship
+// between pprof profiles and reported tracing spans.
+func NewProfilingTracerProvider(tp tracerProvider) tracerProvider {
+	return &profilingTracerProvider{
+		TracerProvider: otelpyroscope.NewTracerProvider(tp),
+		wrappedTp:      tp,
+	}
+}
+
+func (tp *profilingTracerProvider) Shutdown(ctx context.Context) error {
+	return tp.wrappedTp.Shutdown(ctx)
+}