From 09e5b60519b9ee6e793b0766e163ef8ddf449e74 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 26 Apr 2022 07:02:53 -0700 Subject: [PATCH] feat: support opentelemetry for grpc tracing (#7539) * feat: add support for otel grpc tracing Signed-off-by: Ben Ye * chore: rebase code to master Signed-off-by: Ben Ye * feat: add otelgrpc to cmpserver Signed-off-by: Ben Ye * feat: add util trace package Signed-off-by: Ben Ye * fix: fix tracer context Signed-off-by: Ben Ye * chore: move tracer initialization part to the loop Signed-off-by: Ben Ye --- .../commands/argocd_cmp_server.go | 16 ++++++ .../commands/argocd_repo_server.go | 14 +++++ cmd/argocd-server/commands/argocd_server.go | 11 ++++ cmpserver/server.go | 15 +++++- .../server-commands/argocd-repo-server.md | 1 + .../server-commands/argocd-server.md | 1 + go.mod | 24 +++++++-- go.sum | 35 ++++++++++-- reposerver/server.go | 9 +++- server/server.go | 3 ++ util/trace/trace.go | 54 +++++++++++++++++++ 11 files changed, 173 insertions(+), 10 deletions(-) create mode 100644 util/trace/trace.go diff --git a/cmd/argocd-cmp-server/commands/argocd_cmp_server.go b/cmd/argocd-cmp-server/commands/argocd_cmp_server.go index 3616e6289dff4..cc90588c52a1e 100644 --- a/cmd/argocd-cmp-server/commands/argocd_cmp_server.go +++ b/cmd/argocd-cmp-server/commands/argocd_cmp_server.go @@ -1,9 +1,11 @@ package commands import ( + "context" "time" "github.com/argoproj/pkg/stats" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" cmdutil "github.com/argoproj/argo-cd/v2/cmd/util" @@ -11,7 +13,9 @@ import ( "github.com/argoproj/argo-cd/v2/cmpserver/plugin" "github.com/argoproj/argo-cd/v2/common" "github.com/argoproj/argo-cd/v2/util/cli" + "github.com/argoproj/argo-cd/v2/util/env" "github.com/argoproj/argo-cd/v2/util/errors" + traceutil "github.com/argoproj/argo-cd/v2/util/trace" ) const ( @@ -22,6 +26,7 @@ const ( func NewCommand() *cobra.Command { var ( configFilePath string + otlpAddress string ) var command = cobra.Command{ Use: cliName, @@ -35,6 +40,16 @@ func NewCommand() *cobra.Command { config, err := plugin.ReadPluginConfig(configFilePath) errors.CheckError(err) + if otlpAddress != "" { + var closer func() + var err error + closer, err = traceutil.InitTracer(context.Background(), "argocd-cmp-server", otlpAddress) + if err != nil { + log.Fatalf("failed to initialize tracing: %v", err) + } + defer closer() + } + server, err := cmpserver.NewServer(plugin.CMPServerInitConstants{ PluginConfig: *config, }) @@ -54,5 +69,6 @@ func NewCommand() *cobra.Command { command.Flags().StringVar(&cmdutil.LogFormat, "logformat", "text", "Set the logging format. One of: text|json") command.Flags().StringVar(&cmdutil.LogLevel, "loglevel", "info", "Set the logging level. One of: debug|info|warn|error") command.Flags().StringVar(&configFilePath, "config-dir-path", common.DefaultPluginConfigFilePath, "Config management plugin configuration file location, Default is '/home/argocd/cmp-server/config/'") + command.Flags().StringVar(&otlpAddress, "otlp-address", env.StringFromEnv("ARGOCD_REPO_SERVER_OTLP_ADDRESS", ""), "OpenTelemetry collector address to send traces to") return &command } diff --git a/cmd/argocd-repo-server/commands/argocd_repo_server.go b/cmd/argocd-repo-server/commands/argocd_repo_server.go index 1d3c2eaf1ae1d..3c9aaac268370 100644 --- a/cmd/argocd-repo-server/commands/argocd_repo_server.go +++ b/cmd/argocd-repo-server/commands/argocd_repo_server.go @@ -1,6 +1,7 @@ package commands import ( + "context" "fmt" "math" "net" @@ -30,6 +31,7 @@ import ( "github.com/argoproj/argo-cd/v2/util/healthz" ioutil "github.com/argoproj/argo-cd/v2/util/io" "github.com/argoproj/argo-cd/v2/util/tls" + traceutil "github.com/argoproj/argo-cd/v2/util/trace" ) const ( @@ -71,6 +73,7 @@ func NewCommand() *cobra.Command { parallelismLimit int64 listenPort int metricsPort int + otlpAddress string cacheSrc func() (*reposervercache.Cache, error) tlsConfigCustomizer tls.ConfigCustomizer tlsConfigCustomizerSrc func() (tls.ConfigCustomizer, error) @@ -107,6 +110,16 @@ func NewCommand() *cobra.Command { }, askPassServer) errors.CheckError(err) + if otlpAddress != "" { + var closer func() + var err error + closer, err = traceutil.InitTracer(context.Background(), "argocd-repo-server", otlpAddress) + if err != nil { + log.Fatalf("failed to initialize tracing: %v", err) + } + defer closer() + } + grpc := server.CreateGRPC() listener, err := net.Listen("tcp", fmt.Sprintf(":%d", listenPort)) errors.CheckError(err) @@ -167,6 +180,7 @@ func NewCommand() *cobra.Command { command.Flags().Int64Var(¶llelismLimit, "parallelismlimit", int64(env.ParseNumFromEnv("ARGOCD_REPO_SERVER_PARALLELISM_LIMIT", 0, 0, math.MaxInt32)), "Limit on number of concurrent manifests generate requests. Any value less the 1 means no limit.") command.Flags().IntVar(&listenPort, "port", common.DefaultPortRepoServer, "Listen on given port for incoming connections") command.Flags().IntVar(&metricsPort, "metrics-port", common.DefaultPortRepoServerMetrics, "Start metrics server on given port") + command.Flags().StringVar(&otlpAddress, "otlp-address", env.StringFromEnv("ARGOCD_REPO_SERVER_OTLP_ADDRESS", ""), "OpenTelemetry collector address to send traces to") command.Flags().BoolVar(&disableTLS, "disable-tls", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_TLS", false), "Disable TLS on the gRPC endpoint") tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command) diff --git a/cmd/argocd-server/commands/argocd_server.go b/cmd/argocd-server/commands/argocd_server.go index c23b65ee0dbc1..95de950e026a6 100644 --- a/cmd/argocd-server/commands/argocd_server.go +++ b/cmd/argocd-server/commands/argocd_server.go @@ -25,6 +25,7 @@ import ( "github.com/argoproj/argo-cd/v2/util/errors" "github.com/argoproj/argo-cd/v2/util/kube" "github.com/argoproj/argo-cd/v2/util/tls" + traceutil "github.com/argoproj/argo-cd/v2/util/trace" ) const ( @@ -49,6 +50,7 @@ func NewCommand() *cobra.Command { insecure bool listenPort int metricsPort int + otlpAddress string glogLevel int clientConfig clientcmd.ClientConfig repoServerTimeoutSeconds int @@ -151,12 +153,20 @@ func NewCommand() *cobra.Command { stats.StartStatsTicker(10 * time.Minute) stats.RegisterHeapDumper("memprofile") + var closer func() for { ctx := context.Background() ctx, cancel := context.WithCancel(ctx) + if otlpAddress != "" { + closer, err = traceutil.InitTracer(ctx, "argocd-server", otlpAddress) + if err != nil { + log.Fatalf("failed to initialize tracing: %v", err) + } + } argocd := server.NewServer(ctx, argoCDOpts) argocd.Run(ctx, listenPort, metricsPort) cancel() + closer() } }, } @@ -176,6 +186,7 @@ func NewCommand() *cobra.Command { command.AddCommand(cli.NewVersionCmd(cliName)) command.Flags().IntVar(&listenPort, "port", common.DefaultPortAPIServer, "Listen on given port") command.Flags().IntVar(&metricsPort, "metrics-port", common.DefaultPortArgoCDAPIServerMetrics, "Start metrics on given port") + command.Flags().StringVar(&otlpAddress, "otlp-address", env.StringFromEnv("ARGOCD_SERVER_OTLP_ADDRESS", ""), "OpenTelemetry collector address to send traces to") command.Flags().IntVar(&repoServerTimeoutSeconds, "repo-server-timeout-seconds", env.ParseNumFromEnv("ARGOCD_SERVER_REPO_SERVER_TIMEOUT_SECONDS", 60, 0, math.MaxInt64), "Repo server RPC call timeout seconds.") command.Flags().StringVar(&frameOptions, "x-frame-options", env.StringFromEnv("ARGOCD_SERVER_X_FRAME_OPTIONS", "sameorigin"), "Set X-Frame-Options header in HTTP responses to `value`. To disable, set to \"\".") command.Flags().StringVar(&contentSecurityPolicy, "content-security-policy", env.StringFromEnv("ARGOCD_SERVER_CONTENT_SECURITY_POLICY", "frame-ancestors 'self';"), "Set Content-Security-Policy header in HTTP responses to `value`. To disable, set to \"\".") diff --git a/cmpserver/server.go b/cmpserver/server.go index 68d8c84b4c7da..636dbcd1f7339 100644 --- a/cmpserver/server.go +++ b/cmpserver/server.go @@ -2,6 +2,7 @@ package cmpserver import ( "fmt" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "net" "os" "os/signal" @@ -42,8 +43,18 @@ func NewServer(initConstants plugin.CMPServerInitConstants) (*ArgoCDCMPServer, e } serverLog := log.NewEntry(log.StandardLogger()) - streamInterceptors := []grpc.StreamServerInterceptor{grpc_logrus.StreamServerInterceptor(serverLog), grpc_prometheus.StreamServerInterceptor, grpc_util.PanicLoggerStreamServerInterceptor(serverLog)} - unaryInterceptors := []grpc.UnaryServerInterceptor{grpc_logrus.UnaryServerInterceptor(serverLog), grpc_prometheus.UnaryServerInterceptor, grpc_util.PanicLoggerUnaryServerInterceptor(serverLog)} + streamInterceptors := []grpc.StreamServerInterceptor{ + otelgrpc.StreamServerInterceptor(), + grpc_logrus.StreamServerInterceptor(serverLog), + grpc_prometheus.StreamServerInterceptor, + grpc_util.PanicLoggerStreamServerInterceptor(serverLog), + } + unaryInterceptors := []grpc.UnaryServerInterceptor{ + otelgrpc.UnaryServerInterceptor(), + grpc_logrus.UnaryServerInterceptor(serverLog), + grpc_prometheus.UnaryServerInterceptor, + grpc_util.PanicLoggerUnaryServerInterceptor(serverLog), + } serverOpts := []grpc.ServerOption{ grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...)), diff --git a/docs/operator-manual/server-commands/argocd-repo-server.md b/docs/operator-manual/server-commands/argocd-repo-server.md index a72b89d46b5a4..a909e80a65f32 100644 --- a/docs/operator-manual/server-commands/argocd-repo-server.md +++ b/docs/operator-manual/server-commands/argocd-repo-server.md @@ -19,6 +19,7 @@ argocd-repo-server [flags] --logformat string Set the logging format. One of: text|json (default "text") --loglevel string Set the logging level. One of: debug|info|warn|error (default "info") --metrics-port int Start metrics server on given port (default 8084) + --otlp-address string OpenTelemetry collector address to send traces to --parallelismlimit int Limit on number of concurrent manifests generate requests. Any value less the 1 means no limit. --port int Listen on given port for incoming connections (default 8081) --redis string Redis server hostname and port (e.g. argocd-redis:6379). diff --git a/docs/operator-manual/server-commands/argocd-server.md b/docs/operator-manual/server-commands/argocd-server.md index 369c142310aad..fbee0b86a0ffb 100644 --- a/docs/operator-manual/server-commands/argocd-server.md +++ b/docs/operator-manual/server-commands/argocd-server.md @@ -40,6 +40,7 @@ argocd-server [flags] --metrics-port int Start metrics on given port (default 8083) -n, --namespace string If present, the namespace scope for this CLI request --oidc-cache-expiration duration Cache expiration for OIDC state (default 3m0s) + --otlp-address string OpenTelemetry collector address to send traces to --password string Password for basic authentication to the API server --port int Listen on given port (default 8080) --redis string Redis server hostname and port (e.g. argocd-redis:6379). diff --git a/go.mod b/go.mod index 480a5867a21b8..49afec77dd5f6 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/fsnotify/fsnotify v1.5.1 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-git/v5 v5.4.2 - github.com/go-logr/logr v1.2.2 + github.com/go-logr/logr v1.2.3 github.com/go-openapi/loads v0.19.4 github.com/go-openapi/runtime v0.19.4 github.com/go-openapi/spec v0.19.5 // indirect @@ -36,7 +36,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.2.0 github.com/golang/protobuf v1.5.2 github.com/gomodule/redigo v2.0.0+incompatible // indirect - github.com/google/go-cmp v0.5.6 + github.com/google/go-cmp v0.5.7 github.com/google/go-github/v35 v35.3.0 github.com/google/go-jsonnet v0.18.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 @@ -70,7 +70,7 @@ require ( github.com/soheilhy/cmux v0.1.5 github.com/spf13/cobra v1.3.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.1 github.com/valyala/fasttemplate v1.2.1 github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0 github.com/xanzy/go-gitlab v0.60.0 @@ -229,7 +229,23 @@ require ( sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect ) -require github.com/jmespath/go-jmespath v0.4.0 // indirect +require ( + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.31.0 + go.opentelemetry.io/otel v1.6.3 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3 + go.opentelemetry.io/otel/sdk v1.6.3 +) + +require ( + github.com/cenkalti/backoff/v4 v4.1.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3 // indirect + go.opentelemetry.io/otel/trace v1.6.3 // indirect + go.opentelemetry.io/proto/otlp v0.15.0 // indirect +) replace ( // https://github.com/golang/go/issues/33546#issuecomment-519656923 diff --git a/go.sum b/go.sum index 5951a5cf26bb7..ab6c56d62625e 100644 --- a/go.sum +++ b/go.sum @@ -194,6 +194,8 @@ github.com/casbin/casbin/v2 v2.39.1 h1:TatfPL1hByffzPs610HL8+gBjCisAtEhjVhpIsbZ+ github.com/casbin/casbin/v2 v2.39.1/go.mod h1:sEL80qBYTbd+BPeL4iyvwYzFT3qwLaESq5aFKVLbLfA= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= @@ -388,8 +390,11 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.0.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -525,8 +530,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github/v35 v35.3.0 h1:fU+WBzuukn0VssbayTT+Zo3/ESKX9JYWjbZTLOTEyho= github.com/google/go-github/v35 v35.3.0/go.mod h1:yWB7uCcVWaUbUP74Aq3whuMySRMatyRmq5U9FTNlbio= github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg= @@ -598,6 +604,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= @@ -1068,8 +1076,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= @@ -1147,16 +1156,34 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.31.0 h1:li8u9OSMvLau7rMs8bmiL82OazG6MAkwPz2i6eS8TBQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.31.0/go.mod h1:SY9qHHUES6W3oZnO1H2W8NvsSovIoXRg/A1AH9px8+I= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.6.1/go.mod h1:blzUabWHkX6LJewxvadmzafgh/wnvBSDBdOuwkAtrWQ= +go.opentelemetry.io/otel v1.6.3 h1:FLOfo8f9JzFVFVyU+MSRJc2HdEAXQgm7pIv2uFKRSZE= +go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3 h1:nAmg1WgsUXoXf46dJG9eS/AzOcvkCTK4xJSUYpWyHYg= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3/go.mod h1:NEu79Xo32iVb+0gVNV8PMd7GoWqnyDXRlj04yFjqz40= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3 h1:4/UjHWMVVc5VwX/KAtqJOHErKigMCH8NexChMuanb/o= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3/go.mod h1:UJmXdiVVBaZ63umRUTwJuCMAV//GCMvDiQwn703/GoY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3 h1:leYDq5psbM3K4QNcZ2juCj30LjUnvxjuYQj1mkGjXFM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3/go.mod h1:ycItY/esVj8c0dKgYTOztTERXtPzcfDU/0o8EdwCjoA= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.6.3 h1:prSHYdwCQOX5DrsEzxowH3nLhoAzEBdZhvrR79scfLs= +go.opentelemetry.io/otel/sdk v1.6.3/go.mod h1:A4iWF7HTXa+GWL/AaqESz28VuSBIcZ+0CV+IzJ5NMiQ= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.6.1/go.mod h1:RkFRM1m0puWIq10oxImnGEduNBzxiN7TXluRBtE+5j0= +go.opentelemetry.io/otel/trace v1.6.3 h1:IqN4L+5b0mPNjdXIiZ90Ni4Bl5BRkDQywePLWemd9bc= +go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0 h1:h0bKrvdrT/9sBwEJ6iWUqT/N/xPcS66bL4u3isneJ6w= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1167,6 +1194,7 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -1433,6 +1461,7 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/reposerver/server.go b/reposerver/server.go index 75ddbfd2c0605..a6f5e3b04bdc2 100644 --- a/reposerver/server.go +++ b/reposerver/server.go @@ -3,6 +3,7 @@ package reposerver import ( "crypto/tls" "fmt" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "os" "path/filepath" @@ -66,8 +67,14 @@ func NewServer(metricsServer *metrics.MetricsServer, cache *reposervercache.Cach } serverLog := log.NewEntry(log.StandardLogger()) - streamInterceptors := []grpc.StreamServerInterceptor{grpc_logrus.StreamServerInterceptor(serverLog), grpc_prometheus.StreamServerInterceptor, grpc_util.PanicLoggerStreamServerInterceptor(serverLog)} + streamInterceptors := []grpc.StreamServerInterceptor{ + otelgrpc.StreamServerInterceptor(), + grpc_logrus.StreamServerInterceptor(serverLog), + grpc_prometheus.StreamServerInterceptor, + grpc_util.PanicLoggerStreamServerInterceptor(serverLog), + } unaryInterceptors := []grpc.UnaryServerInterceptor{ + otelgrpc.UnaryServerInterceptor(), grpc_logrus.UnaryServerInterceptor(serverLog), grpc_prometheus.UnaryServerInterceptor, grpc_util.PanicLoggerUnaryServerInterceptor(serverLog), diff --git a/server/server.go b/server/server.go index d1160ae3a1891..616838c2ebd7e 100644 --- a/server/server.go +++ b/server/server.go @@ -34,6 +34,7 @@ import ( "github.com/improbable-eng/grpc-web/go/grpcweb" log "github.com/sirupsen/logrus" "github.com/soheilhy/cmux" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" netCtx "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -559,6 +560,7 @@ func (a *ArgoCDServer) newGRPCServer() (*grpc.Server, application.AppResourceTre // NOTE: notice we do not configure the gRPC server here with TLS (e.g. grpc.Creds(creds)) // This is because TLS handshaking occurs in cmux handling sOpts = append(sOpts, grpc.StreamInterceptor(grpc_middleware.ChainStreamServer( + otelgrpc.StreamServerInterceptor(), grpc_logrus.StreamServerInterceptor(a.log), grpc_prometheus.StreamServerInterceptor, grpc_auth.StreamServerInterceptor(a.Authenticate), @@ -572,6 +574,7 @@ func (a *ArgoCDServer) newGRPCServer() (*grpc.Server, application.AppResourceTre ))) sOpts = append(sOpts, grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer( bug21955WorkaroundInterceptor, + otelgrpc.UnaryServerInterceptor(), grpc_logrus.UnaryServerInterceptor(a.log), grpc_prometheus.UnaryServerInterceptor, grpc_auth.UnaryServerInterceptor(a.Authenticate), diff --git a/util/trace/trace.go b/util/trace/trace.go new file mode 100644 index 0000000000000..b7db1326162f0 --- /dev/null +++ b/util/trace/trace.go @@ -0,0 +1,54 @@ +package trace + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.6.1" +) + +// InitTracer initializes the trace provider and the otel grpc exporter. +func InitTracer(ctx context.Context, serviceName, otlpAddress string) (func(), error) { + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String(serviceName), + ), + ) + if err != nil { + return nil, fmt.Errorf("failed to create resource: %w", err) + } + + // Set up a trace exporter + exporter, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithInsecure(), + otlptracegrpc.WithEndpoint(otlpAddress), + ) + if err != nil { + return nil, fmt.Errorf("failed to create trace exporter: %w", err) + } + + // Register the trace exporter with a TracerProvider, using a batch + // span processor to aggregate spans before export. + bsp := sdktrace.NewBatchSpanProcessor(exporter) + provider := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithResource(res), + sdktrace.WithSpanProcessor(bsp), + ) + + // set global propagator to tracecontext (the default is no-op). + otel.SetTextMapPropagator(propagation.TraceContext{}) + otel.SetTracerProvider(provider) + + return func() { + if err := exporter.Shutdown(ctx); err != nil { + log.Errorf("failed to stop exporter: %v", err) + } + }, nil +}