feat: support opentelemetry for grpc tracing (argoproj#7539)

* feat: add support for otel grpc tracing Signed-off-by: Ben Ye <[email protected]> * chore: rebase code to master Signed-off-by: Ben Ye <[email protected]> * feat: add otelgrpc to cmpserver Signed-off-by: Ben Ye <[email protected]> * feat: add util trace package Signed-off-by: Ben Ye <[email protected]> * fix: fix tracer context Signed-off-by: Ben Ye <[email protected]> * chore: move tracer initialization part to the loop Signed-off-by: Ben Ye <[email protected]>
presidenten · Apr 26, 2022 · 09e5b60 · 09e5b60
1 parent 6cfd81b
commit 09e5b60
Show file tree

Hide file tree

Showing 11 changed files with 173 additions and 10 deletions.
diff --git a/cmd/argocd-cmp-server/commands/argocd_cmp_server.go b/cmd/argocd-cmp-server/commands/argocd_cmp_server.go
@@ -1,17 +1,21 @@
 package commands
 
 import (
+ "context"
  "time"
 
  "github.com/argoproj/pkg/stats"
+ log "github.com/sirupsen/logrus"
  "github.com/spf13/cobra"
 
  cmdutil "github.com/argoproj/argo-cd/v2/cmd/util"
  "github.com/argoproj/argo-cd/v2/cmpserver"
  "github.com/argoproj/argo-cd/v2/cmpserver/plugin"
  "github.com/argoproj/argo-cd/v2/common"
  "github.com/argoproj/argo-cd/v2/util/cli"
+ "github.com/argoproj/argo-cd/v2/util/env"
  "github.com/argoproj/argo-cd/v2/util/errors"
+ traceutil "github.com/argoproj/argo-cd/v2/util/trace"
 )
 
 const (
@@ -22,6 +26,7 @@ const (
 func NewCommand() *cobra.Command {
  var (
  configFilePath string
+ otlpAddress string
  )
  var command = cobra.Command{
  Use: cliName,
@@ -35,6 +40,16 @@ func NewCommand() *cobra.Command {
  config, err := plugin.ReadPluginConfig(configFilePath)
  errors.CheckError(err)
 
+ if otlpAddress != "" {
+ var closer func()
+ var err error
+ closer, err = traceutil.InitTracer(context.Background(), "argocd-cmp-server", otlpAddress)
+ if err != nil {
+ log.Fatalf("failed to initialize tracing: %v", err)
+ }
+ defer closer()
+ }
+
  server, err := cmpserver.NewServer(plugin.CMPServerInitConstants{
  PluginConfig: *config,
  })
@@ -54,5 +69,6 @@ func NewCommand() *cobra.Command {
  command.Flags().StringVar(&cmdutil.LogFormat, "logformat", "text", "Set the logging format. One of: text|json")
  command.Flags().StringVar(&cmdutil.LogLevel, "loglevel", "info", "Set the logging level. One of: debug|info|warn|error")
  command.Flags().StringVar(&configFilePath, "config-dir-path", common.DefaultPluginConfigFilePath, "Config management plugin configuration file location, Default is '/home/argocd/cmp-server/config/'")
+ command.Flags().StringVar(&otlpAddress, "otlp-address", env.StringFromEnv("ARGOCD_REPO_SERVER_OTLP_ADDRESS", ""), "OpenTelemetry collector address to send traces to")
  return &command
 }
diff --git a/cmd/argocd-repo-server/commands/argocd_repo_server.go b/cmd/argocd-repo-server/commands/argocd_repo_server.go
@@ -1,6 +1,7 @@
 package commands
 
 import (
+ "context"
  "fmt"
  "math"
  "net"
@@ -30,6 +31,7 @@ import (
  "github.com/argoproj/argo-cd/v2/util/healthz"
  ioutil "github.com/argoproj/argo-cd/v2/util/io"
  "github.com/argoproj/argo-cd/v2/util/tls"
+ traceutil "github.com/argoproj/argo-cd/v2/util/trace"
 )
 
 const (
@@ -71,6 +73,7 @@ func NewCommand() *cobra.Command {
  parallelismLimit int64
  listenPort int
  metricsPort int
+ otlpAddress string
  cacheSrc func() (*reposervercache.Cache, error)
  tlsConfigCustomizer tls.ConfigCustomizer
  tlsConfigCustomizerSrc func() (tls.ConfigCustomizer, error)
@@ -107,6 +110,16 @@ func NewCommand() *cobra.Command {
  }, askPassServer)
  errors.CheckError(err)
 
+ if otlpAddress != "" {
+ var closer func()
+ var err error
+ closer, err = traceutil.InitTracer(context.Background(), "argocd-repo-server", otlpAddress)
+ if err != nil {
+ log.Fatalf("failed to initialize tracing: %v", err)
+ }
+ defer closer()
+ }
+
  grpc := server.CreateGRPC()
  listener, err := net.Listen("tcp", fmt.Sprintf(":%d", listenPort))
  errors.CheckError(err)
@@ -167,6 +180,7 @@ func NewCommand() *cobra.Command {
  command.Flags().Int64Var(&parallelismLimit, "parallelismlimit", int64(env.ParseNumFromEnv("ARGOCD_REPO_SERVER_PARALLELISM_LIMIT", 0, 0, math.MaxInt32)), "Limit on number of concurrent manifests generate requests. Any value less the 1 means no limit.")
  command.Flags().IntVar(&listenPort, "port", common.DefaultPortRepoServer, "Listen on given port for incoming connections")
  command.Flags().IntVar(&metricsPort, "metrics-port", common.DefaultPortRepoServerMetrics, "Start metrics server on given port")
+ command.Flags().StringVar(&otlpAddress, "otlp-address", env.StringFromEnv("ARGOCD_REPO_SERVER_OTLP_ADDRESS", ""), "OpenTelemetry collector address to send traces to")
  command.Flags().BoolVar(&disableTLS, "disable-tls", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_TLS", false), "Disable TLS on the gRPC endpoint")
 
  tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command)

diff --git a/cmd/argocd-server/commands/argocd_server.go b/cmd/argocd-server/commands/argocd_server.go
@@ -25,6 +25,7 @@ import (
  "github.com/argoproj/argo-cd/v2/util/errors"
  "github.com/argoproj/argo-cd/v2/util/kube"
  "github.com/argoproj/argo-cd/v2/util/tls"
+ traceutil "github.com/argoproj/argo-cd/v2/util/trace"
 )
 
 const (
@@ -49,6 +50,7 @@ func NewCommand() *cobra.Command {
  insecure bool
  listenPort int
  metricsPort int
+ otlpAddress string
  glogLevel int
  clientConfig clientcmd.ClientConfig
  repoServerTimeoutSeconds int
@@ -151,12 +153,20 @@ func NewCommand() *cobra.Command {
  stats.StartStatsTicker(10 * time.Minute)
  stats.RegisterHeapDumper("memprofile")
 
+ var closer func()
  for {
  ctx := context.Background()
  ctx, cancel := context.WithCancel(ctx)
+ if otlpAddress != "" {
+ closer, err = traceutil.InitTracer(ctx, "argocd-server", otlpAddress)
+ if err != nil {
+ log.Fatalf("failed to initialize tracing: %v", err)
+ }
+ }
  argocd := server.NewServer(ctx, argoCDOpts)
  argocd.Run(ctx, listenPort, metricsPort)
  cancel()
+ closer()
  }
  },
  }
@@ -176,6 +186,7 @@ func NewCommand() *cobra.Command {
  command.AddCommand(cli.NewVersionCmd(cliName))
  command.Flags().IntVar(&listenPort, "port", common.DefaultPortAPIServer, "Listen on given port")
  command.Flags().IntVar(&metricsPort, "metrics-port", common.DefaultPortArgoCDAPIServerMetrics, "Start metrics on given port")
+ command.Flags().StringVar(&otlpAddress, "otlp-address", env.StringFromEnv("ARGOCD_SERVER_OTLP_ADDRESS", ""), "OpenTelemetry collector address to send traces to")
  command.Flags().IntVar(&repoServerTimeoutSeconds, "repo-server-timeout-seconds", env.ParseNumFromEnv("ARGOCD_SERVER_REPO_SERVER_TIMEOUT_SECONDS", 60, 0, math.MaxInt64), "Repo server RPC call timeout seconds.")
  command.Flags().StringVar(&frameOptions, "x-frame-options", env.StringFromEnv("ARGOCD_SERVER_X_FRAME_OPTIONS", "sameorigin"), "Set X-Frame-Options header in HTTP responses to `value`. To disable, set to \"\".")
  command.Flags().StringVar(&contentSecurityPolicy, "content-security-policy", env.StringFromEnv("ARGOCD_SERVER_CONTENT_SECURITY_POLICY", "frame-ancestors 'self';"), "Set Content-Security-Policy header in HTTP responses to `value`. To disable, set to \"\".")

diff --git a/cmpserver/server.go b/cmpserver/server.go
@@ -2,6 +2,7 @@ package cmpserver
 
 import (
  "fmt"
+ "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
  "net"
  "os"
  "os/signal"
@@ -42,8 +43,18 @@ func NewServer(initConstants plugin.CMPServerInitConstants) (*ArgoCDCMPServer, e
  }
 
  serverLog := log.NewEntry(log.StandardLogger())
- streamInterceptors := []grpc.StreamServerInterceptor{grpc_logrus.StreamServerInterceptor(serverLog), grpc_prometheus.StreamServerInterceptor, grpc_util.PanicLoggerStreamServerInterceptor(serverLog)}
- unaryInterceptors := []grpc.UnaryServerInterceptor{grpc_logrus.UnaryServerInterceptor(serverLog), grpc_prometheus.UnaryServerInterceptor, grpc_util.PanicLoggerUnaryServerInterceptor(serverLog)}
+ streamInterceptors := []grpc.StreamServerInterceptor{
+ otelgrpc.StreamServerInterceptor(),
+ grpc_logrus.StreamServerInterceptor(serverLog),
+ grpc_prometheus.StreamServerInterceptor,
+ grpc_util.PanicLoggerStreamServerInterceptor(serverLog),
+ }
+ unaryInterceptors := []grpc.UnaryServerInterceptor{
+ otelgrpc.UnaryServerInterceptor(),
+ grpc_logrus.UnaryServerInterceptor(serverLog),
+ grpc_prometheus.UnaryServerInterceptor,
+ grpc_util.PanicLoggerUnaryServerInterceptor(serverLog),
+ }
 
  serverOpts := []grpc.ServerOption{
  grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...)),

diff --git a/docs/operator-manual/server-commands/argocd-repo-server.md b/docs/operator-manual/server-commands/argocd-repo-server.md
@@ -19,6 +19,7 @@ argocd-repo-server [flags]
  --logformat string Set the logging format. One of: text|json (default "text")
  --loglevel string Set the logging level. One of: debug|info|warn|error (default "info")
  --metrics-port int Start metrics server on given port (default 8084)
+ --otlp-address string OpenTelemetry collector address to send traces to
  --parallelismlimit int Limit on number of concurrent manifests generate requests. Any value less the 1 means no limit.
  --port int Listen on given port for incoming connections (default 8081)
  --redis string Redis server hostname and port (e.g. argocd-redis:6379). 

diff --git a/docs/operator-manual/server-commands/argocd-server.md b/docs/operator-manual/server-commands/argocd-server.md
@@ -40,6 +40,7 @@ argocd-server [flags]
  --metrics-port int Start metrics on given port (default 8083)
  -n, --namespace string If present, the namespace scope for this CLI request
  --oidc-cache-expiration duration Cache expiration for OIDC state (default 3m0s)
+ --otlp-address string OpenTelemetry collector address to send traces to
  --password string Password for basic authentication to the API server
  --port int Listen on given port (default 8080)
  --redis string Redis server hostname and port (e.g. argocd-redis:6379). 

diff --git a/go.mod b/go.mod
@@ -23,7 +23,7 @@ require (
  github.com/fsnotify/fsnotify v1.5.1
  github.com/ghodss/yaml v1.0.0
  github.com/go-git/go-git/v5 v5.4.2
- github.com/go-logr/logr v1.2.2
+ github.com/go-logr/logr v1.2.3
  github.com/go-openapi/loads v0.19.4
  github.com/go-openapi/runtime v0.19.4
  github.com/go-openapi/spec v0.19.5 // indirect
@@ -36,7 +36,7 @@ require (
  github.com/golang-jwt/jwt/v4 v4.2.0
  github.com/golang/protobuf v1.5.2
  github.com/gomodule/redigo v2.0.0+incompatible // indirect
- github.com/google/go-cmp v0.5.6
+ github.com/google/go-cmp v0.5.7
  github.com/google/go-github/v35 v35.3.0
  github.com/google/go-jsonnet v0.18.0
  github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
@@ -70,7 +70,7 @@ require (
  github.com/soheilhy/cmux v0.1.5
  github.com/spf13/cobra v1.3.0
  github.com/spf13/pflag v1.0.5
- github.com/stretchr/testify v1.7.0
+ github.com/stretchr/testify v1.7.1
  github.com/valyala/fasttemplate v1.2.1
  github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0
  github.com/xanzy/go-gitlab v0.60.0
@@ -229,7 +229,23 @@ require (
  sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect
 )
 
-require github.com/jmespath/go-jmespath v0.4.0 // indirect
+require (
+ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.31.0
+ go.opentelemetry.io/otel v1.6.3
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3
+ go.opentelemetry.io/otel/sdk v1.6.3
+)
+
+require (
+ github.com/cenkalti/backoff/v4 v4.1.2 // indirect
+ github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3 // indirect
+ go.opentelemetry.io/otel/trace v1.6.3 // indirect
+ go.opentelemetry.io/proto/otlp v0.15.0 // indirect
+)
 
 replace (
  // https://github.com/golang/go/issues/33546#issuecomment-519656923