feat: add tls policy to scheduler grpc server (dragonflyoss#1616)

Signed-off-by: Gaius <[email protected]>

client/daemon/daemon.go

@@ -156,7 +156,7 @@ func New(opt *config.DaemonOption, d dfpath.Dfpath) (Daemon, error) {
 				Logger:       zapadapter.New(logger.CoreLogger.Desugar()),
 				Cache: cache.NewCertifyMutliCache(
 					certify.NewMemCache(),
-					certify.DirCache(path.Join(d.CacheDir(), cache.CertifyCacheDirName))),
+					certify.DirCache(path.Join(d.CacheDir(), cache.DfdaemonCertifyCacheDirName))),
 			}
 
 			// issue a certificate to reduce first time delay

client/daemon/rpcserver/seeder_test.go

@@ -29,6 +29,8 @@ import (
 	testifyassert "github.com/stretchr/testify/assert"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 
 	cdnsystemv1 "d7y.io/api/pkg/apis/cdnsystem/v1"
 	commonv1 "d7y.io/api/pkg/apis/common/v1"
@@ -382,7 +384,7 @@ func setupSeederServerAndClient(t *testing.T, srv *server, sd *seeder, assert *t
 	client, err := client.GetClientByAddr(context.Background(), dfnet.NetAddr{
 		Type: dfnet.TCP,
 		Addr: fmt.Sprintf(":%d", port),
-	})
+	}, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		t.Fatal(err)
 	}

hack/ca/gen-manager-ca.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 cat << EOF > manager.conf
 [ ca ]
 # man ca
@@ -146,4 +148,4 @@ echo merge root ca cert
 cat ./manager/certs/manager.cert.pem rootca/certs/rootca.cert.pem > ./manager/certs/manager.cert-chain.pem
 
 echo review manager cert
-openssl x509 -in ./manager/certs/manager.cert.pem -noout -text
+openssl x509 -in ./manager/certs/manager.cert.pem -noout -text

hack/ca/gen-root-ca.sh

@@ -148,4 +148,4 @@ echo generate root ca cer format cert
 openssl x509 -inform PEM -in rootca/certs/rootca.cert.pem -outform DER -out rootca/certs/rootca.cert.cer
 
 echo review ca cert
-openssl x509 -in rootca/certs/rootca.cert.pem -noout -text
+openssl x509 -in rootca/certs/rootca.cert.pem -noout -text

manager/manager.go

@@ -170,7 +170,7 @@ func New(cfg *config.Config, d dfpath.Dfpath) (*Server, error) {
 
 		// Manager GRPC server's tls varify must be false. If ClientCAs are required for client verification,
 		// the client cannot call the IssueCertificate api.
-		transportCredentials, err := rpc.NewServerCredentialsByCertify(cfg.Security.TLSPolicy, false, &cert, &certify.Certify{
+		transportCredentials, err := rpc.NewServerCredentialsByCertify(cfg.Security.TLSPolicy, false, cert.Certificate, &certify.Certify{
 			CommonName:   ip.IPv4,
 			Issuer:       issuer.NewDragonflyManagerIssuer(&cert),
 			RenewBefore:  time.Hour,
@@ -179,7 +179,7 @@ func New(cfg *config.Config, d dfpath.Dfpath) (*Server, error) {
 			Logger:       zapadapter.New(logger.CoreLogger.Desugar()),
 			Cache: pkgcache.NewCertifyMutliCache(
 				certify.NewMemCache(),
-				certify.DirCache(path.Join(d.CacheDir(), pkgcache.CertifyCacheDirName))),
+				certify.DirCache(path.Join(d.CacheDir(), pkgcache.ManagerCertifyCacheDirName))),
 		})
 		if err != nil {
 			return nil, err

pkg/cache/certify.go

@@ -24,8 +24,14 @@ import (
 )
 
 const (
-	// CertifyCacheDirName is dir name of certify cache.
-	CertifyCacheDirName = "certs"
+	// SchedulerCertifyCacheDirName is dir name of scheduler certify cache.
+	SchedulerCertifyCacheDirName = "scheduler-certs"
+
+	// ManagerCertifyCacheDirName is dir name of manager certify cache.
+	ManagerCertifyCacheDirName = "manager-certs"
+
+	// DfdaemonCertifyCacheDirName is dir name of dfdaemon certify cache.
+	DfdaemonCertifyCacheDirName = "dfdaemon-certs"
 )
 
 type certifyCache struct {

pkg/issuer/dragonfly.go

@@ -95,7 +95,8 @@ func fromCertifyCertConfig(commonName string, conf *certify.CertConfig) ([]byte,
 
 	template := &x509.CertificateRequest{
 		Subject: pkix.Name{
-			CommonName: commonName,
+			CommonName:   commonName,
+			Organization: defaultSubjectOrganization,
 		},
 	}
 

pkg/issuer/dragonfly_manager.go

@@ -34,15 +34,9 @@ import (
 )
 
 var (
-	// defaultSubjectCommonName is default common name of subject.
-	defaultSubjectCommonName = "manager"
-
 	// defaultSubjectOrganization is default organization of subject.
 	defaultSubjectOrganization = []string{"Dragonfly"}
 
-	// defaultSubjectOrganizationalUnit is default organizational unit of subject.
-	defaultSubjectOrganizationalUnit = []string{"Manager"}
-
 	// defaultIPAddresses is default ip addresses of certificate.
 	defaultIPAddresses = []net.IP{net.ParseIP(ip.IPv4)}
 
@@ -138,14 +132,13 @@ func (i *dragonflyManagerIssuer) Issue(ctx context.Context, commonName string, c
 	template := x509.Certificate{
 		SerialNumber: serial,
 		Subject: pkix.Name{
-			CommonName:         defaultSubjectCommonName,
-			Organization:       defaultSubjectOrganization,
-			OrganizationalUnit: defaultSubjectOrganizationalUnit,
+			CommonName:   commonName,
+			Organization: defaultSubjectOrganization,
 		},
-		DNSNames:              i.dnsNames,
+		DNSNames:              append(certConfig.SubjectAlternativeNames, i.dnsNames...),
 		EmailAddresses:        i.emailAddresses,
-		IPAddresses:           i.ipAddresses,
-		URIs:                  i.uris,
+		IPAddresses:           append(certConfig.IPSubjectAlternativeNames, i.ipAddresses...),
+		URIs:                  append(certConfig.URISubjectAlternativeNames, i.uris...),
 		NotBefore:             now.Add(-10 * time.Minute).UTC(),
 		NotAfter:              now.Add(i.validityDuration).UTC(),
 		BasicConstraintsValid: true,

pkg/rpc/cdnsystem/client/client.go

@@ -29,7 +29,6 @@ import (
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer"
-	"google.golang.org/grpc/credentials/insecure"
 
 	cdnsystemv1 "d7y.io/api/pkg/apis/cdnsystem/v1"
 	commonv1 "d7y.io/api/pkg/apis/common/v1"
@@ -59,7 +58,6 @@ func GetClientByAddr(ctx context.Context, netAddr dfnet.NetAddr, opts ...grpc.Di
 		ctx,
 		netAddr.Addr,
 		append([]grpc.DialOption{
-			grpc.WithTransportCredentials(insecure.NewCredentials()),
 			grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(
 				rpc.ConvertErrorUnaryClientInterceptor,
 				otelgrpc.UnaryClientInterceptor(),
@@ -98,7 +96,6 @@ func GetClient(ctx context.Context, dynconfig config.DynconfigInterface, opts ..
 		resolver.SeedPeerVirtualTarget,
 		append([]grpc.DialOption{
 			grpc.WithDefaultServiceConfig(pkgbalancer.BalancerServiceConfig),
-			grpc.WithTransportCredentials(insecure.NewCredentials()),
 			grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(
 				rpc.ConvertErrorUnaryClientInterceptor,
 				otelgrpc.UnaryClientInterceptor(),

pkg/rpc/credential.go

@@ -24,6 +24,8 @@ import (
 
 	"github.com/johanbrandhorst/certify"
 	"google.golang.org/grpc/credentials"
+
+	"d7y.io/dragonfly/v2/pkg/net/ip"
 )
 
 const (
@@ -41,9 +43,15 @@ const (
 )
 
 // NewServerCredentialsByCertify returns server transport credentials by certify.
-func NewServerCredentialsByCertify(tlsPolicy string, tlsVerify bool, tlsCACert *tls.Certificate, certifyClient *certify.Certify) (credentials.TransportCredentials, error) {
+func NewServerCredentialsByCertify(tlsPolicy string, tlsVerify bool, clientCAs [][]byte, certifyClient *certify.Certify) (credentials.TransportCredentials, error) {
 	tlsConfig := &tls.Config{
-		GetCertificate: certifyClient.GetCertificate,
+		GetCertificate: func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			if hello.ServerName == "" {
+				hello.ServerName = ip.IPv4
+			}
+
+			return certifyClient.GetCertificate(hello)
+		},
 	}
 
 	switch tlsPolicy {
@@ -56,7 +64,7 @@ func NewServerCredentialsByCertify(tlsPolicy string, tlsVerify bool, tlsCACert *
 		}
 
 		certPool := x509.NewCertPool()
-		for _, cert := range tlsCACert.Certificate {
+		for _, cert := range clientCAs {
 			if !certPool.AppendCertsFromPEM(cert) {
 				return nil, errors.New("invalid CA Cert")
 			}
@@ -69,3 +77,62 @@ func NewServerCredentialsByCertify(tlsPolicy string, tlsVerify bool, tlsCACert *
 		return nil, fmt.Errorf("invalid tlsPolicy: %s", tlsPolicy)
 	}
 }
+
+// NewClientCredentialsByCertify returns client transport credentials by certify.
+func NewClientCredentialsByCertify(tlsPolicy string, tlsVerify bool, rootCAs [][]byte, certifyClient *certify.Certify) (credentials.TransportCredentials, error) {
+	tlsConfig := &tls.Config{
+		GetClientCertificate: certifyClient.GetClientCertificate,
+	}
+
+	if !tlsVerify {
+		tlsConfig.InsecureSkipVerify = true
+	}
+
+	switch tlsPolicy {
+	case DefaultTLSPolicy, PreferTLSPolicy:
+		return NewMuxTransportCredentials(tlsConfig,
+			WithTLSPreferClientHandshake(tlsPolicy == PreferTLSPolicy)), nil
+	case ForceTLSPolicy:
+		certPool := x509.NewCertPool()
+		for _, cert := range rootCAs {
+			if !certPool.AppendCertsFromPEM(cert) {
+				return nil, errors.New("invalid CA Cert")
+			}
+		}
+
+		tlsConfig.RootCAs = certPool
+		return credentials.NewTLS(tlsConfig), nil
+	default:
+		return nil, fmt.Errorf("invalid tlsPolicy: %s", tlsPolicy)
+	}
+}
+
+// NewClientCredentials returns client transport credentials.
+func NewClientCredentials(tlsPolicy string, tlsVerify bool, certs []tls.Certificate, rootCAs [][]byte) (credentials.TransportCredentials, error) {
+	tlsConfig := &tls.Config{}
+	if !tlsVerify {
+		tlsConfig.InsecureSkipVerify = true
+	}
+
+	if len(certs) > 0 {
+		tlsConfig.Certificates = certs
+	}
+
+	switch tlsPolicy {
+	case DefaultTLSPolicy, PreferTLSPolicy:
+		return NewMuxTransportCredentials(tlsConfig,
+			WithTLSPreferClientHandshake(tlsPolicy == PreferTLSPolicy)), nil
+	case ForceTLSPolicy:
+		certPool := x509.NewCertPool()
+		for _, cert := range rootCAs {
+			if !certPool.AppendCertsFromPEM(cert) {
+				return nil, errors.New("invalid CA Cert")
+			}
+		}
+
+		tlsConfig.RootCAs = certPool
+		return credentials.NewTLS(tlsConfig), nil
+	default:
+		return nil, fmt.Errorf("invalid tlsPolicy: %s", tlsPolicy)
+	}
+}

scheduler/config/config.go

@@ -23,6 +23,8 @@ import (
 	"d7y.io/dragonfly/v2/cmd/dependency/base"
 	"d7y.io/dragonfly/v2/pkg/net/fqdn"
 	"d7y.io/dragonfly/v2/pkg/net/ip"
+	"d7y.io/dragonfly/v2/pkg/rpc"
+	"d7y.io/dragonfly/v2/pkg/serialize"
 	"d7y.io/dragonfly/v2/scheduler/storage"
 )
 
@@ -56,6 +58,9 @@ type Config struct {
 
 	// Metrics configuration.
 	Metrics *MetricsConfig `yaml:"metrics" mapstructure:"metrics"`
+
+	// Security configuration.
+	Security *SecurityConfig `yaml:"security" mapstructure:"security"`
 }
 
 // New default configuration.
@@ -122,6 +127,11 @@ func New() *Config {
 			Addr:           DefaultMetricsAddr,
 			EnablePeerHost: false,
 		},
+		Security: &SecurityConfig{
+			AutoIssueCert: false,
+			TLSVerify:     true,
+			TLSPolicy:     rpc.DefaultTLSPolicy,
+		},
 	}
 }
 
@@ -257,6 +267,12 @@ func (cfg *Config) Validate() error {
 		}
 	}
 
+	if cfg.Security.AutoIssueCert {
+		if cfg.Security.CACert == "" {
+			return errors.New("security requires parameter caCert")
+		}
+	}
+
 	return nil
 }
 
@@ -436,3 +452,21 @@ type MetricsConfig struct {
 	// Enable peer host metrics.
 	EnablePeerHost bool `yaml:"enablePeerHost" mapstructure:"enablePeerHost"`
 }
+
+type SecurityConfig struct {
+	// AutoIssueCert indicates to issue client certificates for all grpc call
+	// if AutoIssueCert is false, any other option in Security will be ignored.
+	AutoIssueCert bool `mapstructure:"autoIssueCert" yaml:"autoIssueCert"`
+
+	// CACert is the root CA certificate for all grpc tls handshake, it can be path or PEM format string.
+	CACert serialize.PEMContent `mapstructure:"caCert" yaml:"caCert"`
+
+	// TLSPrefer indicates to verify client certificates for grpc ServerHandshake.
+	TLSVerify bool `mapstructure:"tlsVerify" yaml:"tlsVerify"`
+
+	// TLSPolicy controls the grpc shandshake behaviors:
+	// force: both ClientHandshake and ServerHandshake are only support tls.
+	// prefer: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support tls.
+	// default or empty: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support insecure (non-tls).
+	TLSPolicy string `mapstructure:"tlsPolicy" yaml:"tlsPolicy"`
+}

scheduler/config/config_test.go

@@ -27,6 +27,7 @@ import (
 
 	"d7y.io/dragonfly/v2/pkg/net/fqdn"
 	"d7y.io/dragonfly/v2/pkg/net/ip"
+	"d7y.io/dragonfly/v2/pkg/rpc"
 	"d7y.io/dragonfly/v2/scheduler/storage"
 )
 
@@ -106,6 +107,12 @@ func TestConfig_Load(t *testing.T) {
 			Addr:           ":8000",
 			EnablePeerHost: false,
 		},
+		Security: &SecurityConfig{
+			AutoIssueCert: true,
+			CACert:        "foo",
+			TLSVerify:     true,
+			TLSPolicy:     "force",
+		},
 	}
 
 	schedulerConfigYAML := &Config{}
@@ -188,5 +195,11 @@ func TestConfig_New(t *testing.T) {
 			Addr:           ":8000",
 			EnablePeerHost: false,
 		},
+		Security: &SecurityConfig{
+			AutoIssueCert: false,
+			CACert:        "",
+			TLSVerify:     true,
+			TLSPolicy:     rpc.DefaultTLSPolicy,
+		},
 	})
 }

scheduler/config/testdata/scheduler.yaml

@@ -65,3 +65,9 @@ metrics:
   enable: false
   addr: ":8000"
   enablePeerHost: false
+
+security:
+  autoIssueCert: true
+  caCert: foo
+  tlsVerify: true
+  tlsPolicy: force

scheduler/resource/resource_test.go

@@ -23,6 +23,8 @@ import (
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/resolver"
 
 	"d7y.io/dragonfly/v2/pkg/gc"
@@ -169,7 +171,7 @@ func TestResource_New(t *testing.T) {
 			dynconfig := configmocks.NewMockDynconfigInterface(ctl)
 			tc.mock(gc.EXPECT(), dynconfig.EXPECT())
 
-			resource, err := New(tc.config, gc, dynconfig)
+			resource, err := New(tc.config, gc, dynconfig, grpc.WithTransportCredentials(insecure.NewCredentials()))
 			tc.expect(t, resource, err)
 		})
 	}