Use internal server certificate for peering TLS

A previous commit introduced an internally-managed server certificate
to use for peering-related purposes.

Now the peering token has been updated to match that behavior:
- The server name matches the structure of the server cert
- The CA PEMs correspond to the Connect CA

Note that if Conect is disabled, and by extension the Connect CA, we
fall back to the previous behavior of returning the manually configured
certs and local server SNI.

Several tests were updated to use the gRPC TLS port since they enable
Connect by default. This means that the peering token will embed the
Connect CA, and the dialer will expect a TLS listener.

agent/agent_endpoint_test.go

@@ -1443,8 +1443,8 @@ func TestAgent_Self(t *testing.T) {
 			}
 			ports = {
 				grpc = -1
-			}
-			`,
+				grpc_tls = -1
+			}`,
 			expectXDS: false,
 			grpcTLS:   false,
 		},
@@ -1453,16 +1453,17 @@ func TestAgent_Self(t *testing.T) {
 			node_meta {
 				somekey = "somevalue"
 			}
-			`,
+			ports = {
+				grpc_tls = -1
+			}`,
 			expectXDS: true,
 			grpcTLS:   false,
 		},
 		"tls grpc": {
 			hcl: `
 				node_meta {
 					somekey = "somevalue"
-				}
-				`,
+				}`,
 			expectXDS: true,
 			grpcTLS:   true,
 		},

agent/connect/testing_ca.go

@@ -183,8 +183,7 @@ func TestCAWithKeyType(t testing.T, xc *structs.CARoot, keyType string, keyBits
 	return testCA(t, xc, keyType, keyBits, 0)
 }
 
-func testLeafWithID(t testing.T, spiffeId CertURI, root *structs.CARoot, keyType string, keyBits int, expiration time.Duration) (string, string, error) {
-
+func testLeafWithID(t testing.T, spiffeId CertURI, dnsSAN string, root *structs.CARoot, keyType string, keyBits int, expiration time.Duration) (string, string, error) {
 	if expiration == 0 {
 		// this is 10 years
 		expiration = 10 * 365 * 24 * time.Hour
@@ -238,6 +237,7 @@ func testLeafWithID(t testing.T, spiffeId CertURI, root *structs.CARoot, keyType
 		NotBefore:      time.Now(),
 		AuthorityKeyId: testKeyID(t, caSigner.Public()),
 		SubjectKeyId:   testKeyID(t, pkSigner.Public()),
+		DNSNames:       []string{dnsSAN},
 	}
 
 	// Create the certificate, PEM encode it and return that value.
@@ -263,7 +263,7 @@ func TestAgentLeaf(t testing.T, node string, datacenter string, root *structs.CA
 		Agent:      node,
 	}
 
-	return testLeafWithID(t, spiffeId, root, DefaultPrivateKeyType, DefaultPrivateKeyBits, expiration)
+	return testLeafWithID(t, spiffeId, "", root, DefaultPrivateKeyType, DefaultPrivateKeyBits, expiration)
 }
 
 func testLeaf(t testing.T, service string, namespace string, root *structs.CARoot, keyType string, keyBits int) (string, string, error) {
@@ -275,7 +275,7 @@ func testLeaf(t testing.T, service string, namespace string, root *structs.CARoo
 		Service:    service,
 	}
 
-	return testLeafWithID(t, spiffeId, root, keyType, keyBits, 0)
+	return testLeafWithID(t, spiffeId, "", root, keyType, keyBits, 0)
 }
 
 // TestLeaf returns a valid leaf certificate and it's private key for the named
@@ -305,7 +305,23 @@ func TestMeshGatewayLeaf(t testing.T, partition string, root *structs.CARoot) (s
 		Datacenter: "dc1",
 	}
 
-	certPEM, keyPEM, err := testLeafWithID(t, spiffeId, root, DefaultPrivateKeyType, DefaultPrivateKeyBits, 0)
+	certPEM, keyPEM, err := testLeafWithID(t, spiffeId, "", root, DefaultPrivateKeyType, DefaultPrivateKeyBits, 0)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	return certPEM, keyPEM
+}
+
+func TestServerLeaf(t testing.T, dc string, root *structs.CARoot) (string, string) {
+	t.Helper()
+
+	spiffeID := &SpiffeIDServer{
+		Datacenter: dc,
+		Host:       fmt.Sprintf("%s.consul", TestClusterID),
+	}
+	san := PeeringServerSAN(dc, TestTrustDomain)
+
+	certPEM, keyPEM, err := testLeafWithID(t, spiffeID, san, root, DefaultPrivateKeyType, DefaultPrivateKeyBits, 0)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}

agent/consul/leader_peering_test.go

@@ -22,6 +22,7 @@ import (
 	"google.golang.org/protobuf/proto"
 
 	"github.com/hashicorp/consul/acl"
+	"github.com/hashicorp/consul/agent/connect"
 	"github.com/hashicorp/consul/agent/consul/state"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/api"
@@ -33,29 +34,53 @@ import (
 	"github.com/hashicorp/consul/types"
 )
 
+type tlsMode byte
+
+const (
+	tlsModeNone tlsMode = iota
+	tlsModeManual
+	tlsModeAuto
+)
+
 func TestLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T) {
 	t.Run("without-tls", func(t *testing.T) {
-		testLeader_PeeringSync_Lifecycle_ClientDeletion(t, false)
+		testLeader_PeeringSync_Lifecycle_ClientDeletion(t, tlsModeNone)
+	})
+	t.Run("manual-tls", func(t *testing.T) {
+		testLeader_PeeringSync_Lifecycle_ClientDeletion(t, tlsModeManual)
 	})
-	t.Run("with-tls", func(t *testing.T) {
-		testLeader_PeeringSync_Lifecycle_ClientDeletion(t, true)
+	t.Run("auto-tls", func(t *testing.T) {
+		testLeader_PeeringSync_Lifecycle_ClientDeletion(t, tlsModeAuto)
 	})
 }
 
-func testLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T, enableTLS bool) {
+func testLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T, mode tlsMode) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")
 	}
 
+	ca := connect.TestCA(t, nil)
 	_, acceptor := testServerWithConfig(t, func(c *Config) {
 		c.NodeName = "acceptor"
 		c.Datacenter = "dc1"
 		c.TLSConfig.Domain = "consul"
-		if enableTLS {
+		if mode == tlsModeManual {
+			c.ConnectEnabled = false
 			c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
 			c.TLSConfig.GRPC.CertFile = "../../test/hostname/Bob.crt"
 			c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Bob.key"
 		}
+		if mode == tlsModeAuto {
+			c.CAConfig = &structs.CAConfiguration{
+				ClusterID: connect.TestClusterID,
+				Provider:  structs.ConsulCAProvider,
+				Config: map[string]interface{}{
+					"PrivateKey": ca.SigningKey,
+					"RootCert":   ca.RootCert,
+				},
+			}
+
+		}
 	})
 	testrpc.WaitForLeader(t, acceptor.RPC, "dc1")
 
@@ -94,11 +119,6 @@ func testLeader_PeeringSync_Lifecycle_ClientDeletion(t *testing.T, enableTLS boo
 		c.NodeName = "dialer"
 		c.Datacenter = "dc2"
 		c.PrimaryDatacenter = "dc2"
-		if enableTLS {
-			c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
-			c.TLSConfig.GRPC.CertFile = "../../test/hostname/Betty.crt"
-			c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Betty.key"
-		}
 	})
 	testrpc.WaitForLeader(t, dialer.RPC, "dc2")
 
@@ -345,27 +365,43 @@ func TestLeader_PeeringSync_Lifecycle_UnexportWhileDown(t *testing.T) {
 
 func TestLeader_PeeringSync_Lifecycle_ServerDeletion(t *testing.T) {
 	t.Run("without-tls", func(t *testing.T) {
-		testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, false)
+		testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, tlsModeNone)
+	})
+	t.Run("manual-tls", func(t *testing.T) {
+		testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, tlsModeManual)
 	})
-	t.Run("with-tls", func(t *testing.T) {
-		testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, true)
+	t.Run("auto-tls", func(t *testing.T) {
+		testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t, tlsModeAuto)
 	})
 }
 
-func testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t *testing.T, enableTLS bool) {
+func testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t *testing.T, mode tlsMode) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")
 	}
 
+	ca := connect.TestCA(t, nil)
 	_, acceptor := testServerWithConfig(t, func(c *Config) {
 		c.NodeName = "acceptor"
 		c.Datacenter = "dc1"
 		c.TLSConfig.Domain = "consul"
-		if enableTLS {
+		if mode == tlsModeManual {
+			c.ConnectEnabled = false
 			c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
 			c.TLSConfig.GRPC.CertFile = "../../test/hostname/Bob.crt"
 			c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Bob.key"
 		}
+		if mode == tlsModeAuto {
+			c.CAConfig = &structs.CAConfiguration{
+				ClusterID: connect.TestClusterID,
+				Provider:  structs.ConsulCAProvider,
+				Config: map[string]interface{}{
+					"PrivateKey": ca.SigningKey,
+					"RootCert":   ca.RootCert,
+				},
+			}
+
+		}
 	})
 	testrpc.WaitForLeader(t, acceptor.RPC, "dc1")
 
@@ -399,11 +435,6 @@ func testLeader_PeeringSync_Lifecycle_AcceptorDeletion(t *testing.T, enableTLS b
 		c.NodeName = "dialer"
 		c.Datacenter = "dc2"
 		c.PrimaryDatacenter = "dc2"
-		if enableTLS {
-			c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
-			c.TLSConfig.GRPC.CertFile = "../../test/hostname/Betty.crt"
-			c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Betty.key"
-		}
 	})
 	testrpc.WaitForLeader(t, dialer.RPC, "dc2")
 
@@ -496,6 +527,7 @@ func testLeader_PeeringSync_failsForTLSError(t *testing.T, tokenMutateFn func(to
 		c.Datacenter = "dc1"
 		c.TLSConfig.Domain = "consul"
 
+		c.ConnectEnabled = false
 		c.TLSConfig.GRPC.CAFile = "../../test/hostname/CertAuth.crt"
 		c.TLSConfig.GRPC.CertFile = "../../test/hostname/Bob.crt"
 		c.TLSConfig.GRPC.KeyFile = "../../test/hostname/Bob.key"

agent/consul/peering_backend.go

@@ -9,12 +9,14 @@ import (
 
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/acl/resolver"
+	"github.com/hashicorp/consul/agent/connect"
 	"github.com/hashicorp/consul/agent/consul/state"
 	"github.com/hashicorp/consul/agent/consul/stream"
 	"github.com/hashicorp/consul/agent/grpc-external/services/peerstream"
 	"github.com/hashicorp/consul/agent/rpc/peering"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/ipaddr"
+	"github.com/hashicorp/consul/lib"
 	"github.com/hashicorp/consul/proto/pbpeering"
 )
 
@@ -53,10 +55,39 @@ func (b *PeeringBackend) GetLeaderAddress() string {
 	return b.leaderAddr
 }
 
-// GetAgentCACertificates gets the server's raw CA data from its TLS Configurator.
-func (b *PeeringBackend) GetAgentCACertificates() ([]string, error) {
-	// TODO(peering): handle empty CA pems
-	return b.srv.tlsConfigurator.GRPCManualCAPems(), nil
+// GetTLSMaterials returns the TLS materials for the dialer to dial the acceptor using TLS.
+// It returns the server name to validate, and the CA certificate to validate with.
+func (b *PeeringBackend) GetTLSMaterials() (string, []string, error) {
+	// Do not send TLS materials to the dialer if we to not have TLS configured for gRPC.
+	if b.srv.config.GRPCTLSPort <= 0 && !b.srv.tlsConfigurator.GRPCServerUseTLS() {
+		return "", nil, nil
+	}
+
+	// If the Connect CA is not in use we rely on the manually configured certs.
+	// Otherwise we rely on the internally managed server certificate.
+	if !b.srv.config.ConnectEnabled {
+		serverName := b.srv.tlsConfigurator.ServerSNI(b.srv.config.Datacenter, "")
+		caPems := b.srv.tlsConfigurator.GRPCManualCAPems()
+
+		return serverName, caPems, nil
+	}
+
+	roots, err := b.srv.getCARoots(nil, b.srv.fsm.State())
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to fetch roots: %w", err)
+	}
+	if len(roots.Roots) == 0 {
+		return "", nil, fmt.Errorf("CA has not finished initializing")
+	}
+
+	serverName := connect.PeeringServerSAN(b.srv.config.Datacenter, roots.TrustDomain)
+
+	var caPems []string
+	for _, r := range roots.Roots {
+		caPems = append(caPems, lib.EnsureTrailingNewline(r.RootCert))
+	}
+
+	return serverName, caPems, nil
 }
 
 // GetServerAddresses looks up server or mesh gateway addresses from the state store.
@@ -117,12 +148,6 @@ func serverAddresses(state *state.Store) ([]string, error) {
 	return addrs, nil
 }
 
-// GetServerName returns the SNI to be returned in the peering token data which
-// will be used by peers when establishing peering connections over TLS.
-func (b *PeeringBackend) GetServerName() string {
-	return b.srv.tlsConfigurator.ServerSNI(b.srv.config.Datacenter, "")
-}
-
 // EncodeToken encodes a peering token as a bas64-encoded representation of JSON (for now).
 func (b *PeeringBackend) EncodeToken(tok *structs.PeeringToken) ([]byte, error) {
 	jsonToken, err := json.Marshal(tok)

agent/consul/server_test.go

@@ -232,7 +232,7 @@ func testServerWithConfig(t *testing.T, configOpts ...func(*Config)) (string, *S
 		}
 
 		// Apply config to copied fields because many tests only set the old
-		//values.
+		// values.
 		config.ACLResolverSettings.ACLsEnabled = config.ACLsEnabled
 		config.ACLResolverSettings.NodeName = config.NodeName
 		config.ACLResolverSettings.Datacenter = config.Datacenter
@@ -247,15 +247,32 @@ func testServerWithConfig(t *testing.T, configOpts ...func(*Config)) (string, *S
 	})
 	t.Cleanup(func() { srv.Shutdown() })
 
-	if srv.config.GRPCPort > 0 {
+	for _, grpcPort := range []int{srv.config.GRPCPort, srv.config.GRPCTLSPort} {
+		if grpcPort == 0 {
+			continue
+		}
+
 		// Normally the gRPC server listener is created at the agent level and
 		// passed down into the Server creation.
-		externalGRPCAddr := fmt.Sprintf("127.0.0.1:%d", srv.config.GRPCPort)
-		ln, err := net.Listen("tcp", externalGRPCAddr)
+		ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
 		require.NoError(t, err)
 
-		// Wrap the listener with TLS
-		if deps.TLSConfigurator.GRPCServerUseTLS() {
+		if grpcPort == srv.config.GRPCTLSPort || deps.TLSConfigurator.GRPCServerUseTLS() {
+			// Set the internally managed server certificate. The cert manager is hooked to the Agent, so we need to bypass that here.
+			if srv.config.PeeringEnabled && srv.config.ConnectEnabled {
+				key, _ := srv.config.CAConfig.Config["PrivateKey"].(string)
+				cert, _ := srv.config.CAConfig.Config["RootCert"].(string)
+				if key != "" && cert != "" {
+					ca := &structs.CARoot{
+						SigningKey: key,
+						RootCert:   cert,
+					}
+					require.NoError(t, deps.TLSConfigurator.UpdateAutoTLSCert(connect.TestServerLeaf(t, srv.config.Datacenter, ca)))
+					deps.TLSConfigurator.UpdateAutoTLSPeeringServerName(connect.PeeringServerSAN("dc1", connect.TestTrustDomain))
+				}
+			}
+
+			// Wrap the listener with TLS.
 			ln = tls.NewListener(ln, deps.TLSConfigurator.IncomingGRPCConfig())
 		}