CONSOLE-3241: Use cluster proxy for managed cluster API server reques…

…ts (openshift#11970)

* CONSOLE 3241: Update server to remove original multicluster proxying implementation and instead proxy managed cluster requests to the MCE cluster proxy service

* Add proxy endpoint to proxy.ServeHTTP debug log

* Require public cluster proxy endpoint for off-cluster multicluster environment

cmd/bridge/main.go

@@ -56,6 +56,9 @@ const (
  // Well-known location of the GitOps service. This is only accessible in-cluster
  openshiftGitOpsHost = "cluster.openshift-gitops.svc:8080"
 
+ // Well-known location of the cluster proxy service. This is only accessible in-cluster
+ openshiftClusterProxyHost = "cluster-proxy-addon-user.multicluster-engine.svc:9092"
+
  clusterManagementURL = "https://api.openshift.com/"
 )
 
@@ -91,6 +94,7 @@ func main() {
  fK8sModeOffClusterThanos := fs.String("k8s-mode-off-cluster-thanos", "", "DEV ONLY. URL of the cluster's Thanos server.")
  fK8sModeOffClusterAlertmanager := fs.String("k8s-mode-off-cluster-alertmanager", "", "DEV ONLY. URL of the cluster's AlertManager server.")
  fK8sModeOffClusterMetering := fs.String("k8s-mode-off-cluster-metering", "", "DEV ONLY. URL of the cluster's metering server.")
+ fK8sModeOffClusterManagedClusterProxy := fs.String("k8s-mode-off-cluster-managed-cluster-proxy", "", "DEV ONLY. Public URL of the ACM/MCE cluster proxy.")
 
  fK8sAuth := fs.String("k8s-auth", "service-account", "service-account | bearer-token | oidc | openshift")
  fK8sAuthBearerToken := fs.String("k8s-auth-bearer-token", "", "Authorization token to send with proxied Kubernetes API requests.")
@@ -286,8 +290,6 @@ func main() {
  AddPage: *fAddPage,
  ProjectAccessClusterRoles: *fProjectAccessClusterRoles,
  Perspectives: *fPerspectives,
- K8sProxyConfigs: make(map[string]*proxy.Config),
- K8sClients: make(map[string]*http.Client),
  Telemetry: telemetryFlags,
  ReleaseVersion: *fReleaseVersion,
  NodeArchitectures: nodeArchitectures,
@@ -309,45 +311,6 @@ func main() {
  }
  }
 
- if len(managedClusterConfigs) > 0 {
- for _, managedCluster := range managedClusterConfigs {
- klog.Infof("Configuring managed cluster %s", managedCluster.Name)
- managedClusterAPIEndpointURL, err := url.Parse(managedCluster.APIServer.URL)
- if err != nil {
- klog.Errorf("Error parsing managed cluster URL for cluster %s", managedCluster.Name)
- continue
- }
-
- managedClusterCertPEM, err := ioutil.ReadFile(managedCluster.APIServer.CAFile)
- if err != nil {
- klog.Errorf("Error parsing managed cluster CA file for cluster %s", managedCluster.Name)
- continue
- }
-
- managedClusterRootCAs := x509.NewCertPool()
- if !managedClusterRootCAs.AppendCertsFromPEM(managedClusterCertPEM) {
- klog.Errorf("No CA found for the managed cluster %s", managedCluster.Name)
- continue
- }
-
- managedClusterTLSConfig := oscrypto.SecureTLSConfig(&tls.Config{
- RootCAs: managedClusterRootCAs,
- })
-
- srv.K8sProxyConfigs[managedCluster.Name] = &proxy.Config{
- TLSClientConfig: managedClusterTLSConfig,
- HeaderBlacklist: []string{"Cookie", "X-CSRFToken"},
- Endpoint: managedClusterAPIEndpointURL,
- }
-
- srv.K8sClients[managedCluster.Name] = &http.Client{
- Transport: &http.Transport{
- TLSClientConfig: managedClusterTLSConfig,
- },
- }
- }
- }
-
  // if !in-cluster (dev) we should not pass these values to the frontend
  if *fK8sMode == "in-cluster" {
  srv.GOARCH = runtime.GOARCH
@@ -409,7 +372,7 @@ func main() {
  klog.Fatalf("failed to read bearer token: %v", err)
  }
 
- srv.K8sProxyConfigs[serverutils.LocalClusterName] = &proxy.Config{
+ srv.LocalK8sProxyConfig = &proxy.Config{
  TLSClientConfig: tlsConfig,
  HeaderBlacklist: []string{"Cookie", "X-CSRFToken"},
  Endpoint: k8sEndpoint,
@@ -473,14 +436,20 @@ func main() {
  HeaderBlacklist: []string{"Cookie", "X-CSRFToken"},
  Endpoint: &url.URL{Scheme: "https", Host: openshiftGitOpsHost},
  }
+ srv.ManagedClusterProxyConfig = &proxy.Config{
+ TLSClientConfig: serviceProxyTLSConfig,
+ HeaderBlacklist: []string{"Cookie", "X-CSRFToken"},
+ Endpoint: &url.URL{Scheme: "https", Host: openshiftClusterProxyHost},
+ }
  }
 
  case "off-cluster":
  k8sEndpoint = bridge.ValidateFlagIsURL("k8s-mode-off-cluster-endpoint", *fK8sModeOffClusterEndpoint)
  serviceProxyTLSConfig := oscrypto.SecureTLSConfig(&tls.Config{
  InsecureSkipVerify: *fK8sModeOffClusterSkipVerifyTLS,
  })
- srv.K8sProxyConfigs[serverutils.LocalClusterName] = &proxy.Config{
+
+ srv.LocalK8sProxyConfig = &proxy.Config{
  TLSClientConfig: serviceProxyTLSConfig,
  HeaderBlacklist: []string{"Cookie", "X-CSRFToken"},
  Endpoint: k8sEndpoint,
@@ -548,18 +517,28 @@ func main() {
  }
  }
 
+ // Must have off-cluster cluster proxy endpoint if we have managed clusters
+ if len(managedClusterConfigs) > 0 {
+ offClusterManagedClusterProxyURL := bridge.ValidateFlagIsURL("k8s-mode-off-cluster-managed-cluster-proxy", *fK8sModeOffClusterManagedClusterProxy)
+ srv.ManagedClusterProxyConfig = &proxy.Config{
+ TLSClientConfig: serviceProxyTLSConfig,
+ HeaderBlacklist: []string{"Cookie", "X-CSRFToken"},
+ Endpoint: offClusterManagedClusterProxyURL,
+ }
+ }
+
  default:
  bridge.FlagFatalf("k8s-mode", "must be one of: in-cluster, off-cluster")
  }
 
  apiServerEndpoint := *fK8sPublicEndpoint
  if apiServerEndpoint == "" {
- apiServerEndpoint = srv.K8sProxyConfigs[serverutils.LocalClusterName].Endpoint.String()
+ apiServerEndpoint = srv.LocalK8sProxyConfig.Endpoint.String()
  }
  srv.KubeAPIServerURL = apiServerEndpoint
- srv.K8sClients[serverutils.LocalClusterName] = &http.Client{
+ srv.LocalK8sClient = &http.Client{
  Transport: &http.Transport{
- TLSClientConfig: srv.K8sProxyConfigs[serverutils.LocalClusterName].TLSClientConfig,
+ TLSClientConfig: srv.LocalK8sProxyConfig.TLSClientConfig,
  },
  }
 
@@ -735,7 +714,7 @@ func main() {
  },
  &http.Client{
  Transport: &http.Transport{
- TLSClientConfig: srv.K8sProxyConfigs[serverutils.LocalClusterName].TLSClientConfig,
+ TLSClientConfig: srv.LocalK8sProxyConfig.TLSClientConfig,
  },
  },
  nil,
@@ -753,7 +732,7 @@ func main() {
  },
  &http.Client{
  Transport: &http.Transport{
- TLSClientConfig: srv.K8sProxyConfigs[serverutils.LocalClusterName].TLSClientConfig,
+ TLSClientConfig: srv.LocalK8sProxyConfig.TLSClientConfig,
  },
  },
  knative.EventSourceFilter,
@@ -771,7 +750,7 @@ func main() {
  },
  &http.Client{
  Transport: &http.Transport{
- TLSClientConfig: srv.K8sProxyConfigs[serverutils.LocalClusterName].TLSClientConfig,
+ TLSClientConfig: srv.LocalK8sProxyConfig.TLSClientConfig,
  },
  },
  knative.ChannelFilter,

pkg/proxy/proxy.go

@@ -102,7 +102,7 @@ func CopyRequestHeaders(originalRequest, newRequest *http.Request) {
 func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
  if klog.V(3) {
- klog.Infof("PROXY: %#q\n", r.URL)
+ klog.Infof("PROXY: %#q\n", SingleJoiningSlash(p.config.Endpoint.String(), r.URL.Path))
  }
 
  // Block scripts from running in proxied content for browsers that support Content-Security-Policy.

pkg/server/kube_version.go

@@ -2,7 +2,10 @@ package server
 
 import (
  "errors"
+ "fmt"
+ "net/http"
 
+ "github.com/openshift/console/pkg/serverutils"
  "k8s.io/client-go/discovery"
  "k8s.io/client-go/rest"
  "k8s.io/klog"
@@ -13,9 +16,20 @@ func (s *Server) GetKubeVersion(cluster string) string {
  return s.KubeVersion
  }
  config := &rest.Config{
- Host: s.K8sProxyConfigs[cluster].Endpoint.String(),
- Transport: s.K8sClients[cluster].Transport,
+ Host: s.LocalK8sProxyConfig.Endpoint.String(),
+ Transport: s.LocalK8sClient.Transport,
  }
+
+ if cluster != serverutils.LocalClusterName {
+ config = &rest.Config{
+ Host: s.ManagedClusterProxyConfig.Endpoint.String(),
+ Transport: &http.Transport{
+ TLSClientConfig: s.ManagedClusterProxyConfig.TLSClientConfig,
+ },
+ APIPath: fmt.Sprintf("/%s", cluster),
+ }
+ }
+
  kubeVersion, err := kubeVersion(config)
  if err != nil {
  kubeVersion = ""

pkg/server/server.go

@@ -119,7 +119,6 @@ type jsGlobals struct {
 }
 
 type Server struct {
- K8sProxyConfigs map[string]*proxy.Config
  BaseURL *url.URL
  LogoutRedirect *url.URL
  PublicDir string
@@ -145,7 +144,8 @@ type Server struct {
  I18nNamespaces []string
  PluginProxy string
  // Clients with the correct TLS setup for communicating with the API servers.
- K8sClients map[string]*http.Client
+ LocalK8sClient *http.Client
+ LocalK8sProxyConfig *proxy.Config
  ThanosProxyConfig *proxy.Config
  ThanosTenancyProxyConfig *proxy.Config
  ThanosTenancyProxyForRulesConfig *proxy.Config
@@ -157,6 +157,7 @@ type Server struct {
  PluginsProxyTLSConfig *tls.Config
  GitOpsProxyConfig *proxy.Config
  ClusterManagementProxyConfig *proxy.Config
+ ManagedClusterProxyConfig *proxy.Config
  // A lister for resource listing of a particular kind
  MonitoringDashboardConfigMapLister ResourceLister
  KnativeEventSourceCRDLister ResourceLister
@@ -185,7 +186,7 @@ func (s *Server) authDisabled() bool {
 }
 
 func (s *Server) prometheusProxyEnabled() bool {
- return len(s.K8sProxyConfigs) == 1 && s.ThanosTenancyProxyConfig != nil && s.ThanosTenancyProxyForRulesConfig != nil
+ return len(s.Authers) == 1 && s.ThanosTenancyProxyConfig != nil && s.ThanosTenancyProxyForRulesConfig != nil
 }
 
 func (s *Server) alertManagerProxyEnabled() bool {
@@ -204,31 +205,38 @@ func (s *Server) getLocalAuther() *auth.Authenticator {
  return s.Authers[serverutils.LocalClusterName]
 }
 
-func (s *Server) getLocalK8sProxyConfig() *proxy.Config {
- return s.K8sProxyConfigs[serverutils.LocalClusterName]
+func (s *Server) getK8sProxyConfig(cluster string) *proxy.Config {
+ proxyConfig := s.LocalK8sProxyConfig
+ if cluster != serverutils.LocalClusterName {
+ proxyConfig = s.ManagedClusterProxyConfig
+ path := fmt.Sprintf("/%s", cluster)
+ proxyConfig.Endpoint.Path = path
+ proxyConfig.Endpoint.RawPath = path
+ }
+
+ if len(s.BaseURL.Scheme) > 0 && len(s.BaseURL.Host) > 0 {
+ proxyConfig.Origin = fmt.Sprintf("%s://%s", s.BaseURL.Scheme, s.BaseURL.Host)
+ }
+
+ return proxyConfig
 }
 
-func (s *Server) getLocalK8sClient() *http.Client {
- return s.K8sClients[serverutils.LocalClusterName]
+func (s *Server) getK8sClient(cluster string) *http.Client {
+ if cluster == serverutils.LocalClusterName {
+ return s.LocalK8sClient
+ }
+ return &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: s.ManagedClusterProxyConfig.TLSClientConfig,
+ },
+ }
 }
 
 func (s *Server) HTTPHandler() http.Handler {
  mux := http.NewServeMux()
-
- if len(s.BaseURL.Scheme) > 0 && len(s.BaseURL.Host) > 0 {
- for cluster := range s.K8sProxyConfigs {
- s.K8sProxyConfigs[cluster].Origin = fmt.Sprintf("%s://%s", s.BaseURL.Scheme, s.BaseURL.Host)
- }
- }
-
  localAuther := s.getLocalAuther()
- localK8sProxyConfig := s.getLocalK8sProxyConfig()
- localK8sClient := s.getLocalK8sClient()
- k8sProxies := make(map[string]*proxy.Proxy)
- for cluster, proxyConfig := range s.K8sProxyConfigs {
- k8sProxies[cluster] = proxy.NewProxy(proxyConfig)
- }
-
+ localK8sProxyConfig := s.getK8sProxyConfig(serverutils.LocalClusterName)
+ localK8sProxy := proxy.NewProxy(localK8sProxyConfig)
  handle := func(path string, handler http.Handler) {
  mux.Handle(proxy.SingleJoiningSlash(s.BaseURL.Path, path), handler)
  }
@@ -318,15 +326,9 @@ func (s *Server) HTTPHandler() http.Handler {
  proxy.SingleJoiningSlash(s.BaseURL.Path, k8sProxyEndpoint),
  authHandlerWithUser(func(user *auth.User, w http.ResponseWriter, r *http.Request) {
  cluster := serverutils.GetCluster(r)
- k8sProxy, k8sProxyFound := k8sProxies[cluster]
-
- if !k8sProxyFound {
- klog.Errorf("Bad Request. Invalid cluster: %v", cluster)
- w.WriteHeader(http.StatusBadRequest)
- return
- }
-
  r.Header.Set("Authorization", fmt.Sprintf("Bearer %s", user.Token))
+ proxyConfig := s.getK8sProxyConfig(cluster)
+ k8sProxy := proxy.NewProxy(proxyConfig)
  k8sProxy.ServeHTTP(w, r)
  })),
  )
@@ -348,7 +350,7 @@ func (s *Server) HTTPHandler() http.Handler {
  panic(err)
  }
  opts := []graphql.SchemaOpt{graphql.UseFieldResolvers()}
- k8sResolver := resolver.K8sResolver{K8sProxy: k8sProxies[serverutils.LocalClusterName]}
+ k8sResolver := resolver.K8sResolver{K8sProxy: localK8sProxy}
  rootResolver := resolver.RootResolver{K8sResolver: &k8sResolver}
  schema := graphql.MustParseSchema(string(graphQLSchema), &rootResolver, opts...)
  handler := graphqlws.NewHandler()
@@ -529,7 +531,7 @@ func (s *Server) HTTPHandler() http.Handler {
  // List operator operands endpoint
  operandsListHandler := &OperandsListHandler{
  APIServerURL: s.KubeAPIServerURL,
- Client: localK8sClient,
+ Client: s.LocalK8sClient,
  }
 
  handle(operandsListEndpoint, http.StripPrefix(
@@ -547,14 +549,14 @@ func (s *Server) HTTPHandler() http.Handler {
  // User settings
  userSettingHandler := usersettings.UserSettingsHandler{
  K8sProxyConfig: localK8sProxyConfig,
- Client: localK8sClient,
+ Client: s.LocalK8sClient,
  Endpoint: localK8sProxyConfig.Endpoint.String(),
  ServiceAccountToken: s.ServiceAccountToken,
  }
  handle("/api/console/user-settings", authHandlerWithUser(userSettingHandler.HandleUserSettings))
 
- helmHandlers := helmhandlerspkg.New(localK8sProxyConfig.Endpoint.String(), localK8sClient.Transport, s)
- verifierHandler := helmhandlerspkg.NewVerifierHandler(localK8sProxyConfig.Endpoint.String(), localK8sClient.Transport, s)
+ helmHandlers := helmhandlerspkg.New(localK8sProxyConfig.Endpoint.String(), s.LocalK8sClient.Transport, s)
+ verifierHandler := helmhandlerspkg.NewVerifierHandler(localK8sProxyConfig.Endpoint.String(), s.LocalK8sClient.Transport, s)
  handle("/api/helm/verify", authHandlerWithUser(func(user *auth.User, w http.ResponseWriter, r *http.Request) {
  switch r.Method {
  case http.MethodPost:
@@ -703,8 +705,8 @@ func (s *Server) indexHandler(w http.ResponseWriter, r *http.Request) {
  plugins = append(plugins, plugin)
  }
 
- clusters := make([]string, 0, len(s.K8sProxyConfigs))
- for cluster := range s.K8sProxyConfigs {
+ clusters := make([]string, 0, len(s.Authers))
+ for cluster := range s.Authers {
  clusters = append(clusters, cluster)
  }
 
@@ -813,14 +815,8 @@ func (s *Server) handleOpenShiftTokenDeletion(user *auth.User, w http.ResponseWr
 
  // Proxy request to correct cluster
  cluster := serverutils.GetCluster(r)
- k8sProxy, k8sProxyFound := s.K8sProxyConfigs[cluster]
- k8sClient, k8sClientFound := s.K8sClients[cluster]
- if !k8sProxyFound || !k8sClientFound {
- klog.Errorf("Bad Request. Invalid cluster: %v", cluster)
- w.WriteHeader(http.StatusBadRequest)
- return
- }
-
+ k8sProxy := s.getK8sProxyConfig(cluster)
+ k8sClient := s.getK8sClient(cluster)
  tokenName := user.Token
  if strings.HasPrefix(tokenName, sha256Prefix) {
  tokenName = tokenToObjectName(tokenName)

pkg/serverconfig/types.go

@@ -225,7 +225,7 @@ type Helm struct {
  ChartRepo HelmChartRepo `yaml:"chartRepository"`
 }
 
-// ManagedClusterAPIServerConfig enables proxying managed cluster API server requests
+// TODO Remove this type once the console operator has been updated. It is obsolete now that we are using the MCE cluster proxy.
 type ManagedClusterAPIServerConfig struct {
  URL string `json:"url" yaml:"url"`
  CAFile string `json:"caFile" yaml:"caFile"`
@@ -240,7 +240,7 @@ type ManagedClusterOAuthConfig struct {
 
 // ManagedClusterConfig enables proxying to an ACM managed cluster
 type ManagedClusterConfig struct {
- Name string `json:"name" yaml:"name"` // ManagedCluster name, provided through ACM
- APIServer ManagedClusterAPIServerConfig `json:"apiServer" yaml:"apiServer"`
+ Name string `json:"name" yaml:"name"`  // ManagedCluster name, provided through ACM
+ APIServer ManagedClusterAPIServerConfig `json:"apiServer" yaml:"apiServer"` // TODO Remove this property once conosle operator has been updated
  OAuth ManagedClusterOAuthConfig `json:"oauth" yaml:"oauth"`
 }