WIP Remove nginx unix sockets (kubernetes#4531)

* Remove nginx unix sockets * Use an emptyDir volume for /tmp in PSP e2e tests
mashiro · Sep 8, 2019 · ce3e3d5 · ce3e3d5
1 parent 76e2a5d
commit ce3e3d5
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 25 deletions.
diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go
@@ -795,10 +795,10 @@ type TemplateConfig struct {
 	PublishService           *apiv1.Service
 	EnableMetrics            bool
 
-	PID          string
-	StatusPath   string
-	StatusPort   int
-	StreamSocket string
+	PID        string
+	StatusPath string
+	StatusPort int
+	StreamPort int
 }
 
 // ListenPorts describe the ports required to run the

diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go
@@ -603,11 +603,11 @@ func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressC
 		PublishService:           n.GetPublishService(),
 		EnableMetrics:            n.cfg.EnableMetrics,
 
-		HealthzURI:   nginx.HealthPath,
-		PID:          nginx.PID,
-		StatusPath:   nginx.StatusPath,
-		StatusPort:   nginx.StatusPort,
-		StreamSocket: nginx.StreamSocket,
+		HealthzURI: nginx.HealthPath,
+		PID:        nginx.PID,
+		StatusPath: nginx.StatusPath,
+		StatusPort: nginx.StatusPort,
+		StreamPort: nginx.StreamPort,
 	}
 
 	tc.Cfg.Checksum = ingressCfg.ConfigurationChecksum
@@ -923,16 +923,16 @@ func updateStreamConfiguration(TCPEndpoints []ingress.L4Service, UDPEndpoints []
 		})
 	}
 
-	conn, err := net.Dial("unix", nginx.StreamSocket)
+	buf, err := json.Marshal(streams)
 	if err != nil {
 		return err
 	}
-	defer conn.Close()
 
-	buf, err := json.Marshal(streams)
+	conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%v", nginx.StreamPort))
 	if err != nil {
 		return err
 	}
+	defer conn.Close()
 
 	_, err = conn.Write(buf)
 	if err != nil {

diff --git a/internal/ingress/controller/nginx_test.go b/internal/ingress/controller/nginx_test.go
@@ -151,16 +151,15 @@ func TestIsDynamicConfigurationEnough(t *testing.T) {
 func TestConfigureDynamically(t *testing.T) {
 	listener, err := net.Listen("tcp", fmt.Sprintf(":%v", nginx.StatusPort))
 	if err != nil {
-		t.Fatalf("crating unix listener: %s", err)
+		t.Fatalf("crating tcp listener: %s", err)
 	}
 	defer listener.Close()
 
-	streamListener, err := net.Listen("unix", nginx.StreamSocket)
+	streamListener, err := net.Listen("tcp", fmt.Sprintf(":%v", nginx.StreamPort))
 	if err != nil {
-		t.Fatalf("crating unix listener: %s", err)
+		t.Fatalf("crating tcp listener: %s", err)
 	}
 	defer streamListener.Close()
-	defer os.Remove(nginx.StreamSocket)
 
 	endpointStats := map[string]int{"/configuration/backends": 0, "/configuration/general": 0, "/configuration/servers": 0}
 	resetEndpointStats := func() {
@@ -321,16 +320,15 @@ func TestConfigureDynamically(t *testing.T) {
 func TestConfigureCertificates(t *testing.T) {
 	listener, err := net.Listen("tcp", fmt.Sprintf(":%v", nginx.StatusPort))
 	if err != nil {
-		t.Fatalf("crating unix listener: %s", err)
+		t.Fatalf("crating tcp listener: %s", err)
 	}
 	defer listener.Close()
 
-	streamListener, err := net.Listen("unix", nginx.StreamSocket)
+	streamListener, err := net.Listen("tcp", fmt.Sprintf(":%v", nginx.StreamPort))
 	if err != nil {
-		t.Fatalf("crating unix listener: %s", err)
+		t.Fatalf("crating tcp listener: %s", err)
 	}
 	defer streamListener.Close()
-	defer os.Remove(nginx.StreamSocket)
 
 	servers := []*ingress.Server{{
 		Hostname: "myapp.fake",

diff --git a/internal/nginx/main.go b/internal/nginx/main.go
@@ -50,10 +50,8 @@ var HealthCheckTimeout = 10 * time.Second
 // http://nginx.org/en/docs/http/ngx_http_stub_status_module.html
 var StatusPath = "/nginx_status"
 
-// StreamSocket defines the location of the unix socket used by NGINX for the NGINX stream configuration socket
-var StreamSocket = "/tmp/ingress-stream.sock"
-
-var statusLocation = "nginx-status"
+// StreamPort defines the port used by NGINX for the NGINX stream configuration socket
+var StreamPort = 10257
 
 // NewGetStatusRequest creates a new GET request to the internal NGINX status server
 func NewGetStatusRequest(path string) (int, []byte, error) {

diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl
@@ -669,7 +669,7 @@ stream {
     }
 
     server {
-        listen unix:{{ .StreamSocket }};
+        listen 127.0.0.1:{{ .StreamPort }};
 
         access_log off;
 

diff --git a/test/e2e/settings/pod_security_policy_volumes.go b/test/e2e/settings/pod_security_policy_volumes.go
@@ -71,6 +71,11 @@ var _ = framework.IngressNginxDescribe("Pod Security Policies with volumes", fun
 							EmptyDir: &corev1.EmptyDirVolumeSource{},
 						},
 					},
+					{
+						Name: "tmp", VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
 				}
 
 				fsGroup := int64(33)
@@ -82,6 +87,9 @@ var _ = framework.IngressNginxDescribe("Pod Security Policies with volumes", fun
 					{
 						Name: "ssl", MountPath: "/etc/ingress-controller",
 					},
+					{
+						Name: "tmp", MountPath: "/tmp",
+					},
 				}
 
 				_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(deployment)