[grafana][auth] add nginx sidecar in front of grafana for tls and auth (

#10139)

* [grafana][auth] add nginx sidecar in front of grafana for tls and auth

* use jinja templating instead of bash conditional

auth/auth/auth.py

@@ -8,6 +8,7 @@
 import google.auth.transport.requests
 import google.oauth2.id_token
 import google_auth_oauthlib.flow
+from hailtop.auth import async_get_userinfo
 from hailtop.config import get_deploy_config
 from hailtop.tls import internal_server_ssl_context
 from hailtop.hail_logging import AccessLogger
@@ -558,6 +559,19 @@ async def userinfo(request):
     return web.json_response(user)
 
 
+@routes.get('/api/v1alpha/verify_dev_credentials')
+async def verify_dev_credentials(request):
+    session = await aiohttp_session.get_session(request)
+    session_id = session.get('session_id')
+    if not session_id:
+        raise web.HTTPUnauthorized()
+    userdata = await async_get_userinfo(session_id=session_id)
+    is_developer = userdata is not None and userdata['is_developer'] == 1
+    if not is_developer:
+        raise web.HTTPUnauthorized()
+    return web.Response(status=200)
+
+
 async def on_startup(app):
     db = Database()
     await db.async_init(maxsize=50)

build.yaml

@@ -513,6 +513,41 @@ steps:
    publishAs: batch
    dependsOn:
      - service_base_image
+ - kind: runImage
+   name: render_grafana_nginx_conf
+   image:
+     valueFrom: service_base_image.image
+   script: |
+     set -ex
+     cd /io
+     rm -rf repo
+     mkdir repo
+     cd repo
+     {{ code.checkout_script }}
+     cd grafana
+     {% if deploy %}
+     DEPLOY=true
+     {% else %}
+     DEPLOY=false
+     {% endif %}
+     python3 ../ci/jinja2_render.py '{"deploy": '${DEPLOY}', "default_ns": {"name": "{{ default_ns.name }}"}}' nginx.conf nginx.conf.out
+   outputs:
+     - from: /io/repo/grafana/nginx.conf.out
+       to: /nginx.conf.out
+   dependsOn:
+    - default_ns
+    - service_base_image
+ - kind: buildImage
+   name: grafana_nginx_image
+   dockerFile: grafana/Dockerfile.nginx
+   contextPath: grafana
+   publishAs: grafana
+   inputs:
+     - from: /nginx.conf.out
+       to: /nginx.conf.out
+   dependsOn:
+     - hail_ubuntu_image
+     - render_grafana_nginx_conf
  - kind: buildImage
    name: test_ci_image
    dockerFile: ci/Dockerfile.test
@@ -1644,6 +1679,7 @@ steps:
     - dev
    dependsOn:
     - default_ns
+    - grafana_nginx_image
     - deploy_router
     - create_certs
  - kind: runImage

grafana/Dockerfile.nginx

@@ -0,0 +1,12 @@
+FROM {{ hail_ubuntu_image.image }}
+
+RUN hail-apt-get-install nginx
+
+RUN rm -f /etc/nginx/sites-enabled/default && \
+    rm -f /etc/nginx/nginx.conf
+ADD nginx.conf.out /etc/nginx/nginx.conf
+
+RUN ln -sf /dev/stdout /var/log/nginx/access.log
+RUN ln -sf /dev/stderr /var/log/nginx/error.log
+
+CMD ["nginx", "-g", "daemon off;"]

grafana/Makefile

@@ -1,7 +1,24 @@
 include ../config.mk
 
-.PHONY: deploy
-deploy:
+.PHONY: build push deploy
+
+GRAFANA_NGINX_LATEST = gcr.io/$(PROJECT)/grafana_nginx:latest
+GRAFANA_NGINX_IMAGE = gcr.io/$(PROJECT)/grafana_nginx:$(shell docker images -q --no-trunc grafana_nginx | sed -e 's,[^:]*:,,')
+
+build:
+	$(MAKE) -C ../docker hail-ubuntu
+	-docker pull $(GRAFANA_NGINX_LATEST)
+	python3 ../ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"hail-ubuntu"}}' Dockerfile.nginx Dockerfile.nginx.out
+	python3 ../ci/jinja2_render.py '{"deploy": $(DEPLOY), "default_ns": {"name": "$(NAMESPACE)"}}' nginx.conf nginx.conf.out
+	docker build -t grafana_nginx -f Dockerfile.nginx.out --cache-from grafana_nginx,$(GRAFANA_NGINX_LATEST),hail-ubuntu .
+
+push: build
+	docker tag grafana_nginx $(GRAFANA_NGINX_LATEST)
+	docker push $(GRAFANA_NGINX_LATEST)
+	docker tag grafana_nginx $(GRAFANA_NGINX_IMAGE)
+	docker push $(GRAFANA_NGINX_IMAGE)
+
+deploy: push
 	! [ -z $(NAMESPACE) ]  # call this like: make deploy NAMESPACE=default
-	python3 ../ci/jinja2_render.py '{"deploy":$(DEPLOY),"default_ns":{"name":"$(NAMESPACE)"}}' deployment.yaml deployment.yaml.out
+	python3 ../ci/jinja2_render.py '{"deploy":$(DEPLOY),"default_ns":{"name":"$(NAMESPACE)"}, "grafana_nginx_image": {"image": "$(GRAFANA_NGINX_IMAGE)"}}' deployment.yaml deployment.yaml.out
 	kubectl -n $(NAMESPACE) apply -f deployment.yaml.out

grafana/deployment.yaml

@@ -17,14 +17,14 @@ spec:
         app: grafana
     spec:
       priorityClassName: infrastructure
-      securityContext:
-        fsGroup: 472
-        runAsNonRoot: true
-        runAsUser: 472
       volumes:
         - name: grafana-configmap-volume
           configMap:
             name: grafana-config
+        - name: ssl-config-grafana
+          secret:
+            optional: false
+            secretName: ssl-config-grafana
       containers:
        - name: grafana
          image: grafana/grafana:7.3.7
@@ -54,6 +54,26 @@ spec:
              memory: "1G"
          ports:
           - containerPort: 3000
+       - name: nginx
+         image: {{ grafana_nginx_image.image }}
+         resources:
+           requests:
+             cpu: "20m"
+             memory: "20M"
+           limits:
+             cpu: "1"
+             memory: "1G"
+         ports:
+          - containerPort: 443
+         volumeMounts:
+          - name: ssl-config-grafana
+            mountPath: /ssl-config
+            readOnly: true
+         readinessProbe:
+           tcpSocket:
+             port: 443
+           initialDelaySeconds: 5
+           periodSeconds: 5
   volumeClaimTemplates:
   - metadata:
       name: grafana-storage

grafana/nginx.conf

@@ -0,0 +1,101 @@
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+  worker_connections 768;
+}
+
+http {
+
+  sendfile on;
+  tcp_nopush on;
+  tcp_nodelay on;
+  keepalive_timeout 65;
+  types_hash_max_size 2048;
+  server_names_hash_bucket_size 128;
+
+  include /etc/nginx/mime.types;
+  default_type application/octet-stream;
+
+  ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
+  ssl_prefer_server_ciphers on;
+
+  log_format json-log escape=json '{'
+   '"message":"$scheme $request done in ${request_time}s: $status",'
+   '"response_status":$status,'
+   '"request_duration":$request_time,'
+   '"remote_address":"$remote_addr",'
+   '"x_real_ip":"$http_x_real_ip",'
+   '"request_start_time":"$time_local",'
+   '"body_bytes_sent":"$body_bytes_sent",'
+   '"http_referer":"$http_referer",'
+   '"http_user_agent":"$http_user_agent"'
+ '}';
+
+  access_log /var/log/nginx/access.log json-log;
+  error_log /var/log/nginx/error.log;
+
+  gzip on;
+
+  include /ssl-config/ssl-config-http.conf;
+  map $http_x_forwarded_proto $updated_scheme {
+       default $http_x_forwarded_proto;
+       '' $scheme;
+  }
+  map $http_x_forwarded_host $updated_host {
+       default $http_x_forwarded_host;
+       '' $http_host;
+  }
+  map $http_upgrade $connection_upgrade {
+      default upgrade;
+      ''      close;
+  }
+
+  server {
+    server_name grafana.*;
+
+    location = /auth {
+        internal;
+{% if deploy %}
+        proxy_pass https://auth/api/v1alpha/verify_dev_credentials;
+{% else %}
+        proxy_pass https://auth/{{ default_ns.name }}/auth/api/v1alpha/verify_dev_credentials;
+{% endif %}
+        include /ssl-config/ssl-config-proxy.conf;
+    }
+
+    location = /healthcheck {
+        return 204;
+    }
+
+    location / {
+        auth_request /auth;
+
+        proxy_pass http://127.0.0.1:3000/;
+
+        proxy_set_header Host              $http_host;
+        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host  $updated_host;
+        proxy_set_header X-Forwarded-Proto $updated_scheme;
+        proxy_set_header X-Real-IP         $http_x_real_ip;
+
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+    }
+
+    error_page 401 = @error401;
+
+    location @error401 {
+{% if deploy %}
+        return 302 https://auth.hail.is/login?next=https://$http_host$request_uri;
+{% else %}
+        return 302 https://internal.hail.is/{{ default_ns.name }}/auth/login?next=https://internal.hail.is/{{ default_ns.name }}/grafana;
+{% endif %}
+    }
+
+
+    listen 443 ssl;
+    listen [::]:443 ssl;
+  }
+}

router/deployment.yaml

@@ -263,8 +263,9 @@ metadata:
   name: grafana
 spec:
   ports:
-   - port: 80
-     targetPort: 3000
+   - port: 443
+     protocol: TCP
+     targetPort: 443
   selector:
     app: grafana
 ---

router/router.nginx.conf.in

@@ -103,12 +103,8 @@ server {
     server_name grafana.*;
 
     location / {
-        proxy_pass http://grafana/;
-        proxy_set_header Host              $http_host;
-        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Host  $updated_host;
-        proxy_set_header X-Forwarded-Proto $updated_scheme;
-        proxy_set_header X-Real-IP         $http_x_real_ip;
+        proxy_pass https://grafana/;
+        include /etc/nginx/proxy.conf;
     }
 
     listen 443 ssl;

tls/config.yaml

@@ -97,4 +97,4 @@ principals:
   kind: json
 - name: grafana
   domain: grafana
-  kind: json
+  kind: nginx

web_common/web_common/templates/header.html

@@ -64,9 +64,7 @@
       <span class="disabled-header-dropdown-item">Monitoring</span>
       <div class="header-dropdown-menu">
         <div class="monitoring-caret header-dropdown-menu-caret"></div>
-        <a target="_blank" class="header-dropdown-menu-link" href="https://console.cloud.google.com/monitoring">
-          Monitoring<i class="material-icons text-icon">open_in_new</i>
-	      </a>
+        <a class="header-dropdown-menu-link" href="{{ grafana_base_url }}">Grafana</a>
         <a target="_blank" class="header-dropdown-menu-link" href="https://console.cloud.google.com/logs">
           Log Viewer<i class="material-icons text-icon">open_in_new</i>
 	      </a>

web_common/web_common/web_common.py

@@ -68,6 +68,7 @@ def base_context(session, userdata, service):
         'batch_driver_base_url': deploy_config.external_url('batch-driver', ''),
         'ci_base_url': deploy_config.external_url('ci', ''),
         'scorecard_base_url': deploy_config.external_url('scorecard', ''),
+        'grafana_base_url': deploy_config.external_url('grafana', ''),
         'monitoring_base_url': deploy_config.external_url('monitoring', ''),
         'benchmark_base_url': deploy_config.external_url('benchmark', ''),
         'blog_base_url': deploy_config.external_url('blog', ''),