Skip to content

Commit

Permalink
[notebook] Move notebook and workshop nginx to sidecar (#10204)
Browse files Browse the repository at this point in the history
* [notebook] Move notebook and workshop nginx to sidecar

* [tls] update tls doc

* [notebook] add nginx sidecar to deployment

* add proxy.conf

* move ssl config from python to nginx

* try to fix host header

* add multiple apps to /etc/hosts on the pod

* template auth url
  • Loading branch information
daniel-goldstein authored Mar 26, 2021
1 parent 9380e85 commit 4ed7a2e
Show file tree
Hide file tree
Showing 11 changed files with 305 additions and 103 deletions.
36 changes: 36 additions & 0 deletions build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1049,6 +1049,41 @@ steps:
publishAs: notebook
dependsOn:
- service_base_image
- kind: runImage
name: render_notebook_nginx_conf
image:
valueFrom: service_base_image.image
script: |
set -ex
cd /io
rm -rf repo
mkdir repo
cd repo
{{ code.checkout_script }}
cd notebook
{% if deploy %}
DEPLOY=true
{% else %}
DEPLOY=false
{% endif %}
python3 ../ci/jinja2_render.py '{"deploy": '${DEPLOY}', "default_ns": {"name": "{{ default_ns.name }}"}}' nginx.conf nginx.conf.out
outputs:
- from: /io/repo/notebook/nginx.conf.out
to: /notebook/nginx.conf.out
dependsOn:
- default_ns
- service_base_image
- kind: buildImage
name: notebook_nginx_image
dockerFile: notebook/Dockerfile.nginx
contextPath: notebook
publishAs: notebook_nginx
inputs:
- from: /notebook/nginx.conf.out
to: /nginx.conf.out
dependsOn:
- hail_ubuntu_image
- render_notebook_nginx_conf
- kind: runImage
name: check_notebook
image:
Expand Down Expand Up @@ -3198,6 +3233,7 @@ steps:
- default_ns
- create_session_key
- notebook_image
- notebook_nginx_image
- deploy_router
- deploy_auth
- notebook_database
Expand Down
2 changes: 1 addition & 1 deletion dev-docs/tls.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Currently, all traffic in our cluster should be encrypted except for:
- from the batch-driver to the batch workers
- from the batch workers to the internal-gateway
- to ukbb-rg
- from the router to the notebook workers
- from the notebook service to the notebook workers
- to letsencrypt (oh the irony)

Known issues:
Expand Down
13 changes: 13 additions & 0 deletions notebook/Dockerfile.nginx
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
FROM {{ hail_ubuntu_image.image }}

RUN hail-apt-get-install nginx

RUN rm -f /etc/nginx/sites-enabled/default && \
rm -f /etc/nginx/nginx.conf
ADD nginx.conf.out /etc/nginx/nginx.conf
ADD proxy.conf /etc/nginx/

RUN ln -sf /dev/stdout /var/log/nginx/access.log
RUN ln -sf /dev/stderr /var/log/nginx/error.log

CMD ["nginx", "-g", "daemon off;"]
23 changes: 19 additions & 4 deletions notebook/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ include ../config.mk

NOTEBOOK_LATEST = gcr.io/$(PROJECT)/notebook:latest
NOTEBOOK_IMAGE = gcr.io/$(PROJECT)/notebook:$(shell docker images -q --no-trunc notebook | sed -e 's,[^:]*:,,')
NOTEBOOK_NGINX_LATEST = gcr.io/$(PROJECT)/notebook:latest
NOTEBOOK_NGINX_IMAGE = gcr.io/$(PROJECT)/notebook:$(shell docker images -q --no-trunc notebook_nginx | sed -e 's,[^:]*:,,')

EXTRA_PYTHONPATH := ../hail/python:../gear:../web_common
PYTHON := PYTHONPATH=$${PYTHONPATH:+$${PYTHONPATH}:}$(EXTRA_PYTHONPATH) python3
Expand All @@ -13,21 +15,34 @@ check:
curlylint .
../check-sql.sh

.PHONY: build
build:
.PHONY: build-notebook
build-notebook:
$(MAKE) -C ../docker build
-docker pull $(NOTEBOOK_LATEST)
python3 ../ci/jinja2_render.py '{"service_base_image":{"image":"service-base"}}' Dockerfile Dockerfile.out
docker build -t notebook -f Dockerfile.out --cache-from notebook,$(NOTEBOOK_LATEST),service-base ..

.PHONY: build-nginx
build-nginx:
$(MAKE) -C ../docker hail-ubuntu
-docker pull $(NOTEBOOK_NGINX_LATEST)
python3 ../ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"hail-ubuntu"}}' Dockerfile.nginx Dockerfile.nginx.out
python3 ../ci/jinja2_render.py '{"deploy": $(DEPLOY), "default_ns": {"name": "$(NAMESPACE)"}}' nginx.conf nginx.conf.out
docker build -t notebook_nginx -f Dockerfile.nginx.out --cache-from notebook_nginx,$(NOTEBOOK_NGINX_LATEST),hail-ubuntu .

.PHONY: push
push: build
push: build-notebook build-nginx
docker tag notebook $(NOTEBOOK_LATEST)
docker push $(NOTEBOOK_LATEST)
docker tag notebook $(NOTEBOOK_IMAGE)
docker push $(NOTEBOOK_IMAGE)
docker tag notebook_nginx $(NOTEBOOK_NGINX_LATEST)
docker push $(NOTEBOOK_NGINX_LATEST)
docker tag notebook_nginx $(NOTEBOOK_NGINX_IMAGE)
docker push $(NOTEBOOK_NGINX_IMAGE)


JINJA_ENVIRONMENT = '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"notebook_image":{"image":"$(NOTEBOOK_IMAGE)"},"default_ns":{"name":"$(NAMESPACE)"},"notebook_database":{"user_secret_name":"sql-notebook-user-config"},"global":{"project":"$(PROJECT)","domain":"$(DOMAIN)","k8s_server_url":"$(KUBERNETES_SERVER_URL)"},"scope":"$(SCOPE)"}'
JINJA_ENVIRONMENT = '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"notebook_image":{"image":"$(NOTEBOOK_IMAGE)"},"notebook_nginx_image":{"image":"$(NOTEBOOK_NGINX_IMAGE)"},"default_ns":{"name":"$(NAMESPACE)"},"notebook_database":{"user_secret_name":"sql-notebook-user-config"},"global":{"project":"$(PROJECT)","domain":"$(DOMAIN)","k8s_server_url":"$(KUBERNETES_SERVER_URL)"},"scope":"$(SCOPE)"}'

.PHONY: deploy
deploy: push
Expand Down
29 changes: 29 additions & 0 deletions notebook/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ spec:
hail.is/sha: "{{ code.sha }}"
grafanak8sapp: "true"
spec:
hostAliases:
- ip: "127.0.0.1"
hostnames:
- "notebook.local"
- "workshop.local"
serviceAccountName: notebook
{% if deploy %}
priorityClassName: production
Expand Down Expand Up @@ -93,9 +98,29 @@ spec:
- name: sql-config
mountPath: /sql-config
readOnly: true
- name: ssl-config-notebook-python
mountPath: /ssl-config
readOnly: true
- name: nginx
image: {{ notebook_nginx_image.image }}
resources:
requests:
cpu: "20m"
memory: "20M"
limits:
cpu: "1"
memory: "1G"
ports:
- containerPort: 443
volumeMounts:
- name: ssl-config-notebook
mountPath: /ssl-config
readOnly: true
readinessProbe:
tcpSocket:
port: 443
initialDelaySeconds: 5
periodSeconds: 5
volumes:
- name: deploy-config
secret:
Expand All @@ -113,6 +138,10 @@ spec:
secret:
optional: false
secretName: ssl-config-notebook
- name: ssl-config-notebook-python
secret:
optional: false
secretName: ssl-config-notebook-python
---
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
Expand Down
190 changes: 190 additions & 0 deletions notebook/nginx.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;

events {
worker_connections 768;
}

http {

sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_names_hash_bucket_size 128;

include /etc/nginx/mime.types;
default_type application/octet-stream;

ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;

log_format json-log escape=json '{'
'"message":"$scheme $request done in ${request_time}s: $status",'
'"response_status":$status,'
'"request_duration":$request_time,'
'"remote_address":"$remote_addr",'
'"x_real_ip":"$http_x_real_ip",'
'"request_start_time":"$time_local",'
'"body_bytes_sent":"$body_bytes_sent",'
'"http_referer":"$http_referer",'
'"http_user_agent":"$http_user_agent"'
'}';

access_log /var/log/nginx/access.log json-log;
error_log /var/log/nginx/error.log;

gzip on;

include /ssl-config/ssl-config-http.conf;
map $http_x_forwarded_proto $updated_scheme {
default $http_x_forwarded_proto;
'' $scheme;
}
map $http_x_forwarded_host $updated_host {
default $http_x_forwarded_host;
'' $http_host;
}
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}

server {
server_name notebook.*;

# needed to correctly handle error_page with internal handles
recursive_error_pages on;

location = /auth {
internal;
{% if deploy %}
proxy_pass http://notebook.local:5000/auth/$notebook_token;
{% else %}
proxy_pass http://notebook.local:5000/{{ default_ns.name }}/notebook/auth/$notebook_token;
{% endif %}
}

{% if deploy %}
location ~ /instance/([^/]+)/(.*) {
{% else %}
location ~ {{ default_ns.name }}/notebook/instance/([^/]+)/(.*) {
{% endif %}
set $notebook_token $1;
auth_request /auth;
auth_request_set $auth_pod_ip $upstream_http_pod_ip;

proxy_pass http://$auth_pod_ip$request_uri;

include /etc/nginx/proxy.conf;
proxy_http_version 1.1;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 86400;

proxy_connect_timeout 5s;

proxy_intercept_errors on;
error_page 401 403 502 504 = @error;
}

location @error {
{% if deploy %}
return 302 $updated_scheme://$updated_host/error;
{% else %}
return 302 $updated_scheme://$updated_host/{{ default_ns.name }}/notebook/error;
{% endif %}
}

location / {
proxy_pass http://notebook.local:5000;

# don't set Host, notebook dispatches off domain
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $http_x_real_ip;
proxy_http_version 1.1;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 86400;
}

listen 443 ssl;
listen [::]:443 ssl;
}

server {
server_name workshop.*;

# needed to correctly handle error_page with internal handles
recursive_error_pages on;

location = /auth {
internal;
{% if deploy %}
proxy_pass http://workshop.local:5000/auth/$notebook_token;
{% else %}
proxy_pass http://workshop.local:5000/{{ default_ns.name }}/workshop/auth/$notebook_token;
{% endif %}
}

{% if deploy %}
location ~ /instance/([^/]+)/(.*) {
{% else %}
location ~ {{ default_ns.name }}/workshop/instance/([^/]+)/(.*) {
{% endif %}
set $notebook_token $1;
auth_request /auth;
auth_request_set $auth_pod_ip $upstream_http_pod_ip;

proxy_pass http://$auth_pod_ip$request_uri;

include /etc/nginx/proxy.conf;
proxy_http_version 1.1;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 86400;

proxy_connect_timeout 5s;

proxy_intercept_errors on;
error_page 401 403 502 504 = @error;
}

location @error {
{% if deploy %}
return 302 $updated_scheme://$updated_host/error;
{% else %}
return 302 $updated_scheme://$updated_host/{{ default_ns.name }}/workshop/error;
{% endif %}
}

location / {
proxy_pass http://workshop.local:5000;

# don't set Host, notebook dispatches off domain
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $http_x_real_ip;
proxy_http_version 1.1;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_read_timeout 86400;
}

listen 443 ssl;
listen [::]:443 ssl;
}
}
4 changes: 1 addition & 3 deletions notebook/notebook/notebook.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from prometheus_async.aio.web import server_stats # type: ignore

from hailtop.config import get_deploy_config
from hailtop.tls import internal_server_ssl_context
from hailtop.hail_logging import AccessLogger
from gear import (setup_aiohttp_session, create_database_pool,
web_authenticated_users_only, web_maybe_authenticated_user,
Expand Down Expand Up @@ -828,5 +827,4 @@ def run():
web.run_app(root_app,
host='0.0.0.0',
port=5000,
access_log_class=AccessLogger,
ssl_context=internal_server_ssl_context())
access_log_class=AccessLogger)
6 changes: 6 additions & 0 deletions notebook/proxy.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $updated_host;
proxy_set_header X-Forwarded-Proto $updated_scheme;
proxy_set_header X-Real-IP $http_x_real_ip;
include /ssl-config/ssl-config-proxy.conf;
Loading

0 comments on commit 4ed7a2e

Please sign in to comment.