From f4cf22d5d4e341568a0d9b00eb4093cf31baf6af Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 3 Dec 2021 11:07:53 -0800 Subject: [PATCH 01/13] All attempts at setting keep-alive and timeout --- cmd/proxy/main.go | 15 +++- pkg/proxy/breaker.go | 26 ++++++- pkg/proxy/breaker_test.go | 6 +- pkg/proxy/handler.go | 9 ++- pkg/proxy/proxy.go | 15 ++++ test/apis/realtime/sleep/app/main.py | 51 +++++++++++- test/apis/realtime/sleep/app/requirements.txt | 1 + test/apis/realtime/sleep/cortex_cpu.yaml | 11 +-- test/apis/realtime/sleep/server.yaml | 36 +++++++++ test/apis/realtime/sleepf/app/app.py | 25 ++++++ .../apis/realtime/sleepf/app/requirements.txt | 1 + test/apis/realtime/sleepf/build-cpu.sh | 8 ++ test/apis/realtime/sleepf/cortex_cpu.yaml | 19 +++++ test/apis/realtime/sleepf/cpu.Dockerfile | 14 ++++ test/apis/realtime/sleepfg/app/main.py | 25 ++++++ .../realtime/sleepfg/app/requirements.txt | 2 + test/apis/realtime/sleepfg/build-cpu.sh | 8 ++ test/apis/realtime/sleepfg/cortex_cpu.yaml | 19 +++++ test/apis/realtime/sleepfg/cpu.Dockerfile | 14 ++++ test/apis/realtime/sleepn/build-cpu.sh | 5 ++ test/apis/realtime/sleepn/cortex_cpu.yaml | 19 +++++ test/apis/realtime/sleepn/handler.py | 26 +++++++ .../realtime/sleepn/model-server-config.yaml | 3 + test/apis/realtime/sleepn/nucleus.Dockerfile | 77 +++++++++++++++++++ test/apis/realtime/sleepn/requirements.txt | 0 25 files changed, 420 insertions(+), 15 deletions(-) create mode 100644 test/apis/realtime/sleep/server.yaml create mode 100644 test/apis/realtime/sleepf/app/app.py create mode 100644 test/apis/realtime/sleepf/app/requirements.txt create mode 100755 test/apis/realtime/sleepf/build-cpu.sh create mode 100644 test/apis/realtime/sleepf/cortex_cpu.yaml create mode 100644 test/apis/realtime/sleepf/cpu.Dockerfile create mode 100644 test/apis/realtime/sleepfg/app/main.py create mode 100644 test/apis/realtime/sleepfg/app/requirements.txt create mode 100755 test/apis/realtime/sleepfg/build-cpu.sh create mode 100644 test/apis/realtime/sleepfg/cortex_cpu.yaml create mode 100644 test/apis/realtime/sleepfg/cpu.Dockerfile create mode 100755 test/apis/realtime/sleepn/build-cpu.sh create mode 100644 test/apis/realtime/sleepn/cortex_cpu.yaml create mode 100644 test/apis/realtime/sleepn/handler.py create mode 100644 test/apis/realtime/sleepn/model-server-config.yaml create mode 100644 test/apis/realtime/sleepn/nucleus.Dockerfile create mode 100644 test/apis/realtime/sleepn/requirements.txt diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index ff2af48204..ad3051522e 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -148,10 +148,19 @@ func main() { adminHandler.Handle("/metrics", promStats) adminHandler.Handle("/healthz", readinessTCPHandler(userContainerPort, hasTCPProbe, log)) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) + servers := map[string]*http.Server{ "proxy": { - Addr: ":" + strconv.Itoa(port), - Handler: proxy.Handler(breaker, httpProxy), + Addr: ":" + strconv.Itoa(port), + Handler: proxy.Handler(breaker, httpProxy), + ReadTimeout: 60 * time.Second, + WriteTimeout: 60 * time.Second, + ReadHeaderTimeout: 60 * time.Second, + IdleTimeout: 60 * time.Second, + BaseContext: func(n net.Listener) context.Context { + return ctx + }, }, "admin": { Addr: ":" + strconv.Itoa(adminPort), @@ -159,6 +168,8 @@ func main() { }, } + servers["proxy"].RegisterOnShutdown(cancel) + errCh := make(chan error) for name, server := range servers { go func(name string, server *http.Server) { diff --git a/pkg/proxy/breaker.go b/pkg/proxy/breaker.go index ff00f35c38..0233315f97 100644 --- a/pkg/proxy/breaker.go +++ b/pkg/proxy/breaker.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "time" "go.uber.org/atomic" ) @@ -131,7 +132,7 @@ func (b *Breaker) Reserve(_ context.Context) (func(), bool) { // and queue parameters. If the concurrency limit and queue capacity are // already consumed, Maybe returns immediately without calling thunk. If // the thunk was executed, Maybe returns true, else false. -func (b *Breaker) Maybe(ctx context.Context, thunk func()) error { +func (b *Breaker) Maybe(requestID string, ctx context.Context, thunk func()) error { if !b.tryAcquirePending() { return ErrRequestQueueFull } @@ -149,7 +150,30 @@ func (b *Breaker) Maybe(ctx context.Context, thunk func()) error { defer b.sem.release() // Do the thing. + fmt.Printf("%s | %s | FORWARD to container\n", time.Now().Format(time.StampMilli), requestID) + t := time.Now() + thunk() + + // CATCH TIMEOUT + // done := make(chan struct{}) + // go func() { + // thunk() + // done <- struct{}{} + // }() + + // timer := time.NewTimer(60 * time.Second) + // select { + // // case <-ctx.Done(): + // case <-timer.C: + // fmt.Println("beans1") + // return errors.New("timeout!!") + // case <-done: + // fmt.Println("beans2") + // timer.Stop() + // } + + fmt.Printf("%s | %s | RECEIVE from container: %f\n", time.Now().Format(time.StampMilli), requestID, time.Since(t).Seconds()) // Report success return nil } diff --git a/pkg/proxy/breaker_test.go b/pkg/proxy/breaker_test.go index c94e4851e0..7b959a6497 100644 --- a/pkg/proxy/breaker_test.go +++ b/pkg/proxy/breaker_test.go @@ -368,7 +368,7 @@ func (r *requestor) request() { // or block until processSuccessfully is called. func (r *requestor) requestWithContext(ctx context.Context) { go func() { - err := r.breaker.Maybe(ctx, func() { + err := r.breaker.Maybe("", ctx, func() { r.InProgress.Inc() <-r.barrierCh }) @@ -402,14 +402,14 @@ func BenchmarkBreakerMaybe(b *testing.B) { b.Run(fmt.Sprintf("%d-sequential", c), func(b *testing.B) { for j := 0; j < b.N; j++ { - breaker.Maybe(context.Background(), op) + breaker.Maybe("", context.Background(), op) } }) b.Run(fmt.Sprintf("%d-parallel", c), func(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - breaker.Maybe(context.Background(), op) + breaker.Maybe("", context.Background(), op) } }) }) diff --git a/pkg/proxy/handler.go b/pkg/proxy/handler.go index 06e349ab1f..778d498f3f 100644 --- a/pkg/proxy/handler.go +++ b/pkg/proxy/handler.go @@ -19,7 +19,9 @@ package proxy import ( "context" "errors" + "fmt" "net/http" + "time" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/probe" @@ -32,9 +34,14 @@ func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { return } - if err := breaker.Maybe(r.Context(), func() { + if err := breaker.Maybe(r.Header.Get("x-request-id"), r.Context(), func() { + // This alone caused python to error after 1 min, but it did not return (so semaphore did not release) + newCtx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) + defer cancel() + r = r.Clone(newCtx) next.ServeHTTP(w, r) }); err != nil { + fmt.Println("GOT ERROR:", err) if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrRequestQueueFull) { http.Error(w, err.Error(), http.StatusServiceUnavailable) } else { diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go index 8ec5db589a..c2e02afb5a 100644 --- a/pkg/proxy/proxy.go +++ b/pkg/proxy/proxy.go @@ -17,9 +17,11 @@ limitations under the License. package proxy import ( + "net" "net/http" "net/http/httputil" "net/url" + "time" ) // NewReverseProxy creates a new cortex base reverse proxy @@ -42,5 +44,18 @@ func buildHTTPTransport(maxIdle, maxIdlePerHost int) http.RoundTripper { transport.MaxIdleConnsPerHost = maxIdlePerHost transport.ForceAttemptHTTP2 = false transport.DisableCompression = true + + transport.ExpectContinueTimeout = 60 * time.Second + transport.IdleConnTimeout = 60 * time.Second + transport.ResponseHeaderTimeout = 60 * time.Second + transport.TLSHandshakeTimeout = 60 * time.Second + + transport.Dial = nil + // transport.DialContext = nil + transport.DialContext = (&net.Dialer{ + Timeout: 60 * time.Second, + KeepAlive: 15 * time.Second, + }).DialContext + return transport } diff --git a/test/apis/realtime/sleep/app/main.py b/test/apis/realtime/sleep/app/main.py index 4c44187682..84fcb0a526 100644 --- a/test/apis/realtime/sleep/app/main.py +++ b/test/apis/realtime/sleep/app/main.py @@ -1,17 +1,62 @@ import time +import asyncio +from datetime import datetime -from fastapi import FastAPI +from fastapi import FastAPI, Header, Request, File, UploadFile from fastapi.responses import PlainTextResponse app = FastAPI() +@app.middleware("http") +async def add_process_time_header(request: Request, call_next): + if "x-request-id" in request.headers: + request_id = request.headers["x-request-id"] + else: + request_id = "0" + + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request_id} | middleware start", + flush=True, + ) + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request_id} | middleware finish: {str(process_time)}", + flush=True, + ) + return response + + @app.get("/healthz") def healthz(): return PlainTextResponse("ok") @app.post("/") -def sleep(sleep: float = 0): - time.sleep(sleep) +async def sleep(sleep: float = 0, x_request_id: str = Header(None), image: UploadFile = File(...)): + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {x_request_id} | request start", + flush=True, + ) + start_time = time.time() + image = await image.read() + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {x_request_id} | downloaded image: " + + str(process_time), + flush=True, + ) + # time.sleep(sleep) + await asyncio.sleep(sleep) + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {x_request_id} | request finish: " + + str(process_time), + flush=True, + ) return PlainTextResponse("ok") + + +# r=$((1 + $RANDOM % 100)); echo "request ID: $r"; SECONDS=0; curl -X POST -H "x-request-id: $r" -F image=@wp.jpg http://ad14cde85e57748ff9c384a32617133f-9335e56d3708bc0a.elb.us-west-2.amazonaws.com/sleep?sleep=10 --limit-rate 4k; echo "$SECONDS seconds" diff --git a/test/apis/realtime/sleep/app/requirements.txt b/test/apis/realtime/sleep/app/requirements.txt index 190ccb7716..9673a90c69 100644 --- a/test/apis/realtime/sleep/app/requirements.txt +++ b/test/apis/realtime/sleep/app/requirements.txt @@ -1,2 +1,3 @@ uvicorn[standard] fastapi +python-multipart diff --git a/test/apis/realtime/sleep/cortex_cpu.yaml b/test/apis/realtime/sleep/cortex_cpu.yaml index 029c847e46..108a2654a3 100644 --- a/test/apis/realtime/sleep/cortex_cpu.yaml +++ b/test/apis/realtime/sleep/cortex_cpu.yaml @@ -6,13 +6,14 @@ max_queue_length: 128 containers: - name: api - image: quay.io/cortexlabs-test/realtime-sleep-cpu:latest - readiness_probe: - http_get: - path: "/healthz" - port: 8080 + image: 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest + # readiness_probe: + # http_get: + # path: "/healthz" + # port: 8080 compute: cpu: 200m mem: 128Mi autoscaling: target_in_flight: 1 + max_replicas: 1 diff --git a/test/apis/realtime/sleep/server.yaml b/test/apis/realtime/sleep/server.yaml new file mode 100644 index 0000000000..d20d12c74c --- /dev/null +++ b/test/apis/realtime/sleep/server.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: server + labels: + app: server +spec: + replicas: 1 + selector: + matchLabels: + app: server + template: + metadata: + labels: + app: server + spec: + containers: + - name: server + image: 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest + ports: + - containerPort: 8080 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: server +spec: + type: LoadBalancer + selector: + app: server + ports: + - protocol: TCP + port: 80 + targetPort: 8080 diff --git a/test/apis/realtime/sleepf/app/app.py b/test/apis/realtime/sleepf/app/app.py new file mode 100644 index 0000000000..3bddcd51d0 --- /dev/null +++ b/test/apis/realtime/sleepf/app/app.py @@ -0,0 +1,25 @@ +from flask import Flask, request +from datetime import datetime +import time + +app = Flask(__name__) + + +@app.route("/", methods=["POST"]) +def hello_world(): + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request.headers.get('x-request-id')} | request start", + flush=True, + ) + start_time = time.time() + + file = request.files["image"] + # print(file.content_length, flush=True) + + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request.headers.get('x-request-id')} | request finish: " + + str(process_time), + flush=True, + ) + return "Hello, Docker!" diff --git a/test/apis/realtime/sleepf/app/requirements.txt b/test/apis/realtime/sleepf/app/requirements.txt new file mode 100644 index 0000000000..e3e9a71d9f --- /dev/null +++ b/test/apis/realtime/sleepf/app/requirements.txt @@ -0,0 +1 @@ +Flask diff --git a/test/apis/realtime/sleepf/build-cpu.sh b/test/apis/realtime/sleepf/build-cpu.sh new file mode 100755 index 0000000000..fc7641c05b --- /dev/null +++ b/test/apis/realtime/sleepf/build-cpu.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +image_name="realtime-sleep-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/sleepf/cortex_cpu.yaml b/test/apis/realtime/sleepf/cortex_cpu.yaml new file mode 100644 index 0000000000..108a2654a3 --- /dev/null +++ b/test/apis/realtime/sleepf/cortex_cpu.yaml @@ -0,0 +1,19 @@ +- name: sleep + kind: RealtimeAPI + pod: + port: 8080 + max_concurrency: 1 + max_queue_length: 128 + containers: + - name: api + image: 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest + # readiness_probe: + # http_get: + # path: "/healthz" + # port: 8080 + compute: + cpu: 200m + mem: 128Mi + autoscaling: + target_in_flight: 1 + max_replicas: 1 diff --git a/test/apis/realtime/sleepf/cpu.Dockerfile b/test/apis/realtime/sleepf/cpu.Dockerfile new file mode 100644 index 0000000000..f427a316cc --- /dev/null +++ b/test/apis/realtime/sleepf/cpu.Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.8-slim-buster + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 + +CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0", "--port=8080"] diff --git a/test/apis/realtime/sleepfg/app/main.py b/test/apis/realtime/sleepfg/app/main.py new file mode 100644 index 0000000000..3bddcd51d0 --- /dev/null +++ b/test/apis/realtime/sleepfg/app/main.py @@ -0,0 +1,25 @@ +from flask import Flask, request +from datetime import datetime +import time + +app = Flask(__name__) + + +@app.route("/", methods=["POST"]) +def hello_world(): + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request.headers.get('x-request-id')} | request start", + flush=True, + ) + start_time = time.time() + + file = request.files["image"] + # print(file.content_length, flush=True) + + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request.headers.get('x-request-id')} | request finish: " + + str(process_time), + flush=True, + ) + return "Hello, Docker!" diff --git a/test/apis/realtime/sleepfg/app/requirements.txt b/test/apis/realtime/sleepfg/app/requirements.txt new file mode 100644 index 0000000000..cef5a165b5 --- /dev/null +++ b/test/apis/realtime/sleepfg/app/requirements.txt @@ -0,0 +1,2 @@ +Flask +gunicorn diff --git a/test/apis/realtime/sleepfg/build-cpu.sh b/test/apis/realtime/sleepfg/build-cpu.sh new file mode 100755 index 0000000000..fc7641c05b --- /dev/null +++ b/test/apis/realtime/sleepfg/build-cpu.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +image_name="realtime-sleep-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/sleepfg/cortex_cpu.yaml b/test/apis/realtime/sleepfg/cortex_cpu.yaml new file mode 100644 index 0000000000..108a2654a3 --- /dev/null +++ b/test/apis/realtime/sleepfg/cortex_cpu.yaml @@ -0,0 +1,19 @@ +- name: sleep + kind: RealtimeAPI + pod: + port: 8080 + max_concurrency: 1 + max_queue_length: 128 + containers: + - name: api + image: 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest + # readiness_probe: + # http_get: + # path: "/healthz" + # port: 8080 + compute: + cpu: 200m + mem: 128Mi + autoscaling: + target_in_flight: 1 + max_replicas: 1 diff --git a/test/apis/realtime/sleepfg/cpu.Dockerfile b/test/apis/realtime/sleepfg/cpu.Dockerfile new file mode 100644 index 0000000000..a227b4aaa7 --- /dev/null +++ b/test/apis/realtime/sleepfg/cpu.Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.8-slim-buster + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 + +CMD ["gunicorn", "main:app", "-w", "1", "--threads", "1", "--timeout", "200", "-b", "0.0.0.0:8080"] diff --git a/test/apis/realtime/sleepn/build-cpu.sh b/test/apis/realtime/sleepn/build-cpu.sh new file mode 100755 index 0000000000..7262a43c63 --- /dev/null +++ b/test/apis/realtime/sleepn/build-cpu.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +nucleus generate model-server-config.yaml +docker build -f nucleus.Dockerfile -t 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest . +docker push 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest diff --git a/test/apis/realtime/sleepn/cortex_cpu.yaml b/test/apis/realtime/sleepn/cortex_cpu.yaml new file mode 100644 index 0000000000..108a2654a3 --- /dev/null +++ b/test/apis/realtime/sleepn/cortex_cpu.yaml @@ -0,0 +1,19 @@ +- name: sleep + kind: RealtimeAPI + pod: + port: 8080 + max_concurrency: 1 + max_queue_length: 128 + containers: + - name: api + image: 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest + # readiness_probe: + # http_get: + # path: "/healthz" + # port: 8080 + compute: + cpu: 200m + mem: 128Mi + autoscaling: + target_in_flight: 1 + max_replicas: 1 diff --git a/test/apis/realtime/sleepn/handler.py b/test/apis/realtime/sleepn/handler.py new file mode 100644 index 0000000000..be2c0e9b66 --- /dev/null +++ b/test/apis/realtime/sleepn/handler.py @@ -0,0 +1,26 @@ +import time +from datetime import datetime + + +class Handler: + def __init__(self): + pass + + def handle_post(self, payload, headers): + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {headers['x-request-id']} | request start", + flush=True, + ) + start_time = time.time() + + text = payload["image"].file.read() + print(len(text), flush=True) + + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {headers['x-request-id']} | request finish: " + + str(process_time), + flush=True, + ) + + return "ok" diff --git a/test/apis/realtime/sleepn/model-server-config.yaml b/test/apis/realtime/sleepn/model-server-config.yaml new file mode 100644 index 0000000000..667c2e0212 --- /dev/null +++ b/test/apis/realtime/sleepn/model-server-config.yaml @@ -0,0 +1,3 @@ +type: python +py_version: 3.6.9 +path: handler.py diff --git a/test/apis/realtime/sleepn/nucleus.Dockerfile b/test/apis/realtime/sleepn/nucleus.Dockerfile new file mode 100644 index 0000000000..1dfb527693 --- /dev/null +++ b/test/apis/realtime/sleepn/nucleus.Dockerfile @@ -0,0 +1,77 @@ +# to replace when building the dockerfile +FROM ubuntu:18.04 +ENV CORTEX_MODEL_SERVER_VERSION=0.1.0 + +RUN apt-get update -qq && apt-get install -y -q \ + build-essential \ + pkg-config \ + software-properties-common \ + curl \ + git \ + unzip \ + zlib1g-dev \ + locales \ + nginx=1.14.* \ + && apt-get clean -qq && rm -rf /var/lib/apt/lists/* + +RUN cd /tmp/ && \ + curl -L --output s6-overlay-amd64-installer "https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer" && \ + cd - && \ + chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer + +ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2 + +RUN locale-gen en_US.UTF-8 +ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 +ENV PATH=/opt/conda/bin:$PATH \ + PYTHONVERSION=3.6.9 \ + CORTEX_IMAGE_TYPE=python-handler-cpu \ + CORTEX_TF_SERVING_HOST= + +# conda needs an untainted base environment to function properly +# that's why a new separate conda environment is created +RUN curl "https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh" --output ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm -rf ~/.cache ~/miniconda.sh + +# split the conda installations to reduce memory usage when building the image +RUN /opt/conda/bin/conda create -n env -c conda-forge python=$PYTHONVERSION pip=19.* && \ + /opt/conda/bin/conda clean -a && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" > ~/.env && \ + echo "conda activate env" >> ~/.env && \ + echo "source ~/.env" >> ~/.bashrc + +ENV BASH_ENV=~/.env +SHELL ["/bin/bash", "-c"] +RUN git clone --depth 1 -b ${CORTEX_MODEL_SERVER_VERSION} https://github.com/cortexlabs/nucleus && \ + cp -r nucleus/src/ /src/ && \ + rm -r nucleus/ + +RUN pip install --no-cache-dir \ + -r /src/cortex/serve.requirements.txt \ + -r /src/cortex/cortex_internal.requirements.txt +ENV CORTEX_LOG_CONFIG_FILE=/src/cortex/log_config.yaml \ + CORTEX_TELEMETRY_SENTRY_DSN="https://c334df915c014ffa93f2076769e5b334@sentry.io/1848098" + +RUN mkdir -p /usr/local/cortex/ && \ + cp /src/cortex/init/install-core-dependencies.sh /usr/local/cortex/install-core-dependencies.sh && \ + chmod +x /usr/local/cortex/install-core-dependencies.sh && \ + /usr/local/cortex/install-core-dependencies.sh + +RUN pip install --no-deps /src/cortex/ && \ + mv /src/cortex/init/bootloader.sh /etc/cont-init.d/bootloader.sh + +COPY ./requirements.txt /src/project/ +COPY ./model-server-config.yaml /src/project/model-server-config.yaml + +ENV CORTEX_MODEL_SERVER_CONFIG /src/project/model-server-config.yaml +RUN /opt/conda/envs/env/bin/python /src/cortex/init/expand_server_config.py /src/project/model-server-config.yaml > /tmp/model-server-config.yaml && \ + eval $(/opt/conda/envs/env/bin/python /src/cortex/init/export_env_vars.py /tmp/model-server-config.yaml) && \ + if [ -f "/src/project/${CORTEX_DEPENDENCIES_PIP}" ]; then pip --no-cache-dir install -r "/src/project/${CORTEX_DEPENDENCIES_PIP}"; fi && \ + /usr/local/cortex/install-core-dependencies.sh + +COPY ./ /src/project/ +RUN mv /tmp/model-server-config.yaml /src/project/model-server-config.yaml + +ENTRYPOINT ["/init"] diff --git a/test/apis/realtime/sleepn/requirements.txt b/test/apis/realtime/sleepn/requirements.txt new file mode 100644 index 0000000000..e69de29bb2 From 9e4e751be57dd0bec4fb7f2721b1afb9ac9f6962 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 3 Dec 2021 12:49:12 -0800 Subject: [PATCH 02/13] Comments --- cmd/proxy/main.go | 22 +++++++++++----------- pkg/proxy/breaker.go | 6 +++++- pkg/proxy/handler.go | 7 +++---- pkg/proxy/proxy.go | 20 +++++++++----------- test/apis/realtime/sleep/app/main.py | 2 +- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index ad3051522e..a23a5995b2 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -148,19 +148,19 @@ func main() { adminHandler.Handle("/metrics", promStats) adminHandler.Handle("/healthz", readinessTCPHandler(userContainerPort, hasTCPProbe, log)) - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) + // ctx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) servers := map[string]*http.Server{ "proxy": { - Addr: ":" + strconv.Itoa(port), - Handler: proxy.Handler(breaker, httpProxy), - ReadTimeout: 60 * time.Second, - WriteTimeout: 60 * time.Second, - ReadHeaderTimeout: 60 * time.Second, - IdleTimeout: 60 * time.Second, - BaseContext: func(n net.Listener) context.Context { - return ctx - }, + Addr: ":" + strconv.Itoa(port), + Handler: proxy.Handler(breaker, httpProxy), + ReadTimeout: 60 * time.Second, + // WriteTimeout: 60 * time.Second, + // ReadHeaderTimeout: 60 * time.Second, + // IdleTimeout: 60 * time.Second, + // BaseContext: func(n net.Listener) context.Context { + // return ctx + // }, }, "admin": { Addr: ":" + strconv.Itoa(adminPort), @@ -168,7 +168,7 @@ func main() { }, } - servers["proxy"].RegisterOnShutdown(cancel) + // servers["proxy"].RegisterOnShutdown(cancel) errCh := make(chan error) for name, server := range servers { diff --git a/pkg/proxy/breaker.go b/pkg/proxy/breaker.go index 0233315f97..c7d7230b67 100644 --- a/pkg/proxy/breaker.go +++ b/pkg/proxy/breaker.go @@ -133,6 +133,8 @@ func (b *Breaker) Reserve(_ context.Context) (func(), bool) { // already consumed, Maybe returns immediately without calling thunk. If // the thunk was executed, Maybe returns true, else false. func (b *Breaker) Maybe(requestID string, ctx context.Context, thunk func()) error { + fmt.Printf("%s | %s | RECEIVE request\n", time.Now().Format(time.StampMilli), requestID) + if !b.tryAcquirePending() { return ErrRequestQueueFull } @@ -149,6 +151,8 @@ func (b *Breaker) Maybe(requestID string, ctx context.Context, thunk func()) err // + release calls are equally paired. defer b.sem.release() + // time.Sleep(10 * time.Second) + // Do the thing. fmt.Printf("%s | %s | FORWARD to container\n", time.Now().Format(time.StampMilli), requestID) t := time.Now() @@ -173,7 +177,7 @@ func (b *Breaker) Maybe(requestID string, ctx context.Context, thunk func()) err // timer.Stop() // } - fmt.Printf("%s | %s | RECEIVE from container: %f\n", time.Now().Format(time.StampMilli), requestID, time.Since(t).Seconds()) + fmt.Printf("%s | %s | RESPONSE from container: %f\n", time.Now().Format(time.StampMilli), requestID, time.Since(t).Seconds()) // Report success return nil } diff --git a/pkg/proxy/handler.go b/pkg/proxy/handler.go index 778d498f3f..78e8deb5b1 100644 --- a/pkg/proxy/handler.go +++ b/pkg/proxy/handler.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "net/http" - "time" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/probe" @@ -36,9 +35,9 @@ func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { if err := breaker.Maybe(r.Header.Get("x-request-id"), r.Context(), func() { // This alone caused python to error after 1 min, but it did not return (so semaphore did not release) - newCtx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) - defer cancel() - r = r.Clone(newCtx) + // newCtx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) + // defer cancel() + // r = r.Clone(newCtx) next.ServeHTTP(w, r) }); err != nil { fmt.Println("GOT ERROR:", err) diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go index c2e02afb5a..dcd4ab45c5 100644 --- a/pkg/proxy/proxy.go +++ b/pkg/proxy/proxy.go @@ -17,11 +17,9 @@ limitations under the License. package proxy import ( - "net" "net/http" "net/http/httputil" "net/url" - "time" ) // NewReverseProxy creates a new cortex base reverse proxy @@ -45,17 +43,17 @@ func buildHTTPTransport(maxIdle, maxIdlePerHost int) http.RoundTripper { transport.ForceAttemptHTTP2 = false transport.DisableCompression = true - transport.ExpectContinueTimeout = 60 * time.Second - transport.IdleConnTimeout = 60 * time.Second - transport.ResponseHeaderTimeout = 60 * time.Second - transport.TLSHandshakeTimeout = 60 * time.Second + // transport.ExpectContinueTimeout = 60 * time.Second + // transport.IdleConnTimeout = 60 * time.Second + // transport.ResponseHeaderTimeout = 60 * time.Second + // transport.TLSHandshakeTimeout = 60 * time.Second - transport.Dial = nil + // transport.Dial = nil // transport.DialContext = nil - transport.DialContext = (&net.Dialer{ - Timeout: 60 * time.Second, - KeepAlive: 15 * time.Second, - }).DialContext + // transport.DialContext = (&net.Dialer{ + // Timeout: 60 * time.Second, + // KeepAlive: 15 * time.Second, + // }).DialContext return transport } diff --git a/test/apis/realtime/sleep/app/main.py b/test/apis/realtime/sleep/app/main.py index 84fcb0a526..a76e7bfa9f 100644 --- a/test/apis/realtime/sleep/app/main.py +++ b/test/apis/realtime/sleep/app/main.py @@ -59,4 +59,4 @@ async def sleep(sleep: float = 0, x_request_id: str = Header(None), image: Uploa return PlainTextResponse("ok") -# r=$((1 + $RANDOM % 100)); echo "request ID: $r"; SECONDS=0; curl -X POST -H "x-request-id: $r" -F image=@wp.jpg http://ad14cde85e57748ff9c384a32617133f-9335e56d3708bc0a.elb.us-west-2.amazonaws.com/sleep?sleep=10 --limit-rate 4k; echo "$SECONDS seconds" +# r=$((1 + $RANDOM % 100)); echo "request ID: $r"; SECONDS=0; curl -X POST -H "x-request-id: $r" -F image=@wp.jpg http://ad14cde85e57748ff9c384a32617133f-9335e56d3708bc0a.elb.us-west-2.amazonaws.com/sleep?sleep=1 --limit-rate 4k; echo "$SECONDS seconds" From 6b740c1cb8066c5fa8b7a1aeafb543a3de844545 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 3 Dec 2021 15:54:36 -0800 Subject: [PATCH 03/13] Add timeout to call to user's container --- cmd/proxy/main.go | 6 +++--- pkg/activator/api_activator.go | 2 +- pkg/proxy/breaker.go | 34 ++++++++++++++---------------- pkg/proxy/handler.go | 38 ++++++++++++++++++++++++++-------- 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index a23a5995b2..1df267b9b2 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -152,9 +152,9 @@ func main() { servers := map[string]*http.Server{ "proxy": { - Addr: ":" + strconv.Itoa(port), - Handler: proxy.Handler(breaker, httpProxy), - ReadTimeout: 60 * time.Second, + Addr: ":" + strconv.Itoa(port), + Handler: proxy.Handler(breaker, httpProxy), + // ReadTimeout: 60 * time.Second, // WriteTimeout: 60 * time.Second, // ReadHeaderTimeout: 60 * time.Second, // IdleTimeout: 60 * time.Second, diff --git a/pkg/activator/api_activator.go b/pkg/activator/api_activator.go index 7e01d304bc..ef531668e9 100644 --- a/pkg/activator/api_activator.go +++ b/pkg/activator/api_activator.go @@ -45,7 +45,7 @@ func newAPIActivator(maxQueueLength, maxConcurrency int) *apiActivator { func (a *apiActivator) try(ctx context.Context, fn func() error, tracker *readinessTracker) error { var execErr error - if err := a.breaker.Maybe(ctx, func() { + if err := a.breaker.Maybe("", ctx, func() { ctx, cancel := context.WithTimeout(ctx, consts.WaitForReadyReplicasTimeout) defer cancel() diff --git a/pkg/proxy/breaker.go b/pkg/proxy/breaker.go index c7d7230b67..c14e647661 100644 --- a/pkg/proxy/breaker.go +++ b/pkg/proxy/breaker.go @@ -151,31 +151,29 @@ func (b *Breaker) Maybe(requestID string, ctx context.Context, thunk func()) err // + release calls are equally paired. defer b.sem.release() - // time.Sleep(10 * time.Second) + // time.Sleep(70 * time.Second) // Do the thing. fmt.Printf("%s | %s | FORWARD to container\n", time.Now().Format(time.StampMilli), requestID) t := time.Now() - thunk() + // thunk() // CATCH TIMEOUT - // done := make(chan struct{}) - // go func() { - // thunk() - // done <- struct{}{} - // }() - - // timer := time.NewTimer(60 * time.Second) - // select { - // // case <-ctx.Done(): - // case <-timer.C: - // fmt.Println("beans1") - // return errors.New("timeout!!") - // case <-done: - // fmt.Println("beans2") - // timer.Stop() - // } + done := make(chan struct{}) + go func() { + thunk() + done <- struct{}{} + }() + + timer := time.NewTimer(60 * time.Second) + select { + // case <-ctx.Done(): + case <-timer.C: + return errors.New("timeout!!") + case <-done: + timer.Stop() + } fmt.Printf("%s | %s | RESPONSE from container: %f\n", time.Now().Format(time.StampMilli), requestID, time.Since(t).Seconds()) // Report success diff --git a/pkg/proxy/handler.go b/pkg/proxy/handler.go index 78e8deb5b1..27a439df6d 100644 --- a/pkg/proxy/handler.go +++ b/pkg/proxy/handler.go @@ -21,11 +21,27 @@ import ( "errors" "fmt" "net/http" + "time" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/probe" ) +type doneWriter struct { + http.ResponseWriter + done bool +} + +func (w *doneWriter) WriteHeader(status int) { + w.done = true + w.ResponseWriter.WriteHeader(status) +} + +func (w *doneWriter) Write(b []byte) (int, error) { + w.done = true + return w.ResponseWriter.Write(b) +} + func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if probe.IsRequestKubeletProbe(r) || breaker == nil { @@ -33,19 +49,23 @@ func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { return } + dw := &doneWriter{ResponseWriter: w} + if err := breaker.Maybe(r.Header.Get("x-request-id"), r.Context(), func() { // This alone caused python to error after 1 min, but it did not return (so semaphore did not release) - // newCtx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) - // defer cancel() - // r = r.Clone(newCtx) - next.ServeHTTP(w, r) + newCtx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) + defer cancel() + r = r.Clone(newCtx) + next.ServeHTTP(dw, r) }); err != nil { fmt.Println("GOT ERROR:", err) - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrRequestQueueFull) { - http.Error(w, err.Error(), http.StatusServiceUnavailable) - } else { - w.WriteHeader(http.StatusInternalServerError) - telemetry.Error(err) + if !dw.done { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrRequestQueueFull) { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + } else { + w.WriteHeader(http.StatusInternalServerError) + telemetry.Error(err) + } } } } From 42fd63cb468719a4b3336084137b29b70315bd53 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 3 Dec 2021 16:01:47 -0800 Subject: [PATCH 04/13] Remove DoneWriter (it didn't avoid the double header writing issue) --- pkg/proxy/handler.go | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/pkg/proxy/handler.go b/pkg/proxy/handler.go index 27a439df6d..778d498f3f 100644 --- a/pkg/proxy/handler.go +++ b/pkg/proxy/handler.go @@ -27,21 +27,6 @@ import ( "github.com/cortexlabs/cortex/pkg/probe" ) -type doneWriter struct { - http.ResponseWriter - done bool -} - -func (w *doneWriter) WriteHeader(status int) { - w.done = true - w.ResponseWriter.WriteHeader(status) -} - -func (w *doneWriter) Write(b []byte) (int, error) { - w.done = true - return w.ResponseWriter.Write(b) -} - func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if probe.IsRequestKubeletProbe(r) || breaker == nil { @@ -49,23 +34,19 @@ func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { return } - dw := &doneWriter{ResponseWriter: w} - if err := breaker.Maybe(r.Header.Get("x-request-id"), r.Context(), func() { // This alone caused python to error after 1 min, but it did not return (so semaphore did not release) newCtx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) defer cancel() r = r.Clone(newCtx) - next.ServeHTTP(dw, r) + next.ServeHTTP(w, r) }); err != nil { fmt.Println("GOT ERROR:", err) - if !dw.done { - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrRequestQueueFull) { - http.Error(w, err.Error(), http.StatusServiceUnavailable) - } else { - w.WriteHeader(http.StatusInternalServerError) - telemetry.Error(err) - } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrRequestQueueFull) { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + } else { + w.WriteHeader(http.StatusInternalServerError) + telemetry.Error(err) } } } From fc930be340647d7318bf6ff81a344511d9b41608 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 15 Dec 2021 17:31:19 -0800 Subject: [PATCH 05/13] Comment timeout code --- pkg/proxy/breaker.go | 40 ++++++++++++++++++++-------------------- pkg/proxy/handler.go | 7 +++---- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/pkg/proxy/breaker.go b/pkg/proxy/breaker.go index c14e647661..5c2a5313fa 100644 --- a/pkg/proxy/breaker.go +++ b/pkg/proxy/breaker.go @@ -155,27 +155,27 @@ func (b *Breaker) Maybe(requestID string, ctx context.Context, thunk func()) err // Do the thing. fmt.Printf("%s | %s | FORWARD to container\n", time.Now().Format(time.StampMilli), requestID) - t := time.Now() - - // thunk() - - // CATCH TIMEOUT - done := make(chan struct{}) - go func() { - thunk() - done <- struct{}{} - }() - - timer := time.NewTimer(60 * time.Second) - select { - // case <-ctx.Done(): - case <-timer.C: - return errors.New("timeout!!") - case <-done: - timer.Stop() - } + // t := time.Now() + + thunk() + + // // CATCH TIMEOUT + // done := make(chan struct{}) + // go func() { + // thunk() + // done <- struct{}{} + // }() + + // timer := time.NewTimer(60 * time.Second) + // select { + // // case <-ctx.Done(): + // case <-timer.C: + // return errors.New("timeout!!") + // case <-done: + // timer.Stop() + // } - fmt.Printf("%s | %s | RESPONSE from container: %f\n", time.Now().Format(time.StampMilli), requestID, time.Since(t).Seconds()) + // fmt.Printf("%s | %s | RESPONSE from container: %f\n", time.Now().Format(time.StampMilli), requestID, time.Since(t).Seconds()) // Report success return nil } diff --git a/pkg/proxy/handler.go b/pkg/proxy/handler.go index 778d498f3f..78e8deb5b1 100644 --- a/pkg/proxy/handler.go +++ b/pkg/proxy/handler.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "net/http" - "time" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/probe" @@ -36,9 +35,9 @@ func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { if err := breaker.Maybe(r.Header.Get("x-request-id"), r.Context(), func() { // This alone caused python to error after 1 min, but it did not return (so semaphore did not release) - newCtx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) - defer cancel() - r = r.Clone(newCtx) + // newCtx, cancel := context.WithTimeout(context.Background(), time.Duration(60*time.Second)) + // defer cancel() + // r = r.Clone(newCtx) next.ServeHTTP(w, r) }); err != nil { fmt.Println("GOT ERROR:", err) From d8d640a681687777fe22c2abd58f5d6f99e053c4 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 15 Dec 2021 18:18:18 -0800 Subject: [PATCH 06/13] Add log statement back --- pkg/proxy/breaker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/proxy/breaker.go b/pkg/proxy/breaker.go index 5c2a5313fa..e31073d996 100644 --- a/pkg/proxy/breaker.go +++ b/pkg/proxy/breaker.go @@ -155,7 +155,7 @@ func (b *Breaker) Maybe(requestID string, ctx context.Context, thunk func()) err // Do the thing. fmt.Printf("%s | %s | FORWARD to container\n", time.Now().Format(time.StampMilli), requestID) - // t := time.Now() + t := time.Now() thunk() @@ -175,7 +175,7 @@ func (b *Breaker) Maybe(requestID string, ctx context.Context, thunk func()) err // timer.Stop() // } - // fmt.Printf("%s | %s | RESPONSE from container: %f\n", time.Now().Format(time.StampMilli), requestID, time.Since(t).Seconds()) + fmt.Printf("%s | %s | RESPONSE from container: %f\n", time.Now().Format(time.StampMilli), requestID, time.Since(t).Seconds()) // Report success return nil } From cadccb693a39c1a00df35ec50098c6c8aa25805d Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 15 Dec 2021 19:42:37 -0800 Subject: [PATCH 07/13] Add go example --- test/apis/realtime/sleep-go/app/main.go | 98 +++++++++++++++++++++ test/apis/realtime/sleep-go/build-cpu.sh | 8 ++ test/apis/realtime/sleep-go/cortex_cpu.yaml | 19 ++++ test/apis/realtime/sleep-go/cpu.Dockerfile | 5 ++ 4 files changed, 130 insertions(+) create mode 100644 test/apis/realtime/sleep-go/app/main.go create mode 100755 test/apis/realtime/sleep-go/build-cpu.sh create mode 100644 test/apis/realtime/sleep-go/cortex_cpu.yaml create mode 100644 test/apis/realtime/sleep-go/cpu.Dockerfile diff --git a/test/apis/realtime/sleep-go/app/main.go b/test/apis/realtime/sleep-go/app/main.go new file mode 100644 index 0000000000..162e41c51d --- /dev/null +++ b/test/apis/realtime/sleep-go/app/main.go @@ -0,0 +1,98 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + f, err := os.OpenFile("log.server", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + log.Fatalf("error opening file: %v", err) + } + defer f.Close() + log.SetOutput(f) + + ctx, cancel := context.WithCancel(context.Background()) + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + log.Print("received request\n") + fmt.Print("received request\n") + + r.ParseMultipartForm(32 << 20) // limit your max input length + var buf bytes.Buffer + file, _, err := r.FormFile("image") + if err != nil { + panic(err) + } + defer file.Close() + io.Copy(&buf, file) + log.Print("finished\n") + fmt.Print("finished\n") + + fmt.Fprintf(w, "Hello!") + return + }) + + httpServer := &http.Server{ + Addr: ":8080", + Handler: mux, + BaseContext: func(_ net.Listener) context.Context { return ctx }, + } + // if your BaseContext is more complex you might want to use this instead of doing it manually + // httpServer.RegisterOnShutdown(cancel) + + // Run server + go func() { + log.Println("listening on port 8080") + fmt.Println("listening on port 8080") + if err := httpServer.ListenAndServe(); err != http.ErrServerClosed { + // it is fine to use Fatal here because it is not main gorutine + log.Fatalf("HTTP server ListenAndServe: %v", err) + } + }() + + signalChan := make(chan os.Signal, 1) + + signal.Notify( + signalChan, + syscall.SIGHUP, // kill -SIGHUP XXXX + syscall.SIGINT, // kill -SIGINT XXXX or Ctrl+c + syscall.SIGQUIT, // kill -SIGQUIT XXXX + ) + + <-signalChan + log.Print("os.Interrupt - shutting down...\n") + + go func() { + <-signalChan + log.Fatal("os.Kill - terminating...\n") + }() + + gracefullCtx, cancelShutdown := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelShutdown() + + if err := httpServer.Shutdown(gracefullCtx); err != nil { + log.Printf("shutdown error: %v\n", err) + defer os.Exit(1) + return + } else { + log.Printf("gracefully stopped\n") + } + + // manually cancel context if not using httpServer.RegisterOnShutdown(cancel) + cancel() + + defer os.Exit(0) + return +} diff --git a/test/apis/realtime/sleep-go/build-cpu.sh b/test/apis/realtime/sleep-go/build-cpu.sh new file mode 100755 index 0000000000..fc7641c05b --- /dev/null +++ b/test/apis/realtime/sleep-go/build-cpu.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +image_name="realtime-sleep-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/sleep-go/cortex_cpu.yaml b/test/apis/realtime/sleep-go/cortex_cpu.yaml new file mode 100644 index 0000000000..108a2654a3 --- /dev/null +++ b/test/apis/realtime/sleep-go/cortex_cpu.yaml @@ -0,0 +1,19 @@ +- name: sleep + kind: RealtimeAPI + pod: + port: 8080 + max_concurrency: 1 + max_queue_length: 128 + containers: + - name: api + image: 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest + # readiness_probe: + # http_get: + # path: "/healthz" + # port: 8080 + compute: + cpu: 200m + mem: 128Mi + autoscaling: + target_in_flight: 1 + max_replicas: 1 diff --git a/test/apis/realtime/sleep-go/cpu.Dockerfile b/test/apis/realtime/sleep-go/cpu.Dockerfile new file mode 100644 index 0000000000..0228503ae3 --- /dev/null +++ b/test/apis/realtime/sleep-go/cpu.Dockerfile @@ -0,0 +1,5 @@ +FROM golang:1.17.3 + +COPY app /app + +ENTRYPOINT ["go", "run", "/app/main.go"] From c6ebe37bea43f51d3c3f3311f0649a620af6d822 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 29 Dec 2021 20:23:08 -0800 Subject: [PATCH 08/13] Add nginx example --- test/apis/realtime/sleep-nginx/app/main.py | 62 +++++++++++++++++++ .../realtime/sleep-nginx/app/requirements.txt | 3 + test/apis/realtime/sleep-nginx/build-cpu.sh | 8 +++ test/apis/realtime/sleep-nginx/build-nginx.sh | 8 +++ .../apis/realtime/sleep-nginx/cortex_cpu.yaml | 24 +++++++ test/apis/realtime/sleep-nginx/cpu.Dockerfile | 12 ++++ .../realtime/sleep-nginx/nginx.Dockerfile | 2 + test/apis/realtime/sleep-nginx/nginx.conf | 19 ++++++ test/apis/realtime/sleep-nginx/server.yaml | 36 +++++++++++ .../{sleepn => sleep-nucleus}/build-cpu.sh | 0 .../{sleepn => sleep-nucleus}/cortex_cpu.yaml | 0 .../{sleepn => sleep-nucleus}/handler.py | 0 .../model-server-config.yaml | 0 .../nucleus.Dockerfile | 2 +- .../requirements.txt | 0 15 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 test/apis/realtime/sleep-nginx/app/main.py create mode 100644 test/apis/realtime/sleep-nginx/app/requirements.txt create mode 100755 test/apis/realtime/sleep-nginx/build-cpu.sh create mode 100755 test/apis/realtime/sleep-nginx/build-nginx.sh create mode 100644 test/apis/realtime/sleep-nginx/cortex_cpu.yaml create mode 100644 test/apis/realtime/sleep-nginx/cpu.Dockerfile create mode 100644 test/apis/realtime/sleep-nginx/nginx.Dockerfile create mode 100644 test/apis/realtime/sleep-nginx/nginx.conf create mode 100644 test/apis/realtime/sleep-nginx/server.yaml rename test/apis/realtime/{sleepn => sleep-nucleus}/build-cpu.sh (100%) rename test/apis/realtime/{sleepn => sleep-nucleus}/cortex_cpu.yaml (100%) rename test/apis/realtime/{sleepn => sleep-nucleus}/handler.py (100%) rename test/apis/realtime/{sleepn => sleep-nucleus}/model-server-config.yaml (100%) rename test/apis/realtime/{sleepn => sleep-nucleus}/nucleus.Dockerfile (98%) rename test/apis/realtime/{sleepn => sleep-nucleus}/requirements.txt (100%) diff --git a/test/apis/realtime/sleep-nginx/app/main.py b/test/apis/realtime/sleep-nginx/app/main.py new file mode 100644 index 0000000000..a76e7bfa9f --- /dev/null +++ b/test/apis/realtime/sleep-nginx/app/main.py @@ -0,0 +1,62 @@ +import time +import asyncio +from datetime import datetime + +from fastapi import FastAPI, Header, Request, File, UploadFile +from fastapi.responses import PlainTextResponse + +app = FastAPI() + + +@app.middleware("http") +async def add_process_time_header(request: Request, call_next): + if "x-request-id" in request.headers: + request_id = request.headers["x-request-id"] + else: + request_id = "0" + + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request_id} | middleware start", + flush=True, + ) + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request_id} | middleware finish: {str(process_time)}", + flush=True, + ) + return response + + +@app.get("/healthz") +def healthz(): + return PlainTextResponse("ok") + + +@app.post("/") +async def sleep(sleep: float = 0, x_request_id: str = Header(None), image: UploadFile = File(...)): + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {x_request_id} | request start", + flush=True, + ) + start_time = time.time() + image = await image.read() + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {x_request_id} | downloaded image: " + + str(process_time), + flush=True, + ) + # time.sleep(sleep) + await asyncio.sleep(sleep) + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {x_request_id} | request finish: " + + str(process_time), + flush=True, + ) + return PlainTextResponse("ok") + + +# r=$((1 + $RANDOM % 100)); echo "request ID: $r"; SECONDS=0; curl -X POST -H "x-request-id: $r" -F image=@wp.jpg http://ad14cde85e57748ff9c384a32617133f-9335e56d3708bc0a.elb.us-west-2.amazonaws.com/sleep?sleep=1 --limit-rate 4k; echo "$SECONDS seconds" diff --git a/test/apis/realtime/sleep-nginx/app/requirements.txt b/test/apis/realtime/sleep-nginx/app/requirements.txt new file mode 100644 index 0000000000..9673a90c69 --- /dev/null +++ b/test/apis/realtime/sleep-nginx/app/requirements.txt @@ -0,0 +1,3 @@ +uvicorn[standard] +fastapi +python-multipart diff --git a/test/apis/realtime/sleep-nginx/build-cpu.sh b/test/apis/realtime/sleep-nginx/build-cpu.sh new file mode 100755 index 0000000000..fc7641c05b --- /dev/null +++ b/test/apis/realtime/sleep-nginx/build-cpu.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +image_name="realtime-sleep-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/sleep-nginx/build-nginx.sh b/test/apis/realtime/sleep-nginx/build-nginx.sh new file mode 100755 index 0000000000..817f06a381 --- /dev/null +++ b/test/apis/realtime/sleep-nginx/build-nginx.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +image_name="realtime-sleep-nginx" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/sleep-nginx/cortex_cpu.yaml b/test/apis/realtime/sleep-nginx/cortex_cpu.yaml new file mode 100644 index 0000000000..28a1a722d4 --- /dev/null +++ b/test/apis/realtime/sleep-nginx/cortex_cpu.yaml @@ -0,0 +1,24 @@ +- name: sleep + kind: RealtimeAPI + pod: + port: 8080 + max_concurrency: 1 + max_queue_length: 128 + containers: + - name: api + image: 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest + # readiness_probe: + # http_get: + # path: "/healthz" + # port: 8080 + compute: + cpu: 200m + mem: 128Mi + - name: nginx + image: 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-nginx:latest + compute: + cpu: 200m + mem: 128Mi + autoscaling: + target_in_flight: 1 + max_replicas: 1 diff --git a/test/apis/realtime/sleep-nginx/cpu.Dockerfile b/test/apis/realtime/sleep-nginx/cpu.Dockerfile new file mode 100644 index 0000000000..83f5e5259c --- /dev/null +++ b/test/apis/realtime/sleep-nginx/cpu.Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +CMD uvicorn --workers 1 --limit-concurrency 2 --host 0.0.0.0 --port 9000 main:app diff --git a/test/apis/realtime/sleep-nginx/nginx.Dockerfile b/test/apis/realtime/sleep-nginx/nginx.Dockerfile new file mode 100644 index 0000000000..317ba46fb3 --- /dev/null +++ b/test/apis/realtime/sleep-nginx/nginx.Dockerfile @@ -0,0 +1,2 @@ +FROM nginx +ADD nginx.conf /etc/nginx/conf.d/ diff --git a/test/apis/realtime/sleep-nginx/nginx.conf b/test/apis/realtime/sleep-nginx/nginx.conf new file mode 100644 index 0000000000..8cf1ea1ed4 --- /dev/null +++ b/test/apis/realtime/sleep-nginx/nginx.conf @@ -0,0 +1,19 @@ +limit_conn_zone $server_name zone=servers:1m; +limit_conn servers 1; + +server { + listen 8080; + server_name localhost; + + proxy_read_timeout 600s; + proxy_send_timeout 600s; + proxy_connect_timeout 600s; + + client_body_timeout 60s; + + client_body_buffer_size 1k; + + location / { + proxy_pass http://localhost:9000/; + } +} diff --git a/test/apis/realtime/sleep-nginx/server.yaml b/test/apis/realtime/sleep-nginx/server.yaml new file mode 100644 index 0000000000..d20d12c74c --- /dev/null +++ b/test/apis/realtime/sleep-nginx/server.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: server + labels: + app: server +spec: + replicas: 1 + selector: + matchLabels: + app: server + template: + metadata: + labels: + app: server + spec: + containers: + - name: server + image: 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest + ports: + - containerPort: 8080 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: server +spec: + type: LoadBalancer + selector: + app: server + ports: + - protocol: TCP + port: 80 + targetPort: 8080 diff --git a/test/apis/realtime/sleepn/build-cpu.sh b/test/apis/realtime/sleep-nucleus/build-cpu.sh similarity index 100% rename from test/apis/realtime/sleepn/build-cpu.sh rename to test/apis/realtime/sleep-nucleus/build-cpu.sh diff --git a/test/apis/realtime/sleepn/cortex_cpu.yaml b/test/apis/realtime/sleep-nucleus/cortex_cpu.yaml similarity index 100% rename from test/apis/realtime/sleepn/cortex_cpu.yaml rename to test/apis/realtime/sleep-nucleus/cortex_cpu.yaml diff --git a/test/apis/realtime/sleepn/handler.py b/test/apis/realtime/sleep-nucleus/handler.py similarity index 100% rename from test/apis/realtime/sleepn/handler.py rename to test/apis/realtime/sleep-nucleus/handler.py diff --git a/test/apis/realtime/sleepn/model-server-config.yaml b/test/apis/realtime/sleep-nucleus/model-server-config.yaml similarity index 100% rename from test/apis/realtime/sleepn/model-server-config.yaml rename to test/apis/realtime/sleep-nucleus/model-server-config.yaml diff --git a/test/apis/realtime/sleepn/nucleus.Dockerfile b/test/apis/realtime/sleep-nucleus/nucleus.Dockerfile similarity index 98% rename from test/apis/realtime/sleepn/nucleus.Dockerfile rename to test/apis/realtime/sleep-nucleus/nucleus.Dockerfile index 1dfb527693..dd4999e032 100644 --- a/test/apis/realtime/sleepn/nucleus.Dockerfile +++ b/test/apis/realtime/sleep-nucleus/nucleus.Dockerfile @@ -1,6 +1,6 @@ # to replace when building the dockerfile FROM ubuntu:18.04 -ENV CORTEX_MODEL_SERVER_VERSION=0.1.0 +ENV CORTEX_MODEL_SERVER_VERSION=0.2.0 RUN apt-get update -qq && apt-get install -y -q \ build-essential \ diff --git a/test/apis/realtime/sleepn/requirements.txt b/test/apis/realtime/sleep-nucleus/requirements.txt similarity index 100% rename from test/apis/realtime/sleepn/requirements.txt rename to test/apis/realtime/sleep-nucleus/requirements.txt From b7e4492966e425a6ad97bfbc1d9a9e8c24270c02 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 29 Dec 2021 20:53:21 -0800 Subject: [PATCH 09/13] Updates --- cmd/proxy/main.go | 8 +++---- pkg/workloads/k8s.go | 26 +++++++++++------------ test/apis/realtime/sleep-nginx/nginx.conf | 4 ++-- test/apis/realtime/sleep/cpu.Dockerfile | 2 +- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 1df267b9b2..803d0c4874 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -162,10 +162,10 @@ func main() { // return ctx // }, }, - "admin": { - Addr: ":" + strconv.Itoa(adminPort), - Handler: adminHandler, - }, + // "admin": { + // Addr: ":" + strconv.Itoa(adminPort), + // Handler: adminHandler, + // }, } // servers["proxy"].RegisterOnShutdown(cancel) diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index edbf7beab3..39878f5a6b 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -209,19 +209,19 @@ func realtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { kcore.ResourceMemory: consts.CortexProxyMem, }, }, - ReadinessProbe: &kcore.Probe{ - Handler: kcore.Handler{ - HTTPGet: &kcore.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt(int(consts.AdminPortInt32)), - }, - }, - InitialDelaySeconds: 1, - TimeoutSeconds: 3, - PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, - }, + // ReadinessProbe: &kcore.Probe{ + // Handler: kcore.Handler{ + // HTTPGet: &kcore.HTTPGetAction{ + // Path: "/healthz", + // Port: intstr.FromInt(int(consts.AdminPortInt32)), + // }, + // }, + // InitialDelaySeconds: 1, + // TimeoutSeconds: 3, + // PeriodSeconds: 10, + // SuccessThreshold: 1, + // FailureThreshold: 3, + // }, }, ClusterConfigVolume() } diff --git a/test/apis/realtime/sleep-nginx/nginx.conf b/test/apis/realtime/sleep-nginx/nginx.conf index 8cf1ea1ed4..7a58c51b97 100644 --- a/test/apis/realtime/sleep-nginx/nginx.conf +++ b/test/apis/realtime/sleep-nginx/nginx.conf @@ -9,9 +9,9 @@ server { proxy_send_timeout 600s; proxy_connect_timeout 600s; - client_body_timeout 60s; + client_body_timeout 45s; - client_body_buffer_size 1k; + client_body_buffer_size 16k; location / { proxy_pass http://localhost:9000/; diff --git a/test/apis/realtime/sleep/cpu.Dockerfile b/test/apis/realtime/sleep/cpu.Dockerfile index ea648acdf0..c706bd2d70 100644 --- a/test/apis/realtime/sleep/cpu.Dockerfile +++ b/test/apis/realtime/sleep/cpu.Dockerfile @@ -10,4 +10,4 @@ WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 -CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app +CMD uvicorn --workers 1 --limit-concurrency 2 --host 0.0.0.0 --port $CORTEX_PORT main:app From 7257c337b66819760a4fcc03655e1923be15aa96 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Thu, 30 Dec 2021 21:52:40 -0800 Subject: [PATCH 10/13] indent --- test/apis/realtime/sleep-nginx/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/apis/realtime/sleep-nginx/nginx.conf b/test/apis/realtime/sleep-nginx/nginx.conf index 7a58c51b97..94c941e807 100644 --- a/test/apis/realtime/sleep-nginx/nginx.conf +++ b/test/apis/realtime/sleep-nginx/nginx.conf @@ -14,6 +14,6 @@ server { client_body_buffer_size 16k; location / { - proxy_pass http://localhost:9000/; - } + proxy_pass http://localhost:9000/; + } } From 2a6f9ab9cdef144bc70706a41f256a76d713e465 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 31 Dec 2021 09:15:54 +0200 Subject: [PATCH 11/13] Add apache example --- .../realtime/sleep-apache/apache.Dockerfile | 3 + test/apis/realtime/sleep-apache/app/main.py | 62 ++ .../sleep-apache/app/requirements.txt | 3 + .../realtime/sleep-apache/build-apache.sh | 8 + test/apis/realtime/sleep-apache/build-cpu.sh | 8 + .../realtime/sleep-apache/cortex_cpu.yaml | 20 + .../apis/realtime/sleep-apache/cpu.Dockerfile | 12 + test/apis/realtime/sleep-apache/httpd.conf | 554 ++++++++++++++++++ 8 files changed, 670 insertions(+) create mode 100644 test/apis/realtime/sleep-apache/apache.Dockerfile create mode 100644 test/apis/realtime/sleep-apache/app/main.py create mode 100644 test/apis/realtime/sleep-apache/app/requirements.txt create mode 100755 test/apis/realtime/sleep-apache/build-apache.sh create mode 100755 test/apis/realtime/sleep-apache/build-cpu.sh create mode 100644 test/apis/realtime/sleep-apache/cortex_cpu.yaml create mode 100644 test/apis/realtime/sleep-apache/cpu.Dockerfile create mode 100644 test/apis/realtime/sleep-apache/httpd.conf diff --git a/test/apis/realtime/sleep-apache/apache.Dockerfile b/test/apis/realtime/sleep-apache/apache.Dockerfile new file mode 100644 index 0000000000..2140f51253 --- /dev/null +++ b/test/apis/realtime/sleep-apache/apache.Dockerfile @@ -0,0 +1,3 @@ +FROM httpd:2.4 + +COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf diff --git a/test/apis/realtime/sleep-apache/app/main.py b/test/apis/realtime/sleep-apache/app/main.py new file mode 100644 index 0000000000..a76e7bfa9f --- /dev/null +++ b/test/apis/realtime/sleep-apache/app/main.py @@ -0,0 +1,62 @@ +import time +import asyncio +from datetime import datetime + +from fastapi import FastAPI, Header, Request, File, UploadFile +from fastapi.responses import PlainTextResponse + +app = FastAPI() + + +@app.middleware("http") +async def add_process_time_header(request: Request, call_next): + if "x-request-id" in request.headers: + request_id = request.headers["x-request-id"] + else: + request_id = "0" + + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request_id} | middleware start", + flush=True, + ) + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {request_id} | middleware finish: {str(process_time)}", + flush=True, + ) + return response + + +@app.get("/healthz") +def healthz(): + return PlainTextResponse("ok") + + +@app.post("/") +async def sleep(sleep: float = 0, x_request_id: str = Header(None), image: UploadFile = File(...)): + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {x_request_id} | request start", + flush=True, + ) + start_time = time.time() + image = await image.read() + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {x_request_id} | downloaded image: " + + str(process_time), + flush=True, + ) + # time.sleep(sleep) + await asyncio.sleep(sleep) + process_time = time.time() - start_time + print( + f"{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]} | {x_request_id} | request finish: " + + str(process_time), + flush=True, + ) + return PlainTextResponse("ok") + + +# r=$((1 + $RANDOM % 100)); echo "request ID: $r"; SECONDS=0; curl -X POST -H "x-request-id: $r" -F image=@wp.jpg http://ad14cde85e57748ff9c384a32617133f-9335e56d3708bc0a.elb.us-west-2.amazonaws.com/sleep?sleep=1 --limit-rate 4k; echo "$SECONDS seconds" diff --git a/test/apis/realtime/sleep-apache/app/requirements.txt b/test/apis/realtime/sleep-apache/app/requirements.txt new file mode 100644 index 0000000000..9673a90c69 --- /dev/null +++ b/test/apis/realtime/sleep-apache/app/requirements.txt @@ -0,0 +1,3 @@ +uvicorn[standard] +fastapi +python-multipart diff --git a/test/apis/realtime/sleep-apache/build-apache.sh b/test/apis/realtime/sleep-apache/build-apache.sh new file mode 100755 index 0000000000..c27f92b208 --- /dev/null +++ b/test/apis/realtime/sleep-apache/build-apache.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +image_name="realtime-sleep-apache" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/sleep-apache/build-cpu.sh b/test/apis/realtime/sleep-apache/build-cpu.sh new file mode 100755 index 0000000000..fc7641c05b --- /dev/null +++ b/test/apis/realtime/sleep-apache/build-cpu.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +image_name="realtime-sleep-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/sleep-apache/cortex_cpu.yaml b/test/apis/realtime/sleep-apache/cortex_cpu.yaml new file mode 100644 index 0000000000..d82f5d3622 --- /dev/null +++ b/test/apis/realtime/sleep-apache/cortex_cpu.yaml @@ -0,0 +1,20 @@ +- name: sleep + kind: RealtimeAPI + pod: + port: 8080 + max_concurrency: 1 + max_queue_length: 128 + containers: + - name: api + image: 499593605069.dkr.ecr.eu-central-1.amazonaws.com/cortexlabs/realtime-sleep-cpu:latest + compute: + cpu: 200m + mem: 128Mi + - name: apache + image: 499593605069.dkr.ecr.eu-central-1.amazonaws.com/cortexlabs/realtime-sleep-apache:latest + compute: + cpu: 200m + mem: 128Mi + autoscaling: + target_in_flight: 1 + max_replicas: 1 diff --git a/test/apis/realtime/sleep-apache/cpu.Dockerfile b/test/apis/realtime/sleep-apache/cpu.Dockerfile new file mode 100644 index 0000000000..83f5e5259c --- /dev/null +++ b/test/apis/realtime/sleep-apache/cpu.Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +CMD uvicorn --workers 1 --limit-concurrency 2 --host 0.0.0.0 --port 9000 main:app diff --git a/test/apis/realtime/sleep-apache/httpd.conf b/test/apis/realtime/sleep-apache/httpd.conf new file mode 100644 index 0000000000..f9750b0f45 --- /dev/null +++ b/test/apis/realtime/sleep-apache/httpd.conf @@ -0,0 +1,554 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "logs/access_log" +# with ServerRoot set to "/usr/local/apache2" will be interpreted by the +# server as "/usr/local/apache2/logs/access_log", whereas "/logs/access_log" +# will be interpreted as '/logs/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot "/usr/local/apache2" + +# +# Mutex: Allows you to set the mutex mechanism and mutex file directory +# for individual mutexes, or change the global defaults +# +# Uncomment and change the directory if mutexes are file-based and the default +# mutex file directory is not on a local disk or is not appropriate for some +# other reason. +# +# Mutex default:logs + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen 8080 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +LoadModule mpm_event_module modules/mod_mpm_event.so +#LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +#LoadModule mpm_worker_module modules/mod_mpm_worker.so +LoadModule authn_file_module modules/mod_authn_file.so +#LoadModule authn_dbm_module modules/mod_authn_dbm.so +#LoadModule authn_anon_module modules/mod_authn_anon.so +#LoadModule authn_dbd_module modules/mod_authn_dbd.so +#LoadModule authn_socache_module modules/mod_authn_socache.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_groupfile_module modules/mod_authz_groupfile.so +LoadModule authz_user_module modules/mod_authz_user.so +#LoadModule authz_dbm_module modules/mod_authz_dbm.so +#LoadModule authz_owner_module modules/mod_authz_owner.so +#LoadModule authz_dbd_module modules/mod_authz_dbd.so +LoadModule authz_core_module modules/mod_authz_core.so +#LoadModule authnz_ldap_module modules/mod_authnz_ldap.so +#LoadModule authnz_fcgi_module modules/mod_authnz_fcgi.so +LoadModule access_compat_module modules/mod_access_compat.so +LoadModule auth_basic_module modules/mod_auth_basic.so +#LoadModule auth_form_module modules/mod_auth_form.so +#LoadModule auth_digest_module modules/mod_auth_digest.so +#LoadModule allowmethods_module modules/mod_allowmethods.so +#LoadModule isapi_module modules/mod_isapi.so +#LoadModule file_cache_module modules/mod_file_cache.so +#LoadModule cache_module modules/mod_cache.so +#LoadModule cache_disk_module modules/mod_cache_disk.so +#LoadModule cache_socache_module modules/mod_cache_socache.so +#LoadModule socache_shmcb_module modules/mod_socache_shmcb.so +#LoadModule socache_dbm_module modules/mod_socache_dbm.so +#LoadModule socache_memcache_module modules/mod_socache_memcache.so +#LoadModule socache_redis_module modules/mod_socache_redis.so +#LoadModule watchdog_module modules/mod_watchdog.so +#LoadModule macro_module modules/mod_macro.so +#LoadModule dbd_module modules/mod_dbd.so +#LoadModule bucketeer_module modules/mod_bucketeer.so +#LoadModule dumpio_module modules/mod_dumpio.so +#LoadModule echo_module modules/mod_echo.so +#LoadModule example_hooks_module modules/mod_example_hooks.so +#LoadModule case_filter_module modules/mod_case_filter.so +#LoadModule case_filter_in_module modules/mod_case_filter_in.so +#LoadModule example_ipc_module modules/mod_example_ipc.so +#LoadModule buffer_module modules/mod_buffer.so +#LoadModule data_module modules/mod_data.so +#LoadModule ratelimit_module modules/mod_ratelimit.so +LoadModule reqtimeout_module modules/mod_reqtimeout.so +#LoadModule ext_filter_module modules/mod_ext_filter.so +#LoadModule request_module modules/mod_request.so +#LoadModule include_module modules/mod_include.so +LoadModule filter_module modules/mod_filter.so +#LoadModule reflector_module modules/mod_reflector.so +#LoadModule substitute_module modules/mod_substitute.so +#LoadModule sed_module modules/mod_sed.so +#LoadModule charset_lite_module modules/mod_charset_lite.so +#LoadModule deflate_module modules/mod_deflate.so +#LoadModule xml2enc_module modules/mod_xml2enc.so +#LoadModule proxy_html_module modules/mod_proxy_html.so +#LoadModule brotli_module modules/mod_brotli.so +LoadModule mime_module modules/mod_mime.so +#LoadModule ldap_module modules/mod_ldap.so +LoadModule log_config_module modules/mod_log_config.so +#LoadModule log_debug_module modules/mod_log_debug.so +#LoadModule log_forensic_module modules/mod_log_forensic.so +#LoadModule logio_module modules/mod_logio.so +#LoadModule lua_module modules/mod_lua.so +LoadModule env_module modules/mod_env.so +#LoadModule mime_magic_module modules/mod_mime_magic.so +#LoadModule cern_meta_module modules/mod_cern_meta.so +#LoadModule expires_module modules/mod_expires.so +LoadModule headers_module modules/mod_headers.so +#LoadModule ident_module modules/mod_ident.so +#LoadModule usertrack_module modules/mod_usertrack.so +#LoadModule unique_id_module modules/mod_unique_id.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule version_module modules/mod_version.so +#LoadModule remoteip_module modules/mod_remoteip.so +LoadModule proxy_module modules/mod_proxy.so +#LoadModule proxy_connect_module modules/mod_proxy_connect.so +#LoadModule proxy_ftp_module modules/mod_proxy_ftp.so +#LoadModule proxy_http_module modules/mod_proxy_http.so +#LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so +#LoadModule proxy_scgi_module modules/mod_proxy_scgi.so +#LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so +#LoadModule proxy_fdpass_module modules/mod_proxy_fdpass.so +#LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so +#LoadModule proxy_ajp_module modules/mod_proxy_ajp.so +#LoadModule proxy_balancer_module modules/mod_proxy_balancer.so +#LoadModule proxy_express_module modules/mod_proxy_express.so +#LoadModule proxy_hcheck_module modules/mod_proxy_hcheck.so +#LoadModule session_module modules/mod_session.so +#LoadModule session_cookie_module modules/mod_session_cookie.so +#LoadModule session_crypto_module modules/mod_session_crypto.so +#LoadModule session_dbd_module modules/mod_session_dbd.so +#LoadModule slotmem_shm_module modules/mod_slotmem_shm.so +#LoadModule slotmem_plain_module modules/mod_slotmem_plain.so +#LoadModule ssl_module modules/mod_ssl.so +#LoadModule optional_hook_export_module modules/mod_optional_hook_export.so +#LoadModule optional_hook_import_module modules/mod_optional_hook_import.so +#LoadModule optional_fn_import_module modules/mod_optional_fn_import.so +#LoadModule optional_fn_export_module modules/mod_optional_fn_export.so +#LoadModule dialup_module modules/mod_dialup.so +#LoadModule http2_module modules/mod_http2.so +#LoadModule proxy_http2_module modules/mod_proxy_http2.so +#LoadModule md_module modules/mod_md.so +#LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so +#LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so +#LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so +#LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so +LoadModule unixd_module modules/mod_unixd.so +#LoadModule heartbeat_module modules/mod_heartbeat.so +#LoadModule heartmonitor_module modules/mod_heartmonitor.so +#LoadModule dav_module modules/mod_dav.so +LoadModule status_module modules/mod_status.so +LoadModule autoindex_module modules/mod_autoindex.so +#LoadModule asis_module modules/mod_asis.so +#LoadModule info_module modules/mod_info.so +#LoadModule suexec_module modules/mod_suexec.so + + #LoadModule cgid_module modules/mod_cgid.so + + + #LoadModule cgi_module modules/mod_cgi.so + +#LoadModule dav_fs_module modules/mod_dav_fs.so +#LoadModule dav_lock_module modules/mod_dav_lock.so +#LoadModule vhost_alias_module modules/mod_vhost_alias.so +#LoadModule negotiation_module modules/mod_negotiation.so +LoadModule dir_module modules/mod_dir.so +#LoadModule imagemap_module modules/mod_imagemap.so +#LoadModule actions_module modules/mod_actions.so +#LoadModule speling_module modules/mod_speling.so +#LoadModule userdir_module modules/mod_userdir.so +LoadModule alias_module modules/mod_alias.so +#LoadModule rewrite_module modules/mod_rewrite.so + + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User www-data +Group www-data + + + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin you@example.com + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +#ServerName www.example.com:80 + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# blocks below. +# + + AllowOverride none + Require all denied + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/usr/local/apache2/htdocs" + + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options Indexes FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # AllowOverride FileInfo AuthConfig Limit + # + AllowOverride None + + # + # Controls who can get stuff from this server. + # + Require all granted + + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# + + DirectoryIndex index.html + + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog /proc/self/fd/2 + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + + + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a + # container, they will be logged here. Contrariwise, if you *do* + # define per- access logfiles, transactions will be + # logged therein and *not* in this file. + # + CustomLog /proc/self/fd/1 common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + #CustomLog "logs/access_log" combined + + + + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/" + + + + + # + # ScriptSock: On threaded servers, designate the path to the UNIX + # socket used to communicate with the CGI daemon of mod_cgid. + # + #Scriptsock cgisock + + +# +# "/usr/local/apache2/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Require all granted + + + + # + # Avoid passing HTTP_PROXY environment to CGI's on this or any proxied + # backend servers which have lingering "httpoxy" defects. + # 'Proxy' request header is undefined by the IETF, not listed by IANA + # + RequestHeader unset Proxy early + + + + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig conf/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + #AddType text/html .shtml + #AddOutputFilter INCLUDES .shtml + + +# +# The mod_mime_magic module allows the server to use various hints from the +# contents of the file itself to determine its type. The MIMEMagicFile +# directive tells the module where the hint definitions are located. +# +#MIMEMagicFile conf/magic + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# MaxRanges: Maximum number of Ranges in a request before +# returning the entire resource, or one of the special +# values 'default', 'none' or 'unlimited'. +# Default setting is to accept 200 Ranges. +#MaxRanges unlimited + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +#EnableSendfile on + +# Supplemental configuration +# +# The configuration files in the conf/extra/ directory can be +# included to add extra features or to modify the default configuration of +# the server, or you may simply copy their contents here and change as +# necessary. + +# Server-pool management (MPM specific) +#Include conf/extra/httpd-mpm.conf + +# Multi-language error messages +#Include conf/extra/httpd-multilang-errordoc.conf + +# Fancy directory listings +#Include conf/extra/httpd-autoindex.conf + +# Language settings +#Include conf/extra/httpd-languages.conf + +# User home directories +#Include conf/extra/httpd-userdir.conf + +# Real-time info on requests and configuration +#Include conf/extra/httpd-info.conf + +# Virtual hosts +#Include conf/extra/httpd-vhosts.conf + +# Local access to the Apache HTTP Server Manual +#Include conf/extra/httpd-manual.conf + +# Distributed authoring and versioning (WebDAV) +#Include conf/extra/httpd-dav.conf + +# Various default settings +#Include conf/extra/httpd-default.conf + +# Configure mod_proxy_html to understand HTML4/XHTML1 + +Include conf/extra/proxy-html.conf + + +# Secure (SSL/TLS) connections +#Include conf/extra/httpd-ssl.conf +# +# Note: The following must must be present to support +# starting without SSL on platforms with no /dev/random equivalent +# but a statically compiled-in mod_ssl. +# + +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin + + +RequestReadTimeout body=45 +ProxyPass "/sleep" "http://localhost:9000/" +ProxyPassReverse "/sleep" "http://localhost:9000/" From ec3b9ca1ba5c4c0b56fe59745c65d8f994e0f721 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Mon, 3 Jan 2022 16:45:58 -0800 Subject: [PATCH 12/13] Remove concurrency limit --- test/apis/realtime/sleep/cpu.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/apis/realtime/sleep/cpu.Dockerfile b/test/apis/realtime/sleep/cpu.Dockerfile index c706bd2d70..ea648acdf0 100644 --- a/test/apis/realtime/sleep/cpu.Dockerfile +++ b/test/apis/realtime/sleep/cpu.Dockerfile @@ -10,4 +10,4 @@ WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 -CMD uvicorn --workers 1 --limit-concurrency 2 --host 0.0.0.0 --port $CORTEX_PORT main:app +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app From cbec2a83dbba63a20687dc5c330b866bcd694ce7 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Wed, 5 Jan 2022 19:16:55 +0200 Subject: [PATCH 13/13] Fix sleep apache example --- .../apis/realtime/sleep-apache/cpu.Dockerfile | 2 +- test/apis/realtime/sleep-apache/httpd.conf | 40 ++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/test/apis/realtime/sleep-apache/cpu.Dockerfile b/test/apis/realtime/sleep-apache/cpu.Dockerfile index 83f5e5259c..a6faa3bdef 100644 --- a/test/apis/realtime/sleep-apache/cpu.Dockerfile +++ b/test/apis/realtime/sleep-apache/cpu.Dockerfile @@ -9,4 +9,4 @@ COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app -CMD uvicorn --workers 1 --limit-concurrency 2 --host 0.0.0.0 --port 9000 main:app +CMD uvicorn --workers 1 --host 0.0.0.0 --port 9000 main:app diff --git a/test/apis/realtime/sleep-apache/httpd.conf b/test/apis/realtime/sleep-apache/httpd.conf index f9750b0f45..d0b1b168e1 100644 --- a/test/apis/realtime/sleep-apache/httpd.conf +++ b/test/apis/realtime/sleep-apache/httpd.conf @@ -139,7 +139,7 @@ LoadModule headers_module modules/mod_headers.so LoadModule setenvif_module modules/mod_setenvif.so LoadModule version_module modules/mod_version.so #LoadModule remoteip_module modules/mod_remoteip.so -LoadModule proxy_module modules/mod_proxy.so +#LoadModule proxy_module modules/mod_proxy.so #LoadModule proxy_connect_module modules/mod_proxy_connect.so #LoadModule proxy_ftp_module modules/mod_proxy_ftp.so #LoadModule proxy_http_module modules/mod_proxy_http.so @@ -321,7 +321,7 @@ ErrorLog /proc/self/fd/2 # Possible values include: debug, info, notice, warn, error, crit, # alert, emerg. # -LogLevel warn +LogLevel debug # @@ -549,6 +549,36 @@ SSLRandomSeed startup builtin SSLRandomSeed connect builtin -RequestReadTimeout body=45 -ProxyPass "/sleep" "http://localhost:9000/" -ProxyPassReverse "/sleep" "http://localhost:9000/" +# ProxyPass "/sleep" "http://localhost:9000/" +# ProxyPassReverse "/sleep" "http://localhost:9000/" + +LoadModule proxy_module modules/mod_proxy.so +LoadModule proxy_http_module modules/mod_proxy_http.so + + + RequestReadTimeout body=45 + ProxyPreserveHost On + + # Servers to proxy the connection, or; + # List of application servers: + # Usage: + # ProxyPass / http://[IP Addr.]:[port]/ + # ProxyPassReverse / http://[IP Addr.]:[port]/ + # Example: + ProxyPass / http://localhost:9000/ + ProxyPassReverse / http://localhost:9000/ + + # ServerName localhost + + +# +# BalancerMember http://192.x.x.10:8080/mysite +# BalancerMember http://192.x.x.11:8080/mysite +# +# +# [...] +# ProxyRequests Off +# ProxyPreserveHost On +# ProxyPass "/mysite" "balancer://mysite_cluster" +# ProxyPassReverse "/mysite" "balancer://mysite_cluster" +#