feat: upgrade yatai-deployment to support bentoml configuration, cust…

…om resources, pod topology and affinity (bentoml#344) * feat: bump Go to 1.19 * feat(doc): add v1alpha3 BentoDeployment documentation * feat: upgrade yatai-deployment to support BentoDeployment v1alpha3 * feat: do not overwrite previous annotations, labels and extraPodSpecs * feat: custom resource input * feat: support bentoml configuration * optimize: friendly time format for pod start time * feat: use monaco editor for bentoml configuration * feat: render an icon if pod status is succeeded * feat(dashboard): log component supports the selection of containers * feat: add image-build statuses to deployment * feat: show container status log component * fix(scripts): start-dev with trap * feat(api-server): upgrade yatai-deployment to support extra_pod_metadata * optimize(ci): speed up release ci * fix(ci): make golangci-lint happy * feat(api-server): multipart upload apis * feat(ui): support set transmission strategy for organization * feat: compatible with all versions of BentoDeployment
pluralsh · Oct 13, 2022 · 0c0ecbd · 0c0ecbd
1 parent 982e32d
commit 0c0ecbd
Show file tree

Hide file tree

Showing 56 changed files with 1,674 additions and 935 deletions.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -77,7 +77,7 @@ jobs:
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: 'v1.47.2'
+          version: 'v1.49'
 
   helmchartslint:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -3,6 +3,7 @@ name: Release
 on:
   push:
     tags: [ v\d+\.\d+\.\d+\-d\d+ ]
+
   release:
     types: [ published ]
 
@@ -12,74 +13,96 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+
+      - name: Set vars
+        id: vars
+        run: |
+          echo "::set-output name=dashboard_hash::${{ hashFiles('./dashboard/**') }}"
+
+      - uses: tuler/s3-check-action@master
+        id: check-s3
+        env:
+          FILE: dashboard/${{ steps.vars.outputs.dashboard_hash }}/index.html
+          AWS_REGION: ap-northeast-1
+          AWS_S3_BUCKET: yatai-github-artifacts
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+        if: steps.check-s3.outputs.exists == 'false'
 
       - uses: actions/cache@v2
         with:
           path: '**/dashboard/node_modules'
           key: ${{ runner.os }}-node-modules-${{ hashFiles('**/dashboard/yarn.lock') }}
+        if: steps.check-s3.outputs.exists == 'false'
 
       - name: Install packages
         run: cd dashboard; yarn
+        if: steps.check-s3.outputs.exists == 'false'
 
       - name: Build dashboard
         run: cd dashboard; NODE_OPTIONS="--max_old_space_size=4096" yarn build
+        if: steps.check-s3.outputs.exists == 'false'
         env:
           CI: false
 
-      - name: Set git sha_short outputs
-        id: vars
-        run: echo "::set-output name=sha_short::$(git describe --match=NeVeRmAtCh --tags --always --dirty | cut -c 1-7)"
-
       - name: Upload dashboard artifacts
         uses: prewk/s3-cp-action@v2
         with:
           aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_region: ap-northeast-1
           source: './dashboard/build/'
-          dest: 's3://yatai-github-artifacts/dashboard-artifacts/${{ steps.vars.outputs.sha_short }}'
+          dest: 's3://yatai-github-artifacts/dashboard/${{ steps.vars.outputs.dashboard_hash }}'
           flags: --recursive
+        if: steps.check-s3.outputs.exists == 'false'
 
   build-api-server:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '^1.17.3' # The Go version to download (if necessary) and use.
-
-      - run: go version
-
-      - id: go-cache-paths
+      - name: Set vars
+        id: vars
         run: |
-          echo "::set-output name=go-build::$(go env GOCACHE)"
-          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+          echo "::set-output name=api_server_hash::${{ hashFiles('./api-server/**', './common/**', '**/go.sum', '**go.mod') }}"
 
-      # Cache go build cache, used to speedup go test
-      - name: Go Build Cache
-        uses: actions/cache@v2
+      - uses: tuler/s3-check-action@master
+        id: check-s3
+        env:
+          FILE: api-server/${{ steps.vars.outputs.api_server_hash }}/api-server
+          AWS_REGION: ap-northeast-1
+          AWS_S3_BUCKET: yatai-github-artifacts
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Buildx Cache
+        uses: actions/cache@v3
         with:
-          path: ${{ steps.go-cache-paths.outputs.go-build }}
-          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+          path: /tmp/buildx-cache
+          key: ${{ runner.os }}-buildx-${{ hashFiles('**/go.sum') }}
+        if: steps.check-s3.outputs.exists == 'false'
 
-      # Cache go mod cache, used to speedup builds
-      - name: Go Mod Cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.go-cache-paths.outputs.go-mod }}
-          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        if: steps.check-s3.outputs.exists == 'false'
 
       - name: Build api-server
-        run: make build-api-server
+        run: make docker-build-api-server
+        if: steps.check-s3.outputs.exists == 'false'
 
-      - name: Set git sha_short outputs
-        id: vars
-        run: echo "::set-output name=sha_short::$(git describe --match=NeVeRmAtCh --tags --always --dirty | cut -c 1-7)"
+      - name: Move cache
+        run: |
+          rm -rf /tmp/buildx-cache
+          mv /tmp/buildx-cache-new /tmp/buildx-cache
+        if: steps.check-s3.outputs.exists == 'false'
 
       - name: Upload api-server artifacts
         uses: prewk/s3-cp-action@v2
@@ -88,8 +111,9 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_region: ap-northeast-1
           source: './bin/'
-          dest: 's3://yatai-github-artifacts/api-server-artifacts/${{ steps.vars.outputs.sha_short }}'
+          dest: 's3://yatai-github-artifacts/api-server/${{ steps.vars.outputs.api_server_hash }}'
           flags: --recursive
+        if: steps.check-s3.outputs.exists == 'false'
 
   release-image:
     runs-on: ubuntu-latest
@@ -99,18 +123,13 @@ jobs:
       - build-api-server
 
     steps:
-      - uses: actions/checkout@v2
-
-      - name: Login to Quay.io
-        uses: docker/login-action@v1
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_USERNAME }}
-          password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+      - uses: actions/checkout@v3
 
-      - name: Set git sha_short outputs
+      - name: Set vars
         id: vars
-        run: echo "::set-output name=sha_short::$(git describe --match=NeVeRmAtCh --tags --always --dirty | cut -c 1-7)"
+        run: |
+          echo "::set-output name=dashboard_hash::${{ hashFiles('./dashboard/**') }}"
+          echo "::set-output name=api_server_hash::${{ hashFiles('./api-server/**', './common/**', '**/go.sum', '**go.mod') }}"
 
       - name: Download dashboard artifacts
         uses: prewk/s3-cp-action@v2
@@ -119,7 +138,7 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_region: ap-northeast-1
           dest: './dashboard/build/'
-          source: 's3://yatai-github-artifacts/dashboard-artifacts/${{ steps.vars.outputs.sha_short }}'
+          source: 's3://yatai-github-artifacts/dashboard/${{ steps.vars.outputs.dashboard_hash }}'
           flags: --recursive
 
       - name: Download api-server artifacts
@@ -129,9 +148,16 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_region: ap-northeast-1
           dest: './bin/'
-          source: 's3://yatai-github-artifacts/api-server-artifacts/${{ steps.vars.outputs.sha_short }}'
+          source: 's3://yatai-github-artifacts/api-server/${{ steps.vars.outputs.api_server_hash }}'
           flags: --recursive
 
+      - name: Login to Quay.io
+        uses: docker/login-action@v1
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
       - name: Build yatai image
         run: make build-image
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.17
+FROM alpine:3.16.2
 
 RUN mkdir /app
 

diff --git a/Dockerfile-builder b/Dockerfile-builder
@@ -1,6 +1,19 @@
-FROM golang:1.17
+# syntax = docker/dockerfile:1-experimental
 
-RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.35.2
+FROM golang:1.19-alpine as base
 
-RUN wget -nv https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq && chmod a+x /usr/bin/yq
-RUN go get -u github.com/client9/misspell/cmd/misspell
+WORKDIR /src
+ENV CGO_ENABLED=0
+COPY go.* ./
+RUN --mount=type=cache,target=/go/pkg/mod \
+  go mod download
+
+FROM base AS build
+ARG VERSION_BUILDFLAGS
+RUN --mount=target=. \
+  --mount=type=cache,target=/root/.cache/go-build \
+  --mount=type=cache,target=/go/pkg/mod \
+  go build -ldflags "${VERSION_BUILDFLAGS}" -o /bin/api-server ./api-server/main.go
+
+FROM scratch AS export
+COPY --from=build /bin/api-server .
diff --git a/Makefile b/Makefile
@@ -57,9 +57,10 @@ docker-eslint: pull-ui-builder-image ## Docker eslint
 docker-ui-typecheck: pull-ui-builder-image ## Docker typecheck
 	$(UI_BUILDER_CNTR_CMD) sh -c "cd dashboard; ln -s /cache/node_modules ./node_modules; yarn typecheck"
 
-docker-build-api-server: pull-builder-image ## Build api-server binary
-	mkdir -p ./bin
-	$(BUILDER_CNTR_CMD) go build -ldflags "$(VERSION_BUILDFLAGS)" -o ./bin/api-server ./api-server/main.go
+docker-build-api-server: ## Build api-server binary
+	@mkdir -p ./bin
+	@mkdir -p /tmp/buildx-cache/
+	@docker buildx build -f Dockerfile-builder --build-arg VERSION_BUILDFLAGS="$(VERSION_BUILDFLAGS)" --output bin --cache-from type=local,src=/tmp/buildx-cache --cache-to type=local,mode=max,dest=/tmp/buildx-cache-new .
 
 build-api-server:
 	mkdir -p ./bin
@@ -99,16 +100,15 @@ yatai-dev: ## Run yatai(be and fe) in development mode
 be-deps: ## Fetch Golang deps
 	@echo "Downloading go modules..."
 	@go mod download
-be-run:
-	@echo "Make sure to install postgresql and create yatai DB with 'createdb yatai'"
-	@if [[ ! -f ./yatai-config.dev.yaml ]]; then \
-		echo "yatai-config.dev.yaml not found. Creating one with postgresql user: " $$(whoami); \
-		cp ./yatai-config.sample.yaml ./yatai-config.dev.yaml; \
-		sed -i 's/user: .*/user: '$$(whoami)'/' ./yatai-config.dev.yaml; \
-	fi;
+
+build-api-server-dev: ## Build api-server binary in development mode
 	@go build -gcflags="all=-N -l" -ldflags "$(VERSION_BUILDFLAGS)" -o ./bin/api-server ./api-server/main.go
+
+show-api-server-version:
 	@./bin/api-server version
-	@./bin/api-server serve -c ./yatai-config.dev.yaml
+
+run-api-server: build-api-server-dev show-api-server-version ## Run api-server in development mode
+	@./bin/api-server serve
 
 fe-deps: ## Fetch frontend deps
 	@cd dashboard && yarn

diff --git a/README.md b/README.md
@@ -141,7 +141,7 @@ A Bento Deployment can be created either via Web UI or via a Kubernetes CRD conf
 Define your Bento deployment in a `my_deployment.yaml` file:
 
 ```yaml
-apiVersion: serving.yatai.ai/v1alpha2
+apiVersion: serving.yatai.ai/v1alpha3
 kind: BentoDeployment
 metadata:
     name: my-bento-deployment

diff --git a/api-server/cmd/serve.go b/api-server/cmd/serve.go
@@ -118,7 +118,7 @@ func (opt *ServeOption) Run(ctx context.Context, args []string) error {
 		return errors.Wrap(err, "migrate up db")
 	}
 
-	if !config.YataiConfig.IsSass {
+	if !config.YataiConfig.IsSaaS {
 		err = initSelfHost(ctx)
 		if err != nil {
 			return errors.Wrap(err, "init self host")
@@ -127,6 +127,7 @@ func (opt *ServeOption) Run(ctx context.Context, args []string) error {
 
 	addCron(ctx)
 
+	// nolint: contextcheck
 	router, err := routes.NewRouter()
 	if err != nil {
 		return err

diff --git a/api-server/config/yatai_config.go b/api-server/config/yatai_config.go
@@ -10,11 +10,12 @@ import (
 )
 
 type YataiServerConfigYaml struct {
-	EnableHTTPS       bool   `yaml:"enable_https"`
-	Port              uint   `yaml:"port"`
-	SessionSecretKey  string `yaml:"session_secret_key"`
-	MigrationDir      string `yaml:"migration_dir"`
-	ReadHeaderTimeout int    `yaml:"read_header_timeout"`
+	EnableHTTPS          bool   `yaml:"enable_https"`
+	Port                 uint   `yaml:"port"`
+	SessionSecretKey     string `yaml:"session_secret_key"`
+	MigrationDir         string `yaml:"migration_dir"`
+	ReadHeaderTimeout    int    `yaml:"read_header_timeout"`
+	TransmissionStrategy string `yaml:"transmission_strategy"`
 }
 
 type YataiPostgresqlConfigYaml struct {
@@ -49,7 +50,7 @@ type YataiDockerImageBuilderConfigYaml struct {
 }
 
 type YataiConfigYaml struct {
-	IsSass              bool                      `yaml:"is_sass"`
+	IsSaaS              bool                      `yaml:"is_saas"`
 	SassDomainSuffix    string                    `yaml:"sass_domain_suffix"`
 	InCluster           bool                      `yaml:"in_cluster"`
 	Server              YataiServerConfigYaml     `yaml:"server"`
@@ -62,9 +63,9 @@ type YataiConfigYaml struct {
 var YataiConfig = &YataiConfigYaml{}
 
 func PopulateYataiConfig() error {
-	isSass, ok := os.LookupEnv(consts.EnvIsSass)
+	isSaaS, ok := os.LookupEnv(consts.EnvIsSaaS)
 	if ok {
-		YataiConfig.IsSass = isSass == "true"
+		YataiConfig.IsSaaS = isSaaS == "true"
 	}
 
 	sassDomainSuffix, ok := os.LookupEnv(consts.EnvSassDomainSuffix)
@@ -121,6 +122,11 @@ func PopulateYataiConfig() error {
 		YataiConfig.Server.ReadHeaderTimeout = readHeaderTimeout_
 	}
 
+	transmissionStrategy, ok := os.LookupEnv(consts.EnvTransmissionStrategy)
+	if ok {
+		YataiConfig.Server.TransmissionStrategy = transmissionStrategy
+	}
+
 	initializationToken, ok := os.LookupEnv(consts.EnvInitializationToken)
 	if ok {
 		YataiConfig.InitializationToken = initializationToken