feat: manager embed frontend assets (dragonflyoss#1523)

Signed-off-by: Gaius <[email protected]>

.github/workflows/ci.yml

@@ -26,6 +26,12 @@ jobs:
 
       - name: Checkout code
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Build manager console assets
+        run: |-
+          make build-manager-console
 
       - name: Run Unit tests
         run: |-
@@ -55,6 +61,12 @@ jobs:
 
       - name: Check out code
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Build manager console assets
+        run: |-
+          make build-manager-console
 
       - name: Build
         run: make build

.github/workflows/lint.yml

@@ -17,6 +17,12 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Build manager console assets
+        run: |-
+          make build-manager-console
 
       - name: Golangci lint
         uses: golangci/golangci-lint-action@v2

.github/workflows/release.yml

@@ -21,6 +21,10 @@ jobs:
         with:
           go-version: 1.18
 
+      - name: Build manager console assets
+        run: |-
+          make build-manager-console
+
       - name: Check GoReleaser config
         uses: goreleaser/goreleaser-action@v2
         with:

.gitignore

@@ -20,6 +20,8 @@ vendor/
 *.DS_Store
 target/
 scheduler/test/misc
+manager/dist/*
+!manager/dist/.gitkeep
 .chglog
 
 # mysql

.goreleaser.yml

@@ -1,6 +1,7 @@
 before:
   hooks:
     - hack/update-version-gorelease.sh
+    - make build-manager-console
     - go mod download
 
 builds:
@@ -12,6 +13,7 @@ builds:
       - darwin
     goarch:
       - amd64
+      - arm64
     ldflags:
       - -X d7y.io/dragonfly/v2/version.Major={{ .Major }}
       - -X d7y.io/dragonfly/v2/version.Minor={{ .Minor }}
@@ -29,6 +31,7 @@ builds:
       - darwin
     goarch:
       - amd64
+      - arm64
     ldflags:
       - -X d7y.io/dragonfly/v2/version.Major={{ .Major }}
       - -X d7y.io/dragonfly/v2/version.Minor={{ .Minor }}
@@ -46,6 +49,7 @@ builds:
       - darwin
     goarch:
       - amd64
+      - arm64
     ldflags:
       - -X d7y.io/dragonfly/v2/version.Major={{ .Major }}
       - -X d7y.io/dragonfly/v2/version.Minor={{ .Minor }}
@@ -63,6 +67,7 @@ builds:
       - darwin
     goarch:
       - amd64
+      - arm64
     ldflags:
       - -X d7y.io/dragonfly/v2/version.Major={{ .Major }}
       - -X d7y.io/dragonfly/v2/version.Minor={{ .Minor }}
@@ -80,6 +85,7 @@ builds:
       - darwin
     goarch:
       - amd64
+      - arm64
     ldflags:
       - -X d7y.io/dragonfly/v2/version.Major={{ .Major }}
       - -X d7y.io/dragonfly/v2/version.Minor={{ .Minor }}

Makefile

@@ -132,14 +132,20 @@ build-scheduler: build-dirs
 .PHONY: build-scheduler
 
 # Build manager
-build-manager: build-dirs
+build-manager: build-dirs build-manager-console
 	@echo "Begin to build manager."
+	make build-manager-server
+.PHONY: build-manager
+
+# Build manager server
+build-manager-server: build-dirs
+	@echo "Begin to build manager server."
 	./hack/build.sh manager
 .PHONY: build-manager
 
 # Build manager console
 build-manager-console: build-dirs
-	@echo "Begin to build manager."
+	@echo "Begin to build manager console."
 	./hack/build.sh manager-console
 .PHONY: build-manager-console
 
@@ -396,8 +402,10 @@ help:
 	@echo "make docker-push                    push dragonfly image"
 	@echo "make docker-build-dfdaemon          build dfdaemon image"
 	@echo "make docker-build-scheduler         build scheduler image"
+	@echo "make docker-build-manager           build manager image"
 	@echo "make docker-push-dfdaemon           push dfdaemon image"
 	@echo "make docker-push-scheduler          push scheduler image"
+	@echo "make docker-push-manager            push manager image"
 	@echo "make build                          build dragonfly"
 	@echo "make build-dfget                    build dfget"
 	@echo "make build-linux-dfget              build linux dfget"
@@ -407,6 +415,7 @@ help:
 	@echo "make build-linux-dfstore            build linux dfstore"
 	@echo "make build-scheduler                build scheduler"
 	@echo "make build-manager                  build manager"
+	@echo "make build-manager-server           build manager server"
 	@echo "make build-manager-console          build manager console"
 	@echo "make build-e2e-sha256sum            build sha256sum test tool"
 	@echo "make install-dfget                  install dfget"

build/images/manager/Dockerfile

@@ -1,5 +1,17 @@
 ARG BASE_IMAGE=alpine:3.16
 
+FROM node:12-alpine as console-builder
+
+WORKDIR /build
+
+COPY ./manager/console/package.json /build
+
+RUN npm install --loglevel warn --progress false
+
+COPY ./manager/console /build
+
+RUN npm run build
+
 FROM golang:1.18.3-alpine3.16 as server-builder
 
 ARG GOPROXY
@@ -12,19 +24,9 @@ RUN apk --no-cache add bash make gcc libc-dev git
 
 COPY . /go/src/d7y.io/dragonfly/v2
 
-RUN make build-manager && make install-manager
+COPY --from=console-builder /build/dist /go/src/d7y.io/dragonfly/v2/manager/dist
 
-FROM node:12-alpine as console-builder
-
-WORKDIR /build
-
-COPY ./manager/console/package.json /build
-
-RUN npm install --loglevel warn --progress false
-
-COPY ./manager/console /build
-
-RUN npm run build
+RUN make build-manager-server && make install-manager
 
 FROM ${BASE_IMAGE} as health
 
@@ -43,7 +45,6 @@ RUN mkdir -p /opt/dragonfly/bin/manager/console \
     && echo "hosts: files dns" > /etc/nsswitch.conf
 
 COPY --from=server-builder /opt/dragonfly/bin/manager /opt/dragonfly/bin/server
-COPY --from=console-builder /build/dist /opt/dragonfly/manager/console/dist
 COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe
 
 EXPOSE 8080 65003

build/plugin-builder/images/manager/Dockerfile

@@ -1,20 +1,17 @@
 ARG BASE_IMAGE=alpine:3.16
 
-FROM node:12-alpine as console-builder
-WORKDIR /build
-COPY ./manager/console/package.json /build
-RUN npm install --loglevel warn --progress false
-COPY ./manager/console /build
-RUN npm run build
-
 FROM ${BASE_IMAGE}
+
 WORKDIR /opt/dragonfly
+
 ENV PATH=/opt/dragonfly/bin:$PATH
-RUN mkdir -p /opt/dragonfly/bin/manager/console \
-    && echo "hosts: files dns" > /etc/nsswitch.conf  && \
+
+RUN echo "hosts: files dns" > /etc/nsswitch.conf  && \
     mkdir -p /usr/local/dragonfly/plugins/
+
 COPY ./artifacts/binaries/manager /opt/dragonfly/bin/server
 COPY ./artifacts/plugins/d7y-manager-plugin-* /usr/local/dragonfly/plugins/
-COPY --from=console-builder /build/dist /opt/dragonfly/manager/console/dist
+
 EXPOSE 8080 65003
+
 ENTRYPOINT ["/opt/dragonfly/bin/server"]

deploy/docker-compose/template/manager.template.yaml

@@ -13,8 +13,6 @@ server:
   rest:
     # stand address
     addr: :8080
-  # front-end console resource path
-  # publicPath: /dist
 
 # database info used for server
 database:

hack/build.sh

@@ -26,8 +26,8 @@ GOLDFLAGS="${GOLDFLAGS} -X \"d7y.io/dragonfly/v2/version.Gotags=${GOTAGS:-none}\
 GOLDFLAGS="${GOLDFLAGS} -X \"d7y.io/dragonfly/v2/version.GoVersion=$(go version | grep -o 'go[^ ].*')\""
 GOLDFLAGS="${GOLDFLAGS} -X \"d7y.io/dragonfly/v2/version.Gogcflags=${GOGCFLAGS:-none}\""
 
-curDir=$(cd "$(dirname "$0")" && pwd)
-cd "${curDir}" || return
+CUR_DIR=$(cd "$(dirname "$0")" && pwd)
+cd "${CUR_DIR}" || return
 BUILD_SOURCE_HOME=$(cd ".." && pwd)
 
 . ./env.sh
@@ -114,10 +114,14 @@ build-manager-docker() {
 
 build-manager-console() {
     set -x
-    consoleDir=$(echo $curDir | sed 's#hack#manager/console#')
+    CONSOLE_DIR=$(echo $CUR_DIR | sed 's#hack#manager/console#')
+    MANAGER_DIR=$(echo $CUR_DIR | sed 's#hack#manager#')
+    CONSOLE_ASSETS_DIR=$CONSOLE_DIR/dist/
+    MANAGER_ASSETS_DIR=$MANAGER_DIR/dist/
     docker run --workdir=/build \
-        --rm -v ${consoleDir}:/build node:12-alpine \
+        --rm -v ${CONSOLE_DIR}:/build node:12-alpine \
         sh -c "npm install --loglevel warn --progress false && npm run build"
+    cp -r $CONSOLE_ASSETS_DIR $MANAGER_ASSETS_DIR
 }
 
 main() {

hack/gen-buildx.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-components="scheduler manager"
+components="scheduler manager-server"
 
 set -x
 

manager/config/config.go

@@ -54,9 +54,6 @@ type ServerConfig struct {
 	// Server log directory.
 	LogDir string `yaml:"logDir" mapstructure:"logDir"`
 
-	// Console resource path.
-	PublicPath string `yaml:"publicPath" mapstructure:"publicPath"`
-
 	// GRPC server configuration.
 	GRPC *TCPListenConfig `yaml:"grpc" mapstructure:"grpc"`
 
@@ -249,8 +246,7 @@ type ObjectStorageConfig struct {
 func New() *Config {
 	return &Config{
 		Server: &ServerConfig{
-			Name:       DefaultServerName,
-			PublicPath: DefaultPublicPath,
+			Name: DefaultServerName,
 			GRPC: &TCPListenConfig{
 				PortRange: TCPListenPortRange{
 					Start: DefaultGRPCPort,

manager/config/config_test.go

@@ -32,9 +32,8 @@ func TestManagerConfig_Load(t *testing.T) {
 
 	config := &Config{
 		Server: &ServerConfig{
-			Name:       "foo",
-			LogDir:     "foo",
-			PublicPath: "foo",
+			Name:   "foo",
+			LogDir: "foo",
 			GRPC: &TCPListenConfig{
 				Listen: "127.0.0.1",
 				PortRange: TCPListenPortRange{

manager/config/constants.go

@@ -33,9 +33,6 @@ const (
 	// DefaultServerName is default server name.
 	DefaultServerName = "d7y/manager"
 
-	// DefaultPublicPath is default path for frontend assets.
-	DefaultPublicPath = "manager/console/dist"
-
 	// DefaultGRPCPort is default port for grpc server.
 	DefaultGRPCPort = 65003
 

manager/config/testdata/manager.yaml

@@ -1,7 +1,6 @@
 server:
   name: foo
   logDir: foo
-  publicPath: foo
   grpc:
     listen: 127.0.0.1
     port:

manager/manager.go

@@ -18,9 +18,12 @@ package manager
 
 import (
 	"context"
+	"embed"
+	"io/fs"
 	"net/http"
 	"time"
 
+	"github.com/gin-contrib/static"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 
@@ -42,10 +45,39 @@ import (
 
 const (
 	// gracefulStopTimeout specifies a time limit for
-	// grpc server to complete a graceful shutdown
+	// grpc server to complete a graceful shutdown.
 	gracefulStopTimeout = 10 * time.Second
+
+	// assetsTargetPath is target path of embed assets.
+	assetsTargetPath = "dist"
 )
 
+//go:embed dist/*
+var assets embed.FS
+
+type embedFileSystem struct {
+	http.FileSystem
+}
+
+func (e embedFileSystem) Exists(prefix string, path string) bool {
+	_, err := e.Open(path)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func EmbedFolder(fsEmbed embed.FS, targetPath string) static.ServeFileSystem {
+	fsys, err := fs.Sub(fsEmbed, targetPath)
+	if err != nil {
+		panic(err)
+	}
+
+	return embedFileSystem{
+		FileSystem: http.FS(fsys),
+	}
+}
+
 type Server struct {
 	// Server configuration
 	config *config.Config
@@ -107,7 +139,7 @@ func New(cfg *config.Config, d dfpath.Dfpath) (*Server, error) {
 
 	// Initialize REST server
 	restService := service.New(db, cache, job, enforcer, objectStorage)
-	router, err := router.Init(cfg, d.LogDir(), restService, enforcer)
+	router, err := router.Init(cfg, d.LogDir(), restService, enforcer, EmbedFolder(assets, assetsTargetPath))
 	if err != nil {
 		return nil, err
 	}