Skip to content

Commit

Permalink
helm chart packager (koderover#698)
Browse files Browse the repository at this point in the history
* add basic logic to push images by running jobs

Signed-off-by: allenshen <[email protected]>

* add more detail to progress data

Signed-off-by: allenshen <[email protected]>

* optimize code

Signed-off-by: allenshen <[email protected]>

* fix comments

Signed-off-by: allenshen <[email protected]>

* resolve comments

Signed-off-by: allenshen <[email protected]>

* fix conflicts

Signed-off-by: allenshen <[email protected]>
  • Loading branch information
AllenShen authored Dec 10, 2021
1 parent 8a47d15 commit ec50023
Show file tree
Hide file tree
Showing 29 changed files with 1,401 additions and 224 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
IMAGE_REPOSITORY = ccr.ccs.tencentyun.com/koderover-rc
VERSION ?= $(shell date +'%Y%m%d%H%M%S')

TARGETS = aslan cron hub-agent hub-server jenkins-plugin podexec predator-plugin resource-server ua warpdrive policy user picket config init
TARGETS = aslan cron hub-agent hub-server jenkins-plugin podexec predator-plugin packager-plugin resource-server ua warpdrive policy user picket config init
REAPER_OS= focal xenial bionic

ALL_IMAGES=$(TARGETS:=.image)
Expand Down
29 changes: 29 additions & 0 deletions cmd/packager/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
Copyright 2021 The KodeRover Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"log"

"github.com/koderover/zadig/pkg/microservice/packager/executor"
)

func main() {
if err := executor.Execute(); err != nil {
log.Fatalf("Failed to run packager, the error is: %+v", err)
}
}
11 changes: 11 additions & 0 deletions docker/service/packager-plugin.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#golang.Dockerfile

RUN go build -v -o /packager-plugin ./cmd/packager/main.go

#alpine.Dockerfile

WORKDIR /app

COPY --from=build /packager-plugin .

ENTRYPOINT ["/app/packager-plugin"]
4 changes: 4 additions & 0 deletions pkg/microservice/aslan/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,10 @@ func PredatorImage() string {
return viper.GetString(setting.ENVPredatorImage)
}

func PackagerImage() string {
return viper.GetString(setting.EnvPackagerImage)
}

func DockerHosts() []string {
return strings.Split(viper.GetString(setting.ENVDockerHosts), ",")
}
Expand Down
31 changes: 17 additions & 14 deletions pkg/microservice/aslan/config/consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ const (
TestType PipelineType = "test"
// ServiceType 服务
ServiceType PipelineType = "service"
// ArtifactPackageType package artifact
ArtifactType PipelineType = "artifact"
)

type Status string
Expand Down Expand Up @@ -117,20 +119,21 @@ const (
type TaskType string

const (
TaskPipeline TaskType = "pipeline"
TaskBuild TaskType = "buildv2"
TaskJenkinsBuild TaskType = "jenkins_build"
TaskArtifact TaskType = "artifact"
TaskArtifactDeploy TaskType = "artifact_deploy"
TaskDeploy TaskType = "deploy"
TaskTestingV2 TaskType = "testingv2"
TaskDistributeToS3 TaskType = "distribute2kodo"
TaskReleaseImage TaskType = "release_image"
TaskJira TaskType = "jira"
TaskDockerBuild TaskType = "docker_build"
TaskSecurity TaskType = "security"
TaskResetImage TaskType = "reset_image"
TaskDistribute TaskType = "distribute"
TaskPipeline TaskType = "pipeline"
TaskBuild TaskType = "buildv2"
TaskJenkinsBuild TaskType = "jenkins_build"
TaskArtifact TaskType = "artifact"
TaskArtifactDeploy TaskType = "artifact_deploy"
TaskDeploy TaskType = "deploy"
TaskTestingV2 TaskType = "testingv2"
TaskDistributeToS3 TaskType = "distribute2kodo"
TaskReleaseImage TaskType = "release_image"
TaskJira TaskType = "jira"
TaskDockerBuild TaskType = "docker_build"
TaskSecurity TaskType = "security"
TaskResetImage TaskType = "reset_image"
TaskDistribute TaskType = "distribute"
TaskArtifactPackage TaskType = "artifact_package"
)

type DistributeType string
Expand Down
123 changes: 66 additions & 57 deletions pkg/microservice/aslan/core/common/repository/models/queue.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,63 +25,50 @@ import (
)

type Queue struct {
ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"`
TaskID int64 `bson:"task_id" json:"task_id"`
ProductName string `bson:"product_name" json:"product_name"`
PipelineName string `bson:"pipeline_name" json:"pipeline_name"`
Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"`
Type config.PipelineType `bson:"type" json:"type"`
Status config.Status `bson:"status" json:"status,omitempty"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"`
TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"`
CreateTime int64 `bson:"create_time" json:"create_time,omitempty"`
StartTime int64 `bson:"start_time" json:"start_time,omitempty"`
EndTime int64 `bson:"end_time" json:"end_time,omitempty"`
SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"`
Stages []*Stage `bson:"stages" json:"stages"`
ReqID string `bson:"req_id,omitempty" json:"req_id,omitempty"`
AgentHost string `bson:"agent_host,omitempty" json:"agent_host,omitempty"`
DockerHost string `bson:"-" json:"docker_host,omitempty"`
TeamID int `bson:"team_id,omitempty" json:"team_id,omitempty"`
TeamName string `bson:"team,omitempty" json:"team,omitempty"`
IsDeleted bool `bson:"is_deleted" json:"is_deleted"`
IsArchived bool `bson:"is_archived" json:"is_archived"`
AgentID string `bson:"agent_id" json:"agent_id"`
// 是否允许同时运行多次
MultiRun bool `bson:"multi_run" json:"multi_run"`
// target 服务名称, k8s为容器名称, 物理机为服务名
Target string `bson:"target,omitempty" json:"target"`
// 使用预定义编译管理模块中的内容生成SubTasks,
// 查询条件为 服务模板名称: ServiceTmpl, 版本: BuildModuleVer
// 如果为空,则使用pipeline自定义SubTasks
BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"`
ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"`
// TaskArgs 单服务工作流任务参数
TaskArgs *TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"`
// WorkflowArgs 多服务工作流任务参数
WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args,omitempty"`
// TestArgs 测试任务参数
TestArgs *TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"`
// ServiceTaskArgs 脚本部署工作流任务参数
ServiceTaskArgs *ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"`
ConfigPayload *ConfigPayload `json:"config_payload,omitempty"`
Error string `bson:"error,omitempty" json:"error,omitempty"`
Services [][]*ProductService `bson:"services" json:"services"`
Render *RenderInfo `bson:"render" json:"render"`
StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"`
// interface{} 为types.TestReport
TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"`

RwLock sync.Mutex `bson:"-" json:"-"`

ResetImage bool `json:"resetImage" bson:"resetImage"`

TriggerBy *TriggerBy `json:"trigger_by,omitempty" bson:"trigger_by,omitempty"`

Features []string `bson:"features" json:"features"`
IsRestart bool `bson:"is_restart" json:"is_restart"`
StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"`
ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"`
TaskID int64 `bson:"task_id" json:"task_id"`
ProductName string `bson:"product_name" json:"product_name"`
PipelineName string `bson:"pipeline_name" json:"pipeline_name"`
Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"`
Type config.PipelineType `bson:"type" json:"type"`
Status config.Status `bson:"status" json:"status,omitempty"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"`
TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"`
CreateTime int64 `bson:"create_time" json:"create_time,omitempty"`
StartTime int64 `bson:"start_time" json:"start_time,omitempty"`
EndTime int64 `bson:"end_time" json:"end_time,omitempty"`
SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"`
Stages []*Stage `bson:"stages" json:"stages"`
ReqID string `bson:"req_id,omitempty" json:"req_id,omitempty"`
AgentHost string `bson:"agent_host,omitempty" json:"agent_host,omitempty"`
DockerHost string `bson:"-" json:"docker_host,omitempty"`
TeamID int `bson:"team_id,omitempty" json:"team_id,omitempty"`
TeamName string `bson:"team,omitempty" json:"team,omitempty"`
IsDeleted bool `bson:"is_deleted" json:"is_deleted"`
IsArchived bool `bson:"is_archived" json:"is_archived"`
AgentID string `bson:"agent_id" json:"agent_id"`
MultiRun bool `bson:"multi_run" json:"multi_run"`
Target string `bson:"target,omitempty" json:"target"` // target 服务名称, k8s为容器名称, 物理机为服务名
BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"` // 使用预定义编译管理模块中的内容生成SubTasks, 查询条件为 服务模板名称: ServiceTmpl, 版本: BuildModuleVer 如果为空,则使用pipeline自定义SubTasks
ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"`
TaskArgs *TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"` // TaskArgs 单服务工作流任务参数
WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args,omitempty"` // WorkflowArgs 多服务工作流任务参数
TestArgs *TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` // TestArgs 测试任务参数
ServiceTaskArgs *ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` // ServiceTaskArgs 脚本部署工作流任务参数
ArtifactPackageTaskArgs *ArtifactPackageTaskArgs `bson:"artifact_package_args,omitempty" json:"artifact_package_args,omitempty"`
ConfigPayload *ConfigPayload `bson:"configpayload,omitempty" json:"config_payload"`
Error string `bson:"error,omitempty" json:"error,omitempty"`
Services [][]*ProductService `bson:"services" json:"services"`
Render *RenderInfo `bson:"render" json:"render"`
StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"`
TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"`
RwLock sync.Mutex `bson:"-" json:"-"`
ResetImage bool `bson:"resetImage" json:"resetImage"`
TriggerBy *TriggerBy `bson:"trigger_by,omitempty" json:"trigger_by,omitempty"`
Features []string `bson:"features" json:"features"`
IsRestart bool `bson:"is_restart" json:"is_restart"`
StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"`
}

type TriggerBy struct {
Expand All @@ -108,6 +95,25 @@ type ServiceTaskArgs struct {
Updatable bool `bson:"updatable" json:"updatable"`
}

type ImageData struct {
ImageUrl string `bson:"image_url" json:"image_url"`
ImageName string `bson:"image_name" json:"image_name"`
ImageTag string `bson:"image_tag" json:"image_tag"`
}

type ImagesByService struct {
ServiceName string `bson:"service_name" json:"service_name"`
Images []*ImageData `bson:"images" json:"images"`
}

type ArtifactPackageTaskArgs struct {
ProjectName string `bson:"project_name" json:"project_name"`
EnvName string `bson:"env_name" json:"env_name"`
Images []*ImagesByService `bson:"images" json:"images"`
SourceRegistries []string `json:"source_registries" json:"source_registries"`
TargetRegistries []string `json:"target_registries" json:"target_registries"`
}

type ConfigPayload struct {
Proxy Proxy `json:"proxy"`
S3Storage S3Config `json:"s3_storage"`
Expand Down Expand Up @@ -210,6 +216,9 @@ type ReleaseConfig struct {
// PredatorImage sets docker build image
// e.g. xxx.com/resources/predator-plugin:v0.1.0
PredatorImage string
// PackagerImage sets docker build image
// e.g. xxx.com/resources/predator-plugin:v0.1.0
PackagerImage string
}

type ImageReleaseConfig struct {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
Copyright 2021 The KodeRover Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package task

import (
"fmt"

"github.com/koderover/zadig/pkg/microservice/aslan/config"
"github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models"
)

type ArtifactPackage struct {
TaskType config.TaskType `bson:"type" json:"type"`
TaskStatus config.Status `bson:"status" json:"status"`
Enabled bool `bson:"enabled" json:"enabled"`
Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"`
Error string `bson:"error,omitempty" json:"error,omitempty"`
StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"`
EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"`
LogFile string `bson:"log_file" json:"log_file"`

// source images
Images []*models.ImagesByService `bson:"images" json:"images"`
// source registries need auth to pull images
SourceRegistries []string `bson:"source_registries" json:"source_registries"`
// target registries to push images
TargetRegistries []string `bson:"target_registries" json:"target_registries"`
}

func (ri *ArtifactPackage) ToSubTask() (map[string]interface{}, error) {
var task map[string]interface{}
if err := IToi(ri, &task); err != nil {
return nil, fmt.Errorf("convert ReleaseImageTask to interface error: %v", err)
}
return task, nil
}
Loading

0 comments on commit ec50023

Please sign in to comment.