Skip to content

Commit

Permalink
zadig supports the ability of host projects to execute build and gene…
Browse files Browse the repository at this point in the history
…ral tasks on the host side through zadig agent client (koderover#3043)

Signed-off-by: zhatiai <[email protected]>
Co-authored-by: zhatiai <[email protected]>
  • Loading branch information
zhatiai and zhatiai authored Sep 26, 2023
1 parent c0037c4 commit ee2a0e4
Show file tree
Hide file tree
Showing 97 changed files with 9,042 additions and 130 deletions.
39 changes: 39 additions & 0 deletions cmd/zadig-agent/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
Copyright 2023 The KodeRover Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"os"
"runtime/debug"

"github.com/koderover/zadig/pkg/cli/zadig-agent/command"
"github.com/koderover/zadig/pkg/cli/zadig-agent/helper/log"
)

func main() {
defer func() {
if err := recover(); err != nil {
log.Errorf("Agent panic error: %v", err)
log.Errorf("Agent panic stack: %s", string(debug.Stack()))
os.Exit(1)
}
}()

if err := command.Execute(); err != nil {
log.Fatalf("Failed to run zadig-agent cmd executor, error: %s", err)
}
}
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ require (
github.com/go-ldap/ldap/v3 v3.3.0
github.com/go-resty/resty/v2 v2.7.0
github.com/go-sql-driver/mysql v1.6.0
github.com/gogo/protobuf v1.3.2
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/golang/protobuf v1.5.3
github.com/google/gnostic v0.6.9
Expand Down Expand Up @@ -68,6 +67,7 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/samber/lo v1.37.0
github.com/sashabaranov/go-openai v1.12.0
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/shirou/gopsutil/v3 v3.22.8
github.com/spf13/cobra v1.5.0
github.com/spf13/viper v1.8.1
Expand All @@ -84,6 +84,7 @@ require (
golang.org/x/net v0.10.0
golang.org/x/oauth2 v0.5.0
golang.org/x/sync v0.1.0
google.golang.org/protobuf v1.30.0
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v2 v2.4.0
Expand Down Expand Up @@ -167,6 +168,7 @@ require (
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
Expand Down Expand Up @@ -281,7 +283,6 @@ require (
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03 // indirect
google.golang.org/grpc v1.47.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.66.6 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -845,6 +845,8 @@ github.com/sashabaranov/go-openai v1.12.0 h1:aRNHH0gtVfrpIaEolD0sWrLLRnYQNK4cH/b
github.com/sashabaranov/go-openai v1.12.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil/v3 v3.22.8 h1:a4s3hXogo5mE2PfdfJIonDbstO/P+9JszdfhAHSzD9Y=
github.com/shirou/gopsutil/v3 v3.22.8/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
Expand Down
156 changes: 156 additions & 0 deletions pkg/cli/upgradeassistant/cmd/migrate/1190.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,12 @@ func V1180ToV1190() error {
return err
}

log.Infof("-------- start migrate infrastructure filed in build & build template module and general job --------")
if err := migrateInfrastructureField(); err != nil {
log.Infof("migrate infrastructure filed in build & build template module and general job err: %v", err)
return err
}

return nil
}

Expand Down Expand Up @@ -555,3 +561,153 @@ func migrateApolloIntegration() error {
}
return nil
}

func migrateInfrastructureField() error {
// change build module infrastructure field
cursor, err := mongodb.NewBuildColl().ListByCursor(&mongodb.BuildListOption{})
if err != nil {
return fmt.Errorf("failed to list build module cursor for infrastructure field in migrateInfrastructureField method, err: %v", err)
}

var ms []mongo.WriteModel
for cursor.Next(context.Background()) {
var build models.Build
if err := cursor.Decode(&build); err != nil {
return err
}

if build.Infrastructure == "" {
build.Infrastructure = setting.JobK8sInfrastructure
ms = append(ms,
mongo.NewUpdateOneModel().
SetFilter(bson.D{{"_id", build.ID}}).
SetUpdate(bson.D{{"$set",
bson.D{
{"infrastructure", build.Infrastructure},
}},
}),
)
}

if len(ms) >= 50 {
log.Infof("update %d build", len(ms))
if _, err := mongodb.NewBuildColl().BulkWrite(context.Background(), ms); err != nil {
return fmt.Errorf("update build for infrastructure field in migrateInfrastructureField method, error: %s", err)
}
ms = []mongo.WriteModel{}
}
}

if len(ms) > 0 {
log.Infof("update %d build", len(ms))
if _, err := mongodb.NewBuildColl().BulkWrite(context.Background(), ms); err != nil {
return fmt.Errorf("update build for infrastructure field in migrateInfrastructureField method, error: %s", err)
}
}

// change build template module infrastructure field
cursor, err = mongodb.NewBuildTemplateColl().ListByCursor(&mongodb.ListBuildTemplateOption{})
if err != nil {
return fmt.Errorf("failed to list build template module cursor for infrastructure field in migrateInfrastructureField method, err: %v", err)
}

ms = []mongo.WriteModel{}
for cursor.Next(context.Background()) {
var buildTemplate models.BuildTemplate
if err := cursor.Decode(&buildTemplate); err != nil {
return err
}

if buildTemplate.Infrastructure == "" {
buildTemplate.Infrastructure = setting.JobK8sInfrastructure
ms = append(ms,
mongo.NewUpdateOneModel().
SetFilter(bson.D{{"_id", buildTemplate.ID}}).
SetUpdate(bson.D{{"$set",
bson.D{
{"infrastructure", buildTemplate.Infrastructure},
}},
}),
)
}

if len(ms) >= 50 {
log.Infof("update %d build template", len(ms))
if _, err := mongodb.NewBuildTemplateColl().BulkWrite(context.Background(), ms); err != nil {
return fmt.Errorf("update build template for infrastructure field in migrateInfrastructureField method, error: %s", err)
}
ms = []mongo.WriteModel{}
}
}

if len(ms) > 0 {
log.Infof("update %d build template", len(ms))
if _, err := mongodb.NewBuildTemplateColl().BulkWrite(context.Background(), ms); err != nil {
return fmt.Errorf("update build template for infrastructure field in migrateInfrastructureField method, error: %s", err)
}
}

// change general job module infrastructure field
cursor, err = mongodb.NewWorkflowV4Coll().ListByCursor(&mongodb.ListWorkflowV4Option{
JobTypes: []config.JobType{
config.JobFreestyle,
},
})
if err != nil {
return fmt.Errorf("failed to list general job module cursor for infrastructure field in migrateInfrastructureField method, err: %v", err)
}

ms = []mongo.WriteModel{}
for cursor.Next(context.Background()) {
var workflow models.WorkflowV4
if err := cursor.Decode(&workflow); err != nil {
return err
}

changed := false
for _, stage := range workflow.Stages {
for _, job := range stage.Jobs {
if job.JobType == config.JobFreestyle {
spec := &models.FreestyleJobSpec{}
if err := models.IToi(job.Spec, spec); err != nil {
return err
}

if spec.Properties != nil && spec.Properties.Infrastructure == "" {
spec.Properties.Infrastructure = setting.JobK8sInfrastructure
job.Spec = spec
changed = true
}
}
}
}

if changed {
ms = append(ms,
mongo.NewUpdateOneModel().
SetFilter(bson.D{{"_id", workflow.ID}}).
SetUpdate(bson.D{{"$set",
bson.D{
{"stages", workflow.Stages},
}},
}),
)
}

if len(ms) >= 50 {
log.Infof("update %d workflowV4", len(ms))
if _, err := mongodb.NewWorkflowV4Coll().BulkWrite(context.Background(), ms); err != nil {
return fmt.Errorf("update workflowV4 for infrastructure field in migrateInfrastructureField method, error: %s", err)
}
ms = []mongo.WriteModel{}
}
}
if len(ms) > 0 {
log.Infof("update %d workflowV4", len(ms))
if _, err := mongodb.NewWorkflowV4Coll().BulkWrite(context.Background(), ms); err != nil {
return fmt.Errorf("update workflowV4 for infrastructure field in migrateInfrastructureField method, error: %s", err)
}
}

return nil
}
138 changes: 138 additions & 0 deletions pkg/cli/zadig-agent/command/cmd/agent/agent.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
/*
Copyright 2023 The KodeRover Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package agent

import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"time"

"github.com/koderover/zadig/pkg/cli/zadig-agent/config"
"github.com/koderover/zadig/pkg/cli/zadig-agent/helper/log"
"github.com/koderover/zadig/pkg/cli/zadig-agent/internal/agent"
"github.com/koderover/zadig/pkg/cli/zadig-agent/internal/common"
"github.com/koderover/zadig/pkg/cli/zadig-agent/pkg/monitor"
)

type Agent struct {
Ctx context.Context
Cancel context.CancelFunc
SignalsStopChan chan struct{}
}

func newAgent() *Agent {
ctx, cancel := context.WithCancel(context.Background())
return &Agent{
Ctx: ctx,
Cancel: cancel,
SignalsStopChan: make(chan struct{}, 1),
}
}

func (a *Agent) start(stop chan struct{}) {
log.Infof("================================ Zadig Agent ================================")

// Initialize the agent
InitAgent()

// Start the agent core service
agentCtl := agent.NewAgentController()
go agentCtl.Start(a.Ctx)

// Start the heartbeat service
go monitor.NewHeartbeatService(agentCtl, 3, 10, stop).Start(a.Ctx)
}

func (a *Agent) handleSignals() {
defer close(a.SignalsStopChan)

// Listen for system signals
sigCh := make(chan os.Signal, 1)
defer close(sigCh)

signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)

// Wait for signals in another goroutine
sig := <-sigCh
log.Infof("Received signal: %v", sig)

a.SignalsStopChan <- struct{}{}
}

func StartAgent() {
zadigAgent := newAgent()
stopAgentChan := make(chan struct{}, 1)

// Start the agent
go zadigAgent.start(stopAgentChan)

// Handle system signals
go zadigAgent.handleSignals()

// Wait for the agent to exit
for {
select {
case <-stopAgentChan:
msg := fmt.Sprintf("agent stopped by stop agent channel, time: %s", time.Now().Format("2006-01-02 15:04:05"))
zadigAgent.Stop(msg)
time.Sleep(5 * time.Second)
return
case <-zadigAgent.SignalsStopChan:
msg := fmt.Sprintf("agent stopped by signal, time: %s", time.Now().Format("2006-01-02 15:04:05"))
zadigAgent.Stop(msg)
return
default:
if checkStopSignalFile() {
msg := fmt.Sprintf("agent stopped by stop signal file, time: %s", time.Now().Format("2006-01-02 15:04:05"))
zadigAgent.Stop(msg)
return
}
time.Sleep(1 * time.Second)
}
}
}

func (a *Agent) Stop(msg string) {
// Stop the agent
config.SetAgentStatus(common.AGENT_STATUS_STOP)
config.SetAgentErrMsg(msg)

// Cancel the context
a.Cancel()
// Wait for the agent core components to exit
time.Sleep(5 * time.Second)
}

func checkStopSignalFile() bool {
stopFilePath, err := config.GetStopFilePath()
if err != nil {
log.Errorf("failed to get stop file path: %v", err)
return false
}
if _, err := os.Stat(stopFilePath); err == nil {
err = os.Remove(stopFilePath)
if err != nil {
log.Errorf("failed to remove stop file: %v", err)
return false
}
return true
}
return false
}
Loading

0 comments on commit ee2a0e4

Please sign in to comment.