Merge pull request koderover#203 from 27149chen/load_chart_from_sourc…

…e_when_cache_failed

load chart from source when cache failed

pkg/cli/upgradeassistant/cmd/migrate/140.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 
-	"github.com/koderover/zadig/pkg/cli/upgradeassistant/internal/upgradepath"
 	configbase "github.com/koderover/zadig/pkg/config"
 	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb"
 	s3service "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3"
@@ -16,8 +15,8 @@ import (
 )
 
 func init() {
-	upgradepath.AddHandler(upgradepath.V131, upgradepath.V140, V131ToV140)
-	upgradepath.AddHandler(upgradepath.V140, upgradepath.V131, V140ToV131)
+	//upgradepath.AddHandler(upgradepath.V131, upgradepath.V140, V131ToV140)
+	//upgradepath.AddHandler(upgradepath.V140, upgradepath.V131, V140ToV131)
 }
 
 func V131ToV140() error {

pkg/cli/upgradeassistant/internal/upgradepath/versions.go

@@ -10,13 +10,11 @@ const (
 	Latest = iota + 1
 	V130
 	V131
-	V140
 )
 
 var versionMap versions = map[string]int{
 	"1.3.0": V130,
 	"1.3.1": V131,
-	"1.4.0": V140,
 }
 
 type versions map[string]int

pkg/microservice/aslan/core/common/service/helm.go

@@ -18,20 +18,26 @@ package service
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 
+	"github.com/27149chen/afero"
 	"go.uber.org/zap"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/koderover/zadig/pkg/microservice/aslan/config"
 	commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models"
 	commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb"
+	githubservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github"
+	gitlabservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/gitlab"
 	s3service "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3"
 	"github.com/koderover/zadig/pkg/setting"
+	"github.com/koderover/zadig/pkg/shared/codehost"
 	e "github.com/koderover/zadig/pkg/tool/errors"
 	"github.com/koderover/zadig/pkg/tool/log"
 	s3tool "github.com/koderover/zadig/pkg/tool/s3"
-	"github.com/koderover/zadig/pkg/util/fs"
+	fsutil "github.com/koderover/zadig/pkg/util/fs"
 )
 
 func ListHelmRepos(log *zap.SugaredLogger) ([]*commonmodels.HelmRepo, error) {
@@ -44,8 +50,8 @@ func ListHelmRepos(log *zap.SugaredLogger) ([]*commonmodels.HelmRepo, error) {
 	return helmRepos, nil
 }
 
-func PreLoadServiceManifests(base, productName, serviceName string) error {
-	ok, err := fs.DirExists(base)
+func PreLoadServiceManifests(base string, svc *commonmodels.Service) error {
+	ok, err := fsutil.DirExists(base)
 	if err != nil {
 		log.Errorf("Failed to check if dir %s is exiting, err: %s", base, err)
 		return err
@@ -54,12 +60,13 @@ func PreLoadServiceManifests(base, productName, serviceName string) error {
 		return nil
 	}
 
-	if err = DownloadServiceManifests(base, productName, serviceName); err != nil {
-		log.Errorf("Failed to download service from s3, err: %s", err)
-		return err
+	if err = DownloadServiceManifests(base, svc.ProductName, svc.ServiceName); err == nil {
+		return nil
 	}
 
-	return nil
+	log.Warnf("Failed to download service from s3, err: %s", err)
+
+	return preLoadServiceManifestsFromSource(svc)
 }
 
 func DownloadServiceManifests(base, projectName, serviceName string) error {
@@ -86,7 +93,7 @@ func DownloadServiceManifests(base, projectName, serviceName string) error {
 		log.Errorf("Failed to download file from s3, err: %s", err)
 		return err
 	}
-	if err = fs.Untar(tarFilePath, base); err != nil {
+	if err = fsutil.Untar(tarFilePath, base); err != nil {
 		log.Errorf("Untar err: %s", err)
 		return err
 	}
@@ -96,3 +103,159 @@ func DownloadServiceManifests(base, projectName, serviceName string) error {
 
 	return nil
 }
+
+type DownloadFromSourceParams struct {
+	CodehostID                int
+	Owner, Repo, Path, Branch string
+}
+
+func DownloadServiceManifestsFromSource(svc *DownloadFromSourceParams, serviceNameGetter func(afero.Fs) (string, error)) (fs.FS, error) {
+	getter, err := getTreeGetter(svc.CodehostID)
+	if err != nil {
+		log.Errorf("Failed to get tree getter, err: %s", err)
+		return nil, err
+	}
+
+	chartTree, err := getter.GetTreeContents(svc.Owner, svc.Repo, svc.Path, svc.Branch)
+	if err != nil {
+		log.Errorf("Failed to get tree contents for service %+v, err: %s", svc, err)
+		return nil, err
+	}
+
+	serviceName, err := serviceNameGetter(chartTree)
+	if err != nil {
+		log.Errorf("Failed to get service name, err: %s", err)
+		return nil, err
+	}
+
+	// rename the root path of the chart to the service name
+	f, _ := fs.ReadDir(afero.NewIOFS(chartTree), "")
+	if len(f) == 1 {
+		if err = chartTree.Rename(f[0].Name(), serviceName); err != nil {
+			log.Errorf("Failed to rename dir name from %s to %s, err: %s", f[0].Name(), serviceName, err)
+			return nil, err
+		}
+	}
+
+	return afero.NewIOFS(chartTree), nil
+}
+
+type treeGetter interface {
+	GetTreeContents(owner, repo, path, branch string) (afero.Fs, error)
+}
+
+func getTreeGetter(codeHostID int) (treeGetter, error) {
+	ch, err := codehost.GetCodeHostInfoByID(codeHostID)
+	if err != nil {
+		log.Errorf("Failed to get codeHost by id %d, err: %s", codeHostID, err)
+		return nil, err
+	}
+
+	switch ch.Type {
+	case setting.SourceFromGithub:
+		return githubservice.NewClient(ch.AccessToken, config.ProxyHTTPSAddr()), nil
+	case setting.SourceFromGitlab:
+		return gitlabservice.NewClient(ch.Address, ch.AccessToken)
+	default:
+		// should not have happened here
+		log.DPanicf("invalid source: %s", ch.Type)
+		return nil, fmt.Errorf("invalid source: %s", ch.Type)
+	}
+}
+
+func SaveAndUploadService(projectName, serviceName string, fileTree fs.FS) error {
+	var wg wait.Group
+	var err error
+
+	wg.Start(func() {
+		err1 := saveInMemoryFilesToDisk(projectName, serviceName, fileTree)
+		if err1 != nil {
+			log.Errorf("Failed to save files to disk, err: %s", err1)
+			err = err1
+		}
+	})
+	wg.Start(func() {
+		err2 := UploadFilesToS3(projectName, serviceName, fileTree)
+		if err2 != nil {
+			log.Errorf("Failed to upload files to s3, err: %s", err2)
+			err = err2
+		}
+	})
+
+	wg.Wait()
+
+	return err
+}
+
+func preLoadServiceManifestsFromSource(svc *commonmodels.Service) error {
+	tree, err := DownloadServiceManifestsFromSource(
+		&DownloadFromSourceParams{CodehostID: svc.CodehostID, Owner: svc.RepoOwner, Repo: svc.RepoName, Path: svc.LoadPath, Branch: svc.BranchName},
+		func(afero.Fs) (string, error) {
+			return svc.ServiceName, nil
+		})
+
+	// save files to disk and upload them to s3
+	if err = SaveAndUploadService(svc.ProductName, svc.ServiceName, tree); err != nil {
+		log.Errorf("Failed to save or upload files for service %s in project %s, error: %s", svc.ServiceName, svc.ProductName, err)
+		return err
+	}
+
+	return nil
+}
+
+func saveInMemoryFilesToDisk(projectName, serviceName string, fileTree fs.FS) error {
+	root := config.LocalServicePath(projectName, serviceName)
+
+	tmpRoot := root + ".bak"
+	if err := os.Rename(root, tmpRoot); err != nil {
+		return err
+	}
+
+	if err := fsutil.SaveToDisk(fileTree, root); err != nil {
+		if err = os.RemoveAll(root); err != nil {
+			log.Warnf("Failed to delete path %s, err: %s", root, err)
+		}
+		return os.Rename(tmpRoot, root)
+	}
+
+	if err := os.RemoveAll(tmpRoot); err != nil {
+		log.Warnf("Failed to delete path %s, err: %s", tmpRoot, err)
+	}
+
+	return nil
+}
+
+func UploadFilesToS3(projectName, serviceName string, fileTree fs.FS) error {
+	fileName := fmt.Sprintf("%s.tar.gz", serviceName)
+	tmpDir := os.TempDir()
+	tarball := filepath.Join(tmpDir, fileName)
+	if err := fsutil.Tar(fileTree, tarball); err != nil {
+		log.Errorf("Failed to archive tarball %s, err: %s", tarball, err)
+		return err
+	}
+	s3Storage, err := s3service.FindDefaultS3()
+	if err != nil {
+		log.Errorf("Failed to find default s3, err:%v", err)
+		return err
+	}
+	forcedPathStyle := true
+	if s3Storage.Provider == setting.ProviderSourceAli {
+		forcedPathStyle = false
+	}
+	client, err := s3tool.NewClient(s3Storage.Endpoint, s3Storage.Ak, s3Storage.Sk, s3Storage.Insecure, forcedPathStyle)
+	if err != nil {
+		log.Errorf("Failed to get s3 client, err: %s", err)
+		return err
+	}
+	s3Storage.Subfolder = filepath.Join(s3Storage.Subfolder, config.ObjectStorageServicePath(projectName, serviceName))
+	objectKey := s3Storage.GetObjectPath(fileName)
+	if err = client.Upload(s3Storage.Bucket, tarball, objectKey); err != nil {
+		log.Errorf("Failed to upload file %s to s3, err: %s", tarball, err)
+		return err
+	}
+	if err = os.Remove(tarball); err != nil {
+		log.Errorf("Failed to remove file %s, err: %s", tarball, err)
+	}
+
+	return nil
+}

pkg/microservice/aslan/core/environment/service/environment.go

@@ -2153,7 +2153,7 @@ func installOrUpdateHelmChart(user, envName, requestID string, args *commonmodel
 					defer wg.Done()
 
 					base := config.LocalServicePath(currentService.ProductName, currentService.ServiceName)
-					if err = commonservice.PreLoadServiceManifests(base, currentService.ProductName, currentService.ServiceName); err != nil {
+					if err = commonservice.PreLoadServiceManifests(base, currentService); err != nil {
 						log.Errorf("Failed to load service menifests for service %s in project %s, err: %s", currentService.ServiceName, currentService.ProductName, err)
 						return
 					}
@@ -2320,7 +2320,7 @@ func updateProductGroup(productName, envName, updateType string, productResp *co
 						Timeout:     Timeout * time.Second * 10,
 					}
 					base := config.LocalServicePath(currentService.ProductName, currentService.ServiceName)
-					if err = commonservice.PreLoadServiceManifests(base, currentService.ProductName, currentService.ServiceName); err != nil {
+					if err = commonservice.PreLoadServiceManifests(base, currentService); err != nil {
 						return
 					}
 					err = helmClient.InstallOrUpgradeChart(context.Background(), &chartSpec,
@@ -2528,7 +2528,7 @@ func updateProductVariable(productName, envName string, productResp *commonmodel
 					defer wg.Done()
 
 					base := config.LocalServicePath(currentService.ProductName, currentService.ServiceName)
-					if err = commonservice.PreLoadServiceManifests(base, currentService.ProductName, currentService.ServiceName); err != nil {
+					if err = commonservice.PreLoadServiceManifests(base, currentService); err != nil {
 						return
 					}
 					chartSpec := helmclient.ChartSpec{