feat: add objectstorage pkg (dragonflyoss#1366)

* feat: add objectstorage pkg

Signed-off-by: Gaius <[email protected]>

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/agiledragon/gomonkey/v2 v2.3.1
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.4+incompatible
 	github.com/appleboy/gin-jwt/v2 v2.8.0
+	github.com/aws/aws-sdk-go v1.44.24
 	github.com/bits-and-blooms/bitset v1.2.2
 	github.com/casbin/casbin/v2 v2.47.1
 	github.com/casbin/gorm-adapter/v3 v3.3.2
@@ -94,7 +95,6 @@ require (
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae // indirect
-	github.com/aws/aws-sdk-go v1.44.24 // indirect
 	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect

manager/config/config.go

@@ -24,6 +24,7 @@ import (
 	"github.com/docker/go-connections/tlsconfig"
 
 	"d7y.io/dragonfly/v2/cmd/dependency/base"
+	"d7y.io/dragonfly/v2/pkg/objectstorage"
 )
 
 type Config struct {
@@ -39,8 +40,11 @@ type Config struct {
 	// Cache configuration
 	Cache *CacheConfig `yaml:"cache" mapstructure:"cache"`
 
+	// ObjectStorage configuration
+	ObjectStorage *ObjectStorageConfig `yaml:"objectStorage" mapstructure:"objectStorage"`
+
 	// Metrics configuration
-	Metrics *RestConfig `yaml:"metrics" mapstructure:"metrics"`
+	Metrics *MetricsConfig `yaml:"metrics" mapstructure:"metrics"`
 }
 
 type ServerConfig struct {
@@ -162,6 +166,14 @@ type RestConfig struct {
 	Addr string `yaml:"addr" mapstructure:"addr"`
 }
 
+type MetricsConfig struct {
+	// Enable metrics service.
+	Enable bool `yaml:"enable" mapstructure:"enable"`
+
+	// Metrics service address.
+	Addr string `yaml:"addr" mapstructure:"addr"`
+}
+
 type TCPListenConfig struct {
 	// Listen stands listen interface, like: 0.0.0.0, 192.168.0.1
 	Listen string `mapstructure:"listen" yaml:"listen"`
@@ -175,6 +187,26 @@ type TCPListenPortRange struct {
 	End   int
 }
 
+type ObjectStorageConfig struct {
+	// Enable object storage.
+	Enable bool `yaml:"enable" mapstructure:"enable"`
+
+	// Object storage name of type, it can be s3 or oss.
+	Name string `mapstructure:"name" yaml:"name"`
+
+	// Storage region.
+	Region string `mapstructure:"region" yaml:"region"`
+
+	// Datacenter endpoint.
+	Endpoint string `mapstructure:"endpoint" yaml:"endpoint"`
+
+	// Access key ID.
+	AccessKey string `mapstructure:"accessKey" yaml:"accessKey"`
+
+	// Access key secret
+	SecretKey string `mapstructure:"secretKey" yaml:"secretKey"`
+}
+
 // New config instance
 func New() *Config {
 	return &Config{
@@ -210,119 +242,145 @@ func New() *Config {
 				TTL:  30 * time.Second,
 			},
 		},
+		ObjectStorage: &ObjectStorageConfig{
+			Enable: false,
+		},
+		Metrics: &MetricsConfig{
+			Enable: false,
+		},
 	}
 }
 
 // Validate config values
 func (cfg *Config) Validate() error {
 	if cfg.Server == nil {
-		return errors.New("empty server config is not specified")
+		return errors.New("config requires parameter server")
 	}
 
 	if cfg.Server.Name == "" {
-		return errors.New("empty server name config is not specified")
+		return errors.New("server requires parameter name")
 	}
 
 	if cfg.Server.GRPC == nil {
-		return errors.New("empty grpc server config is not specified")
+		return errors.New("server requires parameter grpc")
 	}
 
 	if cfg.Server.REST == nil {
-		return errors.New("empty rest server config is not specified")
+		return errors.New("server requires parameter rest")
 	}
 
 	if cfg.Database == nil {
-		return errors.New("empty database config is not specified")
+		return errors.New("config requires parameter database")
 	}
 
 	if cfg.Database.Redis == nil {
-		return errors.New("empty database redis config is not specified")
+		return errors.New("database requires parameter redis")
 	}
 
 	if cfg.Database.Redis.Host == "" {
-		return errors.New("empty database redis host is not specified")
+		return errors.New("redis requires parameter host")
 	}
 
 	if cfg.Database.Redis.Port <= 0 {
-		return errors.New("empty database redis port is not specified")
+		return errors.New("redis requires parameter port")
 	}
 
 	if cfg.Database.Redis.CacheDB < 0 {
-		return errors.New("empty database redis cacheDB is not specified")
+		return errors.New("redis requires parameter cacheDB")
 	}
 
 	if cfg.Database.Redis.BrokerDB < 0 {
-		return errors.New("empty database redis brokerDB is not specified")
+		return errors.New("redis requires parameter brokerDB")
 	}
 
 	if cfg.Database.Redis.BackendDB < 0 {
-		return errors.New("empty database redis backendDB is not specified")
+		return errors.New("redis requires parameter backendDB")
 	}
 
 	if cfg.Database.Mysql == nil {
-		return errors.New("empty database mysql config is not specified")
+		return errors.New("database requires parameter mysql")
 	}
 
 	if cfg.Database.Mysql.User == "" {
-		return errors.New("empty database mysql user is not specified")
+		return errors.New("mysql requires parameter user")
 	}
 
 	if cfg.Database.Mysql.Password == "" {
-		return errors.New("empty database mysql password is not specified")
+		return errors.New("mysql requires parameter password")
 	}
 
 	if cfg.Database.Mysql.Host == "" {
-		return errors.New("empty database mysql host is not specified")
+		return errors.New("mysql requires parameter host")
 	}
 
 	if cfg.Database.Mysql.Port <= 0 {
-		return errors.New("empty database mysql port is not specified")
+		return errors.New("mysql requires parameter port")
 	}
 
 	if cfg.Database.Mysql.DBName == "" {
-		return errors.New("empty database mysql dbName is not specified")
+		return errors.New("mysql requires parameter dbname")
 	}
 
 	if cfg.Database.Mysql.TLS != nil {
 		if cfg.Database.Mysql.TLS.Cert == "" {
-			return errors.New("empty database mysql tls cert is not specified")
+			return errors.New("tls requires parameter cert")
 		}
 
 		if cfg.Database.Mysql.TLS.Key == "" {
-			return errors.New("empty database mysql tls key is not specified")
+			return errors.New("tls requires parameter key")
 		}
 
 		if cfg.Database.Mysql.TLS.CA == "" {
-			return errors.New("empty database mysql tls ca is not specified")
+			return errors.New("tls requires parameter ca")
 		}
 	}
 
 	if cfg.Cache == nil {
-		return errors.New("empty cache config is not specified")
+		return errors.New("config requires parameter cache")
 	}
 
 	if cfg.Cache.Redis == nil {
-		return errors.New("empty cache redis config is not specified")
+		return errors.New("cache requires parameter redis")
 	}
 
 	if cfg.Cache.Redis.TTL == 0 {
-		return errors.New("empty cache redis TTL is not specified")
+		return errors.New("redis requires parameter ttl")
 	}
 
 	if cfg.Cache.Local == nil {
-		return errors.New("empty cache local config is not specified")
+		return errors.New("cache requires parameter local")
 	}
 
 	if cfg.Cache.Local.Size == 0 {
-		return errors.New("empty cache local size is not specified")
+		return errors.New("local requires parameter size")
 	}
 
 	if cfg.Cache.Local.TTL == 0 {
-		return errors.New("empty cache local TTL is not specified")
+		return errors.New("local requires parameter ttl")
+	}
+
+	if cfg.ObjectStorage != nil && cfg.ObjectStorage.Enable {
+		if cfg.ObjectStorage.Name == "" {
+			return errors.New("objectStorage requires parameter name")
+		}
+
+		if cfg.ObjectStorage.Name != objectstorage.ServiceNameS3 && cfg.ObjectStorage.Name != objectstorage.ServiceNameOSS {
+			return errors.New("objectStorage requires parameter name")
+		}
+
+		if cfg.ObjectStorage.AccessKey == "" {
+			return errors.New("objectStorage requires parameter accessKey")
+		}
+
+		if cfg.ObjectStorage.SecretKey == "" {
+			return errors.New("objectStorage requires parameter secretKey")
+		}
 	}
 
-	if cfg.Metrics != nil && cfg.Metrics.Addr == "" {
-		return errors.New("empty metrics addr is not specified")
+	if cfg.Metrics != nil && cfg.Metrics.Enable {
+		if cfg.Metrics.Addr == "" {
+			return errors.New("metrics requires parameter addr")
+		}
 	}
 
 	return nil

manager/config/config_test.go

@@ -23,6 +23,8 @@ import (
 	"github.com/mitchellh/mapstructure"
 	testifyassert "github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
+
+	"d7y.io/dragonfly/v2/pkg/objectstorage"
 )
 
 func TestManagerConfig_Load(t *testing.T) {
@@ -76,8 +78,16 @@ func TestManagerConfig_Load(t *testing.T) {
 				TTL:  1000,
 			},
 		},
-		Metrics: &RestConfig{
-			Addr: ":8000",
+		ObjectStorage: &ObjectStorageConfig{
+			Enable:    true,
+			Name:      objectstorage.ServiceNameS3,
+			Endpoint:  "127.0.0.1",
+			AccessKey: "foo",
+			SecretKey: "bar",
+		},
+		Metrics: &MetricsConfig{
+			Enable: true,
+			Addr:   ":8000",
 		},
 	}
 

manager/config/testdata/manager.yaml

@@ -37,5 +37,13 @@ cache:
     size: 10000
     ttl: 1000
 
+objectStorage:
+  enable: true
+  name: s3
+  endpoint: 127.0.0.1
+  accessKey: foo
+  secretKey: bar
+
 metrics:
+  enable: true
   addr: :8000

manager/manager.go

@@ -118,7 +118,7 @@ func New(cfg *config.Config, d dfpath.Dfpath) (*Server, error) {
 	s.grpcServer = grpcServer
 
 	// Initialize prometheus
-	if cfg.Metrics != nil {
+	if cfg.Metrics.Enable {
 		s.metricsServer = metrics.New(cfg.Metrics, grpcServer)
 	}
 

manager/metrics/metrics.go

@@ -29,7 +29,7 @@ import (
 	"d7y.io/dragonfly/v2/manager/config"
 )
 
-func New(cfg *config.RestConfig, grpcServer *grpc.Server) *http.Server {
+func New(cfg *config.MetricsConfig, grpcServer *grpc.Server) *http.Server {
 	grpc_prometheus.Register(grpcServer)
 
 	mux := http.NewServeMux()

pkg/objectstorage/constants.go

@@ -0,0 +1,25 @@
+/*
+ *     Copyright 2022 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package objectstorage
+
+const (
+	// ServiceNameS3 is name of s3 storage
+	ServiceNameS3 = "s3"
+
+	// ServiceNameOSS is name of oss storage
+	ServiceNameOSS = "oss"
+)