forked from dragonflyoss/dragonfly
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: define and implement new dfdaemon APIs to make dragonfly2 work …
…as a distributed cache (dragonflyoss#1227) Introduce a new dfcache command which uses dfcache SDK interface to interactive with dfdaemon and operate on P2P cache system. For example: - add a file into cache system dfcache import --cid sha256:xxxxxx --tag testtag /path/to/file - check if a file exists in cache system dfcache stat --cid testid --local # only check local cache dfcache stat --cid testid # check other peers as well - export/download a file from cache system dfcache export --cid testid -O /path/to/output - delete a file from cache system, both local cache and P2P network dfcache delete -i testid -t testtag Signed-off-by: Gaius <[email protected]> Signed-off-by: Eryu Guan <[email protected]> Co-authored-by: Jim Ma <[email protected]> Co-authored-by: Gaius <[email protected]>
- Loading branch information
3 people
authored
Apr 14, 2022
1 parent
0d10ee4
commit a666680
Showing
53 changed files
with
4,305 additions
and
237 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,236 @@ | ||
| /* | ||
| * Copyright 2022 The Dragonfly Authors | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package config | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "syscall" | ||
| "time" | ||
|
|
||
| "github.com/pkg/errors" | ||
| "golang.org/x/time/rate" | ||
|
|
||
| "d7y.io/dragonfly/v2/cmd/dependency/base" | ||
| "d7y.io/dragonfly/v2/internal/dferrors" | ||
| "d7y.io/dragonfly/v2/pkg/basic" | ||
| "d7y.io/dragonfly/v2/pkg/util/stringutils" | ||
| ) | ||
|
|
||
| type DfcacheConfig = CacheOption | ||
|
|
||
| // CacheOption holds all the runtime config information. | ||
| type CacheOption struct { | ||
| base.Options `yaml:",inline" mapstructure:",squash"` | ||
|
|
||
| // Cid content/cache ID | ||
| Cid string `yaml:"cid,omitempty" mapstructure:"cid,omitempty"` | ||
|
|
||
| // Tag identify task | ||
| Tag string `yaml:"tag,omitempty" mapstructure:"tag,omitempty"` | ||
|
|
||
| // Timeout operation timeout(second). | ||
| Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"` | ||
|
|
||
| // CallSystem optional system name | ||
| CallSystem string `yaml:"callSystem,omitempty" mapstructure:"callSystem,omitempty"` | ||
|
|
||
| // LogDir is log directory of dfcache. | ||
| LogDir string `yaml:"logDir,omitempty" mapstructure:"logDir,omitempty"` | ||
|
|
||
| // WorkHome is working directory of dfcache. | ||
| WorkHome string `yaml:"workHome,omitempty" mapstructure:"workHome,omitempty"` | ||
|
|
||
| // Output full output path for export task | ||
| Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"` | ||
|
|
||
| // Path full input path for import task | ||
| // TODO: change to Input | ||
| Path string `yaml:"path,omitempty" mapstructure:"path,omitempty"` | ||
|
|
||
| // RateLimit limits export task | ||
| RateLimit rate.Limit `yaml:"rateLimit,omitempty" mapstructure:"rateLimit,omitempty"` | ||
|
|
||
| // LocalOnly indicates check local cache only | ||
| LocalOnly bool `yaml:"localOnly,omitempty" mapstructure:"localOnly,omitempty"` | ||
| } | ||
|
|
||
| func NewDfcacheConfig() *CacheOption { | ||
| return &CacheOption{} | ||
| } | ||
|
|
||
| func validateCacheStat(cfg *CacheOption) error { | ||
| return nil | ||
| } | ||
|
|
||
| func validateCacheImport(cfg *CacheOption) error { | ||
| if err := cfg.checkInput(); err != nil { | ||
| return errors.Wrapf(dferrors.ErrInvalidArgument, "input path: %v", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func ValidateCacheExport(cfg *CacheOption) error { | ||
| if err := cfg.checkOutput(); err != nil { | ||
| return errors.Wrapf(dferrors.ErrInvalidArgument, "output: %v", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func ValidateCacheDelete(cfg *CacheOption) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (cfg *CacheOption) Validate(cmd string) error { | ||
| // Some common validations | ||
| if cfg == nil { | ||
| return errors.Wrap(dferrors.ErrInvalidArgument, "runtime config") | ||
| } | ||
| if cfg.Cid == "" { | ||
| return errors.Wrap(dferrors.ErrInvalidArgument, "missing Cid") | ||
| } | ||
| if stringutils.IsBlank(cfg.Cid) { | ||
| return errors.Wrap(dferrors.ErrInvalidArgument, "Cid are all blanks") | ||
| } | ||
|
|
||
| // cmd specific validations | ||
| switch cmd { | ||
| case CmdStat: | ||
| return validateCacheStat(cfg) | ||
| case CmdImport: | ||
| return validateCacheImport(cfg) | ||
| case CmdExport: | ||
| return ValidateCacheExport(cfg) | ||
| case CmdDelete: | ||
| return ValidateCacheDelete(cfg) | ||
| default: | ||
| return errors.Wrapf(dferrors.ErrInvalidArgument, "unknown cache subcommand: %s", cmd) | ||
| } | ||
| } | ||
|
|
||
| func ConvertCacheStat(cfg *CacheOption, args []string) error { | ||
| return nil | ||
| } | ||
|
|
||
| func convertCacheImport(cfg *CacheOption, args []string) error { | ||
| var err error | ||
| if cfg.Path == "" && len(args) > 0 { | ||
| cfg.Path = args[0] | ||
| } | ||
| if cfg.Path == "" { | ||
| return errors.Wrap(dferrors.ErrInvalidArgument, "missing input file") | ||
| } | ||
|
|
||
| if cfg.Path, err = filepath.Abs(cfg.Path); err != nil { | ||
| return errors.Wrapf(err, "get absulate path for %s", cfg.Path) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func ConvertCacheExport(cfg *CacheOption, args []string) error { | ||
| var err error | ||
| if cfg.Output == "" && len(args) > 0 { | ||
| cfg.Output = args[0] | ||
| } | ||
| if cfg.Output == "" { | ||
| return errors.Wrap(dferrors.ErrInvalidArgument, "missing output file") | ||
| } | ||
|
|
||
| if cfg.Output, err = filepath.Abs(cfg.Output); err != nil { | ||
| return errors.Wrapf(err, "get absulate path for %s", cfg.Output) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func ConvertCacheDelete(cfg *CacheOption, args []string) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (cfg *CacheOption) Convert(cmd string, args []string) error { | ||
| if cfg == nil { | ||
| return errors.Wrap(dferrors.ErrInvalidArgument, "runtime config") | ||
| } | ||
|
|
||
| switch cmd { | ||
| case CmdStat: | ||
| return ConvertCacheStat(cfg, args) | ||
| case CmdImport: | ||
| return convertCacheImport(cfg, args) | ||
| case CmdExport: | ||
| return ConvertCacheExport(cfg, args) | ||
| case CmdDelete: | ||
| return ConvertCacheDelete(cfg, args) | ||
| default: | ||
| return errors.Wrapf(dferrors.ErrInvalidArgument, "unknown cache subcommand: %s", cmd) | ||
| } | ||
| } | ||
|
|
||
| func (cfg *CacheOption) String() string { | ||
| js, _ := json.Marshal(cfg) | ||
| return string(js) | ||
| } | ||
|
|
||
| func (cfg *CacheOption) checkInput() error { | ||
| stat, err := os.Stat(cfg.Path) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "stat input path %q", cfg.Path) | ||
| } | ||
| if stat.IsDir() { | ||
| return fmt.Errorf("path[%q] is directory but requires file path", cfg.Path) | ||
| } | ||
| if err := syscall.Access(cfg.Path, syscall.O_RDONLY); err != nil { | ||
| return errors.Wrapf(err, "access %q", cfg.Path) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (cfg *CacheOption) checkOutput() error { | ||
| if cfg.Output == "" { | ||
| return errors.New("no output file path specified") | ||
| } | ||
|
|
||
| if !filepath.IsAbs(cfg.Output) { | ||
| absPath, err := filepath.Abs(cfg.Output) | ||
| if err != nil { | ||
| return fmt.Errorf("get absolute path[%s] error: %v", cfg.Output, err) | ||
| } | ||
| cfg.Output = absPath | ||
| } | ||
|
|
||
| outputDir, _ := path.Split(cfg.Output) | ||
| if err := MkdirAll(outputDir, 0777, basic.UserID, basic.UserGroup); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| f, err := os.Stat(cfg.Output) | ||
| if err == nil && f.IsDir() { | ||
| return fmt.Errorf("path[%s] is directory but requires file path", cfg.Output) | ||
| } | ||
|
|
||
| // check permission | ||
| for dir := cfg.Output; !stringutils.IsBlank(dir); dir = filepath.Dir(dir) { | ||
| if err := syscall.Access(dir, syscall.O_RDWR); err == nil { | ||
| break | ||
| } else if os.IsPermission(err) || dir == "/" { | ||
| return fmt.Errorf("user[%s] path[%s] %v", basic.Username, cfg.Output, err) | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.