Skip to content

Commit

Permalink
Change TempDir to use a hash of Task information for the directory name.
Browse files Browse the repository at this point in the history
  • Loading branch information
dwmunster authored and samuell committed Jun 14, 2019
1 parent d700a84 commit 56e188f
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 9 deletions.
20 changes: 12 additions & 8 deletions task.go
Original file line number Diff line number Diff line change
Expand Up @@ -374,21 +374,25 @@ var tempDirPrefix = "_scipipe_tmp"
// in file paths. It is built up by merging all input filenames and parameter
// values that a task takes as input, joined with dots.
func (t *Task) TempDir() string {
pathPcs := []string{tempDirPrefix + "." + sanitizePathFragment(t.Name)}
pathPrefix := tempDirPrefix + "." + sanitizePathFragment(t.Name)
hashPcs := []string{}
for _, ipName := range sortedFileIPMapKeys(t.InIPs) {
pathPcs = append(pathPcs, filepath.Base(t.InIP(ipName).Path()))
hashPcs = append(hashPcs, splitAllPaths(t.InIP(ipName).Path())...)
}
for _, paramName := range sortedStringMapKeys(t.Params) {
pathPcs = append(pathPcs, paramName+"_"+t.Param(paramName))
hashPcs = append(hashPcs, paramName+"_"+t.Param(paramName))
}
for _, tagName := range sortedStringMapKeys(t.Tags) {
pathPcs = append(pathPcs, tagName+"_"+t.Tag(tagName))
hashPcs = append(hashPcs, tagName+"_"+t.Tag(tagName))
}
pathSegment := strings.Join(pathPcs, ".")
if len(pathSegment) > 255 {
sha1sum := sha1.Sum([]byte(pathSegment))
pathSegment = t.Name + "." + hex.EncodeToString(sha1sum[:])

// If resulting name is longer than 255
if len(pathPrefix) > (255 - 40 - 1) {
hashPcs = append(hashPcs, pathPrefix)
pathPrefix = tempDirPrefix
}
sha1sum := sha1.Sum([]byte(strings.Join(hashPcs, "")))
pathSegment := pathPrefix + "." + hex.EncodeToString(sha1sum[:])
return pathSegment
}

Expand Down
2 changes: 1 addition & 1 deletion task_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ func TestTempDirsExist(t *testing.T) {
func TestTempDir(t *testing.T) {
tsk := NewTask(nil, nil, "test_task", "echo foo", map[string]*FileIP{"in1": NewFileIP("infile.txt"), "in2": NewFileIP("infile2.txt")}, nil, nil, map[string]string{"p1": "p1val", "p2": "p2val"}, nil, "", nil, 4)

expected := tempDirPrefix + ".test_task.infile.txt.infile2.txt.p1_p1val.p2_p2val"
expected := tempDirPrefix + ".test_task.aaa94846ee057056e7f2d4d3aa2236bdf353d5a1"
actual := tsk.TempDir()
if actual != expected {
t.Errorf("TempDir() was %s Expected: %s", actual, expected)
Expand Down
13 changes: 13 additions & 0 deletions utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"math/rand"
"os"
"os/exec"
"path/filepath"
re "regexp"
"time"

Expand Down Expand Up @@ -80,3 +81,15 @@ func errWrap(err error, msg string) error {
func errWrapf(err error, msg string, v ...interface{}) error {
return errors.New(fmt.Sprintf(msg, v...) + "\nOriginal error: " + err.Error())
}

// splitAllPaths takes in a filepath and returns a list of all its parts
// e.g. "/a/b/c" becomes ["a" "b" "c'"]
func splitAllPaths(path string) []string {
dir, file := filepath.Dir(path), filepath.Base(path)
parts := []string{}
for dir != file {
parts = append([]string{file}, parts...)
dir, file = filepath.Dir(dir), filepath.Base(dir)
}
return parts
}

0 comments on commit 56e188f

Please sign in to comment.