Refractor of accessors and path manipulations (Velocidex#1545)

Filesystems are inherently directed graphs (usually trees). An abstract "path" is a list of components specifying a path from the root to traverse. When we call an OS API we need to provide a string to the OS we term an "OS Path".

One way to think about paths is that the OS path is a serialisation of an abstract "OSPath". Different operating systems have a different way of actually serialising the same abstract OSPath.

For example, a the following components based abstract path ["C:", "Windows", "System32"] will be serialised by windows into C:\\Windows\\System32 and by Linux conventions as /C:/Windows/System32

This refactor converts all path handling internally into an abstract representation which get serialised by the various accessors. This allows us to deal with paths in an OS agnostic way.

This change also refactors the accessors to separate them in the code base.

Also introduced the concept of a device manager which handles accessor registration in a specified way - this allows accessors to be remapped for dead disk forensic applications.

accessors/api.go

@@ -0,0 +1,192 @@
+package accessors
+
+import (
+	"io"
+	"os"
+	"time"
+
+	"github.com/Velocidex/ordereddict"
+	"www.velocidex.com/golang/velociraptor/utils"
+	"www.velocidex.com/golang/vfilter"
+)
+
+// An OS Path can be thought of as a sequence of components. The OS
+// APIs receive a single string, which is the serialization of the OS
+// Path. Different operating systems have different serialization
+// methods to arrive at the same OS Path.
+
+// For example, on Windows components are separated by the backslash
+// characted and the first component can be a device name (which may
+// contain path separators):
+
+// \\.\C:\Windows\System32 -> ["\\.\C:", "Windows", "System32"]
+// C:\Windows\System32 -> ["C:", "Windows", "System32"]
+
+// On Linux, the path separator is / and serializations start with "/"
+
+// /usr/bin/ls -> ["usr", "bin", "ls"]
+
+// In Velociraptor we try to keep the OS Path object intact as far as
+// possible and only serialize to OS representation when necessary.
+
+type PathManipulator interface {
+	PathParse(path string, result *OSPath)
+	PathJoin(path *OSPath) string
+	AsPathSpec(path *OSPath) *PathSpec
+}
+
+type OSPath struct {
+	Components []string
+
+	// Some paths need more information. They store an additional path
+	// spec here.
+	pathspec    *PathSpec
+	Manipulator PathManipulator
+}
+
+// Make a copy of the OSPath
+func (self OSPath) Copy() *OSPath {
+	pathspec := self.pathspec
+	if pathspec != nil {
+		pathspec = pathspec.Copy()
+	}
+	return &OSPath{
+		Components:  utils.CopySlice(self.Components),
+		pathspec:    pathspec,
+		Manipulator: self.Manipulator,
+	}
+}
+
+func (self *OSPath) SetPathSpec(pathspec *PathSpec) {
+	self.Manipulator.PathParse(pathspec.Path, self)
+	self.pathspec = pathspec
+}
+
+func (self *OSPath) PathSpec() *PathSpec {
+	return self.Manipulator.AsPathSpec(self)
+}
+
+func (self *OSPath) DelegatePath() string {
+	return self.Manipulator.AsPathSpec(self).DelegatePath
+}
+
+func (self *OSPath) DelegateAccessor() string {
+	return self.Manipulator.AsPathSpec(self).DelegateAccessor
+}
+
+func (self *OSPath) Path() string {
+	return self.Manipulator.AsPathSpec(self).Path
+}
+
+func (self *OSPath) String() string {
+	return self.Manipulator.PathJoin(self)
+}
+
+func (self *OSPath) Parse(path string) *OSPath {
+	result := &OSPath{
+		Manipulator: self.Manipulator,
+	}
+
+	self.Manipulator.PathParse(path, result)
+
+	return result
+}
+
+func (self *OSPath) Basename() string {
+	return self.Components[len(self.Components)-1]
+}
+
+func (self *OSPath) Dirname() *OSPath {
+	result := self.Copy()
+	result.Components = result.Components[:len(self.Components)-1]
+
+	return result
+}
+
+func (self *OSPath) TrimComponents(components ...string) *OSPath {
+	if components == nil {
+		return self.Copy()
+	}
+
+	result := self.Copy()
+	for idx, c := range result.Components {
+		if idx >= len(components) || c != components[idx] {
+			result := &OSPath{
+				Components:  self.Components[idx:],
+				pathspec:    self.pathspec,
+				Manipulator: self.Manipulator,
+			}
+			return result
+		}
+	}
+	return self
+}
+
+// Make a copy
+func (self *OSPath) Append(children ...string) *OSPath {
+	result := self.Copy()
+	result.Components = append(result.Components, children...)
+
+	return result
+}
+
+func (self *OSPath) Clear() *OSPath {
+	return &OSPath{
+		Manipulator: self.Manipulator,
+	}
+}
+
+// A FileInfo represents information about a file. It is similar to
+// os.FileInfo but not identical.
+type FileInfo interface {
+	Name() string
+	ModTime() time.Time
+
+	// Path as OS serialization.
+	FullPath() string
+
+	OSPath() *OSPath
+
+	// Time the file was birthed (initially created)
+	Btime() time.Time
+	Mtime() time.Time
+
+	// Time the inode was changed.
+	Ctime() time.Time
+	Atime() time.Time
+
+	// Arbitrary key/value for storing file metadata. This is accessor
+	// dependent can be nil.
+	Data() *ordereddict.Dict
+	Size() int64
+
+	IsDir() bool
+	IsLink() bool
+	GetLink() (*OSPath, error)
+	Mode() os.FileMode
+}
+
+// A File reader with
+type ReadSeekCloser interface {
+	io.ReadSeeker
+	io.Closer
+
+	//	Stat() (FileInfo, error)
+}
+
+// Interface for accessing the filesystem.
+type FileSystemAccessor interface {
+	// List a directory.
+	ReadDir(path string) ([]FileInfo, error)
+
+	// Open a file for reading
+	Open(path string) (ReadSeekCloser, error)
+	Lstat(filename string) (FileInfo, error)
+
+	ParsePath(filename string) *OSPath
+}
+
+// A factory for new accessors
+type FileSystemAccessorFactory interface {
+	New(scope vfilter.Scope) (FileSystemAccessor, error)
+}

glob/data.go → accessors/data/data.go

@@ -17,53 +17,44 @@
 */
 // A data filesystem accessor - allows data to be read as a file.
 
-package glob
+package data
 
 import (
-	"path/filepath"
-	"regexp"
 	"strings"
 
 	errors "github.com/pkg/errors"
-	"www.velocidex.com/golang/velociraptor/utils"
+	"www.velocidex.com/golang/velociraptor/accessors"
 	"www.velocidex.com/golang/vfilter"
 )
 
 type DataFilesystemAccessor struct{}
 
-func (self DataFilesystemAccessor) New(scope vfilter.Scope) (FileSystemAccessor, error) {
+func (self DataFilesystemAccessor) New(scope vfilter.Scope) (accessors.FileSystemAccessor, error) {
 	return DataFilesystemAccessor{}, nil
 }
 
-func (self DataFilesystemAccessor) Lstat(filename string) (FileInfo, error) {
-	return utils.NewDataFileInfo(filename), nil
+func (self DataFilesystemAccessor) ParsePath(path string) *accessors.OSPath {
+	return accessors.NewLinuxOSPath(path)
 }
 
-func (self DataFilesystemAccessor) ReadDir(path string) ([]FileInfo, error) {
-	return nil, errors.New("Not implemented")
-}
-
-func (self DataFilesystemAccessor) Open(path string) (ReadSeekCloser, error) {
-	return utils.DataReadSeekCloser{
-		ReadSeeker: strings.NewReader(path),
-		Data:       path,
+func (self DataFilesystemAccessor) Lstat(filename string) (accessors.FileInfo, error) {
+	return &accessors.VirtualFileInfo{
+		RawData: []byte(filename),
+		Path:    self.ParsePath(filename),
 	}, nil
 }
 
-func (self DataFilesystemAccessor) PathSplit(path string) []string {
-	re := regexp.MustCompile("/")
-	return re.Split(path, -1)
-}
-
-func (self DataFilesystemAccessor) PathJoin(root, stem string) string {
-	return filepath.Join(root, stem)
+func (self DataFilesystemAccessor) ReadDir(path string) ([]accessors.FileInfo, error) {
+	return nil, errors.New("Not implemented")
 }
 
-func (self DataFilesystemAccessor) GetRoot(path string) (string, string, error) {
-	return "/", path, nil
+func (self DataFilesystemAccessor) Open(path string) (accessors.ReadSeekCloser, error) {
+	return accessors.VirtualReadSeekCloser{
+		ReadSeeker: strings.NewReader(path),
+	}, nil
 }
 
 func init() {
-	Register("data", &DataFilesystemAccessor{},
+	accessors.Register("data", &DataFilesystemAccessor{},
 		`Makes a string appears as an in memory file. Path is taken as a literal string to use as the file's data`)
 }

accessors/data/data_test.go

@@ -0,0 +1,41 @@
+package data
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/Velocidex/ordereddict"
+	"www.velocidex.com/golang/velociraptor/accessors"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+	"www.velocidex.com/golang/velociraptor/vtesting/assert"
+)
+
+func TestAccessorData(t *testing.T) {
+	scope := vql_subsystem.MakeScope()
+	accessor, err := accessors.GetAccessor("data", scope)
+	assert.NoError(t, err)
+
+	fd, err := accessor.Open("Hello world")
+	assert.NoError(t, err)
+
+	data, err := ioutil.ReadAll(fd)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "Hello world", string(data))
+}
+
+func TestAccessorScope(t *testing.T) {
+	scope := vql_subsystem.MakeScope().AppendVars(ordereddict.NewDict().
+		Set("Foobar", "Hello world"))
+
+	accessor, err := accessors.GetAccessor("scope", scope)
+	assert.NoError(t, err)
+
+	fd, err := accessor.Open("Foobar")
+	assert.NoError(t, err)
+
+	data, err := ioutil.ReadAll(fd)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "Hello world", string(data))
+}

accessors/data/scope.go

@@ -0,0 +1,75 @@
+package data
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/pkg/errors"
+	"www.velocidex.com/golang/velociraptor/accessors"
+	"www.velocidex.com/golang/vfilter"
+)
+
+type ScopeFilesystemAccessor struct {
+	scope vfilter.Scope
+}
+
+func (self ScopeFilesystemAccessor) New(scope vfilter.Scope) (
+	accessors.FileSystemAccessor, error) {
+	return ScopeFilesystemAccessor{scope}, nil
+}
+
+func (self ScopeFilesystemAccessor) getData(variable string) (string, error) {
+	variable_data, pres := self.scope.Resolve(variable)
+	if !pres {
+		return "", os.ErrNotExist
+	}
+
+	switch t := variable_data.(type) {
+	case string:
+		return t, nil
+
+	case []byte:
+		return string(t), nil
+
+	default:
+		return fmt.Sprintf("%v", variable_data), nil
+	}
+}
+
+func (self ScopeFilesystemAccessor) ParsePath(path string) *accessors.OSPath {
+	return accessors.NewLinuxOSPath(path)
+}
+
+func (self ScopeFilesystemAccessor) Lstat(variable string) (
+	accessors.FileInfo, error) {
+	str, err := self.getData(variable)
+	if err != nil {
+		return nil, err
+	}
+	return &accessors.VirtualFileInfo{
+		RawData: []byte(str),
+		Path:    self.ParsePath(variable),
+	}, nil
+}
+
+func (self ScopeFilesystemAccessor) ReadDir(path string) (
+	[]accessors.FileInfo, error) {
+	return nil, errors.New("Not implemented")
+}
+
+func (self ScopeFilesystemAccessor) Open(path string) (
+	accessors.ReadSeekCloser, error) {
+	str, err := self.getData(path)
+	if err != nil {
+		return nil, err
+	}
+	return accessors.VirtualReadSeekCloser{
+		ReadSeeker: strings.NewReader(str),
+	}, nil
+}
+
+func init() {
+	accessors.Register("scope", &ScopeFilesystemAccessor{},
+		`Similar to the "data" accessor, this makes a string appears as a file. However, instead of the Filename containing the file content itself, the Filename refers to the name of a variable in the current scope that contains the data. This is useful when the binary data is not unicode safe and can not be properly represented by JSON.`)
+}