feat(rules): Bound fields (rabbitstack#143)

* introduce bound fields in sequence rules

* improve sequence joining and expression evaluation

* minor adjustments

pkg/filter/_fixtures/default/default.yml

@@ -6,7 +6,7 @@
 #
 # Obviously, some events can't be directly translated from the corresponding
 # sysmon expressions, since Fibratus doesn't support them yet. In the same way,
-# some filter fields are still missing in Fibratus, so that sysmon rules were
+# some filter fields are still missing in Fibratus, so those sysmon rules were
 # omitted.
 #
 # ======================= Process creation ================================

pkg/filter/_fixtures/sequence_and_simple_rule_mix.yml

@@ -1,4 +1,4 @@
-- group: command shell execution and temp files
+- group: Command shell execution and temp files
   enabled: true
   rules:
     - name: Process spawned by powershell

pkg/filter/_fixtures/sequence_rule_bound_fields.yml

@@ -0,0 +1,15 @@
+- group: Command shell execution and temp files with network outbound
+  enabled: true
+  rules:
+    - name: Command shell created a temp file with network outbound
+      condition: >
+        sequence
+        maxspan 200ms
+        |kevt.name = 'CreateProcess' and ps.name = 'cmd.exe'| as e1
+        |kevt.name = 'CreateFile'
+            and
+         file.name icontains 'temp'
+            and
+         $e1.ps.sid = ps.sid
+        | as e2
+        |kevt.name = 'Connect' and ps.sid != $e2.ps.sid and ps.sid = $e1.ps.sid|

pkg/filter/_fixtures/sequence_rule_simple.yml

@@ -1,4 +1,4 @@
-- group: command shell execution and temp files
+- group: Command shell execution and temp files
   enabled: true
   rules:
     - name: Command shell created a temp file

pkg/filter/_fixtures/sequence_rule_simple_max_span.yml

@@ -1,4 +1,4 @@
-- group: command shell execution and temp files
+- group: Command shell execution and temp files
   enabled: true
   rules:
     - name: Command shell created a temp file

pkg/filter/filter.go

@@ -50,7 +50,7 @@ type Filter interface {
 	// RunSequence runs a filter with sequence expressions. Sequence rules depend
 	// on the state machine transitions and partial matches to decide whether the
 	// rule is fired.
-	RunSequence(kevt *kevent.Kevent, seqID uint16) bool
+	RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uint16][]*kevent.Kevent) bool
 	// GetStringFields returns field names mapped to their string values.
 	GetStringFields() map[fields.Field][]string
 	// GetSequence returns the sequence descriptor or nil if this filter is not a sequence.
@@ -96,35 +96,25 @@ func (f *filter) Compile() error {
 
 	// traverse the expression tree
 	walk := func(n ql.Node) {
-		if expr, ok := n.(*ql.BinaryExpr); ok {
+		switch expr := n.(type) {
+		case *ql.BinaryExpr:
 			if lhs, ok := expr.LHS.(*ql.FieldLiteral); ok {
 				field := fields.Field(lhs.Value)
 				f.addField(field)
-				switch v := expr.RHS.(type) {
-				case *ql.StringLiteral:
-					f.stringFields[field] = append(f.stringFields[field], v.Value)
-				case *ql.ListLiteral:
-					f.stringFields[field] = append(f.stringFields[field], v.Values...)
-				}
+				f.addStringFields(field, expr.RHS)
 			}
 			if rhs, ok := expr.RHS.(*ql.FieldLiteral); ok {
 				field := fields.Field(rhs.Value)
 				f.addField(field)
-				switch v := expr.LHS.(type) {
-				case *ql.StringLiteral:
-					f.stringFields[field] = append(f.stringFields[field], v.Value)
-				case *ql.ListLiteral:
-					f.stringFields[field] = append(f.stringFields[field], v.Values...)
-				}
+				f.addStringFields(field, expr.LHS)
 			}
-		}
-		if expr, ok := n.(*ql.Function); ok {
-			f.useFuncValuer = true
+		case *ql.Function:
 			for _, arg := range expr.Args {
-				if fld, ok := arg.(*ql.FieldLiteral); ok {
-					f.addField(fields.Field(fld.Value))
+				if field, ok := arg.(*ql.FieldLiteral); ok {
+					f.addField(fields.Field(field.Value))
 				}
 			}
+			f.useFuncValuer = true
 		}
 	}
 	if f.expr != nil {
@@ -140,7 +130,6 @@ func (f *filter) Compile() error {
 			}
 		}
 	}
-
 	if len(f.fields) == 0 && !f.useFuncValuer {
 		return ErrNoFields
 	}
@@ -162,25 +151,95 @@ func (f *filter) Run(kevt *kevent.Kevent) bool {
 	return ql.Eval(f.expr, f.mapValuer(kevt), f.useFuncValuer)
 }
 
-func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16) bool {
-	if f.seq == nil || seqID > uint16(len(f.seq.Expressions))-1 {
+func (f *filter) RunSequence(kevt *kevent.Kevent, seqID uint16, partials map[uint16][]*kevent.Kevent) bool {
+	nseqs := uint16(len(f.seq.Expressions))
+	if f.seq == nil || seqID > nseqs-1 {
 		return false
 	}
-
 	valuer := f.mapValuer(kevt)
 	expr := f.seq.Expressions[seqID]
-	ok := ql.Eval(expr.Expr, valuer, f.useFuncValuer)
-	if !ok {
-		return false
-	}
 
-	by := f.seq.By
-	if by.IsEmpty() {
-		by = expr.By
+	var match bool
+	if seqID >= 1 && expr.HasBoundFields() {
+		// if a sequence expression contains references to
+		// bound fields we map all partials to their sequence
+		// aliases
+		pls := make(map[string][]*kevent.Kevent)
+		for i := uint16(0); i < seqID; i++ {
+			alias := f.seq.Expressions[i].Alias
+			if alias == "" {
+				continue
+			}
+			pls[alias] = partials[i+1]
+		}
+		// process until partials from all slots are consumed
+		n := 0
+		nslots := len(pls)
+		for nslots > 0 {
+			for _, field := range expr.BoundFields {
+				for _, accessor := range f.accessors {
+					evts := pls[field.Alias()]
+					if n > len(evts)-1 {
+						nslots--
+						continue
+					}
+					evt := evts[n]
+					if !accessor.canAccess(evt, f) {
+						continue
+					}
+					v, err := accessor.get(field.Field(), evt)
+					if err != nil && !kerrors.IsKparamNotFound(err) {
+						accessorErrors.Add(err.Error(), 1)
+						continue
+					}
+					if v != nil {
+						valuer[field.String()] = v
+						break
+					}
+				}
+			}
+			n++
+			match = ql.Eval(expr.Expr, valuer, f.useFuncValuer)
+			if match {
+				break
+			}
+		}
+	} else {
+		by := f.seq.By
+		if by.IsEmpty() {
+			by = expr.By
+		}
+		if seqID >= 1 && !by.IsEmpty() {
+			// traverse upstream partials for join equality
+			joins := make([]bool, seqID)
+			joinID := valuer[by.String()]
+		outer:
+			for i := uint16(0); i < seqID; i++ {
+				for _, p := range partials[i+1] {
+					if compareSeqJoin(joinID, p.SequenceBy()) {
+						joins[i] = true
+						continue outer
+					}
+				}
+			}
+			match = joinsEqual(joins) && ql.Eval(expr.Expr, valuer, f.useFuncValuer)
+		} else {
+			match = ql.Eval(expr.Expr, valuer, f.useFuncValuer)
+		}
+		if match && !by.IsEmpty() {
+			if v := valuer[by.String()]; v != nil {
+				kevt.AddMeta(kevent.RuleSequenceByKey, v)
+			}
+		}
 	}
-	v := valuer[by.String()]
-	if v != nil {
-		kevt.AddMeta(kevent.RuleSequenceByKey, v)
+	return match
+}
+
+func joinsEqual(joins []bool) bool {
+	for _, j := range joins {
+		if !j {
+			return false
+		}
 	}
 	return true
 }
@@ -276,3 +335,13 @@ func (f *filter) addField(field fields.Field) {
 	}
 	f.fields = append(f.fields, field)
 }
+
+// addStringFields appends values for all string field expressions.
+func (f *filter) addStringFields(field fields.Field, expr ql.Expr) {
+	switch v := expr.(type) {
+	case *ql.StringLiteral:
+		f.stringFields[field] = append(f.stringFields[field], v.Value)
+	case *ql.ListLiteral:
+		f.stringFields[field] = append(f.stringFields[field], v.Values...)
+	}
+}

pkg/filter/filter_test.go

@@ -55,7 +55,7 @@ func TestFilterCompile(t *testing.T) {
 	f = New(`ps.name`, cfg)
 	require.EqualError(t, f.Compile(), "expected at least one field or operator but zero found")
 	f = New(`ps.name =`, cfg)
-	require.EqualError(t, f.Compile(), "ps.name =\n╭─────────^\n|\n|\n╰─────────────────── expected field, string, number, bool, ip, function")
+	require.EqualError(t, f.Compile(), "ps.name =\n╭─────────^\n|\n|\n╰─────────────────── expected field, bound field, string, number, bool, ip, function")
 }
 
 func TestSeqFilterCompile(t *testing.T) {

pkg/filter/ql/ast.go

@@ -171,6 +171,12 @@ func (v *ValuerEval) Eval(expr Expr) interface{} {
 			return nil
 		}
 		return val
+	case *BoundFieldLiteral:
+		val, ok := v.Valuer.Value(expr.Value)
+		if !ok {
+			return nil
+		}
+		return val
 	case *IPLiteral:
 		return expr.Value
 	case *Function:

pkg/filter/ql/error_test.go

@@ -86,9 +86,9 @@ func TestParseError(t *testing.T) {
 |	          'user shell folders\\startup'
 |	        )
 |
-╰─────────────────── expected field, string, number, bool, ip, function, pattern binding`
+╰─────────────────── expected field, bound field, string, number, bool, ip, function`
 
-	e := newParseError("[", []string{"field, string, number, bool, ip, function, pattern binding"}, 145, expr)
+	e := newParseError("[", []string{"field, bound field, string, number, bool, ip, function"}, 145, expr)
 	require.Equal(t, expected, e.Error())
 
 	expr = `ps.name = 'cmd.exe' aand ps.cmdline contains 'ss'`

pkg/filter/ql/function_test.go

@@ -34,6 +34,7 @@ func TestParseFunction(t *testing.T) {
 		{expr: "cidr_contains(net.dip)", err: errors.New("CIDR_CONTAINS function requires 2 argument(s) but 1 argument(s) given")},
 		{expr: "cidr_contains(net.dip, 12)", err: errors.New("argument #2 (cidr) in function CIDR_CONTAINS should be one of: string")},
 		{expr: "cidr_contains(net.dip, '172.17.12.4/24')"},
+		{expr: "cidr_contains($e1.net.dip, '172.17.12.4/24')"},
 		{expr: "md('172.17.12.4')", err: errors.New("md function is undefined")},
 		{expr: "concat('hello ', 'world')"},
 		{expr: "concat('hello')", err: errors.New("CONCAT function requires 2 argument(s) but 1 argument(s) given")},
@@ -49,7 +50,7 @@ func TestParseFunction(t *testing.T) {
 		_, err := p.ParseExpr()
 		if err == nil && tt.err != nil {
 			t.Errorf("%d. exp=%s expected error=%v", i, tt.expr, tt.err)
-		} else if err != nil {
+		} else if err != nil && tt.err != nil {
 			assert.True(t, strings.Contains(err.Error(), tt.err.Error()))
 		} else if err != nil && tt.err == nil {
 			t.Errorf("%d. exp=%s got error=%v", i, tt.expr, err)

pkg/filter/ql/lexer.go

@@ -52,7 +52,7 @@ func (s *scanner) scan() (tok token, pos int, lit string) {
 	// as an ident or reserved word.
 	if isWhitespace(ch0) {
 		return s.scanWhitespace()
-	} else if isLetter(ch0) || ch0 == '_' || ch0 == '$' {
+	} else if isLetter(ch0) || ch0 == '_' {
 		s.r.unread()
 		return s.scanIdent()
 	} else if isDigit(ch0) {
@@ -61,7 +61,7 @@ func (s *scanner) scan() (tok token, pos int, lit string) {
 
 	// Otherwise, parse individual characters.
 	switch ch0 {
-	case reof:
+	case eof:
 		return EOF, pos, ""
 	case '"':
 		s.r.unread()
@@ -109,6 +109,12 @@ func (s *scanner) scan() (tok token, pos int, lit string) {
 		return Pipe, pos, ""
 	case ',':
 		return Comma, pos, ""
+	case '$':
+		tok, _, lit = s.scanIdent()
+		if tok != Ident {
+			return tok, pos, "$" + lit
+		}
+		return BoundField, pos, "$" + lit
 	}
 	return Illegal, pos, string(ch0)
 }
@@ -124,7 +130,7 @@ func (s *scanner) scanWhitespace() (tok token, pos int, lit string) {
 	// Non-whitespace characters and EOF will cause the loop to exit.
 	for {
 		ch, _ = s.r.read()
-		if ch == reof {
+		if ch == eof {
 			break
 		} else if !isWhitespace(ch) {
 			s.r.unread()
@@ -144,7 +150,7 @@ func (s *scanner) scanIdent() (tok token, pos int, lit string) {
 
 	var buf bytes.Buffer
 	for {
-		if ch, _ := s.r.read(); ch == reof {
+		if ch, _ := s.r.read(); ch == eof {
 			break
 		} else if ch == '"' {
 			tok0, pos0, lit0 := s.scanString()
@@ -430,7 +436,7 @@ type reader struct {
 // Note that this function does not return size.
 func (r *reader) ReadRune() (ch rune, size int, err error) {
 	ch, _ = r.read()
-	if ch == reof {
+	if ch == eof {
 		err = io.EOF
 	}
 	return
@@ -443,7 +449,7 @@ func (r *reader) UnreadRune() error {
 	return nil
 }
 
-var reof = rune(0)
+var eof = rune(0)
 
 // read reads the next rune from the reader.
 func (r *reader) read() (ch rune, pos int) {
@@ -457,7 +463,7 @@ func (r *reader) read() (ch rune, pos int) {
 	// Any error (including io.EOF) should return as EOF.
 	ch, _, err := r.r.ReadRune()
 	if err != nil {
-		ch = reof
+		ch = eof
 	} else if ch == '\r' {
 		if ch, _, err := r.r.ReadRune(); err != nil {
 			// nop
@@ -479,7 +485,7 @@ func (r *reader) read() (ch rune, pos int) {
 
 	// Mark the reader as EOF.
 	// This is used to avoid doubling the count of EOF characters.
-	if ch == reof {
+	if ch == eof {
 		r.eof = true
 	}