e2e: add pure go stateful fuzz test generator

Drop dependency to fMBT in e2e fuzz test generation. This patch adds go implementations of necessary parts of fMBT and replaces AAL/Python models with go models for topology-aware fuzz test. Signed-off-by: Antti Kervinen <[email protected]>
containers · Jan 15, 2025 · 400e12b · 400e12b
1 parent f09cb0e
commit 400e12b
Show file tree

Hide file tree

Showing 10 changed files with 1,065 additions and 19 deletions.
diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go
@@ -0,0 +1,185 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"sort"
+	"strings"
+
+	m "github.com/containers/nri-plugins/test/gofmbt"
+)
+
+type PodResources struct {
+	cpu  int // total CPU allocated for all containers in the pod
+	rcpu int // total reserved CPU allocated for all containers in the pod
+	mem  int // total memory allocated for all containers in the pod
+}
+
+type TestState struct {
+	cpu    int                      // free CPU on node for non-reserved pods
+	rcpu   int                      // free CPU on node for reserved pods
+	mem    int                      // free memory on node
+	podRes map[string]*PodResources // map running pod name to resources allocated to it
+}
+
+func (s *TestState) String() string {
+	pr := []string{}
+	pods := make([]string, 0, len(s.podRes))
+	for pod := range s.podRes {
+		pods = append(pods, pod)
+	}
+	sort.Strings(pods)
+	for _, pod := range pods {
+		res := s.podRes[pod]
+		switch {
+		case res.rcpu == 0:
+			pr = append(pr, fmt.Sprintf("%s:%dmCPU/%dM", pod, res.cpu, res.mem))
+		case res.cpu == 0:
+			pr = append(pr, fmt.Sprintf("%s:%dmRCPU/%dM", pod, res.rcpu, res.mem))
+		default:
+			pr = append(pr, fmt.Sprintf("%s:%dmCPU/%dmRCPU/%dM", pod, res.cpu, res.rcpu, res.mem))
+		}
+	}
+	return fmt.Sprintf("[free:%dmCPU/%dmRCPU/%dM pods:[%s]]", s.cpu, s.rcpu, s.mem, strings.Join(pr, " "))
+}
+
+func createPod(pod string, cpu, rcpu, mem int) m.StateChange {
+	return func(current m.State) m.State {
+		s := current.(*TestState)
+		if s.cpu < cpu || s.rcpu < rcpu || s.mem < mem {
+			// refuse from state change if not enough resources
+			return nil
+		}
+		if _, ok := s.podRes[pod]; ok {
+			// refuse to create pod if it is already running
+			return nil
+		}
+		newPodRes := make(map[string]*PodResources)
+		for k, v := range s.podRes {
+			newPodRes[k] = v
+		}
+		newPodRes[pod] = &PodResources{cpu, rcpu, mem}
+		return &TestState{
+			cpu:    s.cpu - cpu,
+			rcpu:   s.rcpu - rcpu,
+			mem:    s.mem - mem,
+			podRes: newPodRes,
+		}
+	}
+}
+
+func deletePod(pod string) m.StateChange {
+	return func(current m.State) m.State {
+		s := current.(*TestState)
+		res, ok := s.podRes[pod]
+		if !ok {
+			// refuse to delete pod if it is not running
+			return nil
+		}
+		newPodRes := make(map[string]*PodResources)
+		for k, v := range s.podRes {
+			if k != pod {
+				newPodRes[k] = v
+			}
+		}
+		return &TestState{
+			cpu:    s.cpu + res.cpu,
+			rcpu:   s.rcpu + res.rcpu,
+			mem:    s.mem + res.mem,
+			podRes: newPodRes,
+		}
+	}
+}
+
+var (
+	maxMem         int
+	maxCpu         int
+	maxReservedCpu int
+	maxTestSteps   int
+)
+
+func newModel() *m.Model {
+	podNames := []string{"gu0", "gu1", "gu2", "gu3", "gu4", "bu0", "bu1", "be0", "be1"}
+	rPodNames := []string{"rbe0", "rgu0", "rbu0"}
+
+	model := m.NewModel()
+
+	model.From(func(current m.State) []*m.Transition {
+		s := current.(*TestState)
+		return m.When(true,
+			m.OnAction("NAME=be0 CONTCOUNT=1 CPU=0 MEM=0 create besteffort").Do(createPod("be0", 1*0, 0, 1*0)),
+			m.OnAction("NAME=be1 CONTCOUNT=3 CPU=0 MEM=0 create besteffort").Do(createPod("be1", 3*0, 0, 3*0)),
+			m.OnAction("NAME=rbe0 CONTCOUNT=2 CPU=0 MEM=0 namespace=kube-system create besteffort").Do(createPod("rbe0", 0, 2*0, 2*0)),
+			m.When(s.mem > 0,
+				m.When(s.cpu >= 200,
+					m.OnAction("NAME=gu0 CONTCOUNT=1 CPU=200m MEM=1500M create guaranteed").Do(createPod("gu0", 1*200, 0, 1*1500)),
+					m.OnAction("NAME=gu1 CONTCOUNT=2 CPU=1000m MEM=500M create guaranteed").Do(createPod("gu1", 2*1000, 0, 2*500)),
+					m.OnAction("NAME=gu2 CONTCOUNT=2 CPU=1200m MEM=4500M create guaranteed").Do(createPod("gu2", 2*1200, 0, 2*4500)),
+					m.OnAction("NAME=gu3 CONTCOUNT=3 CPU=2000m MEM=500M create guaranteed").Do(createPod("gu3", 3*2000, 0, 3*500)),
+					m.OnAction("NAME=gu4 CONTCOUNT=1 CPU=4200m MEM=100M create guaranteed").Do(createPod("gu4", 1*4200, 0, 1*100)),
+					m.OnAction("NAME=bu0 CONTCOUNT=1 CPU=1200m MEM=50M CPUREQ=900m MEMREQ=49M CPULIM=1200m MEMLIM=50M create burstable").Do(createPod("bu0", 1*1200, 0, 1*50)),
+					m.OnAction("NAME=bu1 CONTCOUNT=2 CPU=1900m MEM=300M CPUREQ=1800m MEMREQ=299M CPULIM=1900m MEMLIM=300M create burstable").Do(createPod("bu1", 2*1900, 0, 2*300)),
+				),
+				m.When(s.rcpu > 99,
+					m.OnAction("NAME=rgu0 CONTCOUNT=2 CPU=100m MEM=1000M namespace=kube-system create guaranteed").Do(createPod("rgu0", 0, 2*100, 2*1000)),
+					m.OnAction("NAME=rbu0 CONTCOUNT=1 CPU=100m MEM=100M CPUREQ=99m MEMREQ=99M CPULIM=100m MEMLIM=100M namespace=kube-system create burstable").Do(createPod("rbu0", 0, 1*100, 1*100)),
+				),
+			),
+		)
+	})
+
+	model.From(func(current m.State) []*m.Transition {
+		s := current.(*TestState)
+		ts := []*m.Transition{}
+		for _, pod := range podNames {
+			if _, ok := s.podRes[pod]; ok {
+				ts = append(ts, m.OnAction("NAME=%s vm-command 'kubectl delete pod %s --now'", pod, pod).Do(deletePod(pod))...)
+			}
+		}
+		for _, pod := range rPodNames {
+			if _, ok := s.podRes[pod]; ok {
+				ts = append(ts, m.OnAction("NAME=%s vm-command 'kubectl delete pod --namespace kube-system %s --now'", pod, pod).Do(deletePod(pod))...)
+			}
+		}
+		return ts
+	})
+
+	return model
+}
+
+func main() {
+	flag.IntVar(&maxMem, "mem", 7500, "memory available for test pods")
+	flag.IntVar(&maxCpu, "cpu", 15000, "non-reserved milli-CPU available for test pods")
+	flag.IntVar(&maxReservedCpu, "reserved-cpu", 1000, "reserved milli-CPU availble for test pods")
+	flag.IntVar(&maxTestSteps, "test-steps", 3000, "number of test steps")
+	flag.Parse()
+
+	model := newModel()
+	coverer := m.NewCoverer()
+	coverer.CoverActionCombinations(3)
+
+	var state m.State
+
+	state = &TestState{
+		cpu:  maxCpu,
+		rcpu: maxReservedCpu,
+		mem:  maxMem,
+	}
+	testStep := 0
+	for testStep < maxTestSteps {
+		path, covStats := coverer.BestPath(model, state, 4)
+		if len(path) == 0 {
+			fmt.Printf("# did not find anything to cover\n")
+			break
+		}
+		for i := 0; i < covStats.MaxStep+1; i++ {
+			testStep++
+			step := path[i]
+			fmt.Printf("echo ==== step %d, coverage: %d, state: %v\n", testStep, coverer.Coverage(), state)
+			fmt.Println(step.Action())
+			state = step.EndState()
+			coverer.MarkCovered(step)
+			coverer.UpdateCoverage()
+		}
+	}
+}
diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh
@@ -11,8 +11,6 @@ Configuring test generation with environment variables:
   RESERVED_CPU=<NUM>    Reserved CPU [mCPU] available for test pods in the system.
   STEPS=<NUM>           Total number of test steps in all parallel tests.
 
-  FMBT_IMAGE=<IMG:TAG>  Generate the test using fmbt from docker image IMG:TAG.
-                        The default is fmbt-cli:latest.
 EOF
     exit 0
 }
@@ -28,36 +26,28 @@ MEM=${MEM:-7500}
 CPU=${CPU:-14050}
 RESERVED_CPU=${RESERVED_CPU:-1000}
 STEPS=${STEPS:-100}
-FMBT_IMAGE=${FMBT_IMAGE:-"fmbt-cli:latest"}
 
 mem_per_test=$(( MEM / TESTCOUNT ))
 cpu_per_test=$(( CPU / TESTCOUNT ))
 reserved_cpu_per_test=$(( RESERVED_CPU / TESTCOUNT ))
 steps_per_test=$(( STEPS / TESTCOUNT ))
 
-# Check fmbt Docker image
-docker run "$FMBT_IMAGE" fmbt --version 2>&1 | grep ^Version: || {
-    echo "error: cannot run fmbt from Docker image '$FMBT_IMAGE'"
-    echo "You can build the image locally by running:"
-    echo "( cd /tmp && git clone --branch devel https://github.com/intel/fmbt && cd fmbt && docker build . -t $FMBT_IMAGE -f Dockerfile.fmbt-cli )"
-    exit 1
-}
-
 cd "$(dirname "$0")" || {
     echo "cannot cd to the directory of $0"
     exit 1
 }
 
+
 for testnum in $(seq 1 "$TESTCOUNT"); do
     testid=$(( testnum - 1))
-    sed -e "s/max_mem=.*/max_mem=${mem_per_test}/" \
-        -e "s/max_cpu=.*/max_cpu=${cpu_per_test}/" \
-        -e "s/max_reserved_cpu=.*/max_reserved_cpu=${reserved_cpu_per_test}/" \
-        < fuzz.aal > tmp.fuzz.aal
-    sed -e "s/fuzz\.aal/tmp.fuzz.aal/" \
-        -e "s/pass = steps(.*/pass = steps(${steps_per_test})/" \
-        < fuzz.fmbt.conf > tmp.fuzz.fmbt.conf
     OUTFILE=generated${testid}.sh
+    prefix_pods_with_testid="s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g"
     echo "generating $OUTFILE..."
-    docker run -v "$(pwd):/mnt/models" "$FMBT_IMAGE" sh -c 'cd /mnt/models; fmbt tmp.fuzz.fmbt.conf 2>/dev/null | fmbt-log -f STEP\$sn\$as\$al' | grep -v AAL | sed -e 's/^, /  /g' -e '/^STEP/! s/\(^.*\)/echo "TESTGEN: \1"/g' -e 's/^STEP\([0-9]*\)i:\(.*\)/echo "TESTGEN: STEP \1"; vm-command "date +%T.%N"; \2; vm-command "date +%T.%N"; kubectl get pods -A/g' | sed "s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" > "$OUTFILE"
+    go run ./generate.go \
+       --mem $mem_per_test \
+       --cpu $cpu_per_test \
+       --reserved-cpu $reserved_cpu_per_test \
+       --test-steps $steps_per_test \
+        | sed -e "$prefix_pods_with_testid" \
+              > "$OUTFILE"
 done
diff --git a/test/gofmbt/action.go b/test/gofmbt/action.go
@@ -0,0 +1,51 @@
+// Copyright The NRI Plugins Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gofmbt
+
+import (
+	"fmt"
+)
+
+type Action struct {
+	name   string
+	format string
+	args   []interface{}
+}
+
+func NewAction(format string, args ...interface{}) *Action {
+	return &Action{
+		format: format,
+		args:   args,
+		name:   fmt.Sprintf(format, args...),
+	}
+}
+
+func (a *Action) String() string {
+	return a.name
+}
+
+func OnAction(format string, args ...interface{}) *Action {
+	return NewAction(format, args...)
+}
+
+func (a *Action) Do(stateChanges ...StateChange) []*Transition {
+	stateChange := func(s State) State {
+		for _, sc := range stateChanges {
+			s = sc(s)
+		}
+		return s
+	}
+	return []*Transition{NewTransition(a, stateChange)}
+}