Merge pull request #3599 from Slava0135/fix-modmul

vm: fix modmul operation

pkg/compiler/syscall_test.go

@@ -343,20 +343,22 @@ func TestOpcode(t *testing.T) {
 		src := `package foo
 		import "github.com/nspcc-dev/neo-go/pkg/interop/math"
 		func Main() []int {
-			r := make([]int, 5)
+			r := make([]int, 6)
 			r[0] = math.ModMul(3, 4, 5)
 			r[1] = math.ModMul(-3, 4, 5)
 			r[2] = math.ModMul(3, 4, -5)
 			r[3] = math.ModMul(-3, 4, -5)
 			r[4] = math.ModMul(0, 4, 5)
+			r[5] = math.ModMul(100, -1, -91)
 			return r
 		}`
 		eval(t, src, []stackitem.Item{
 			stackitem.Make(2),
-			stackitem.Make(3),
+			stackitem.Make(-2),
 			stackitem.Make(2),
-			stackitem.Make(3),
+			stackitem.Make(-2),
 			stackitem.Make(0),
+			stackitem.Make(-9),
 		})
 	})
 	t.Run("MODPOW", func(t *testing.T) {

pkg/vm/vm.go

@@ -1030,7 +1030,7 @@ func (v *VM) execute(ctx *Context, op opcode.Opcode, parameter []byte) (err erro
 		x1 := v.estack.Pop().BigInt()
 
 		res := new(big.Int).Mul(x1, x2)
-		v.estack.PushItem(stackitem.NewBigInteger(res.Mod(res, modulus)))
+		v.estack.PushItem(stackitem.NewBigInteger(res.Rem(res, modulus)))
 
 	case opcode.MODPOW:
 		modulus := v.estack.Pop().BigInt()

pkg/vm/vm_test.go

@@ -738,9 +738,10 @@ func TestMODMUL(t *testing.T) {
 	t.Run("bad, zero mod", getTestFuncForVM(prog, nil, 1, 2, 0))
 	t.Run("good, positive base", getTestFuncForVM(prog, 2, 3, 4, 5))
 	t.Run("good, zero base", getTestFuncForVM(prog, 0, 0, 4, 5))
-	t.Run("good, negative base", getTestFuncForVM(prog, 3, -3, 4, 5))
+	t.Run("good, negative base", getTestFuncForVM(prog, -2, -3, 4, 5))
 	t.Run("good, positive base, negative mod", getTestFuncForVM(prog, 2, 3, 4, -5))
-	t.Run("good, negative base, negative mod", getTestFuncForVM(prog, 3, -3, 4, -5))
+	t.Run("good, negative base, negative mod", getTestFuncForVM(prog, -2, -3, 4, -5))
+	t.Run("good, positive base, negative multiplier, negative mod", getTestFuncForVM(prog, -9, 100, -1, -91))
 }
 
 func TestMODPOW(t *testing.T) {