Update mixer rate limit test to latest fortio (istio#2655)

Automatic merge from submit-queue.

Update mixer rate limit test to latest fortio

.gitignore

@@ -22,6 +22,7 @@ pilot/proxy/envoy/envoy
 # lint
 lintconfig.gen.json
 .istiorc
+.istiorc.mk
 # codegen stuff
 bin/protoc-gen-gogoslick*
 bin/protoc-min-version*

Gopkg.toml

@@ -107,7 +107,7 @@ required = [
 
 [[constraint]]
   name = "istio.io/fortio"
-  version = ">= v0.6.2"
+  version = ">= v0.6.4"
 
 ### DO NOT PIN these repos to master. It will result in inconsistent builds in circleci
 ### as we cache dep ensure.

Makefile

@@ -40,7 +40,7 @@ export GOOS ?= linux
 export GOARCH ?= amd64
 
 # Optional file including user-specific settings (HUB, TAG, etc)
--include .istiorc
+-include .istiorc.mk
 
 
 # @todo allow user to run for a single $PKG only?

istio.VERSION

@@ -8,11 +8,12 @@ export PILOT_HUB="gcr.io/istio-testing"
 export PILOT_TAG="3101ea9d82a5f83b699c2d3245b371a19fa6bef4"
 export ISTIOCTL_URL="https://storage.googleapis.com/istio-artifacts/pilot/3101ea9d82a5f83b699c2d3245b371a19fa6bef4/artifacts/istioctl"
 export PROXY_TAG="62ea2e2c70df6a08fd981ae0c0bf3945f93ccd72"
+export PROXY_DEBUG=""
 export ISTIO_NAMESPACE="istio-system"
 export AUTH_DEBIAN_URL="https://storage.googleapis.com/istio-artifacts/auth/d10e0c4aa05ca726cae71aa5d033dea4f7bc26e4/artifacts/debs"
 export PILOT_DEBIAN_URL="https://storage.googleapis.com/istio-artifacts/pilot/3101ea9d82a5f83b699c2d3245b371a19fa6bef4/artifacts/debs"
 export PROXY_DEBIAN_URL="https://storage.googleapis.com/istio-artifacts/proxy/2458e4e27bc588df858711b66de0970183e0cf66/artifacts/debs"
 export FORTIO_HUB="docker.io/istio"
-export FORTIO_TAG="0.6.2"
+export FORTIO_TAG="0.6.4"
 export HYPERKUBE_HUB="quay.io/coreos/hyperkube"
 export HYPERKUBE_TAG="v1.7.6_coreos.0"

tests/e2e/README.md

@@ -78,13 +78,17 @@ kubectl create clusterrolebinding myname-cluster-admin-binding    --clusterrole=
 **Option 1:** Build your own images.
 
 ```
-# Customize .istiorc with your HUB and optional TAG (example: HUB=costinm TAG=mybranch)
+# Customize .istiorc.mk (at the top of the istio.io/istio source tree) with your HUB and optional TAG
+# it allows you to customize Makefile rules. For example:
+cat .istiorc.mk
+HUB=costinm
+TAG=mybranch
 
-# Build images on the local docker. 
+# Build images on the local docker.
 make docker
 
 # Push images to docker registry
-# If you use minikube and its docker environment, images will be  available in minikube for use, 
+# If you use minikube and its docker environment, images will be  available in minikube for use,
 # you can skip this step.
 make push
 

tests/e2e/tests/mixer/mixer_test.go

@@ -36,6 +36,7 @@ import (
 	"github.com/prometheus/common/model"
 
 	"istio.io/fortio/fhttp"
+	// flog "istio.io/fortio/log"
 	"istio.io/fortio/periodic"
 	"istio.io/istio/pkg/log"
 	"istio.io/istio/tests/e2e/framework"
@@ -493,16 +494,20 @@ func TestRateLimit(t *testing.T) {
 
 	url := fmt.Sprintf("%s/productpage", tc.gateway)
 
-	// run at a large QPS (here 100) for a minute to ensure that enough
-	// traffic is generated to trigger 429s from the rate limit rule
+	// run at a high enough QPS (here 10) to ensure that enough
+	// traffic is generated to trigger 429s from the 1 QPS rate limit rule
 	opts := fhttp.HTTPRunnerOptions{
 		RunnerOptions: periodic.RunnerOptions{
 			QPS:        10,
-			Duration:   1 * time.Minute,
-			NumThreads: 8,
+			Exactly:    200,       // will make exactly 200 calls, so run for about 20 seconds
+			NumThreads: 5,         // get the same number of calls per connection (200/5=40)
+			Out:        os.Stderr, // Only needed because of log capture issue
+		},
+		HTTPOptions: fhttp.HTTPOptions{
+			URL: url,
 		},
 	}
-	opts.Init(url)
+
 	// productpage should still return 200s when ratings is rate-limited.
 	res, err := fhttp.RunHTTPTest(&opts)
 	if err != nil {
@@ -517,14 +522,14 @@ func TestRateLimit(t *testing.T) {
 	actualDuration := res.ActualDuration.Seconds() // can be a bit more than requested
 
 	log.Info("Successfully sent request(s) to /productpage; checking metrics...")
-	t.Logf("Fortio Summary: %d reqs (%f rps, %f 200s (%f rps), %d 400s)",
-		totalReqs, res.ActualQPS, succReqs, succReqs/actualDuration, badReqs)
+	t.Logf("Fortio Summary: %d reqs (%f rps, %f 200s (%f rps), %d 400s - %+v)",
+		totalReqs, res.ActualQPS, succReqs, succReqs/actualDuration, badReqs, res.RetCodes)
 
 	// consider only successful requests (as recorded at productpage service)
 	callsToRatings := succReqs
 
 	// the rate-limit is 1 rps
-	want200s := 1. * opts.Duration.Seconds()
+	want200s := 1. * actualDuration
 
 	// everything in excess of 200s should be 429s (ideally)
 	want429s := callsToRatings - want200s
@@ -589,6 +594,7 @@ func TestRateLimit(t *testing.T) {
 		errorf(t, "Bad metric value for successful requests (200s): got %f, want at least %f", got, want)
 	}
 
+	want200s = math.Ceil(want200s * 1.1) // timing is short, allow 10% extra for rounding errors and quota bucket size
 	if got > want200s {
 		t.Logf("prometheus values for istio_request_count:\n%s", promDump(promAPI, "istio_request_count"))
 		errorf(t, "Bad metric value for successful requests (200s): got %f, want at most %f", got, want200s)