Feat/k6 executor (Blazemeter#1450)

* K6 load tool support

Dockerfile

@@ -36,6 +36,12 @@ RUN $APT_INSTALL ./packages-microsoft-prod.deb \
    && $APT_UPDATE \
    && $APT_INSTALL dotnet-sdk-3.1
 
+# Install K6
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 \
+   && echo "deb https://dl.bintray.com/loadimpact/deb stable main" | tee -a /etc/apt/sources.list \
+   && $APT_UPDATE \
+   && $APT_INSTALL k6
+
 # Install Taurus & tools
 RUN $PIP_INSTALL ./bzt*whl \
   && mkdir -p /etc/bzt.d \

bzt/modules/k6.py

@@ -0,0 +1,164 @@
+"""
+Copyright 2021 BlazeMeter Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from bzt import TaurusConfigError, ToolError
+from bzt.engine import HavingInstallableTools
+from bzt.modules import ScenarioExecutor, FileLister, SelfDiagnosable
+from bzt.modules.console import WidgetProvider, ExecutorWidget
+from bzt.modules.aggregator import ResultsReader, ConsolidatingAggregator
+from bzt.utils import RequiredTool, CALL_PROBLEMS, FileReader, shutdown_process
+
+
+class K6Executor(ScenarioExecutor, FileLister, WidgetProvider, HavingInstallableTools, SelfDiagnosable):
+    def __init__(self):
+        super(K6Executor, self).__init__()
+        self.output_file = None
+        self.log_file = None
+        self.script = None
+        self.process = None
+        self.k6 = None
+        self.kpi_file = None
+
+    def prepare(self):
+        super(K6Executor, self).prepare()
+        self.install_required_tools()
+
+        self.script = self.get_script_path()
+        if not self.script:
+            raise TaurusConfigError("'script' should be present for k6 executor")
+
+        self.stdout = open(self.engine.create_artifact("k6", ".out"), "w")
+        self.stderr = open(self.engine.create_artifact("k6", ".err"), "w")
+
+        self.kpi_file = self.engine.create_artifact("kpi", ".csv")
+        self.reader = K6LogReader(self.kpi_file, self.log)
+        if isinstance(self.engine.aggregator, ConsolidatingAggregator):
+            self.engine.aggregator.add_underling(self.reader)
+
+    def startup(self):
+        cmdline = [self.k6.tool_name, "run", "--out", f"csv={self.kpi_file}"]
+
+        load = self.get_load()
+        if load.concurrency:
+            cmdline += ['--vus', str(load.concurrency)]
+
+        if load.hold:
+            cmdline += ['--duration', str(int(load.hold)) + "s"]
+
+        if load.iterations:
+            cmdline += ['--iterations', str(load.iterations)]
+
+        cmdline += [self.script]
+        self.process = self._execute(cmdline)
+
+    def get_widget(self):
+        if not self.widget:
+            label = "%s" % self
+            self.widget = ExecutorWidget(self, "K6: " + label.split('/')[1])
+        return self.widget
+
+    def check(self):
+        retcode = self.process.poll()
+        if retcode is not None:
+            ToolError(f"K6 tool exited with non-zero code: {retcode}")
+            return True
+        return False
+
+    def shutdown(self):
+        shutdown_process(self.process, self.log)
+
+    def post_process(self):
+        if self.kpi_file:
+            self.engine.existing_artifact(self.kpi_file)
+        super(K6Executor, self).post_process()
+
+    def install_required_tools(self):
+        self.k6 = self._get_tool(K6, config=self.settings)
+        self.k6.tool_name = self.k6.tool_name.lower()
+        if not self.k6.check_if_installed():
+            self.k6.install()
+
+
+class K6LogReader(ResultsReader):
+    def __init__(self, filename, parent_logger):
+        super(K6LogReader, self).__init__()
+        self.log = parent_logger.getChild(self.__class__.__name__)
+        self.file = FileReader(filename=filename, parent_logger=self.log)
+        self.data = {'timestamp': [], 'label': [], 'r_code': [], 'error_msg': [], 'http_req_duration': [],
+                     'http_req_connecting': [], 'http_req_tls_handshaking': [], 'http_req_waiting': [], 'vus': [],
+                     'data_received': []}
+
+    def _read(self, last_pass=False):
+        self.lines = list(self.file.get_lines(size=1024 * 1024, last_pass=last_pass))
+
+        for line in self.lines:
+            if line.startswith("http_reqs"):
+                self.data['timestamp'].append(int(line.split(',')[1]))
+                self.data['label'].append(line.split(',')[8])
+                self.data['r_code'].append(line.split(',')[12])
+                self.data['error_msg'].append(line.split(',')[4])
+            elif line.startswith("http_req_duration"):
+                self.data['http_req_duration'].append(float(line.split(',')[2]))
+            elif line.startswith("http_req_connecting"):
+                self.data['http_req_connecting'].append(float(line.split(',')[2]))
+            elif line.startswith("http_req_tls_handshaking"):
+                self.data['http_req_tls_handshaking'].append(float(line.split(',')[2]))
+            elif line.startswith("http_req_waiting"):
+                self.data['http_req_waiting'].append(float(line.split(',')[2]))
+            elif line.startswith("vus") and not line.startswith("vus_max"):
+                self.data['vus'].append(int(float(line.split(',')[2])))
+            elif line.startswith("data_received"):
+                self.data['data_received'].append(float(line.split(',')[2]))
+
+            if self.data['vus'] and len(self.data['data_received']) >= self.data['vus'][0] and \
+                    len(self.data['http_req_waiting']) >= self.data['vus'][0]:
+                for i in range(self.data['vus'][0]):
+                    kpi_set = (
+                        self.data['timestamp'][0],
+                        self.data['label'][0],
+                        self.data['vus'][0],
+                        self.data['http_req_duration'][0] / 1000,
+                        (self.data['http_req_connecting'][0] + self.data['http_req_tls_handshaking'][0]) / 1000,
+                        self.data['http_req_waiting'][0] / 1000,
+                        self.data['r_code'][0],
+                        None if not self.data['error_msg'][0] else self.data['error_msg'][0],
+                        '',
+                        self.data['data_received'][0])
+
+                    for key in self.data.keys():
+                        if key != 'vus':
+                            self.data[key].pop(0)
+
+                    yield kpi_set
+
+                self.data['vus'].pop(0)
+
+
+class K6(RequiredTool):
+    def __init__(self, config=None, **kwargs):
+        super(K6, self).__init__(installable=False, **kwargs)
+
+    def check_if_installed(self):
+        self.log.debug('Checking K6 Framework: %s' % self.tool_path)
+        try:
+            out, err = self.call(['k6', 'version'])
+        except CALL_PROBLEMS as exc:
+            self.log.warning("%s check failed: %s", self.tool_name, exc)
+            return False
+
+        if err:
+            out += err
+        self.log.debug("K6 output: %s", out)
+        return True

bzt/resources/10-base-config.yml

@@ -20,6 +20,8 @@ modules:
     class: bzt.modules.siege.SiegeExecutor
   tsung:
     class: bzt.modules.tsung.TsungExecutor
+  k6:
+    class: bzt.modules.k6.K6Executor
 
   # selenium & functional executors
   selenium:

examples/all-executors.yml

@@ -148,6 +148,13 @@ execution:
   scenario:
     script: functional/postman-sample-collection.json
 
+- executor: k6
+  concurrency: 1
+  hold-for: 10s
+  iterations: 10
+  scenario:
+    script: k6/k6_example.js
+
 ---
 # all of load-style executors
 execution:

examples/k6/k6_example.js

@@ -0,0 +1,7 @@
+import http from 'k6/http';
+import { sleep } from 'k6';
+
+export default function () {
+  http.get('https://blazedemo.com/');
+  sleep(1);
+}

site/dat/docs/ExecutionSettings.md

@@ -47,6 +47,7 @@ Taurus tool may use different underlying tools as executors for scenarios. Curre
   - [WebdriverIO](WebdriverIO.md), executor type `wdio`
   - [Robot](Robot.md), executor type `robot`
   - [Postman/Newman](Postman.md), executor type `newman`
+  - [K6](K6.md), executor type `k6`
 
 Default executor is `jmeter` and can be changed under [general settings](ConfigSyntax.md#Top-Level-Settings) section.
 ```yaml

site/dat/docs/Index.md

@@ -15,6 +15,7 @@
     1. [Grinder Executor](Grinder.md)
     1. [JMeter Executor](JMeter.md)
     1. [JUnit Executor](JUnit.md)
+    1. [K6 Executor](K6.md)
     1. [Locust Executor](Locust.md)
     1. [Mocha Executor](Mocha.md)
     1. [Molotov Executor](Molotov.md)

site/dat/docs/K6.md

@@ -0,0 +1,32 @@
+# K6 Executor
+
+`k6` executor allows you to run [K6](https://k6.io/) based test suites.
+
+In order to launch K6 executor, you can use yaml config like in the example below
+
+Example:
+```yaml
+execution:
+- executor: k6
+  concurrency: 10  # number of K6 workers
+  hold-for: 1m  # execution duration
+  iterations: 20  # number of iterations
+  scenario:
+    script: k6-test.js  # has to be a valid K6 script
+```
+
+Please keep in mind that it is necessary to provide a valid script to run tests.
+
+## Script Example
+
+This is an example of a valid script from [K6 website](https://k6.io/docs/getting-started/running-k6):
+
+```javascript
+import http from 'k6/http';
+import { sleep } from 'k6';
+
+export default function () {
+  http.get('https://blazedemo.com/');
+  sleep(1);
+}
+```

tests/resources/k6/k6_kpi.csv

@@ -0,0 +1,60 @@
+metric_name,timestamp,metric_value,check,error,error_code,group,method,name,proto,scenario,service,status,subproto,tls_version,url,extra_tags
+http_reqs,1611843422,1.000000,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_duration,1611843422,382.011488,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_blocked,1611843422,173.974899,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_connecting,1611843422,25.734642,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_tls_handshaking,1611843422,66.800798,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_sending,1611843422,0.076907,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_waiting,1611843422,380.503510,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_receiving,1611843422,1.431071,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+vus,1611843423,1.000000,,,,,,,,,,,,,,
+vus_max,1611843423,1.000000,,,,,,,,,,,,,,
+data_sent,1611843423,606.000000,,,,,,,,default,,,,,,
+data_received,1611843423,6295.000000,,,,,,,,default,,,,,,
+iteration_duration,1611843423,1556.448014,,,,,,,,default,,,,,,
+iterations,1611843423,1.000000,,,,,,,,default,,,,,,
+vus,1611843424,1.000000,,,,,,,,,,,,,,
+vus_max,1611843424,1.000000,,,,,,,,,,,,,,
+http_reqs,1611843424,1.000000,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_duration,1611843424,449.236493,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_blocked,1611843424,0.000917,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_connecting,1611843424,0.000000,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_tls_handshaking,1611843424,0.000000,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_sending,1611843424,0.113203,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_waiting,1611843424,447.395546,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+http_req_receiving,1611843424,1.727744,,,,,GET,https://blazedemo.com/,HTTP/2.0,default,,200,,tls1.3,https://blazedemo.com/,
+vus,1611843425,1.000000,,,,,,,,,,,,,,
+vus_max,1611843425,1.000000,,,,,,,,,,,,,,
+data_sent,1611843425,110.000000,,,,,,,,default,,,,,,
+data_received,1611843425,3179.000000,,,,,,,,default,,,,,,
+iteration_duration,1611843425,1449.807750,,,,,,,,default,,,,,,
+iterations,1611843425,1.000000,,,,,,,,default,,,,,,
+metric_name,timestamp,metric_value,check,error,error_code,group,method,name,proto,scenario,service,status,subproto,tls_version,url,extra_tags
+http_reqs,1611843450,1.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_duration,1611843450,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_blocked,1611843450,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_connecting,1611843450,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_tls_handshaking,1611843450,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_sending,1611843450,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_waiting,1611843450,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_receiving,1611843450,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+vus,1611843451,1.000000,,,,,,,,,,,,,,
+vus_max,1611843451,1.000000,,,,,,,,,,,,,,
+data_sent,1611843451,0.000000,,,,,,,,default,,,,,,
+data_received,1611843451,0.000000,,,,,,,,default,,,,,,
+iteration_duration,1611843451,1004.632409,,,,,,,,default,,,,,,
+iterations,1611843451,1.000000,,,,,,,,default,,,,,,
+http_reqs,1611843451,1.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_duration,1611843451,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_blocked,1611843451,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_connecting,1611843451,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_tls_handshaking,1611843451,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_sending,1611843451,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_waiting,1611843451,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+http_req_receiving,1611843451,0.000000,,lookup: no such host,1101,,GET,https://non-blazedemo.com/,,default,,0,,,https://non-blazedemo.com/,
+vus,1611843452,1.000000,,,,,,,,,,,,,,
+vus_max,1611843452,1.000000,,,,,,,,,,,,,,
+data_sent,1611843452,0.000000,,,,,,,,default,,,,,,
+data_received,1611843452,0.000000,,,,,,,,default,,,,,,
+iteration_duration,1611843452,1005.109973,,,,,,,,default,,,,,,
+iterations,1611843452,1.000000,,,,,,,,default,,,,,,

tests/resources/k6/k6_mock.bat

@@ -0,0 +1,2 @@
+@echo off
+echo v0.30.0

tests/resources/k6/k6_mock.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo v0.30.0

tests/resources/k6/k6_script.js

@@ -0,0 +1,7 @@
+import http from 'k6/http';
+import { sleep } from 'k6';
+
+export default function () {
+  http.get('https://blazedemo.com/');
+  sleep(1);
+}