improves tests for stateful_set

- adds test to verify that spot request are cancelled
  by AwsEC2Terminator during related instances termination
- adds command line option for pytest to specify ec2 instances
  market type (spot|on-demand) used by default

Signed-off-by: Andrey Kononykhin <[email protected]>

pool_automation/roles/aws_manage/library/conftest.py

@@ -0,0 +1,57 @@
+import boto3
+from collections import defaultdict
+
+import pytest
+
+from stateful_set import AWS_REGIONS
+
+
+AWS_REGIONS_TEST = [r.code for r in AWS_REGIONS.values() if not r.expensive]
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--market-type", choices=['on-demand', 'spot'], default='on-demand',
+        help="EC2 instances market type to use by default"
+    )
+
+
+# parametrization spec for tests
+def pytest_generate_tests(metafunc):
+
+    def idfn(regions):
+        return '_'.join(regions)
+
+    if 'regions' in metafunc.fixturenames:
+        # TODO
+        # as of now (pytest v.4.0.1) pytest doesn't provide public API
+        # to access markers from this hook (e.g. using Metafunc), but according
+        # to comments in code they are going to change that
+        # https://github.com/pytest-dev/pytest/blob/4.0.1/src/_pytest/python.py#L1447),
+        # thus the following code should be reviewed later
+        marker = metafunc.definition.get_closest_marker('regions')
+
+        metafunc.parametrize(
+            "regions",
+            marker.args[0] if marker else [[r] for r in AWS_REGIONS_TEST],
+            ids=marker.kwargs.get('ids', idfn) if marker else idfn)
+
+
+@pytest.fixture(scope="session")
+def ec2_all():
+    return {
+        r: {
+            'rc': boto3.resource('ec2', region_name=r),
+            'cl': boto3.client('ec2', region_name=r)
+        }
+        for r in AWS_REGIONS_TEST
+    }
+
+
+@pytest.fixture(scope="session")
+def ec2_prices():
+    return {
+        r: {
+            'on-demand': defaultdict(dict)
+        } for r in AWS_REGIONS_TEST
+    }

pool_automation/roles/aws_manage/library/test_stateful_set.py

@@ -2,7 +2,6 @@
 import string
 import json
 import boto3
-from collections import defaultdict
 
 import pytest
 
@@ -13,9 +12,6 @@
 )
 
 
-AWS_REGIONS_TEST = [r.code for r in AWS_REGIONS.values() if not r.expensive]
-
-
 class EC2TestCtx(object):
     def __init__(self, region, resource, client, prices=None):
         self.region = region
@@ -29,17 +25,6 @@ def __init__(self, region, resource, client, prices=None):
 ############
 
 
-@pytest.fixture(scope="session")
-def ec2_all():
-    return {
-        r: {
-            'rc': boto3.resource('ec2', region_name=r),
-            'cl': boto3.client('ec2', region_name=r)
-        }
-        for r in AWS_REGIONS_TEST
-    }
-
-
 @pytest.fixture
 def ec2(regions, ec2_all):
     return [ec2_all[r]['rc'] for r in regions]
@@ -75,7 +60,8 @@ def _random(N=7):
                                    if security_group_suffix
                                    else _random()),
             type_name='t2.micro',
-            market_spot=False,
+            # TODO docs
+            market_spot=(request.config.getoption("--market-type") == 'spot'),
             spot_max_price=None
         )
 
@@ -143,15 +129,6 @@ def pricing_client():
     return boto3.client('pricing', region_name='us-east-1')
 
 
-@pytest.fixture(scope="session")
-def ec2_prices():
-    return {
-        r: {
-            'on-demand': defaultdict(dict)
-        } for r in AWS_REGIONS_TEST
-    }
-
-
 @pytest.fixture
 def on_demand_prices(request, pricing_client, ec2_prices,
                      regions, ec2_resources):
@@ -220,27 +197,6 @@ def ec2ctx(ec2ctxs):
     return ec2ctxs[0]
 
 
-# parametrization spec for tests
-def pytest_generate_tests(metafunc):
-
-    def idfn(regions):
-        return '_'.join(regions)
-
-    if 'regions' in metafunc.fixturenames:
-        # TODO
-        # as of now (pytest v.4.0.1) pytest doesn't provide public API
-        # to access markers from this hook (e.g. using Metafunc), but according
-        # to comments in code they are going to change that
-        # https://github.com/pytest-dev/pytest/blob/4.0.1/src/_pytest/python.py#L1447),
-        # thus the following code should be reviewed later
-        marker = metafunc.definition.get_closest_marker('regions')
-
-        metafunc.parametrize(
-            "regions",
-            marker.args[0] if marker else [[r] for r in AWS_REGIONS_TEST],
-            ids=marker.kwargs.get('ids', idfn) if marker else idfn)
-
-
 #########
 # TESTS #
 #########
@@ -338,19 +294,27 @@ def test_AwsEC2Launcher_spot(ec2ctx, ec2_resources, max_price_factor):
     instances = launcher.launch(
         params, 1, region=ec2ctx.region, ec2=ec2ctx.resource)
 
-    assert len(instances) == 1
-
     launcher.wait()
 
     for inst in instances:
         check_instance_params(inst, params, ec2ctx.client, price)
 
 
-def test_AwsEC2Terminator(ec2ctx, ec2_resources):
+@pytest.mark.regions([['us-east-1', 'us-east-2']])
+def test_AwsEC2Terminator_wait(ec2ctxs, ec2_resources):
     launcher = AwsEC2Launcher()
     terminator = AwsEC2Terminator()
 
-    instances = launcher.launch(ec2_resources, 2, ec2=ec2ctx.resource)
+    instances = []
+
+    params = ec2_resources._replace(market_spot=False)
+
+    for ctx in ec2ctxs:
+        _instances = launcher.launch(
+            params, 1, region=ctx.region, ec2=ctx.resource)
+        assert len(_instances) == 1
+        instances += _instances
+
     launcher.wait()
 
     for instance in instances:
@@ -364,6 +328,35 @@ def test_AwsEC2Terminator(ec2ctx, ec2_resources):
         assert instance.state['Name'] == 'terminated'
 
 
+@pytest.mark.regions([['us-east-1'], ['us-east-2']])
+def test_AwsEC2Terminator_spot(ec2ctx, ec2_resources):
+    launcher = AwsEC2Launcher()
+    terminator = AwsEC2Terminator()
+
+    params = ec2_resources._replace(market_spot=True, spot_max_price=None)
+    instances = launcher.launch(
+        params, 1, region=ec2ctx.region, ec2=ec2ctx.resource)
+
+    launcher.wait()
+
+    for instance in instances:
+        terminator.terminate(instance)
+
+    for instance in instances:
+        assert instance.spot_instance_request_id is not None
+        spot_params = ec2ctx.client.describe_spot_instance_requests(
+            SpotInstanceRequestIds=[instance.spot_instance_request_id])
+        # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html#get-spot-instance-bid-status
+        assert (spot_params['SpotInstanceRequests'][0]['State'] in
+                ('closed', 'cancelled'))
+        assert (spot_params['SpotInstanceRequests'][0]['Status']['Code'] in (
+            'instance-terminated-by-user',
+            'request-canceled-and-instance-running'
+        ))
+
+    terminator.wait()
+
+
 @pytest.mark.regions([['us-east-1']])
 def test_find_instances(ec2ctx, ec2_resources):
     launcher = AwsEC2Launcher()