Skip to content

Commit

Permalink
Merge pull request grpc#24757 from voidzcy/bugfix/create_extra_backen…
Browse files Browse the repository at this point in the history 
…d_services_for_circuit_breaking_test

The backend service API is not able to unset backend service's circuit breakers configuration, which causes trouble for restoring target proxy's validate_for_proxyless flag. So we create two dedicated backend services (still using existing instance groups) for performing the circuit breaking test. Existing backend services, which are shared among other test cases will not be polluted.
  • Loading branch information
@voidzcy
voidzcy authored Nov 17, 2020
2 parents 20c350f + 972978c commit 4fb1d56
Showing 1 changed file with 81 additions and 37 deletions.
118 changes: 81 additions & 37 deletions tools/run_tests/run_xds_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1077,51 +1077,79 @@ def test_header_matching(gcp, original_backend_service, instance_group,
patch_backend_service(gcp, alternate_backend_service, [])


def test_circuit_breaking(gcp, original_backend_service, instance_group,
alternate_backend_service, same_zone_instance_group):
def test_circuit_breaking(gcp, original_backend_service, extra_backend_service,
more_extra_backend_service, instance_group,
same_zone_instance_group):
'''
Since backend service circuit_breakers configuration cannot be unset,
which causes trouble for restoring validate_for_proxy flag in target
proxy/global forwarding rule. This test uses dedicated backend sevices.
The url_map and backend services undergoes the following state changes:
Before test:
original_backend_service -> [instance_group]
extra_backend_service -> []
more_extra_backend_service -> []
url_map -> [original_backend_service]
In test:
extra_backend_service (with circuit_breakers) -> [instance_group]
more_extra_backend_service (with circuit_breakers) -> [same_zone_instance_group]
url_map -> [extra_backend_service, more_extra_backend_service]
After test:
original_backend_service -> [instance_group]
extra_backend_service (with circuit_breakers) -> []
more_extra_backend_service (with circuit_breakers) -> []
url_map -> [original_backend_service]
'''
logger.info('Running test_circuit_breaking')
# The config validation for proxyless doesn't allow setting
# circuit_breakers. Disable validate validate_for_proxyless
# for this test. This can be removed when validation
# accepts circuit_breakers.
logger.info('disabling validate_for_proxyless in target proxy')
set_validate_for_proxyless(gcp, False)
original_backend_service_max_requests = 500
alternate_backend_service_max_requests = 1000
extra_backend_service_max_requests = 500
more_extra_backend_service_max_requests = 1000
patch_backend_service(
gcp,
original_backend_service, [instance_group],
circuit_breakers={'maxRequests': original_backend_service_max_requests})
logger.info('Waiting for original backends to become healthy')
wait_for_healthy_backends(gcp, original_backend_service, instance_group)
extra_backend_service, [instance_group],
circuit_breakers={'maxRequests': extra_backend_service_max_requests})
logger.info('Waiting for extra backends to become healthy')
wait_for_healthy_backends(gcp, extra_backend_service, instance_group)
patch_backend_service(gcp,
alternate_backend_service, [same_zone_instance_group],
more_extra_backend_service,
[same_zone_instance_group],
circuit_breakers={
'maxRequests':
alternate_backend_service_max_requests
more_extra_backend_service_max_requests
})
logger.info('Waiting for alternate to become healthy')
wait_for_healthy_backends(gcp, alternate_backend_service,
logger.info('Waiting for more extra backend to become healthy')
wait_for_healthy_backends(gcp, more_extra_backend_service,
same_zone_instance_group)
original_backend_instances = get_instance_names(gcp, instance_group)
alternate_backend_instances = get_instance_names(gcp,
same_zone_instance_group)
extra_backend_instances = get_instance_names(gcp, instance_group)
more_extra_backend_instances = get_instance_names(gcp,
same_zone_instance_group)
route_rules = [
{
'priority': 0,
# UnaryCall -> original_backend_service
# UnaryCall -> extra_backend_service
'matchRules': [{
'fullPathMatch': '/grpc.testing.TestService/UnaryCall'
}],
'service': original_backend_service.url
'service': extra_backend_service.url
},
{
'priority': 1,
# EmptyCall -> alternate_backend_service
# EmptyCall -> more_extra_backend_service
'matchRules': [{
'fullPathMatch': '/grpc.testing.TestService/EmptyCall'
}],
'service': alternate_backend_service.url
'service': more_extra_backend_service.url
},
]
try:
Expand All @@ -1132,11 +1160,11 @@ def test_circuit_breaking(gcp, original_backend_service, instance_group,
], [])
logger.info('Patching url map with %s', route_rules)
patch_url_map_backend_service(gcp,
original_backend_service,
extra_backend_service,
route_rules=route_rules)
logger.info('Waiting for traffic to go to all backends')
wait_until_all_rpcs_go_to_given_backends(
original_backend_instances + alternate_backend_instances,
extra_backend_instances + more_extra_backend_instances,
_WAIT_FOR_STATS_SEC)

# Make all calls keep-open.
Expand All @@ -1148,33 +1176,32 @@ def test_circuit_breaking(gcp, original_backend_service, instance_group,
(messages_pb2.ClientConfigureRequest.RpcType.EMPTY_CALL,
'rpc-behavior', 'keep-open')])
wait_until_rpcs_in_flight(
'UNARY_CALL',
(_WAIT_FOR_BACKEND_SEC +
int(original_backend_service_max_requests / args.qps)),
original_backend_service_max_requests, 1)
'UNARY_CALL', (_WAIT_FOR_BACKEND_SEC +
int(extra_backend_service_max_requests / args.qps)),
extra_backend_service_max_requests, 1)
wait_until_rpcs_in_flight(
'EMPTY_CALL',
(_WAIT_FOR_BACKEND_SEC +
int(alternate_backend_service_max_requests / args.qps)),
alternate_backend_service_max_requests, 1)
int(more_extra_backend_service_max_requests / args.qps)),
more_extra_backend_service_max_requests, 1)

# Increment circuit breakers max_requests threshold.
original_backend_service_max_requests = 800
extra_backend_service_max_requests = 800
patch_backend_service(gcp,
original_backend_service, [instance_group],
extra_backend_service, [instance_group],
circuit_breakers={
'maxRequests':
original_backend_service_max_requests
extra_backend_service_max_requests
})
wait_until_rpcs_in_flight(
'UNARY_CALL',
(_WAIT_FOR_BACKEND_SEC +
int(original_backend_service_max_requests / args.qps)),
original_backend_service_max_requests, 1)
'UNARY_CALL', (_WAIT_FOR_BACKEND_SEC +
int(extra_backend_service_max_requests / args.qps)),
extra_backend_service_max_requests, 1)
finally:
patch_url_map_backend_service(gcp, original_backend_service)
patch_backend_service(gcp, original_backend_service, [instance_group])
patch_backend_service(gcp, alternate_backend_service, [])
patch_backend_service(gcp, extra_backend_service, [])
patch_backend_service(gcp, extra_backend_service, [])
set_validate_for_proxyless(gcp, True)


Expand Down Expand Up @@ -1908,6 +1935,13 @@ def __init__(self, compute, alpha_compute, project):
firewall_name = _BASE_FIREWALL_RULE_NAME + gcp_suffix
backend_service_name = _BASE_BACKEND_SERVICE_NAME + gcp_suffix
alternate_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-alternate' + gcp_suffix
# TODO(chengyuanzhang): Dedicated backend services created for circuit
# breaking test. Other tests should avoid using them. Once the issue
# for unsetting backend service circuit breakers is resolved or
# configuring backend service circuit breakers is enabled for config
# validation, these dedicated backend services can be eliminated.
extra_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-extra' + gcp_suffix
more_extra_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-more-extra' + gcp_suffix
url_map_name = _BASE_URL_MAP_NAME + gcp_suffix
service_host_name = _BASE_SERVICE_HOST + gcp_suffix
target_proxy_name = _BASE_TARGET_PROXY_NAME + gcp_suffix
Expand All @@ -1929,6 +1963,10 @@ def __init__(self, compute, alpha_compute, project):
backend_service = get_backend_service(gcp, backend_service_name)
alternate_backend_service = get_backend_service(
gcp, alternate_backend_service_name)
extra_backend_service = get_backend_service(gcp,
extra_backend_service_name)
more_extra_backend_service = get_backend_service(
gcp, more_extra_backend_service_name)
get_url_map(gcp, url_map_name)
get_target_proxy(gcp, target_proxy_name)
get_global_forwarding_rule(gcp, forwarding_rule_name)
Expand All @@ -1943,6 +1981,10 @@ def __init__(self, compute, alpha_compute, project):
backend_service = add_backend_service(gcp, backend_service_name)
alternate_backend_service = add_backend_service(
gcp, alternate_backend_service_name)
extra_backend_service = add_backend_service(gcp,
extra_backend_service_name)
more_extra_backend_service = add_backend_service(
gcp, more_extra_backend_service_name)
create_url_map(gcp, url_map_name, backend_service, service_host_name)
create_target_proxy(gcp, target_proxy_name)
potential_service_ports = list(args.service_port_range)
Expand Down Expand Up @@ -2089,8 +2131,10 @@ def __init__(self, compute, alpha_compute, project):
alternate_backend_service,
same_zone_instance_group)
elif test_case == 'circuit_breaking':
test_circuit_breaking(gcp, backend_service, instance_group,
alternate_backend_service,
test_circuit_breaking(gcp, backend_service,
extra_backend_service,
more_extra_backend_service,
instance_group,
same_zone_instance_group)
else:
logger.error('Unknown test case: %s', test_case)
Expand Down

0 comments on commit 4fb1d56

Please sign in to comment.