Skip to content

Commit

Permalink
Merge pull request ceph#38897 from Daniel-Pivonka/mgroktostop
Browse files Browse the repository at this point in the history 
mgr/cephadm: ok-to-stop for mgr

Reviewed-by: Sebastian Wagner <[email protected]>
  • Loading branch information
@tchaikov
tchaikov authored Feb 8, 2021
2 parents b0eb012 + 96dcb15 commit 3d01222
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 76 deletions.
29 changes: 25 additions & 4 deletions src/pybind/mgr/cephadm/services/cephadmservice.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import errno
import json
import re
import logging
Expand Down Expand Up @@ -231,7 +232,7 @@ def ok_to_stop(self, daemon_ids: List[str], force: bool = False) -> HandleComman
logger.info(out)
return HandleCommandResult(r.retval, out, r.stderr)

def _enough_daemons_to_stop(self, daemon_type: str, daemon_ids: List[str], service: str, low_limit: int) -> Tuple[bool, str]:
def _enough_daemons_to_stop(self, daemon_type: str, daemon_ids: List[str], service: str, low_limit: int, alert: bool = False) -> Tuple[bool, str]:
# Provides a warning about if it possible or not to stop <n> daemons in a service
names = [f'{daemon_type}.{d_id}' for d_id in daemon_ids]
number_of_running_daemons = len(
Expand All @@ -247,9 +248,14 @@ def plural(count: int) -> str:
daemon_count = "only" if number_of_running_daemons == 1 else number_of_running_daemons
left_count = "no" if num_daemons_left == 0 else num_daemons_left

out = (f'WARNING: Stopping {len(daemon_ids)} out of {number_of_running_daemons} daemons in {service} service. '
f'Service will not be operational with {left_count} {plural(num_daemons_left)} left. '
f'At least {low_limit} {plural(low_limit)} must be running to guarantee service. ')
if alert:
out = (f'ALERT: Cannot stop {names} in {service} service. '
f'Not enough remaining {service} daemons. '
f'Please deploy at least {low_limit + 1} {service} daemons before stopping {names}. ')
else:
out = (f'WARNING: Stopping {len(daemon_ids)} out of {number_of_running_daemons} daemons in {service} service. '
f'Service will not be operational with {left_count} {plural(num_daemons_left)} left. '
f'At least {low_limit} {plural(low_limit)} must be running to guarantee service. ')
return True, out

def pre_remove(self, daemon: DaemonDescription) -> None:
Expand Down Expand Up @@ -521,6 +527,21 @@ def mgr_map_has_standby(self) -> bool:
num = len(mgr_map.get('standbys'))
return bool(num)

def ok_to_stop(self, daemon_ids: List[str], force: bool = False) -> HandleCommandResult:
# ok to stop if there is more than 1 mgr and not trying to stop the active mgr

warn, warn_message = self._enough_daemons_to_stop(self.TYPE, daemon_ids, 'Mgr', 1, True)
if warn:
return HandleCommandResult(-errno.EBUSY, '', warn_message)

mgr_daemons = self.mgr.cache.get_daemons_by_type(self.TYPE)
active = self.get_active_daemon(mgr_daemons).daemon_id
if active in daemon_ids:
warn_message = 'ALERT: Cannot stop active Mgr daemon, Please switch active Mgrs with \'ceph mgr fail %s\'' % active
return HandleCommandResult(-errno.EBUSY, '', warn_message)

return HandleCommandResult(0, warn_message, '')


class MdsService(CephService):
TYPE = 'mds'
Expand Down
146 changes: 74 additions & 72 deletions src/pybind/mgr/cephadm/tests/test_upgrade.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import pytest

from ceph.deployment.service_spec import ServiceSpec
from ceph.deployment.service_spec import PlacementSpec, ServiceSpec
from cephadm import CephadmOrchestrator
from cephadm.upgrade import CephadmUpgrade
from cephadm.serve import CephadmServe
Expand Down Expand Up @@ -33,78 +33,80 @@ def test_upgrade_start(cephadm_module: CephadmOrchestrator):
True
])
def test_upgrade_run(use_repo_digest, cephadm_module: CephadmOrchestrator):
with with_host(cephadm_module, 'test'):
cephadm_module.set_container_image('global', 'from_image')
if use_repo_digest:
cephadm_module.use_repo_digest = True
with with_service(cephadm_module, ServiceSpec('mgr'), CephadmOrchestrator.apply_mgr, 'test'),\
mock.patch("cephadm.module.CephadmOrchestrator.lookup_release_name",
return_value='foo'),\
mock.patch("cephadm.module.CephadmOrchestrator.version",
new_callable=mock.PropertyMock) as version_mock,\
mock.patch("cephadm.module.CephadmOrchestrator.get",
return_value={
# capture fields in both mon and osd maps
"require_osd_release": "pacific",
"min_mon_release": 16,
}):
version_mock.return_value = 'ceph version 18.2.1 (somehash)'
assert wait(cephadm_module, cephadm_module.upgrade_start(
'to_image', None)) == 'Initiating upgrade to to_image'

assert wait(cephadm_module, cephadm_module.upgrade_status()).target_image == 'to_image'

def _versions_mock(cmd):
return json.dumps({
'mgr': {
'ceph version 1.2.3 (asdf) blah': 1
}
})

cephadm_module._mon_command_mock_versions = _versions_mock

with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(json.dumps({
'image_id': 'image_id',
'repo_digests': ['to_image@repo_digest'],
'ceph_version': 'ceph version 18.2.3 (hash)',
}))):

cephadm_module.upgrade._do_upgrade()

assert cephadm_module.upgrade_status is not None

with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(
json.dumps([
dict(
name=list(cephadm_module.cache.daemons['test'].keys())[0],
style='cephadm',
fsid='fsid',
container_id='container_id',
container_image_id='image_id',
container_image_digests=['to_image@repo_digest'],
version='version',
state='running',
)
])
)):
CephadmServe(cephadm_module)._refresh_hosts_and_daemons()

with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(json.dumps({
'image_id': 'image_id',
'repo_digests': ['to_image@repo_digest'],
'ceph_version': 'ceph version 18.2.3 (hash)',
}))):
cephadm_module.upgrade._do_upgrade()

_, image, _ = cephadm_module.check_mon_command({
'prefix': 'config get',
'who': 'global',
'key': 'container_image',
})
with with_host(cephadm_module, 'host1'):
with with_host(cephadm_module, 'host2'):
cephadm_module.set_container_image('global', 'from_image')
if use_repo_digest:
assert image == 'to_image@repo_digest'
else:
assert image == 'to_image'
cephadm_module.use_repo_digest = True
with with_service(cephadm_module, ServiceSpec('mgr', placement=PlacementSpec(host_pattern='*', count=2)), CephadmOrchestrator.apply_mgr, ''),\
mock.patch("cephadm.module.CephadmOrchestrator.lookup_release_name",
return_value='foo'),\
mock.patch("cephadm.module.CephadmOrchestrator.version",
new_callable=mock.PropertyMock) as version_mock,\
mock.patch("cephadm.module.CephadmOrchestrator.get",
return_value={
# capture fields in both mon and osd maps
"require_osd_release": "pacific",
"min_mon_release": 16,
}):
version_mock.return_value = 'ceph version 18.2.1 (somehash)'
assert wait(cephadm_module, cephadm_module.upgrade_start(
'to_image', None)) == 'Initiating upgrade to to_image'

assert wait(cephadm_module, cephadm_module.upgrade_status()
).target_image == 'to_image'

def _versions_mock(cmd):
return json.dumps({
'mgr': {
'ceph version 1.2.3 (asdf) blah': 1
}
})

cephadm_module._mon_command_mock_versions = _versions_mock

with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(json.dumps({
'image_id': 'image_id',
'repo_digests': ['to_image@repo_digest'],
'ceph_version': 'ceph version 18.2.3 (hash)',
}))):

cephadm_module.upgrade._do_upgrade()

assert cephadm_module.upgrade_status is not None

with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(
json.dumps([
dict(
name=list(cephadm_module.cache.daemons['host1'].keys())[0],
style='cephadm',
fsid='fsid',
container_id='container_id',
container_image_id='image_id',
container_image_digests=['to_image@repo_digest'],
version='version',
state='running',
)
])
)):
CephadmServe(cephadm_module)._refresh_hosts_and_daemons()

with mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm(json.dumps({
'image_id': 'image_id',
'repo_digests': ['to_image@repo_digest'],
'ceph_version': 'ceph version 18.2.3 (hash)',
}))):
cephadm_module.upgrade._do_upgrade()

_, image, _ = cephadm_module.check_mon_command({
'prefix': 'config get',
'who': 'global',
'key': 'container_image',
})
if use_repo_digest:
assert image == 'to_image@repo_digest'
else:
assert image == 'to_image'


def test_upgrade_state_null(cephadm_module: CephadmOrchestrator):
Expand Down

0 comments on commit 3d01222

Please sign in to comment.