Skip to content

Commit

Permalink
Merge pull request ceph#52650 from guits/cv-refactor-osd-objectstore
Browse files Browse the repository at this point in the history 
ceph-volume: osd objectstore refactor
  • Loading branch information
@guits
guits authored Apr 15, 2024
2 parents 87dbfb6 + bcf9803 commit e4c1c9a
Show file tree
Hide file tree
Showing 38 changed files with 2,593 additions and 1,146 deletions.
31 changes: 14 additions & 17 deletions src/ceph-volume/ceph_volume/activate/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import argparse

from ceph_volume import terminal
from ceph_volume.devices.lvm.activate import Activate as LVMActivate
from ceph_volume.devices.raw.activate import Activate as RAWActivate
from ceph_volume.objectstore.lvmbluestore import LvmBlueStore as LVMActivate
from ceph_volume.objectstore.rawbluestore import RawBlueStore as RAWActivate
from ceph_volume.devices.simple.activate import Activate as SimpleActivate


Expand Down Expand Up @@ -44,27 +44,24 @@ def main(self):

# first try raw
try:
RAWActivate([]).activate(
devs=None,
start_osd_id=self.args.osd_id,
start_osd_uuid=self.args.osd_uuid,
tmpfs=not self.args.no_tmpfs,
systemd=not self.args.no_systemd,
)
raw_activate = RAWActivate([])
raw_activate.activate(None,
self.args.osd_id,
self.args.osd_uuid,
not self.args.no_tmpfs)
return
except Exception as e:
terminal.info(f'Failed to activate via raw: {e}')

# then try lvm
try:
LVMActivate([]).activate(
argparse.Namespace(
osd_id=self.args.osd_id,
osd_fsid=self.args.osd_uuid,
no_tmpfs=self.args.no_tmpfs,
no_systemd=self.args.no_systemd,
)
)
lvm_activate = LVMActivate(argparse.Namespace(
no_tmpfs=self.args.no_tmpfs,
no_systemd=self.args.no_systemd,
osd_fsid=self.args.osd_uuid))
lvm_activate.activate(None,
self.args.osd_id,
self.args.osd_uuid)
return
except Exception as e:
terminal.info(f'Failed to activate via LVM: {e}')
Expand Down
226 changes: 21 additions & 205 deletions src/ceph-volume/ceph_volume/devices/lvm/activate.py
View file
Original file line number Diff line number Diff line change
@@ -1,216 +1,20 @@
from __future__ import print_function
import argparse
import logging
import os
from textwrap import dedent
from ceph_volume import process, conf, decorators, terminal, configuration
from ceph_volume.util import system, disk
from ceph_volume.util import prepare as prepare_utils
from ceph_volume.util import encryption as encryption_utils
from ceph_volume.systemd import systemctl
from ceph_volume.api import lvm as api
from .listing import direct_report
from ceph_volume import objectstore


logger = logging.getLogger(__name__)



def get_osd_device_path(osd_lvs, device_type, dmcrypt_secret=None):
"""
``device_type`` can be one of ``db``, ``wal`` or ``block`` so that we can
query LVs on system and fallback to querying the uuid if that is not
present.
Return a path if possible, failing to do that a ``None``, since some of
these devices are optional.
"""
osd_block_lv = None
for lv in osd_lvs:
if lv.tags.get('ceph.type') == 'block':
osd_block_lv = lv
break
if osd_block_lv:
is_encrypted = osd_block_lv.tags.get('ceph.encrypted', '0') == '1'
logger.debug('Found block device (%s) with encryption: %s', osd_block_lv.name, is_encrypted)
uuid_tag = 'ceph.%s_uuid' % device_type
device_uuid = osd_block_lv.tags.get(uuid_tag)
if not device_uuid:
return None

device_lv = None
for lv in osd_lvs:
if lv.tags.get('ceph.type') == device_type:
device_lv = lv
break
if device_lv:
if is_encrypted:
encryption_utils.luks_open(dmcrypt_secret, device_lv.lv_path, device_uuid)
return '/dev/mapper/%s' % device_uuid
return device_lv.lv_path

# this could be a regular device, so query it with blkid
physical_device = disk.get_device_from_partuuid(device_uuid)
if physical_device:
if is_encrypted:
encryption_utils.luks_open(dmcrypt_secret, physical_device, device_uuid)
return '/dev/mapper/%s' % device_uuid
return physical_device

raise RuntimeError('could not find %s with uuid %s' % (device_type, device_uuid))


def activate_bluestore(osd_lvs, no_systemd=False, no_tmpfs=False):
for lv in osd_lvs:
if lv.tags.get('ceph.type') == 'block':
osd_block_lv = lv
break
else:
raise RuntimeError('could not find a bluestore OSD to activate')

is_encrypted = osd_block_lv.tags.get('ceph.encrypted', '0') == '1'
if is_encrypted and conf.dmcrypt_no_workqueue is None:
encryption_utils.set_dmcrypt_no_workqueue()
dmcrypt_secret = None
osd_id = osd_block_lv.tags['ceph.osd_id']
conf.cluster = osd_block_lv.tags['ceph.cluster_name']
osd_fsid = osd_block_lv.tags['ceph.osd_fsid']
configuration.load_ceph_conf_path(osd_block_lv.tags['ceph.cluster_name'])
configuration.load()

# mount on tmpfs the osd directory
osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
if not system.path_is_mounted(osd_path):
# mkdir -p and mount as tmpfs
prepare_utils.create_osd_path(osd_id, tmpfs=not no_tmpfs)
# XXX This needs to be removed once ceph-bluestore-tool can deal with
# symlinks that exist in the osd dir
for link_name in ['block', 'block.db', 'block.wal']:
link_path = os.path.join(osd_path, link_name)
if os.path.exists(link_path):
os.unlink(os.path.join(osd_path, link_name))
# encryption is handled here, before priming the OSD dir
if is_encrypted:
osd_lv_path = '/dev/mapper/%s' % osd_block_lv.lv_uuid
lockbox_secret = osd_block_lv.tags['ceph.cephx_lockbox_secret']
encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret)
dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid)
encryption_utils.luks_open(dmcrypt_secret, osd_block_lv.lv_path, osd_block_lv.lv_uuid)
else:
osd_lv_path = osd_block_lv.lv_path

db_device_path = get_osd_device_path(osd_lvs, 'db', dmcrypt_secret=dmcrypt_secret)
wal_device_path = get_osd_device_path(osd_lvs, 'wal', dmcrypt_secret=dmcrypt_secret)

# Once symlinks are removed, the osd dir can be 'primed again. chown first,
# regardless of what currently exists so that ``prime-osd-dir`` can succeed
# even if permissions are somehow messed up
system.chown(osd_path)
prime_command = [
'ceph-bluestore-tool', '--cluster=%s' % conf.cluster,
'prime-osd-dir', '--dev', osd_lv_path,
'--path', osd_path, '--no-mon-config']

process.run(prime_command)
# always re-do the symlink regardless if it exists, so that the block,
# block.wal, and block.db devices that may have changed can be mapped
# correctly every time
process.run(['ln', '-snf', osd_lv_path, os.path.join(osd_path, 'block')])
system.chown(os.path.join(osd_path, 'block'))
system.chown(osd_path)
if db_device_path:
destination = os.path.join(osd_path, 'block.db')
process.run(['ln', '-snf', db_device_path, destination])
system.chown(db_device_path)
system.chown(destination)
if wal_device_path:
destination = os.path.join(osd_path, 'block.wal')
process.run(['ln', '-snf', wal_device_path, destination])
system.chown(wal_device_path)
system.chown(destination)

if no_systemd is False:
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'lvm')

# enable the OSD
systemctl.enable_osd(osd_id)

# start the OSD
systemctl.start_osd(osd_id)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id)


class Activate(object):

help = 'Discover and mount the LVM device associated with an OSD ID and start the Ceph OSD'

def __init__(self, argv):
def __init__(self, argv, args=None):
self.objectstore = None
self.argv = argv

@decorators.needs_root
def activate_all(self, args):
listed_osds = direct_report()
osds = {}
for osd_id, devices in listed_osds.items():
# the metadata for all devices in each OSD will contain
# the FSID which is required for activation
for device in devices:
fsid = device.get('tags', {}).get('ceph.osd_fsid')
if fsid:
osds[fsid] = osd_id
break
if not osds:
terminal.warning('Was unable to find any OSDs to activate')
terminal.warning('Verify OSDs are present with "ceph-volume lvm list"')
return
for osd_fsid, osd_id in osds.items():
if not args.no_systemd and systemctl.osd_is_active(osd_id):
terminal.warning(
'OSD ID %s FSID %s process is active. Skipping activation' % (osd_id, osd_fsid)
)
else:
terminal.info('Activating OSD ID %s FSID %s' % (osd_id, osd_fsid))
self.activate(args, osd_id=osd_id, osd_fsid=osd_fsid)

@decorators.needs_root
def activate(self, args, osd_id=None, osd_fsid=None):
"""
:param args: The parsed arguments coming from the CLI
:param osd_id: When activating all, this gets populated with an
existing OSD ID
:param osd_fsid: When activating all, this gets populated with an
existing OSD FSID
"""
osd_id = osd_id if osd_id else args.osd_id
osd_fsid = osd_fsid if osd_fsid else args.osd_fsid

if osd_id and osd_fsid:
tags = {'ceph.osd_id': osd_id, 'ceph.osd_fsid': osd_fsid}
elif not osd_id and osd_fsid:
tags = {'ceph.osd_fsid': osd_fsid}
elif osd_id and not osd_fsid:
raise RuntimeError('could not activate osd.{}, please provide the '
'osd_fsid too'.format(osd_id))
else:
raise RuntimeError('Please provide both osd_id and osd_fsid')
lvs = api.get_lvs(tags=tags)
if not lvs:
raise RuntimeError('could not find osd.%s with osd_fsid %s' %
(osd_id, osd_fsid))

# This argument is only available when passed in directly or via
# systemd, not when ``create`` is being used
# placeholder when a new objectstore support will be added
if getattr(args, 'auto_detect_objectstore', False):
logger.info('auto detecting objectstore')
return activate_bluestore(lvs, args.no_systemd)

# explicit 'objectstore' flags take precedence
if getattr(args, 'bluestore', False):
activate_bluestore(lvs, args.no_systemd, getattr(args, 'no_tmpfs', False))
elif any('ceph.block_device' in lv.tags for lv in lvs):
activate_bluestore(lvs, args.no_systemd, getattr(args, 'no_tmpfs', False))
self.args = args

def main(self):
sub_command_help = dedent("""
Expand Down Expand Up @@ -256,6 +60,14 @@ def main(self):
action='store_true',
help='force bluestore objectstore activation',
)
parser.add_argument(
'--objectstore',
dest='objectstore',
help='The OSD objectstore.',
default='bluestore',
choices=['bluestore', 'seastore'],
type=str,
)
parser.add_argument(
'--all',
dest='activate_all',
Expand All @@ -273,11 +85,15 @@ def main(self):
action='store_true',
help='Do not use a tmpfs mount for OSD data dir'
)
if len(self.argv) == 0:
if len(self.argv) == 0 and self.args is None:
print(sub_command_help)
return
args = parser.parse_args(self.argv)
if args.activate_all:
self.activate_all(args)
if self.args is None:
self.args = parser.parse_args(self.argv)
if self.args.bluestore:
self.args.objectstore = 'bluestore'
self.objectstore = objectstore.mapping['LVM'][self.args.objectstore](args=self.args)
if self.args.activate_all:
self.objectstore.activate_all()
else:
self.activate(args)
self.objectstore.activate()
Loading

0 comments on commit e4c1c9a

Please sign in to comment.