ceph-disk: OSD hotplug fixes for Centos

Two fixes for Centos 6.3 and other systems with udev versions prior to 172. The disk peristant name using the GPT UUID does not exist, so use the by_path persistent name instead for the journal symlink. The gpt label fields are not available for use in udev rules. Add ceph-disk-udev wrapper script that extracts the partition type guid from the label and calls ceph-disk-activate if it is a ceph guid type. (Bug ceph#4632) Signed-off-by: Gary Lowell <[email protected]>
jhesketh · Apr 23, 2013 · 7ad63d2 · 7ad63d2
1 parent 3dd9574
commit 7ad63d2
Show file tree

Hide file tree

Showing 7 changed files with 131 additions and 23 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -10,7 +10,8 @@ EXTRA_DIST += \
 	src/test/cli \
 	src/test/downloads \
 	udev/50-rbd.rules \
-	udev/95-ceph-osd.rules
+	udev/95-ceph-osd.rules \
+	udev/95-ceph-osd-alt.rules
 
 
 all-local:

diff --git a/ceph.spec.in b/ceph.spec.in
@@ -25,10 +25,11 @@ Requires:	librbd1 = %{version}-%{release}
 Requires:	librados2 = %{version}-%{release}
 Requires:	libcephfs1 = %{version}-%{release}
 Requires:	python
+Requires:	python-argparse
+Requires:	python-lockfile
 Requires:	cryptsetup
 Requires:	parted
 Requires:	util-linux
-Requires:	python-argparse
 Requires:	hdparm
 Requires(post):	binutils
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
@@ -301,7 +302,11 @@ chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config
 
 # udev rules
 install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules
+%if 0%{?centos}
+install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
+%else
 install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules
+%endif
 
 #set up placeholder directories
 mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/ceph
@@ -386,6 +391,7 @@ fi
 %{_sbindir}/ceph-disk
 %{_sbindir}/ceph-disk-activate
 %{_sbindir}/ceph-disk-prepare
+%{_sbindir}/ceph-disk-udev
 %{_sbindir}/ceph-create-keys
 %{_sbindir}/rcceph
 /sbin/mkcephfs

diff --git a/debian/control b/debian/control
@@ -45,6 +45,8 @@ Depends: binutils,
          gdisk,
          parted,
          python,
+         python-argparse,
+         python-lockfile,
          sdparm | hdparm,
          uuid-runtime,
          xfsprogs,

diff --git a/src/Makefile.am b/src/Makefile.am
@@ -33,6 +33,7 @@ ceph_sbin_SCRIPTS = \
 	ceph-disk \
 	ceph-disk-prepare \
 	ceph-disk-activate \
+	ceph-disk-udev \
 	ceph-create-keys
 
 sbin_SCRIPTS = \
@@ -1192,6 +1193,7 @@ EXTRA_DIST += \
 	ceph-disk \
 	ceph-disk-prepare \
 	ceph-disk-activate \
+	ceph-disk-udev \
 	ceph-create-keys \
 	mount.fuse.ceph
 

diff --git a/src/ceph-disk b/src/ceph-disk
@@ -12,6 +12,7 @@ import stat
 import sys
 import tempfile
 import uuid
+import lockfile
 
 CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026'
 
@@ -62,6 +63,7 @@ if LOG_NAME == '__main__':
     LOG_NAME = os.path.basename(sys.argv[0])
 LOG = logging.getLogger(LOG_NAME)
 
+lock = lockfile.FileLock('/var/lib/ceph/tmp/ceph-disk.lock')
 
 ###### exceptions ########
 
@@ -683,6 +685,16 @@ def zap(dev):
         raise Error(e)
 
 
+def get_udev_version():
+    version = _check_output(
+        args=[
+            'udevadm',
+            '--version',
+        ],
+    )   
+    return int(version)
+
+
 def prepare_journal_dev(
     data,
     journal,
@@ -759,9 +771,24 @@ def prepare_journal_dev(
                 ],
             )
 
-        journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format(
-            journal_uuid=journal_uuid,
-            )
+        if get_udev_version() >= 172:
+            journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format(
+                journal_uuid=journal_uuid,
+                )
+        else:
+            # udev prior to version 172 doesn't create by-partuuid directory
+            # use by-path symlink instead.  This is the third symlink returned
+            # by udevadm when queried.
+            LOG.debug('Using alternate persistant name for journal symlink')
+            symlinks = _check_output(
+                args=[
+                    'udevadm',
+                    'info',
+                    '--query=symlink',
+                    '--name={name}'.format(name=os.path.basename(journal)),
+                    ],
+                )   
+            journal_symlink='/dev/{symlink}-part{num}'.format(symlink=symlinks.split()[2], num=num)
 
         journal_dmcrypt = None
         if journal_dm_keypath:
@@ -1033,6 +1060,7 @@ def main_prepare(args):
     osd_dm_keypath = None
 
     try:
+        lock.acquire()
         if not os.path.exists(args.data):
             raise Error('data path does not exist', args.data)
 
@@ -1161,12 +1189,14 @@ def main_prepare(args):
                 )
         else:
             raise Error('not a dir or block device', args.data)
+        lock.release()
 
     except Error as e:
         if journal_dm_keypath:
             os.unlink(journal_dm_keypath)
         if osd_dm_keypath:
             os.unlink(osd_dm_keypath)
+        lock.release()
         raise e
 
 
@@ -1557,26 +1587,32 @@ def main_activate(args):
     if not os.path.exists(args.path):
         raise Error('%s does not exist', args.path)
 
-    mode = os.stat(args.path).st_mode
-    if stat.S_ISBLK(mode):
-        (cluster, osd_id) = mount_activate(
-            dev=args.path,
-            activate_key_template=args.activate_key_template,
-            init=args.mark_init,
-            )
-    elif stat.S_ISDIR(mode):
-        (cluster, osd_id) = activate_dir(
-            path=args.path,
-            activate_key_template=args.activate_key_template,
-            init=args.mark_init,
+    lock.acquire()
+    try:
+        mode = os.stat(args.path).st_mode
+        if stat.S_ISBLK(mode):
+            (cluster, osd_id) = mount_activate(
+                dev=args.path,
+                activate_key_template=args.activate_key_template,
+                init=args.mark_init,
+                )
+        elif stat.S_ISDIR(mode):
+            (cluster, osd_id) = activate_dir(
+                path=args.path,
+                activate_key_template=args.activate_key_template,
+                init=args.mark_init,
+                )
+        else:
+            raise Error('%s is not a directory or block device', args.path)
+
+        start_daemon(
+            cluster=cluster,
+            osd_id=osd_id,
             )
-    else:
-        raise Error('%s is not a directory or block device', args.path)
+        lock.release()
 
-    start_daemon(
-        cluster=cluster,
-        osd_id=osd_id,
-        )
+    except:
+        lock.release()
 
 
 

diff --git a/src/ceph-disk-udev b/src/ceph-disk-udev
@@ -0,0 +1,56 @@
+#! /bin/sh
+
+# Wrapper for the ceph udev rules.  Since older versions of udev do not support gpt label fields, this shell
+# script is invoked from the udev rule to read the needed gpt label fields and call the appropriate ceph
+# OSD functions.
+
+PARTNO=$1
+NAME=$2
+PARENT_NAME=$3
+
+# Get GPT partition type guid
+ID_PART_ENTRY_TYPE=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition GUID code" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+case $ID_PART_ENTRY_TYPE in
+
+45b0969e-9b03-4f30-b4c6-b4b80ceff106)
+    # JOURNAL_UUID
+    ;;
+
+45b0969e-9b03-4f30-b4c6-5ec00ceff106)
+    # DMCRYPT_JOURNAL_UUID
+    # Map journal if using dm-crypt
+    ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+    /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME}
+    ;;
+
+4fbd7e29-9d25-41b8-afd0-062c0ceff05d)
+    # OSD_UUID
+    # activate ceph-tagged partitions.
+    /usr/sbin/ceph-disk -v activate --mount /dev/${NAME}
+    ;;
+
+4fbd7e29-9d25-41b8-afd0-5ec00ceff05d)
+    # DMCRYPT_OSD_UUID
+    # Map data device and activate ceph-tagged partitions
+    # for dm-crypted data devices
+    ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]')
+    /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME}
+    bash -c 'while [ ! -e /dev/mapper/${ID_PART_ENTRY_UUID} ];do sleep 1; done'
+    /usr/sbin/ceph-disk-activate --mount /dev/mapper/${ID_PART_ENTRY_UUID}
+    ;;
+
+89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be)
+    # TOBE_UUID
+    ;;
+
+89c57f98-2fe5-4dc0-89c1-5ec00ceff2be)
+    # DMCRYPT_TOBE_UUID
+    ;;
+
+*)
+    # Not a Ceph device
+    ;;
+
+esac
+
+exit
diff --git a/udev/95-ceph-osd-alt.rules b/udev/95-ceph-osd-alt.rules
@@ -0,0 +1,5 @@
+#  Check gpt partion for ceph tags and activate
+ACTION=="add", SUBSYSTEM=="block", \
+  ENV{DEVTYPE}=="partition", \
+  ENV{ID_PART_TABLE_TYPE}=="gpt", \
+  RUN+="/usr/sbin/ceph-disk-udev $number $name"