Merge pull request ceph#10328 from jcsp/wip-ceph-mgr

Create ceph-mgr

Reviewed-by: Alfredo Deza <[email protected]>
Reviewed-by: Samuel Just <[email protected]>
Reviewed-by: Kefu Chai <[email protected]>

CMakeLists.txt

@@ -329,6 +329,14 @@ option(WITH_RADOSGW "Rados Gateway is enabled" ON)
 #option for CephFS
 option(WITH_CEPHFS "CephFS is enabled" ON)
 
+#option for Mgr
+option(WITH_MGR "ceph-mgr is enabled" ON)
+if(WITH_MGR)
+    set(Python_ADDITIONAL_VERSIONS 2.7)
+    find_package(PythonLibs 2.7 REQUIRED)
+    find_package(Boost COMPONENTS python REQUIRED)
+endif(WITH_MGR)
+
 option(WITH_THREAD_SAFE_RES_QUERY "res_query is thread safe" OFF)
 if(WITH_THREAD_SAFE_RES_QUERY)
     set(HAVE_THREAD_SAFE_RES_QUERY 1 CACHE INTERNAL "Thread safe res_query supported.")

ceph.spec.in

@@ -83,6 +83,7 @@ ExclusiveArch:  x86_64 aarch64
 #################################################################################
 Requires:       ceph-osd = %{epoch}:%{version}-%{release}
 Requires:       ceph-mds = %{epoch}:%{version}-%{release}
+Requires:       ceph-mgr = %{epoch}:%{version}-%{release}
 Requires:       ceph-mon = %{epoch}:%{version}-%{release}
 Requires(post):	binutils
 %if 0%{with cephfs_java}
@@ -95,6 +96,9 @@ BuildRequires:	selinux-policy-devel
 BuildRequires:	/usr/share/selinux/devel/policyhelp
 %endif
 BuildRequires:	boost-devel
+%if ! 0%{?suse_version}
+BuildRequires:	boost-python
+%endif
 BuildRequires:  cmake
 BuildRequires:	cryptsetup
 BuildRequires:	fuse-devel
@@ -289,6 +293,18 @@ system. One or more instances of ceph-mon form a Paxos part-time
 parliament cluster that provides extremely reliable and durable storage
 of cluster membership, configuration, and state.
 
+%package mgr
+Summary:        Ceph Manager Daemon
+License:        LGPL-2.1 and CC-BY-SA-1.0 and GPL-2.0 and BSL-1.0 and GPL-2.0-with-autoconf-exception and BSD-3-Clause and MIT
+Group:          System Environment/Base
+Requires:       ceph-base = %{version}-%{release}
+
+%description mgr
+ceph-mgr enables python modules that provide services (such as the REST
+module derived from Calamari) and expose CLI hooks.  ceph-mgr gathers
+the cluster maps, the daemon metadata, and performance counters, and
+exposes all these to the python modules.
+
 %package fuse
 Summary:	Ceph fuse-based client
 Group:		System Environment/Base
@@ -767,6 +783,7 @@ mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/tmp
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/mon
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/osd
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/mds
+mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/mgr
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/radosgw
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-osd
 mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-mds
@@ -1014,6 +1031,55 @@ if [ $FIRST_ARG -ge 1 ] ; then
   fi
 fi
 
+%files mgr
+%{_bindir}/ceph-mgr
+%{_libdir}/ceph/mgr
+%{_unitdir}/[email protected]
+%{_unitdir}/ceph-mgr.target
+%attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/mgr
+
+%post mgr
+%if 0%{?suse_version}
+if [ $1 -eq 1 ] ; then
+  /usr/bin/systemctl preset ceph-mgr@\*.service ceph-mgr.target >/dev/null 2>&1 || :
+fi
+%endif
+%if 0%{?fedora} || 0%{?rhel}
+%systemd_post ceph-mgr@\*.service ceph-mgr.target
+%endif
+if [ $1 -eq 1 ] ; then
+/usr/bin/systemctl start ceph-mgr.target >/dev/null 2>&1 || :
+fi
+
+%preun mgr
+%if 0%{?suse_version}
+%service_del_preun ceph-mgr@\*.service ceph-mgr.target
+%endif
+%if 0%{?fedora} || 0%{?rhel}
+%systemd_preun ceph-mgr@\*.service ceph-mgr.target
+%endif
+
+%postun mgr
+test -n "$FIRST_ARG" || FIRST_ARG=$1
+%if 0%{?suse_version}
+DISABLE_RESTART_ON_UPDATE="yes"
+%service_del_postun ceph-mgr@\*.service ceph-mgr.target
+%endif
+%if 0%{?fedora} || 0%{?rhel}
+%systemd_postun ceph-mgr@\*.service ceph-mgr.target
+%endif
+if [ $FIRST_ARG -ge 1 ] ; then
+  # Restart on upgrade, but only if "CEPH_AUTO_RESTART_ON_UPGRADE" is set to
+  # "yes". In any case: if units are not running, do not touch them.
+  SYSCONF_CEPH=%{_sysconfdir}/sysconfig/ceph
+  if [ -f $SYSCONF_CEPH -a -r $SYSCONF_CEPH ] ; then
+    source $SYSCONF_CEPH
+  fi
+  if [ "X$CEPH_AUTO_RESTART_ON_UPGRADE" = "Xyes" ] ; then
+    /usr/bin/systemctl try-restart ceph-mgr@\*.service > /dev/null 2>&1 || :
+  fi
+fi
+
 %files mon
 %{_bindir}/ceph-mon
 %{_bindir}/ceph-rest-api

debian/.gitignore

@@ -8,6 +8,8 @@
 /ceph-fs-common
 /ceph-mds-dbg
 /ceph-mds
+/ceph-mgr-dbg
+/ceph-mgr
 /ceph-mon-dbg
 /ceph-mon
 /ceph-osd-dbg

debian/ceph-mgr.dirs

@@ -0,0 +1 @@
+var/lib/ceph/mgr

debian/ceph-mgr.install

@@ -0,0 +1,2 @@
+usr/bin/ceph-mgr
+usr/lib/ceph/mgr

debian/ceph-mgr.postinst

@@ -0,0 +1,51 @@
+#!/bin/sh
+# vim: set noet ts=8:
+# postinst script for ceph-mgr
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+#
+# 	postinst configure <most-recently-configured-version>
+# 	old-postinst abort-upgrade <new-version>
+# 	conflictor's-postinst abort-remove in-favour <package> <new-version>
+# 	postinst abort-remove
+# 	deconfigured's-postinst abort-deconfigure in-favour <failed-install-package> <version> [<removing conflicting-package> <version>]
+#
+
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+[ -f "/etc/default/ceph" ] && . /etc/default/ceph
+[ -z "$SERVER_USER" ] && SERVER_USER=ceph
+[ -z "$SERVER_GROUP" ] && SERVER_GROUP=ceph
+
+case "$1" in
+    configure)
+	[ -x /sbin/start ] && start ceph-mgr-all || :
+
+	if ! dpkg-statoverride --list /var/lib/ceph/mgr >/dev/null
+	then
+            chown $SERVER_USER:$SERVER_GROUP /var/lib/ceph/mgr
+	fi
+    ;;
+    abort-upgrade|abort-remove|abort-deconfigure)
+	:
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
+
+

debian/ceph-mgr.prerm

@@ -0,0 +1,23 @@
+#!/bin/sh
+# vim: set noet ts=8:
+
+set -e
+
+case "$1" in
+    remove)
+	[ -x /sbin/stop ] && stop ceph-mgr-all || :
+	invoke-rc.d ceph stop mgr || {
+	    RESULT=$?
+	    if [ $RESULT != 100 ]; then
+		exit $RESULT
+	    fi
+	}
+	;;
+
+    *)
+	;;
+esac
+
+#DEBHELPER#
+
+exit 0

debian/control

@@ -34,6 +34,7 @@ Build-Depends: btrfs-tools,
 	       libboost-regex-dev,
 	       libboost-random-dev,
                libboost-iostreams-dev (>= 1.42),
+               libboost-python-dev,
                libcurl4-gnutls-dev,
                libexpat1-dev,
                libfcgi-dev,
@@ -72,6 +73,7 @@ Standards-Version: 3.9.3
 Package: ceph
 Architecture: linux-any
 Depends: ceph-mon (= ${binary:Version}),
+         ceph-mgr (= ${binary:Version}),
          ceph-osd (= ${binary:Version})
 Recommends: ceph-mds (= ${binary:Version})
 Description: distributed storage and file system
@@ -110,8 +112,8 @@ Description: common ceph daemon libraries and management tools
  block and file system storage.
  .
  This package contains the libraries and management tools that are common among
- the three Ceph server daemons (ceph-mon, ceph-osd, ceph-mds). These tools are
- necessary for creating, running, and administering a Ceph storage cluster.
+ the Ceph server daemons (ceph-mon, ceph-mgr, ceph-osd, ceph-mds). These tools
+ are necessary for creating, running, and administering a Ceph storage cluster.
 
 Package: ceph-mds
 Architecture: linux-any
@@ -140,6 +142,31 @@ Description: debugging symbols for ceph-mds
  .
  This package contains the debugging symbols for ceph-mds.
 
+Package: ceph-mgr
+Architecture: linux-any
+Depends: ceph-base (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}
+Replaces: ceph (<< 0.93-417)
+Breaks: ceph (<< 0.93-417)
+Description: metadata server for the ceph distributed file system
+ Ceph is a massively scalable, open-source, distributed
+ storage system that runs on commodity hardware and delivers object,
+ block and file system storage.
+ .
+ This package contains the manager daemon, which is used to expose high
+ level management and monitoring functionality.
+
+Package: ceph-mgr-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: ceph-mgr (= ${binary:Version}), ${misc:Depends}
+Description: debugging symbols for ceph-mgr
+ Ceph is a massively scalable, open-source, distributed
+ storage system that runs on commodity hardware and delivers object,
+ block and file system storage.
+ .
+ This package contains the debugging symbols for ceph-mgr.
+
 Package: ceph-mon
 Architecture: linux-any
 Depends: ceph-base (= ${binary:Version}),

debian/rules

@@ -65,6 +65,8 @@ override_dh_installinit:
 	install -m0644 src/upstart/rbdmap.conf debian/ceph-common/etc/init
 	install -d -m0755 debian/ceph-mds/etc/init
 	install -m0644 src/upstart/ceph-mds*.conf debian/ceph-mds/etc/init
+	install -d -m0755 debian/ceph-mgr/etc/init
+	install -m0644 src/upstart/ceph-mgr*.conf debian/ceph-mgr/etc/init
 	install -d -m0755 debian/radosgw/etc/init
 	install -m0644 src/upstart/radosgw*.conf debian/radosgw/etc/init
 	install -d -m0755 debian/rbd-mirror/etc/init
@@ -97,6 +99,11 @@ override_dh_installinit:
 	sed -i s./etc/sysconfig/./etc/default/.g debian/ceph-mds/lib/systemd/system/[email protected]
 	install -m0644 systemd/ceph-mds.target debian/ceph-mds/lib/systemd/system
 
+	install -d -m0755 debian/ceph-mgr/lib/systemd/system
+	install -m0644 systemd/[email protected] debian/ceph-mgr/lib/systemd/system
+	sed -i s./etc/sysconfig/./etc/default/.g debian/ceph-mgr/lib/systemd/system/[email protected]
+	install -m0644 systemd/ceph-mgr.target debian/ceph-mgr/lib/systemd/system
+
 	install -d -m0755 debian/radosgw/lib/systemd/system
 	install -m0644 systemd/[email protected] debian/radosgw/lib/systemd/system
 	sed -i s./etc/sysconfig/./etc/default/.g debian/radosgw/lib/systemd/system/[email protected]
@@ -115,6 +122,7 @@ override_dh_systemd_start:
 
 override_dh_strip:
 	dh_strip -pceph-mds --dbg-package=ceph-mds-dbg
+	dh_strip -pceph-mgr --dbg-package=ceph-mgr-dbg
 	dh_strip -pceph-mon --dbg-package=ceph-mon-dbg
 	dh_strip -pceph-osd --dbg-package=ceph-osd-dbg
 	dh_strip -pceph-fuse --dbg-package=ceph-fuse-dbg

doc/index.rst

@@ -98,6 +98,7 @@ about Ceph, see our `Architecture`_ section.
    cephfs/index
    rbd/rbd
    radosgw/index
+   mgr/index
    api/index
    architecture
    Development <dev/index>

doc/mgr/administrator.rst

@@ -0,0 +1,94 @@
+
+ceph-mgr administrator's guide
+==============================
+
+Setup
+-----
+
+Create an authentication key for your daemon:
+
+::
+    ceph auth get-or-create mgr.$name mon 'allow *'
+
+Place that key into ``mgr data`` path, which for a cluster "ceph"
+and mgr $name "foo" would be ``/var/lib/ceph/mgr/ceph-foo``.
+
+Start the ceph-mgr daemon:
+
+::
+    ceph-mgr -i $name
+
+Check that the mgr has come up by looking at the output
+of ``ceph status``, which should now include a mgr status line:
+
+::
+    mgr active: $name
+
+High availability
+-----------------
+
+In general, you should set up a ceph-mgr on each of the hosts
+running a ceph-mon daemon to achieve the same level of availability. 
+
+By default, whichever ceph-mgr instance comes up first will be made
+active by the monitors, and the others will be standbys.  There is
+no requirement for quorum among the ceph-mgr daemons.
+
+If the active daemon fails to send a beacon to the monitors for
+more than ``mgr beacon period`` (default 30s), then it will be replaced
+by a standby.
+
+If you want to pre-empt failover, you can explicitly mark a ceph-mgr
+daemon as failed using ``ceph mgr fail <mgr name>``.
+
+Calling module commands
+-----------------------
+
+Where a module implements command line hooks, using the Ceph CLI's
+``tell`` command to call them like this:
+
+::
+
+    ceph tell mgr <command | help>
+
+Note that it is not necessary to address a particular mgr instance,
+simply ``mgr`` will pick the current active daemon.
+
+Use the ``help`` command to get a list of available commands from all
+modules.
+
+Configuration
+-------------
+
+OPTION(mgr_module_path, OPT_STR, CEPH_PKGLIBDIR "/mgr") // where to load python modules from
+
+``mgr module path``
+
+:Description: Path to load modules from
+:Type: String
+:Default: ``"<library dir>/mgr"``
+
+``mgr modules``
+
+:Description: List of python modules to load
+:Type: String
+:Default: ``"rest"`` (Load the REST API module only)
+
+``mgr data``
+
+:Description: Path to load daemon data (such as keyring)
+:Type: String
+:Default: ``"/var/lib/ceph/mgr/$cluster-$id"``
+
+``mgr beacon period``
+
+:Description: How many seconds between mgr beacons to monitors
+:Type: Integer
+:Default: ``5``
+
+``mon mgr beacon grace``
+
+:Description: How long after last beacon should a mgr be considered failed
+:Type: Integer
+:Default: ``30``
+