Merge pull request ceph#41912 from trociny/wip-crushdiff

tools/crushdiff: new tool to test crushmap change

Reviewed-by: Neha Ojha <[email protected]>

.mailmap

@@ -416,6 +416,7 @@ My Do <[email protected]>
 My Do <[email protected]> <[email protected]>
 Mykola Golub <[email protected]> <[email protected]>
 Mykola Golub <[email protected]>
+Mykola Golub <[email protected]>
 Mykola Golub <[email protected]>
 Myna Vajha <[email protected]>
 Myoungwon Oh <[email protected]>

.organizationmap

@@ -134,6 +134,7 @@ Cloudwatt <[email protected]> Christophe Courtaut <christophe.courta
 Cloudwatt <[email protected]> Florent Flament <[email protected]>
 Cloudwatt <[email protected]> Loic Dachary <[email protected]>
 Cloudwatt <[email protected]> Sahid Orentino Ferdjaoui <[email protected]>
+Clyso GmbH <[email protected]> Mykola Golub <[email protected]>
 CohortFS, LLC <[email protected]> Casey Bodley <[email protected]>
 CohortFS, LLC <[email protected]> Matt Benjamin <[email protected]>
 Commerce Guys <[email protected]> Nikola Kotur <[email protected]>

ceph.spec.in

@@ -1565,6 +1565,7 @@ exit 0
 %{_bindir}/cephfs-data-scan
 %{_bindir}/cephfs-journal-tool
 %{_bindir}/cephfs-table-tool
+%{_bindir}/crushdiff
 %{_bindir}/rados
 %{_bindir}/radosgw-admin
 %{_bindir}/rbd
@@ -1590,6 +1591,7 @@ exit 0
 %{_mandir}/man8/ceph-syn.8*
 %{_mandir}/man8/ceph-post-file.8*
 %{_mandir}/man8/ceph.8*
+%{_mandir}/man8/crushdiff.8*
 %{_mandir}/man8/mount.ceph.8*
 %{_mandir}/man8/rados.8*
 %{_mandir}/man8/radosgw-admin.8*

debian/ceph-common.install

@@ -15,6 +15,7 @@ usr/bin/ceph-syn
 usr/bin/cephfs-data-scan
 usr/bin/cephfs-journal-tool
 usr/bin/cephfs-table-tool
+usr/bin/crushdiff
 usr/bin/rados
 usr/bin/radosgw-admin
 usr/bin/rbd
@@ -32,6 +33,7 @@ usr/share/man/man8/ceph-rbdnamer.8
 usr/share/man/man8/ceph-syn.8
 usr/share/man/man8/ceph-post-file.8
 usr/share/man/man8/ceph.8
+usr/share/man/man8/crushdiff.8
 usr/share/man/man8/mount.ceph.8
 usr/share/man/man8/rados.8
 usr/share/man/man8/radosgw-admin.8

doc/man/8/CMakeLists.txt

@@ -24,6 +24,7 @@ set(osd_srcs
   ceph-volume.rst
   ceph-volume-systemd.rst
   ceph-osd.rst
+  crushdiff.rst
   osdmaptool.rst
   ceph-bluestore-tool.rst)
 

doc/man/8/crushdiff.rst

@@ -0,0 +1,118 @@
+:orphan:
+
+.. _crushdiff:
+
+=======================================
+ crushdiff -- ceph crush map test tool
+=======================================
+
+.. program:: crushdiff
+
+Synopsis
+========
+
+| **crushdiff** [ --osdmap *osdmap* ] [ --pg-dump *pg-dump* ]
+  [ --compiled ] [ --verbose ] *command* *crushmap*
+
+
+Description
+===========
+
+**crushdiff** is a utility that lets you test the effect of a crushmap
+change: number of pgs, objects, bytes moved. This is a wrapper around
+:doc:`osdmaptool <osdmaptool>`\(8), relying on its **--test-map-pgs-dump**
+option to get the list of changed pgs. Additionally it uses pg stats
+to calculate the numbers of objects and bytes moved.
+
+By default, **crushdiff** will use the cluster current osdmap and pg
+stats, which requires access to the cluster. Though one can use the
+**--osdmap** and **--pg-dump** options to test against previously
+obtained data.
+
+Options
+=======
+
+.. option:: --compiled
+
+   The input/output crushmap is compiled. If the options is not
+   specified the expected/returned crushmap is in txt (decompiled)
+   format.
+
+.. option:: --pg-dump <pg-dump>
+
+   JSON output of **ceph pg dump**. If not specified **crushdiff**
+   will try to get data running the command itself.
+
+.. option:: --osdmap <osdmap>
+
+   The cluster osdmap, obtained with **ceph osd getmap** command. If
+   not specified **crushdiff** will try to get data running the
+   command itself.
+
+.. option:: --verbose
+
+   Produce diagnostic output.
+
+Commands
+========
+
+:command:`compare` *crushmap*
+  Compare the crushmap from *crushmap* file with the crushmap from
+  the cluster osdmap. The output will show the expected number of pgs,
+  objects, bytes moved when the new crushmap is installed.
+
+:command:`export` *crushmap*
+  Export crushmap to *crushmap* file from the cluster osdmap.
+
+:command:`import` *crushmap*
+  Import crushmap from *crushmap* file to the cluster osdmap.
+
+Example
+=======
+
+Get the current crushmap::
+
+        crushdiff export cm.txt
+
+Edit the map::
+
+        $EDITOR cm.txt
+
+Check the result::
+
+        crushdiff compare cm.txt
+
+        79/416 (18.99%) pgs affected
+        281/1392 (20.19%) objects affected
+        80/1248 (6.41%) pg shards to move
+        281/4176 (6.73%) pg object shards to move
+        730.52Mi/10.55Gi (6.76%) bytes to move
+
+When running with **--verbose** option the output will also contain
+detailed information about the affected pgs, like below::
+
+        4.3	[0, 2, 1] -> [1, 4, 2]
+        4.b	[0, 1, 3] -> [2, 1, 3]
+        4.c	[4, 0, 1] -> [4, 1, 2]
+
+i.e. a pg number, and its old and the new osd active sets.
+
+If the result is satisfactory install the updated map::
+
+        crushdiff import cm.txt
+
+
+Availability
+============
+
+**crushdiff** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please
+refer to the Ceph documentation at https://docs.ceph.com for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`crushtool <crushtool>`\(8),
+:doc:`osdmaptool <osdmaptool>`\(8),

doc/man_index.rst

@@ -26,6 +26,7 @@
    man/8/cephfs-top
    man/8/cephfs-mirror
    man/8/cephfs-shell
+   man/8/crushdiff
    man/8/crushtool
    man/8/librados-config
    man/8/monmaptool

doc/rados/man/index.rst

@@ -18,6 +18,7 @@
    ../../man/8/ceph-kvstore-tool.rst
    ../../man/8/ceph-run.rst
    ../../man/8/ceph-syn.rst
+   ../../man/8/crushdiff.rst
    ../../man/8/crushtool.rst
    ../../man/8/librados-config.rst
    ../../man/8/monmaptool.rst

qa/suites/rados/singleton-nomsgr/all/crushdiff.yaml

@@ -0,0 +1,24 @@
+openstack:
+  - volumes: # attached to each instance
+      count: 4
+      size: 10 # GB
+roles:
+- [mon.a, mgr.x, osd.0, osd.1, osd.2, osd.3, client.0]
+
+overrides:
+  ceph:
+    pre-mgr-commands:
+      - sudo ceph config set mgr mgr_pool false --force
+    log-ignorelist:
+    - but it is still running
+    - overall HEALTH_
+    - \(POOL_APP_NOT_ENABLED\)
+    - \(PG_DEGRADED\)
+
+tasks:
+- install:
+- ceph:
+- workunit:
+    clients:
+      all:
+        - rados/test_crushdiff.sh

qa/workunits/rados/test_crushdiff.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+set -ex
+
+REP_POOL=
+EC_POOL=
+TEMPDIR=
+
+OSD_NUM=$(ceph osd ls | wc -l)
+test ${OSD_NUM} -gt 0
+
+setup() {
+    local pool
+
+    TEMPDIR=`mktemp -d`
+
+    pool=test-crushdiff-rep-$$
+    ceph osd pool create ${pool} 32
+    REP_POOL=${pool}
+    rados -p ${REP_POOL} bench 5 write --no-cleanup
+
+    if [ ${OSD_NUM} -gt 3 ]; then
+        pool=test-crushdiff-ec-$$
+        ceph osd pool create ${pool} 32 32 erasure
+        EC_POOL=${pool}
+        rados -p ${EC_POOL} bench 5 write --no-cleanup
+    fi
+}
+
+cleanup() {
+    set +e
+
+    test -n "${EC_POOL}" &&
+        ceph osd pool delete "${EC_POOL}" "${EC_POOL}" \
+             --yes-i-really-really-mean-it
+    EC_POOL=
+
+    test -n "${REP_POOL}" &&
+        ceph osd pool delete "${REP_POOL}" "${REP_POOL}" \
+             --yes-i-really-really-mean-it
+    REP_POOL=
+
+    test -n "${TEMPDIR}" && rm -Rf ${TEMPDIR}
+    TEMPDIR=
+}
+
+trap "cleanup" INT TERM EXIT
+
+setup
+
+# test without crushmap modification
+
+crushdiff export ${TEMPDIR}/cm.txt --verbose
+crushdiff compare ${TEMPDIR}/cm.txt --verbose
+crushdiff import ${TEMPDIR}/cm.txt --verbose
+
+# test using a compiled crushmap
+
+crushdiff export ${TEMPDIR}/cm --compiled --verbose
+crushdiff compare ${TEMPDIR}/cm --compiled --verbose
+crushdiff import ${TEMPDIR}/cm --compiled --verbose
+
+# test using "offline" osdmap and pg-dump
+
+ceph osd getmap -o ${TEMPDIR}/osdmap
+ceph pg dump --format json > ${TEMPDIR}/pg-dump
+
+crushdiff export ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap --verbose
+crushdiff compare ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap \
+          --pg-dump ${TEMPDIR}/pg-dump --verbose | tee ${TEMPDIR}/compare.txt
+
+# test the diff is zero when the crushmap is not modified
+
+grep '^0/[0-9]* (0\.00%) pgs affected' ${TEMPDIR}/compare.txt
+grep '^0/[0-9]* (0\.00%) objects affected' ${TEMPDIR}/compare.txt
+grep '^0/[0-9]* (0\.00%) pg shards to move' ${TEMPDIR}/compare.txt
+grep '^0/[0-9]* (0\.00%) pg object shards to move' ${TEMPDIR}/compare.txt
+grep '^0\.00/.* (0\.00%) bytes to move' ${TEMPDIR}/compare.txt
+crushdiff import ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap --verbose
+
+if [ ${OSD_NUM} -gt 3 ]; then
+
+    # test the diff is non-zero when the crushmap is modified
+
+    cat ${TEMPDIR}/cm.txt >&2
+
+    weight=$(awk '/item osd\.0 weight ([0-9.]+)/ {print $4 * 3}' \
+                 ${TEMPDIR}/cm.txt)
+    test -n "${weight}"
+    sed -i -Ee 's/^(.*item osd\.0 weight )[0-9.]+/\1'${weight}'/' \
+        ${TEMPDIR}/cm.txt
+    crushdiff compare ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap \
+        --pg-dump ${TEMPDIR}/pg-dump --verbose | tee ${TEMPDIR}/compare.txt
+    grep '^[1-9][0-9]*/[0-9]* (.*%) pgs affected' ${TEMPDIR}/compare.txt
+    grep '^[1-9][0-9]*/[0-9]* (.*%) objects affected' ${TEMPDIR}/compare.txt
+    grep '^[1-9][0-9]*/[0-9]* (.*%) pg shards to move' ${TEMPDIR}/compare.txt
+    grep '^[1-9][0-9]*/[0-9]* (.*%) pg object shards to move' \
+         ${TEMPDIR}/compare.txt
+    grep '^.*/.* (.*%) bytes to move' ${TEMPDIR}/compare.txt
+    crushdiff import ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap --verbose
+fi
+
+echo OK

src/tools/CMakeLists.txt

@@ -96,6 +96,8 @@ add_executable(osdmaptool ${osdomaptool_srcs})
 target_link_libraries(osdmaptool global)
 install(TARGETS osdmaptool DESTINATION bin)
 
+install(PROGRAMS crushdiff DESTINATION bin)
+
 set(ceph-diff-sorted_srcs ceph-diff-sorted.cc)
 add_executable(ceph-diff-sorted ${ceph-diff-sorted_srcs})
 set_target_properties(ceph-diff-sorted PROPERTIES