Merge pull request ceph#36397 from dzafman/wip-39012

distinguish unfound + impossible to find, vs start some down OSDs to get Reviewed-by: Neha Ojha <[email protected]> Reviewed-by: Josh Durgin <[email protected]>
senax · Sep 18, 2020 · a97a1cd · a97a1cd
2 parents 251b940 + bb50580
commit a97a1cd
Show file tree

Hide file tree

Showing 48 changed files with 501 additions and 19 deletions.
diff --git a/PendingReleaseNotes b/PendingReleaseNotes
@@ -160,3 +160,7 @@
 
 * fs: "fs authorize" now sets MON cap to "allow <perm> fsname=<fsname>"
       instead of setting it to "allow r" all the time.
+
+* ``ceph pg #.# list_unfound`` output has been enhanced to provide
+  might_have_unfound information which indicates which OSDs may
+  contain the unfound objects.
diff --git a/doc/rados/troubleshooting/troubleshooting-pg.rst b/doc/rados/troubleshooting/troubleshooting-pg.rst
@@ -215,28 +215,55 @@ First, you can identify which objects are unfound with::
 
 .. code-block:: javascript
 
- { "offset": { "oid": "",
-      "key": "",
-      "snapid": 0,
-      "hash": 0,
-      "max": 0},
-  "num_missing": 0,
-  "num_unfound": 0,
-  "objects": [
-     { "oid": "object 1",
-       "key": "",
-       "hash": 0,
-       "max": 0 },
-     ...
-  ],
-  "more": 0}
+  {
+    "num_missing": 1,
+    "num_unfound": 1,
+    "objects": [
+        {
+            "oid": {
+                "oid": "object",
+                "key": "",
+                "snapid": -2,
+                "hash": 2249616407,
+                "max": 0,
+                "pool": 2,
+                "namespace": ""
+            },
+            "need": "43'251",
+            "have": "0'0",
+            "flags": "none",
+            "clean_regions": "clean_offsets: [], clean_omap: 0, new_object: 1",
+            "locations": [
+                "0(3)",
+                "4(2)"
+            ]
+        }
+    ],
+    "state": "NotRecovering",
+    "available_might_have_unfound": true,
+    "might_have_unfound": [
+        {
+            "osd": "2(4)",
+            "status": "osd is down"
+        }
+    ],
+    "more": false
+  }
 
 If there are too many objects to list in a single result, the ``more``
 field will be true and you can query for more.  (Eventually the
 command line tool will hide this from you, but not yet.)
 
 Second, you can identify which OSDs have been probed or might contain
-data::
+data.
+
+At the end of the listing (before ``more`` is false), ``might_have_unfound`` is provided
+when ``available_might_have_unfound`` is true.  This is equivalent to the output
+of ``ceph pg #.# query``.  This eliminates the need to use ``query`` directly.
+The ``might_have_unfound`` information given behaves the same way as described below for ``query``.
+The only difference is that OSDs that have ``already probed`` status are ignored.
+
+Use of ``query``::
 
 	ceph pg 2.4 query
 

diff --git a/qa/standalone/erasure-code/test-erasure-eio.sh b/qa/standalone/erasure-code/test-erasure-eio.sh
@@ -524,6 +524,7 @@ function TEST_ec_backfill_unfound() {
     ceph pg dump pgs
 
     rados_put $dir $poolname $objname || return 1
+    local primary=$(get_primary $poolname $objname)
 
     local -a initial_osds=($(get_osds $poolname $objname))
     local last_osd=${initial_osds[-1]}
@@ -558,8 +559,26 @@ function TEST_ec_backfill_unfound() {
     done
 
     ceph pg dump pgs
+    kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1
+    sleep 5
+
+    ceph pg dump pgs
+    ceph pg 2.0 list_unfound
+    ceph pg 2.0 query
+
     ceph pg 2.0 list_unfound | grep -q $testobj || return 1
 
+    check=$(ceph pg 2.0 list_unfound | jq ".available_might_have_unfound")
+    test "$check" == "true" || return 1
+
+    eval check=$(ceph pg 2.0 list_unfound | jq .might_have_unfound[0].status)
+    test "$check" == "osd is down" || return 1
+
+    eval check=$(ceph pg 2.0 list_unfound | jq .might_have_unfound[0].osd)
+    test "$check" == "2(4)" || return 1
+
+    activate_osd $dir ${last_osd} || return 1
+
     # Command should hang because object is unfound
     timeout 5 rados -p $poolname get $testobj $dir/CHECK
     test $? = "124" || return 1
@@ -638,8 +657,17 @@ function TEST_ec_recovery_unfound() {
     done
 
     ceph pg dump pgs
+    ceph pg 2.0 list_unfound
+    ceph pg 2.0 query
+
     ceph pg 2.0 list_unfound | grep -q $testobj || return 1
 
+    check=$(ceph pg 2.0 list_unfound | jq ".available_might_have_unfound")
+    test "$check" == "true" || return 1
+
+    check=$(ceph pg 2.0 list_unfound | jq ".might_have_unfound |  length")
+    test $check == 0 || return 1
+
     # Command should hang because object is unfound
     timeout 5 rados -p $poolname get $testobj $dir/CHECK
     test $? = "124" || return 1

diff --git a/qa/tasks/ec_lost_unfound.py b/qa/tasks/ec_lost_unfound.py
@@ -14,7 +14,7 @@ def task(ctx, config):
     """
     Test handling of lost objects on an ec pool.
 
-    A pretty rigid cluster is brought up andtested by this task
+    A pretty rigid cluster is brought up and tested by this task
     """
     if config is None:
         config = {}

diff --git a/qa/tasks/lost_unfound.py b/qa/tasks/lost_unfound.py
@@ -14,7 +14,7 @@ def task(ctx, config):
     """
     Test handling of lost objects.
 
-    A pretty rigid cluseter is brought up andtested by this task
+    A pretty rigid cluster is brought up and tested by this task
     """
     POOL = 'unfound_pool'
     if config is None:
@@ -142,6 +142,9 @@ def task(ctx, config):
             m = manager.list_pg_unfound(pg['pgid'])
             #log.info('%s' % m)
             assert m['num_unfound'] == pg['stat_sum']['num_objects_unfound']
+            assert m['available_might_have_unfound'] == True
+            assert m['might_have_unfound'][0]['osd'] == "1"
+            assert m['might_have_unfound'][0]['status'] == "osd is down"
             num_unfound=0
             for o in m['objects']:
                 if len(o['locations']) == 0:

diff --git a/qa/tasks/rep_lost_unfound_delete.py b/qa/tasks/rep_lost_unfound_delete.py
@@ -15,7 +15,7 @@ def task(ctx, config):
     """
     Test handling of lost objects.
 
-    A pretty rigid cluseter is brought up andtested by this task
+    A pretty rigid cluster is brought up and tested by this task
     """
     POOL = 'unfounddel_pool'
     if config is None:

diff --git a/src/crimson/admin/osd_admin.cc b/src/crimson/admin/osd_admin.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "crimson/admin/osd_admin.h"
 #include <string>
 #include <string_view>

diff --git a/src/crimson/admin/pg_commands.cc b/src/crimson/admin/pg_commands.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "crimson/admin/pg_commands.h"
 
 #include <memory>

diff --git a/src/crimson/osd/backfill_state.cc b/src/crimson/osd/backfill_state.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include <algorithm>
 #include <boost/type_index.hpp>
 

diff --git a/src/crimson/osd/ec_backend.cc b/src/crimson/osd/ec_backend.cc
@@ -1,3 +1,8 @@
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "ec_backend.h"
 
 #include "crimson/osd/shard_services.h"

diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "heartbeat.h"
 
 #include <boost/range/join.hpp>

diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include <sys/types.h>
 #include <unistd.h>
 

diff --git a/src/crimson/osd/objclass.cc b/src/crimson/osd/objclass.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include <cstdarg>
 #include <cstring>
 #include "common/ceph_context.h"

diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "ops_executer.h"
 
 #include <boost/range/adaptor/filtered.hpp>

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "osd.h"
 
 #include <sys/utsname.h>

diff --git a/src/crimson/osd/osd_operations/background_recovery.cc b/src/crimson/osd/osd_operations/background_recovery.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include <seastar/core/future.hh>
 
 #include "messages/MOSDOp.h"

diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include <seastar/core/future.hh>
 
 #include "messages/MOSDOp.h"

diff --git a/src/crimson/osd/osd_operations/compound_peering_request.cc b/src/crimson/osd/osd_operations/compound_peering_request.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include <seastar/core/future.hh>
 
 #include "osd/PeeringState.h"

diff --git a/src/crimson/osd/osd_operations/peering_event.cc b/src/crimson/osd/osd_operations/peering_event.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include <seastar/core/future.hh>
 
 #include "messages/MOSDPGLog.h"

diff --git a/src/crimson/osd/osd_operations/pg_advance_map.cc b/src/crimson/osd/osd_operations/pg_advance_map.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "crimson/osd/osd_operations/pg_advance_map.h"
 
 #include <boost/smart_ptr/local_shared_ptr.hpp>

diff --git a/src/crimson/osd/osd_operations/replicated_request.cc b/src/crimson/osd/osd_operations/replicated_request.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "replicated_request.h"
 
 #include "common/Formatter.h"

diff --git a/src/crimson/osd/osdmap_gate.cc b/src/crimson/osd/osdmap_gate.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "crimson/common/exception.h"
 #include "crimson/osd/osdmap_gate.h"
 #include "crimson/osd/shard_services.h"

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "pg.h"
 
 #include <functional>

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include "crimson/osd/pg_map.h"
 
 #include "crimson/osd/pg.h"

diff --git a/src/crimson/osd/pg_recovery.cc b/src/crimson/osd/pg_recovery.cc
@@ -1,6 +1,11 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#define BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
+#if !defined(BOOST_MPL_LIMIT_LIST_SIZE)
+#   define BOOST_MPL_LIMIT_LIST_SIZE 30
+#endif
+
 #include <fmt/format.h>
 #include <fmt/ostream.h>