Merge pull request ceph#42735 from amathuria/wip-amathuria-scrub-stats

osd/scrub: Add stats to PG dump for number of objects scrubbed

Reviewed-by: Ronen Friedman <[email protected]>

PendingReleaseNotes

@@ -93,10 +93,11 @@
 
   https://docs.ceph.com/en/latest/rados/operations/placement-groups/
 
-* The ``ceph pg dump`` command now prints two additional columns:
+* The ``ceph pg dump`` command now prints three additional columns:
   `LAST_SCRUB_DURATION` shows the duration (in seconds) of the last completed scrub;
   `SCRUB_SCHEDULING` conveys whether a PG is scheduled to be scrubbed at a specified
-  time, queued for scrubbing, or being scrubbed.
+  time, queued for scrubbing, or being scrubbed;
+  `OBJECTS_SCRUBBED` shows the number of objects scrubbed in a PG after scrub begins.
 
 * A health warning will now be reported if the ``require-osd-release`` flag is not
   set to the appropriate release after a cluster upgrade.

qa/standalone/scrub/osd-scrub-test.sh

@@ -618,6 +618,43 @@ function TEST_dump_scrub_schedule() {
     wait_any_cond $pgid 10 $saved_last_stamp cond_active_dmp "WaitingActive " sched_data || return 1
 }
 
+function TEST_pg_dump_objects_scrubbed() {
+    local dir=$1
+    local poolname=test
+    local OSDS=3
+    local objects=15
+    local timeout=10
+
+    TESTDATA="testdata.$$"
+
+    setup $dir || return 1
+    run_mon $dir a --osd_pool_default_size=$OSDS || return 1
+    run_mgr $dir x || return 1
+    for osd in $(seq 0 $(expr $OSDS - 1))
+    do
+      run_osd $dir $osd || return 1
+    done
+
+    # Create a pool with a single pg
+    create_pool $poolname 1 1
+    wait_for_clean || return 1
+    poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }')
+
+    dd if=/dev/urandom of=$TESTDATA bs=1032 count=1
+    for i in `seq 1 $objects`
+    do
+        rados -p $poolname put obj${i} $TESTDATA
+    done
+    rm -f $TESTDATA
+
+    local pgid="${poolid}.0"
+    #Trigger a scrub on a PG
+    pg_scrub $pgid || return 1
+    test "$(ceph pg $pgid query | jq '.info.stats.objects_scrubbed')" '=' $objects || return 1
+
+    teardown $dir || return 1
+}
+
 main osd-scrub-test "$@"
 
 # Local Variables:

src/mon/PGMap.cc

@@ -1675,6 +1675,7 @@ void PGMap::dump_pg_stats_plain(
     tab.define_column("SNAPTRIMQ_LEN", TextTable::LEFT, TextTable::RIGHT);
     tab.define_column("LAST_SCRUB_DURATION", TextTable::LEFT, TextTable::RIGHT);
     tab.define_column("SCRUB_SCHEDULING", TextTable::LEFT, TextTable::LEFT);
+    tab.define_column("OBJECTS_SCRUBBED", TextTable::LEFT, TextTable::RIGHT);
   }
 
   for (const auto& [pg, st] : pg_stats) {
@@ -1716,6 +1717,7 @@ void PGMap::dump_pg_stats_plain(
           << st.snaptrimq_len
           << st.last_scrub_duration
           << st.dump_scrub_schedule()
+          << st.objects_scrubbed
           << TextTable::endrow;
     }
   }

src/osd/PG.h

@@ -282,6 +282,31 @@ class PG : public DoutPrefixProvider, public PeeringState::PeeringListener {
       });
   }
 
+  static void add_objects_scrubbed_count(
+    int64_t count, pg_stat_t &stats) {
+    stats.objects_scrubbed += count;
+  }
+
+  void add_objects_scrubbed_count(int64_t count) {
+    recovery_state.update_stats(
+      [=](auto &history, auto &stats) {
+	add_objects_scrubbed_count(count, stats);
+	return true;
+      });
+  }
+
+  static void reset_objects_scrubbed(pg_stat_t &stats) {
+    stats.objects_scrubbed = 0;
+  }
+
+  void reset_objects_scrubbed() {
+    recovery_state.update_stats(
+      [=](auto &history, auto &stats) {
+  reset_objects_scrubbed(stats);
+  return true;
+      });
+  }
+
   bool is_deleting() const {
     return recovery_state.is_deleting();
   }

src/osd/osd_types.cc

@@ -2850,6 +2850,7 @@ void pg_stat_t::dump(Formatter *f) const
   f->dump_stream("last_deep_scrub") << last_deep_scrub;
   f->dump_stream("last_deep_scrub_stamp") << last_deep_scrub_stamp;
   f->dump_stream("last_clean_scrub_stamp") << last_clean_scrub_stamp;
+  f->dump_int("objects_scrubbed", objects_scrubbed);
   f->dump_int("log_size", log_size);
   f->dump_int("ondisk_log_size", ondisk_log_size);
   f->dump_bool("stats_invalid", stats_invalid);
@@ -3012,14 +3013,16 @@ void pg_stat_t::encode(ceph::buffer::list &bl) const
   encode(scrub_sched_status.m_is_active, bl);
   encode((scrub_sched_status.m_is_deep==scrub_level_t::deep), bl);
   encode(scrub_sched_status.m_is_periodic, bl);
+  encode(objects_scrubbed, bl);
+
   ENCODE_FINISH(bl);
 }
 
 void pg_stat_t::decode(ceph::buffer::list::const_iterator &bl)
 {
   bool tmp;
   uint32_t old_state;
-  DECODE_START(26, bl);
+  DECODE_START(27, bl);
   decode(version, bl);
   decode(reported_seq, bl);
   decode(reported_epoch, bl);
@@ -3099,6 +3102,7 @@ void pg_stat_t::decode(ceph::buffer::list::const_iterator &bl)
       scrub_sched_status.m_is_deep = tmp ? scrub_level_t::deep : scrub_level_t::shallow;
       decode(tmp, bl);
       scrub_sched_status.m_is_periodic = tmp;
+      decode(objects_scrubbed, bl);
     }
   }
   DECODE_FINISH(bl);
@@ -3134,6 +3138,7 @@ void pg_stat_t::generate_test_instances(list<pg_stat_t*>& o)
   a.last_clean_scrub_stamp = utime_t(17, 18);
   a.last_scrub_duration = 3617;
   a.snaptrimq_len = 1048576;
+  a.objects_scrubbed = 0;
   list<object_stat_collection_t*> l;
   object_stat_collection_t::generate_test_instances(l);
   a.stats = *l.back();
@@ -3208,7 +3213,8 @@ bool operator==(const pg_stat_t& l, const pg_stat_t& r)
     l.purged_snaps == r.purged_snaps &&
     l.snaptrimq_len == r.snaptrimq_len &&
     l.last_scrub_duration == r.last_scrub_duration &&
-    l.scrub_sched_status == r.scrub_sched_status;
+    l.scrub_sched_status == r.scrub_sched_status &&
+    l.objects_scrubbed == r.objects_scrubbed;
 }
 
 // -- store_statfs_t --

src/osd/osd_types.h

@@ -2246,6 +2246,7 @@ struct pg_stat_t {
 
   int64_t log_size;
   int64_t ondisk_log_size;    // >= active_log_size
+  int64_t objects_scrubbed;
 
   std::vector<int32_t> up, acting;
   std::vector<pg_shard_t> avail_no_missing;
@@ -2286,6 +2287,7 @@ struct pg_stat_t {
       created(0), last_epoch_clean(0),
       parent_split_bits(0),
       log_size(0), ondisk_log_size(0),
+      objects_scrubbed(0),
       mapping_epoch(0),
       up_primary(-1),
       acting_primary(-1),

src/osd/scrubber/pg_scrubber.cc

@@ -950,7 +950,7 @@ void PgScrubber::on_init()
 {
   // going upwards from 'inactive'
   ceph_assert(!is_scrub_active());
-
+  m_pg->reset_objects_scrubbed();
   preemption_data.reset();
   m_pg->publish_stats_to_osd();
   m_interval_start = m_pg->get_history().same_interval_since;
@@ -1410,6 +1410,7 @@ void PgScrubber::scrub_compare_maps()
 
     dout(2) << __func__ << ": primary (" << m_pg->get_primary() << ") has "
 	    << m_primary_scrubmap.objects.size() << " items" << dendl;
+    m_pg->add_objects_scrubbed_count(m_primary_scrubmap.objects.size());
 
     ss.str("");
     ss.clear();