Merge pull request ceph#50403 from yuvalif/wip-yuval-dont-start-notif…

…-thread rgw: don't start the bucket notification thread in radosgw-admin reviewewd-by: cbodley
mitsu-ko · Mar 7, 2023 · 430edd3 · 430edd3
2 parents 5841654 + a94f5b3
commit 430edd3
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 50 deletions.
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
@@ -1065,7 +1065,9 @@ void RGWRados::finalize()
   delete reshard;
   delete index_completion_manager;
 
-  rgw::notify::shutdown();
+  if (run_notification_thread) {
+    rgw::notify::shutdown();
+  }
 }
 
 /** 
@@ -1314,9 +1316,12 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp)
   }
 
   index_completion_manager = new RGWIndexCompletionManager(this);
-  ret = rgw::notify::init(cct, driver, dpp);
-  if (ret < 0 ) {
-    ldpp_dout(dpp, 1) << "ERROR: failed to initialize notification manager" << dendl;
+
+  if (run_notification_thread) {
+    ret = rgw::notify::init(cct, driver, dpp);
+    if (ret < 0 ) {
+      ldpp_dout(dpp, 1) << "ERROR: failed to initialize notification manager" << dendl;
+    }
   }
 
   return ret;

diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
@@ -355,49 +355,50 @@ class RGWRados
 		    bool mostly_omap);
 
 
-  ceph::mutex lock = ceph::make_mutex("rados_timer_lock");
-  SafeTimer *timer;
-
-  rgw::sal::RadosStore* driver = nullptr;
-  RGWGC *gc = nullptr;
-  RGWLC *lc;
-  RGWObjectExpirer *obj_expirer;
-  bool use_gc_thread;
-  bool use_lc_thread;
-  bool quota_threads;
-  bool run_sync_thread;
-  bool run_reshard_thread;
-
-  RGWMetaNotifier *meta_notifier;
-  RGWDataNotifier *data_notifier;
-  RGWMetaSyncProcessorThread *meta_sync_processor_thread;
-  RGWSyncTraceManager *sync_tracer = nullptr;
+  ceph::mutex lock{ceph::make_mutex("rados_timer_lock")};
+  SafeTimer* timer{nullptr};
+
+  rgw::sal::RadosStore* driver{nullptr};
+  RGWGC* gc{nullptr};
+  RGWLC* lc{nullptr};
+  RGWObjectExpirer* obj_expirer{nullptr};
+  bool use_gc_thread{false};
+  bool use_lc_thread{false};
+  bool quota_threads{false};
+  bool run_sync_thread{false};
+  bool run_reshard_thread{false};
+  bool run_notification_thread{false};
+
+  RGWMetaNotifier* meta_notifier{nullptr};
+  RGWDataNotifier* data_notifier{nullptr};
+  RGWMetaSyncProcessorThread* meta_sync_processor_thread{nullptr};
+  RGWSyncTraceManager* sync_tracer{nullptr};
   std::map<rgw_zone_id, RGWDataSyncProcessorThread *> data_sync_processor_threads;
 
   boost::optional<rgw::BucketTrimManager> bucket_trim;
-  RGWSyncLogTrimThread *sync_log_trimmer{nullptr};
+  RGWSyncLogTrimThread* sync_log_trimmer{nullptr};
 
-  ceph::mutex meta_sync_thread_lock = ceph::make_mutex("meta_sync_thread_lock");
-  ceph::mutex data_sync_thread_lock = ceph::make_mutex("data_sync_thread_lock");
+  ceph::mutex meta_sync_thread_lock{ceph::make_mutex("meta_sync_thread_lock")};
+  ceph::mutex data_sync_thread_lock{ceph::make_mutex("data_sync_thread_lock")};
 
   librados::IoCtx root_pool_ctx;      // .rgw
 
-  double inject_notify_timeout_probability = 0;
-  unsigned max_notify_retries = 0;
+  double inject_notify_timeout_probability{0.0};
+  unsigned max_notify_retries{0};
 
   friend class RGWWatcher;
 
-  ceph::mutex bucket_id_lock = ceph::make_mutex("rados_bucket_id");
+  ceph::mutex bucket_id_lock{ceph::make_mutex("rados_bucket_id")};
 
   // This field represents the number of bucket index object shards
-  uint32_t bucket_index_max_shards;
+  uint32_t bucket_index_max_shards{0};
 
   std::string get_cluster_fsid(const DoutPrefixProvider *dpp, optional_yield y);
 
   int get_obj_head_ref(const DoutPrefixProvider *dpp, const rgw_placement_rule& target_placement_rule, const rgw_obj& obj, rgw_rados_ref *ref);
   int get_obj_head_ref(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_rados_ref *ref);
   int get_system_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref);
-  uint64_t max_bucket_id;
+  uint64_t max_bucket_id{0};
 
   int get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx& rctx,
 			   RGWBucketInfo& bucket_info, const rgw_obj& obj,
@@ -416,26 +417,26 @@ class RGWRados
   void cls_obj_check_prefix_exist(librados::ObjectOperation& op, const std::string& prefix, bool fail_if_exist);
   void cls_obj_check_mtime(librados::ObjectOperation& op, const real_time& mtime, bool high_precision_time, RGWCheckMTimeType type);
 protected:
-  CephContext *cct;
+  CephContext* cct{nullptr};
 
   librados::Rados rados;
 
   using RGWChainedCacheImpl_bucket_info_entry = RGWChainedCacheImpl<bucket_info_entry>;
-  RGWChainedCacheImpl_bucket_info_entry *binfo_cache;
+  RGWChainedCacheImpl_bucket_info_entry* binfo_cache{nullptr};
 
-  tombstone_cache_t *obj_tombstone_cache;
+  tombstone_cache_t* obj_tombstone_cache{nullptr};
 
   librados::IoCtx gc_pool_ctx;        // .rgw.gc
   librados::IoCtx lc_pool_ctx;        // .rgw.lc
   librados::IoCtx objexp_pool_ctx;
   librados::IoCtx reshard_pool_ctx;
   librados::IoCtx notif_pool_ctx;     // .rgw.notif
 
-  bool pools_initialized;
+  bool pools_initialized{false};
 
-  RGWQuotaHandler *quota_handler;
+  RGWQuotaHandler* quota_handler{nullptr};
 
-  RGWCoroutinesManagerRegistry *cr_registry;
+  RGWCoroutinesManagerRegistry* cr_registry{nullptr};
 
   RGWSyncModuleInstanceRef sync_module;
   bool writeable_zone{false};
@@ -447,19 +448,9 @@ class RGWRados
   bool use_datacache{false};
 
   int get_obj_head_ioctx(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::IoCtx *ioctx);
+
 public:
-  RGWRados(): timer(NULL),
-               gc(NULL), lc(NULL), obj_expirer(NULL), use_gc_thread(false), use_lc_thread(false), quota_threads(false),
-               run_sync_thread(false), run_reshard_thread(false), meta_notifier(NULL),
-               data_notifier(NULL), meta_sync_processor_thread(NULL),
-               bucket_index_max_shards(0),
-               max_bucket_id(0), cct(NULL),
-               binfo_cache(NULL), obj_tombstone_cache(nullptr),
-               pools_initialized(false),
-               quota_handler(NULL),
-               cr_registry(NULL),
-               pctl(&ctl),
-               reshard(NULL) {}
+  RGWRados() = default;
 
   RGWRados& set_use_cache(bool status) {
     use_cache = status;
@@ -512,6 +503,11 @@ class RGWRados
     run_reshard_thread = _run_reshard_thread;
     return *this;
   }
+
+  RGWRados& set_run_notification_thread(bool _run_notification_thread) {
+    run_notification_thread = _run_notification_thread;
+    return *this;
+  }
 
   librados::IoCtx* get_lc_pool_ctx() {
     return &lc_pool_ctx;
@@ -531,15 +527,15 @@ class RGWRados
   RGWServices svc;
   RGWCtl ctl;
 
-  RGWCtl *pctl{nullptr};
+  RGWCtl* const pctl{&ctl};
 
   /**
    * AmazonS3 errors contain a HostId string, but is an opaque base64 blob; we
    * try to be more transparent. This has a wrapper so we can update it when zonegroup/zone are changed.
    */
   std::string host_id;
 
-  RGWReshard *reshard;
+  RGWReshard* reshard{nullptr};
   std::shared_ptr<RGWReshardWait> reshard_wait;
 
   virtual ~RGWRados() = default;

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
@@ -4242,6 +4242,7 @@ int main(int argc, const char **argv)
 					false,
 					false,
 					false,
+                                        false,
 					need_cache && g_conf()->rgw_cache_enabled,
 					need_gc);
     }

diff --git a/src/rgw/rgw_appmain.cc b/src/rgw/rgw_appmain.cc
@@ -222,6 +222,7 @@ void rgw::AppMain::init_storage()
           run_quota,
           run_sync,
           g_conf().get_val<bool>("rgw_dynamic_resharding"),
+          true, // run notification thread
           g_conf()->rgw_cache_enabled);
 
 } /* init_storage */

diff --git a/src/rgw/rgw_object_expirer.cc b/src/rgw/rgw_object_expirer.cc
@@ -83,7 +83,7 @@ int main(const int argc, const char **argv)
   DriverManager::Config cfg;
   cfg.store_name = "rados";
   cfg.filter_name = "none";
-  driver = DriverManager::get_storage(&dp, g_ceph_context, cfg, false, false, false, false, false);
+  driver = DriverManager::get_storage(&dp, g_ceph_context, cfg, false, false, false, false, false, false);
   if (!driver) {
     std::cerr << "couldn't init storage provider" << std::endl;
     return EIO;

diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc
@@ -121,6 +121,7 @@ void RGWRealmReloader::reload()
 				   cct->_conf->rgw_enable_quota_threads,
 				   cct->_conf->rgw_run_sync_thread,
 				   cct->_conf.get_val<bool>("rgw_dynamic_resharding"),
+                                   true, // run notification thread
 				   cct->_conf->rgw_cache_enabled);
 
     ldpp_dout(&dp, 1) << "Creating new driver" << dendl;

diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc
@@ -100,6 +100,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
 						     bool quota_threads,
 						     bool run_sync_thread,
 						     bool run_reshard_thread,
+                                                     bool run_notification_thread,
 						     bool use_cache,
 						     bool use_gc)
 {
@@ -117,6 +118,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
                 .set_run_quota_threads(quota_threads)
                 .set_run_sync_thread(run_sync_thread)
                 .set_run_reshard_thread(run_reshard_thread)
+                .set_run_notification_thread(run_notification_thread)
                 .init_begin(cct, dpp) < 0) {
       delete driver;
       return nullptr;
@@ -143,6 +145,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
                 .set_run_quota_threads(quota_threads)
                 .set_run_sync_thread(run_sync_thread)
                 .set_run_reshard_thread(run_reshard_thread)
+                .set_run_notification_thread(run_notification_thread)
                 .init_begin(cct, dpp) < 0) {
       delete driver;
       return nullptr;

diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
@@ -1576,13 +1576,15 @@ class DriverManager {
 				      bool quota_threads,
 				      bool run_sync_thread,
 				      bool run_reshard_thread,
+				      bool run_notification_thread,
 				      bool use_cache = true,
 				      bool use_gc = true) {
     rgw::sal::Driver* driver = init_storage_provider(dpp, cct, cfg, use_gc_thread,
 						   use_lc_thread,
 						   quota_threads,
 						   run_sync_thread,
 						   run_reshard_thread,
+                                                   run_notification_thread,
 						   use_cache, use_gc);
     return driver;
   }
@@ -1601,6 +1603,7 @@ class DriverManager {
 						bool quota_threads,
 						bool run_sync_thread,
 						bool run_reshard_thread,
+                                                bool run_notification_thread,
 						bool use_metadata_cache,
 						bool use_gc);
   /** Initialize a new raw Driver */
-Original file line number
+Diff line change
@@ Expand Up / @@ -4242,6 +4242,7 @@ int main(int argc, const char **argv) @@
     					false,
     					false,
     					false,
+                                            false,
     					need_cache && g_conf()->rgw_cache_enabled,
     					need_gc);
         }
@@ Expand Down @@