Skip to content

Commit

Permalink
Merge PR ceph#57274 into main
Browse files Browse the repository at this point in the history 
* refs/pull/57274/head:
	mds: don't stall the asok thread for flush commands
	qa/quiescer: relax some timing requirements in the quiescer

Reviewed-by: Xiubo Li <[email protected]>
Reviewed-by: Patrick Donnelly <[email protected]>
Reviewed-by: Venky Shankar <[email protected]>
  • Loading branch information
@batrick
batrick committed May 16, 2024
2 parents 3e92f50 + a4dc881 commit 70ed382
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 81 deletions.
16 changes: 8 additions & 8 deletions qa/tasks/quiescer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class Quiescer(ThrasherGreenlet):
"""
The Quiescer does periodic quiescing of the configured paths, by default - the root '/'.
quiesce_timeout: [1..) default: 60
quiesce_timeout: [1..) default: 90
:: maximum time in seconds to wait for the quiesce to succeed
quiesce_factor: [0.005..0.5] default: 0.35
:: the fraction of the total runtime we want the system quiesced
Expand All @@ -42,7 +42,7 @@ class Quiescer(ThrasherGreenlet):
split_if_longer: int default: mean(min_quiesce, max_quiesce)
:: if the duration is longer than this,
it will be split into two back-to-back half durations
ops_dump_interval: [1..quiesce_timeout) default: 0.7*quiesce_timeout
ops_dump_interval: [1..quiesce_timeout) default: 0.5*quiesce_timeout
:: during the quiesce phase, the quiescer will dump current ops from all
ranks until the quiesce terminates. values outside the allowed
range (1 <= x < quiesce_timeout) disable the dump
Expand All @@ -54,7 +54,7 @@ class Quiescer(ThrasherGreenlet):

def __init__(self, ctx, fscid,
cluster_name='ceph',
quiesce_timeout=60,
quiesce_timeout=90,
quiesce_factor=0.35,
min_quiesce=10,
max_quiesce=60,
Expand Down Expand Up @@ -104,7 +104,7 @@ def __init__(self, ctx, fscid,
self.cancelations_cap = cancelations_cap

if ops_dump_interval is None:
ops_dump_interval = 0.7 * self.quiesce_timeout
ops_dump_interval = 0.5 * self.quiesce_timeout

if ops_dump_interval < 1 or ops_dump_interval >= self.quiesce_timeout:
self.logger.warn(f"ops_dump_interval ({ops_dump_interval}) is outside the valid range [1..{self.quiesce_timeout}), disabling the dump")
Expand Down Expand Up @@ -275,7 +275,7 @@ def rcinfo(rc):
rc, stdout = self.tell_quiesce_leader(
*self.roots,
"--timeout", str(self.quiesce_timeout),
"--expiration", str(duration + 60), # give us a minute to run the release command
"--expiration", str(duration + 120), # give us 2 minutes (!) to run the release command
*await_args
)
else:
Expand Down Expand Up @@ -310,8 +310,8 @@ def rcinfo(rc):
raise RuntimeError(f"Error quiescing set '{set_id}': {rcinfo(rc)}")

elapsed = round(time.time() - start_time, 1)
self.logger.info(f"Successfully quiesced set '{set_id}', quiesce took {elapsed} seconds. Will release after: {duration}")
self.sleep_unless_stopped(duration)
self.logger.info(f"Successfully quiesced set '{set_id}', quiesce took {elapsed} seconds. Will release after: {duration - elapsed}")
self.sleep_unless_stopped(duration - elapsed)

# release the root
rc, stdout = self.tell_quiesce_leader(
Expand All @@ -330,7 +330,7 @@ def rcinfo(rc):
raise RuntimeError(f"Error releasing set '{set_id}': {rcinfo(rc)}")
else:
elapsed = round(time.time() - start_time, 1)
self.logger.info(f"Successfully released set '{set_id}', seconds elapsed: {elapsed}")
self.logger.info(f"Successfully released set '{set_id}', total seconds elapsed: {elapsed}")


def _run(self):
Expand Down
111 changes: 42 additions & 69 deletions src/mds/MDSRank.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2656,6 +2656,28 @@ void MDSRankDispatcher::handle_asok_command(
CachedStackStringStream css;
bufferlist outbl;
dout(10) << __func__ << ": " << command << dendl;

struct AsyncResponse : Context {
Formatter* f;
decltype(on_finish) do_respond;
std::basic_ostringstream<char> css;

AsyncResponse(Formatter* f, decltype(on_finish)&& respond_action)
: f(f), do_respond(std::forward<decltype(on_finish)>(respond_action)) {}

void finish(int rc) override {
f->open_object_section("result");
if (!css.view().empty()) {
f->dump_string("message", css.view());
}
f->dump_int("return_code", rc);
f->close_section();

bufferlist outbl;
do_respond(rc, {}, outbl);
}
};

if (command == "dump_ops_in_flight") {
if (!op_tracker.dump_ops_in_flight(f)) {
*css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
Expand Down Expand Up @@ -2888,20 +2910,13 @@ void MDSRankDispatcher::handle_asok_command(
goto out;
}

auto respond = new AsyncResponse(f, std::move(on_finish));
finisher->queue(
new LambdaContext(
[this, on_finish, f](int r) {
command_scrub_abort(
f,
new LambdaContext(
[on_finish, f](int r) {
bufferlist outbl;
f->open_object_section("result");
f->dump_int("return_code", r);
f->close_section();
on_finish(r, {}, outbl);
}));
}));
[this, respond](int r) {
std::lock_guard l(mds_lock);
scrubstack->scrub_abort(respond);
}));
return;
} else if (command == "scrub pause") {
if (whoami != 0) {
Expand All @@ -2910,20 +2925,13 @@ void MDSRankDispatcher::handle_asok_command(
goto out;
}

auto respond = new AsyncResponse(f, std::move(on_finish));
finisher->queue(
new LambdaContext(
[this, on_finish, f](int r) {
command_scrub_pause(
f,
new LambdaContext(
[on_finish, f](int r) {
bufferlist outbl;
f->open_object_section("result");
f->dump_int("return_code", r);
f->close_section();
on_finish(r, {}, outbl);
}));
}));
[this, respond](int r) {
std::lock_guard l(mds_lock);
scrubstack->scrub_pause(respond);
}));
return;
} else if (command == "scrub resume") {
if (whoami != 0) {
Expand All @@ -2948,9 +2956,17 @@ void MDSRankDispatcher::handle_asok_command(
} else if (command == "flush_path") {
string path;
cmd_getval(cmdmap, "path", path);
command_flush_path(f, path);

std::lock_guard l(mds_lock);
mdcache->flush_dentry(path, new AsyncResponse(f, std::move(on_finish)));
return;
} else if (command == "flush journal") {
command_flush_journal(f);
auto respond = new AsyncResponse(f, std::move(on_finish));
C_Flush_Journal* flush_journal = new C_Flush_Journal(mdcache, mdlog, this, &respond->css, respond);

std::lock_guard locker(mds_lock);
flush_journal->send();
return;
} else if (command == "get subtrees") {
command_get_subtrees(f);
} else if (command == "export dir") {
Expand Down Expand Up @@ -3170,16 +3186,6 @@ void MDSRank::command_tag_path(Formatter *f,
scond.wait();
}

void MDSRank::command_scrub_abort(Formatter *f, Context *on_finish) {
std::lock_guard l(mds_lock);
scrubstack->scrub_abort(on_finish);
}

void MDSRank::command_scrub_pause(Formatter *f, Context *on_finish) {
std::lock_guard l(mds_lock);
scrubstack->scrub_pause(on_finish);
}

void MDSRank::command_scrub_resume(Formatter *f) {
std::lock_guard l(mds_lock);
int r = scrubstack->scrub_resume();
Expand All @@ -3194,39 +3200,6 @@ void MDSRank::command_scrub_status(Formatter *f) {
scrubstack->scrub_status(f);
}

void MDSRank::command_flush_path(Formatter *f, std::string_view path)
{
C_SaferCond scond;
{
std::lock_guard l(mds_lock);
mdcache->flush_dentry(path, &scond);
}
int r = scond.wait();
f->open_object_section("results");
f->dump_int("return_code", r);
f->close_section(); // results
}

// synchronous wrapper around "journal flush" asynchronous context
// execution.
void MDSRank::command_flush_journal(Formatter *f) {
ceph_assert(f != NULL);

C_SaferCond cond;
CachedStackStringStream css;
{
std::lock_guard locker(mds_lock);
C_Flush_Journal *flush_journal = new C_Flush_Journal(mdcache, mdlog, this, css.get(), &cond);
flush_journal->send();
}
int r = cond.wait();

f->open_object_section("result");
f->dump_string("message", css->strv());
f->dump_int("return_code", r);
f->close_section();
}

void MDSRank::command_get_subtrees(Formatter *f)
{
ceph_assert(f != NULL);
Expand Down
4 changes: 0 additions & 4 deletions src/mds/MDSRank.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,13 +504,9 @@ class MDSRank {
void command_tag_path(Formatter *f, std::string_view path,
std::string_view tag);
// scrub control commands
void command_scrub_abort(Formatter *f, Context *on_finish);
void command_scrub_pause(Formatter *f, Context *on_finish);
void command_scrub_resume(Formatter *f);
void command_scrub_status(Formatter *f);

void command_flush_path(Formatter *f, std::string_view path);
void command_flush_journal(Formatter *f);
void command_get_subtrees(Formatter *f);
void command_export_dir(Formatter *f,
std::string_view path, mds_rank_t dest);
Expand Down

0 comments on commit 70ed382

Please sign in to comment.