[Enhancement] Dump tablet metadata of cloud native table (StarRocks#2…

…2756)

Add a new HTTP interface /api/cloudnative/dump_tablet_metadata/<tablet_id>
to dump the tablet metadata in json format, for all tablets within the same
directory of the specified tablet.

Signed-off-by: sduzh <[email protected]>

be/src/http/CMakeLists.txt

@@ -25,6 +25,7 @@ add_library(Webserver STATIC
   http_channel.cpp
   http_status.cpp
   http_parser.cpp
+  http_stream_channel.cpp
   web_page_handler.cpp
   monitor_action.cpp
   default_path_handlers.cpp
@@ -47,7 +48,8 @@ add_library(Webserver STATIC
   action/update_config_action.cpp
   action/runtime_filter_cache_action.cpp
   action/query_cache_action.cpp
-  action/pipeline_blocking_drivers_action.cpp)
+  action/pipeline_blocking_drivers_action.cpp
+  action/lake/dump_tablet_metadata_action.cpp)
 
 # target_link_libraries(Webserver pthread dl Util)
 #ADD_BE_TEST(integer-array-test)

be/src/http/action/lake/dump_tablet_metadata_action.cpp

@@ -0,0 +1,108 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "http/action/lake/dump_tablet_metadata_action.h"
+
+#include <event2/buffer.h>
+#include <event2/http.h>
+#include <json2pb/pb_to_json.h>
+
+#include "fs/fs.h"
+#include "http/http_channel.h"
+#include "http/http_headers.h"
+#include "http/http_request.h"
+#include "http/http_status.h"
+#include "http/http_stream_channel.h"
+#include "runtime/exec_env.h"
+#include "storage/lake/filenames.h"
+#include "storage/lake/join_path.h"
+#include "storage/lake/location_provider.h"
+#include "storage/lake/tablet_manager.h"
+#include "util/string_parser.hpp"
+
+namespace starrocks::lake {
+
+static const char* const kParamPretty = "pretty";
+
+void DumpTabletMetadataAction::handle(HttpRequest* req) {
+    std::string tablet_id_str = req->param("TabletId");
+    StringParser::ParseResult result;
+    auto tablet_id = StringParser::string_to_int<int64_t>(tablet_id_str.data(), tablet_id_str.size(), &result);
+    if (result != StringParser::PARSE_SUCCESS) {
+        HttpChannel::send_error(req, HttpStatus::BAD_REQUEST);
+        return;
+    }
+    const auto& pretty_str = req->param(kParamPretty);
+    bool pretty = true;
+    if (!pretty_str.empty()) {
+        pretty = StringParser::string_to_bool(pretty_str.data(), pretty_str.size(), &result);
+        if (result != StringParser::PARSE_SUCCESS) {
+            HttpChannel::send_error(req, HttpStatus::BAD_REQUEST);
+            return;
+        }
+    }
+
+    TabletManager* tablet_mgr = _exec_env->lake_tablet_manager();
+    if (tablet_mgr == nullptr) {
+        HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, "Not built with --use-staros");
+        return;
+    }
+
+    auto location = tablet_mgr->location_provider()->metadata_root_location(tablet_id);
+    auto fs_or = FileSystem::CreateSharedFromString(location);
+    if (!fs_or.ok()) {
+        HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, fs_or.status().to_string());
+        return;
+    }
+    auto fs = std::move(fs_or).value();
+
+    HttpStreamChannel response(req);
+    response.start();
+    response.write("[\n");
+    bool first_object = true;
+    auto st = fs->iterate_dir(location, [&](std::string_view name) {
+        if (is_tablet_metadata(name)) {
+            if (!first_object) {
+                response.write(",\n");
+            } else {
+                first_object = false;
+            }
+            auto path = join_path(location, name);
+            auto metadata_or = tablet_mgr->get_tablet_metadata(path, false);
+            if (!metadata_or.ok() && !metadata_or.status().is_not_found()) {
+                response.write("{\"error\": \"").write(metadata_or.status().to_string()).write("\"}");
+            } else if (metadata_or.ok()) {
+                auto metadata = std::move(metadata_or).value();
+                json2pb::Pb2JsonOptions options;
+                options.pretty_json = pretty;
+                std::string json;
+                std::string error;
+                if (!json2pb::ProtoMessageToJson(*metadata, &json, options, &error)) {
+                    response.write("{\"error\": \"").write(error).write("\"}");
+                } else {
+                    response.write(json);
+                }
+            }
+        }
+        return true;
+    });
+
+    if (!st.ok()) {
+        response.write("{\"error\": \"").write(st.to_string()).write("\"}");
+    }
+
+    response.write("\n]\n");
+}
+
+} // namespace starrocks::lake

be/src/http/action/lake/dump_tablet_metadata_action.h

@@ -0,0 +1,40 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "gutil/macros.h"
+#include "http/http_handler.h"
+
+namespace starrocks {
+class ExecEnv;
+class HttpRequest;
+} // namespace starrocks
+
+namespace starrocks::lake {
+
+class DumpTabletMetadataAction : public HttpHandler {
+public:
+    explicit DumpTabletMetadataAction(ExecEnv* exec_env) : _exec_env(exec_env) {}
+    ~DumpTabletMetadataAction() override = default;
+
+    DISALLOW_COPY_AND_MOVE(DumpTabletMetadataAction);
+
+    void handle(HttpRequest* req) override;
+
+private:
+    ExecEnv* _exec_env;
+};
+
+} // namespace starrocks::lake

be/src/http/http_stream_channel.cpp

@@ -0,0 +1,52 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "http/http_stream_channel.h"
+
+#include <event2/buffer.h>
+#include <event2/http.h>
+
+#include "http/http_request.h"
+
+namespace starrocks {
+
+HttpStreamChannel::HttpStreamChannel(HttpRequest* req) : _req(req), _buffer(nullptr) {}
+
+HttpStreamChannel::~HttpStreamChannel() {
+    end();
+}
+
+HttpStreamChannel& HttpStreamChannel::start() {
+    CHECK(_buffer == nullptr);
+    _buffer = evbuffer_new();
+    evhttp_send_reply_start(_req->get_evhttp_request(), 200, "OK");
+    return *this;
+}
+
+HttpStreamChannel& HttpStreamChannel::write(const void* chunk, size_t size) {
+    CHECK(_buffer != nullptr);
+    evbuffer_add(_buffer, chunk, size);
+    evhttp_send_reply_chunk(_req->get_evhttp_request(), _buffer);
+    return *this;
+}
+
+void HttpStreamChannel::end() {
+    if (_buffer != nullptr) {
+        evbuffer_free(_buffer);
+        evhttp_send_reply_end(_req->get_evhttp_request());
+        _buffer = nullptr;
+    }
+}
+
+} // namespace starrocks

be/src/http/http_stream_channel.h

@@ -0,0 +1,47 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string_view>
+
+#include "gutil/macros.h"
+#include "http/http_request.h"
+
+struct evbuffer;
+
+namespace starrocks {
+
+class HttpStreamChannel {
+public:
+    explicit HttpStreamChannel(HttpRequest* req);
+
+    ~HttpStreamChannel();
+
+    DISALLOW_COPY_AND_MOVE(HttpStreamChannel);
+
+    HttpStreamChannel& start();
+
+    HttpStreamChannel& write(std::string_view chunk) { return write(chunk.data(), chunk.size()); }
+
+    HttpStreamChannel& write(const void* chunk, size_t size);
+
+    void end();
+
+private:
+    HttpRequest* _req;
+    evbuffer* _buffer;
+};
+
+} // namespace starrocks

be/src/service/service_be/http_service.cpp

@@ -41,6 +41,7 @@
 #include "http/action/compaction_action.h"
 #include "http/action/greplog_action.h"
 #include "http/action/health_action.h"
+#include "http/action/lake/dump_tablet_metadata_action.h"
 #include "http/action/meta_action.h"
 #include "http/action/metrics_action.h"
 #include "http/action/pipeline_blocking_drivers_action.h"
@@ -242,6 +243,13 @@ Status HttpServiceBE::start() {
     _ev_http_server->register_handler(HttpMethod::GET, "/greplog", greplog_action);
     _http_handlers.emplace_back(greplog_action);
 
+#ifdef USE_STAROS
+    auto* lake_dump_metadata_action = new lake::DumpTabletMetadataAction(_env);
+    _ev_http_server->register_handler(HttpMethod::GET, "/api/cloudnative/dump_tablet_metadata/{TabletId}",
+                                      lake_dump_metadata_action);
+    _http_handlers.emplace_back(lake_dump_metadata_action);
+#endif
+
     RETURN_IF_ERROR(_ev_http_server->start());
     return Status::OK();
 }