Skip to content

Commit

Permalink
Meta index/parts/schema log and comments (vesoft-inc#3918)
Browse files Browse the repository at this point in the history 
* change GetIndex succeed log to VLOG(2)

* meta/processor/index change log level

* meta/process/parts change log level

* change log level

* change log level, read info to VLOG(2), errors to LOG(INFO)

* change back to INFO

* add comments for meta index

* add more comments

* fix typo

* fix comments

* fix rebase

* change part info vlog

Co-authored-by: Nivras <[email protected]>
Co-authored-by: Sophie <[email protected]>
  • Loading branch information
@pengweisong @Nivras @Sophie-Xie
3 people authored Feb 28, 2022
1 parent 65e396b commit e89491a
Show file tree
Hide file tree
Showing 57 changed files with 550 additions and 279 deletions.
48 changes: 26 additions & 22 deletions src/meta/processors/index/CreateEdgeIndexProcessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,28 +22,29 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) {
columnSet.emplace(field.get_name());
}
if (fields.size() != columnSet.size()) {
LOG(ERROR) << "Conflict field in the edge index.";
LOG(INFO) << "Conflict field in the edge index.";
handleErrorCode(nebula::cpp2::ErrorCode::E_CONFLICT);
onFinished();
return;
}

// A maximum of 16 columns are allowed in the index
if (columnSet.size() > maxIndexLimit) {
LOG(ERROR) << "The number of index columns exceeds maximum limit " << maxIndexLimit;
LOG(INFO) << "The number of index columns exceeds maximum limit " << maxIndexLimit;
handleErrorCode(nebula::cpp2::ErrorCode::E_CONFLICT);
onFinished();
return;
}

folly::SharedMutex::ReadHolder rHolder(LockUtils::snapshotLock());
folly::SharedMutex::WriteHolder holder(LockUtils::lock());
// check if the space already exist index has the same index name
auto ret = getIndexID(space, indexName);
if (nebula::ok(ret)) {
if (req.get_if_not_exists()) {
handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED);
} else {
LOG(ERROR) << "Create Edge Index Failed: " << indexName << " has existed";
LOG(INFO) << "Create Edge Index Failed: " << indexName << " has existed";
handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED);
}
resp_.id_ref() = to(nebula::value(ret), EntryType::INDEX);
Expand All @@ -52,8 +53,8 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) {
} else {
auto retCode = nebula::error(ret);
if (retCode != nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND) {
LOG(ERROR) << "Create Edge Index Failed, index name " << indexName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Create Edge Index Failed, index name " << indexName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
handleErrorCode(retCode);
onFinished();
return;
Expand All @@ -63,8 +64,8 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) {
auto edgeTypeRet = getEdgeType(space, edgeName);
if (!nebula::ok(edgeTypeRet)) {
auto retCode = nebula::error(edgeTypeRet);
LOG(ERROR) << "Create Edge Index Failed, Edge " << edgeName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Create Edge Index Failed, Edge " << edgeName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
handleErrorCode(retCode);
onFinished();
return;
Expand All @@ -75,14 +76,15 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) {
auto iterRet = doPrefix(prefix);
if (!nebula::ok(iterRet)) {
auto retCode = nebula::error(iterRet);
LOG(ERROR) << "Edge indexes prefix failed, space id " << space
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Edge indexes prefix failed, space id " << space
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
handleErrorCode(retCode);
onFinished();
return;
}
auto checkIter = nebula::value(iterRet).get();

// check if the index having same fields exist
while (checkIter->valid()) {
auto val = checkIter->val();
auto item = MetaKeyUtils::parseIndex(val);
Expand All @@ -104,13 +106,14 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) {
auto schemaRet = getLatestEdgeSchema(space, edgeType);
if (!nebula::ok(schemaRet)) {
auto retCode = nebula::error(schemaRet);
LOG(ERROR) << "Get edge schema failed, space id " << space << " edgeName " << edgeName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Get edge schema failed, space id " << space << " edgeName " << edgeName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
handleErrorCode(retCode);
onFinished();
return;
}

// check if all given fields valid to create index on
auto latestEdgeSchema = std::move(nebula::value(schemaRet));
const auto& schemaCols = latestEdgeSchema.get_columns();
std::vector<cpp2::ColumnDef> columns;
Expand All @@ -120,62 +123,63 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) {
});

if (iter == schemaCols.end()) {
LOG(ERROR) << "Field " << field.get_name() << " not found in Edge " << edgeName;
LOG(INFO) << "Field " << field.get_name() << " not found in Edge " << edgeName;
handleErrorCode(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND);
onFinished();
return;
}
cpp2::ColumnDef col = *iter;
if (col.type.get_type() == nebula::cpp2::PropertyType::DURATION) {
LOG(ERROR) << "Field " << field.get_name() << " in Edge " << edgeName << " is duration."
<< "It can not be indexed.";
LOG(INFO) << "Field " << field.get_name() << " in Edge " << edgeName << " is duration."
<< "It can not be indexed.";
handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM);
onFinished();
return;
}
if (col.type.get_type() == nebula::cpp2::PropertyType::FIXED_STRING) {
if (*col.type.get_type_length() > MAX_INDEX_TYPE_LENGTH) {
LOG(ERROR) << "Unsupport index type lengths greater than " << MAX_INDEX_TYPE_LENGTH << " : "
<< field.get_name();
LOG(INFO) << "Unsupported index type lengths greater than " << MAX_INDEX_TYPE_LENGTH
<< " : " << field.get_name();
handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED);
onFinished();
return;
}
} else if (col.type.get_type() == nebula::cpp2::PropertyType::STRING) {
if (!field.type_length_ref().has_value()) {
LOG(ERROR) << "No type length set : " << field.get_name();
LOG(INFO) << "No type length set : " << field.get_name();
handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM);
onFinished();
return;
}
if (*field.get_type_length() > MAX_INDEX_TYPE_LENGTH) {
LOG(ERROR) << "Unsupport index type lengths greater than " << MAX_INDEX_TYPE_LENGTH << " : "
<< field.get_name();
LOG(INFO) << "Unsupported index type lengths greater than " << MAX_INDEX_TYPE_LENGTH
<< " : " << field.get_name();
handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED);
onFinished();
return;
}
col.type.type_ref() = nebula::cpp2::PropertyType::FIXED_STRING;
col.type.type_length_ref() = *field.get_type_length();
} else if (field.type_length_ref().has_value()) {
LOG(ERROR) << "No need to set type length : " << field.get_name();
LOG(INFO) << "No need to set type length : " << field.get_name();
handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM);
onFinished();
return;
} else if (col.type.get_type() == nebula::cpp2::PropertyType::GEOGRAPHY && fields.size() > 1) {
// TODO(jie): Support joint index for geography
LOG(ERROR) << "Only support to create index on a single geography column currently";
LOG(INFO) << "Only support to create index on a single geography column currently";
handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED);
onFinished();
return;
}
columns.emplace_back(col);
}

// add index item
std::vector<kvstore::KV> data;
auto edgeIndexRet = autoIncrementIdInSpace(space);
if (!nebula::ok(edgeIndexRet)) {
LOG(ERROR) << "Create edge index failed: Get edge index ID failed";
LOG(INFO) << "Create edge index failed: Get edge index ID failed";
handleErrorCode(nebula::error(edgeIndexRet));
onFinished();
return;
Expand Down
13 changes: 13 additions & 0 deletions src/meta/processors/index/CreateEdgeIndexProcessor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,19 @@
namespace nebula {
namespace meta {

/**
* @brief Create edge index on given space with:
* 1. given index name
* 2. given edge name
* 3. fields in the given edge
*
* It will first check if all the parameters valid, then create index item
* to save these meta without building the index on existing data actually.
* But when new data inserted, it will generate index data according to the index
* created before.
* Or user could call `rebuild index` to generate indexes for all existing data.
*
*/
class CreateEdgeIndexProcessor : public BaseProcessor<cpp2::ExecResp> {
public:
static CreateEdgeIndexProcessor* instance(kvstore::KVStore* kvstore) {
Expand Down
48 changes: 26 additions & 22 deletions src/meta/processors/index/CreateTagIndexProcessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,28 +22,30 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
columnSet.emplace(field.get_name());
}
if (fields.size() != columnSet.size()) {
LOG(ERROR) << "Conflict field in the tag index.";
LOG(INFO) << "Conflict field in the tag index.";
handleErrorCode(nebula::cpp2::ErrorCode::E_CONFLICT);
onFinished();
return;
}

// A maximum of 16 columns are allowed in the index.
if (columnSet.size() > maxIndexLimit) {
LOG(ERROR) << "The number of index columns exceeds maximum limit " << maxIndexLimit;
LOG(INFO) << "The number of index columns exceeds maximum limit " << maxIndexLimit;
handleErrorCode(nebula::cpp2::ErrorCode::E_CONFLICT);
onFinished();
return;
}

folly::SharedMutex::ReadHolder rHolder(LockUtils::snapshotLock());
folly::SharedMutex::WriteHolder holder(LockUtils::lock());

// check if the space has the index with the same name
auto ret = getIndexID(space, indexName);
if (nebula::ok(ret)) {
if (req.get_if_not_exists()) {
handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED);
} else {
LOG(ERROR) << "Create Tag Index Failed: " << indexName << " has existed";
LOG(INFO) << "Create Tag Index Failed: " << indexName << " has existed";
handleErrorCode(nebula::cpp2::ErrorCode::E_EXISTED);
}
resp_.id_ref() = to(nebula::value(ret), EntryType::INDEX);
Expand All @@ -52,8 +54,8 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
} else {
auto retCode = nebula::error(ret);
if (retCode != nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND) {
LOG(ERROR) << "Create Tag Index Failed, index name " << indexName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Create Tag Index Failed, index name " << indexName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
handleErrorCode(retCode);
onFinished();
return;
Expand All @@ -63,8 +65,8 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
auto tagIDRet = getTagId(space, tagName);
if (!nebula::ok(tagIDRet)) {
auto retCode = nebula::error(tagIDRet);
LOG(ERROR) << "Create Tag Index Failed, Tag " << tagName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Create Tag Index Failed, Tag " << tagName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
handleErrorCode(retCode);
onFinished();
return;
Expand All @@ -75,14 +77,15 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
auto iterRet = doPrefix(prefix);
if (!nebula::ok(iterRet)) {
auto retCode = nebula::error(iterRet);
LOG(ERROR) << "Tag indexes prefix failed, space id " << space
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Tag indexes prefix failed, space id " << space
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
handleErrorCode(retCode);
onFinished();
return;
}
auto checkIter = nebula::value(iterRet).get();

// check if the tag index with the same fields exist
while (checkIter->valid()) {
auto val = checkIter->val();
auto item = MetaKeyUtils::parseIndex(val);
Expand All @@ -103,13 +106,14 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
auto schemaRet = getLatestTagSchema(space, tagID);
if (!nebula::ok(schemaRet)) {
auto retCode = nebula::error(schemaRet);
LOG(ERROR) << "Get tag schema failed, space id " << space << " tagName " << tagName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Get tag schema failed, space id " << space << " tagName " << tagName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
handleErrorCode(retCode);
onFinished();
return;
}

// check if all the given fields valid for building index in latest tag schema
auto latestTagSchema = std::move(nebula::value(schemaRet));
const auto& schemaCols = latestTagSchema.get_columns();
std::vector<cpp2::ColumnDef> columns;
Expand All @@ -118,51 +122,51 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
return field.get_name() == col.get_name();
});
if (iter == schemaCols.end()) {
LOG(ERROR) << "Field " << field.get_name() << " not found in Tag " << tagName;
LOG(INFO) << "Field " << field.get_name() << " not found in Tag " << tagName;
handleErrorCode(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND);
onFinished();
return;
}
cpp2::ColumnDef col = *iter;
if (col.type.get_type() == nebula::cpp2::PropertyType::DURATION) {
LOG(ERROR) << "Field " << field.get_name() << " in Tag " << tagName << " is duration."
<< "It can not be indexed.";
LOG(INFO) << "Field " << field.get_name() << " in Tag " << tagName << " is duration."
<< "It can not be indexed.";
handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM);
onFinished();
return;
}
if (col.type.get_type() == nebula::cpp2::PropertyType::FIXED_STRING) {
if (*col.type.get_type_length() > MAX_INDEX_TYPE_LENGTH) {
LOG(ERROR) << "Unsupport index type lengths greater than " << MAX_INDEX_TYPE_LENGTH << " : "
<< field.get_name();
LOG(INFO) << "Unsupported index type lengths greater than " << MAX_INDEX_TYPE_LENGTH
<< " : " << field.get_name();
handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED);
onFinished();
return;
}
} else if (col.type.get_type() == nebula::cpp2::PropertyType::STRING) {
if (!field.type_length_ref().has_value()) {
LOG(ERROR) << "No type length set : " << field.get_name();
LOG(INFO) << "No type length set : " << field.get_name();
handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM);
onFinished();
return;
}
if (*field.get_type_length() > MAX_INDEX_TYPE_LENGTH) {
LOG(ERROR) << "Unsupport index type lengths greater than " << MAX_INDEX_TYPE_LENGTH << " : "
<< field.get_name();
LOG(INFO) << "Unsupported index type lengths greater than " << MAX_INDEX_TYPE_LENGTH
<< " : " << field.get_name();
handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED);
onFinished();
return;
}
col.type.type_ref() = nebula::cpp2::PropertyType::FIXED_STRING;
col.type.type_length_ref() = *field.get_type_length();
} else if (field.type_length_ref().has_value()) {
LOG(ERROR) << "No need to set type length : " << field.get_name();
LOG(INFO) << "No need to set type length : " << field.get_name();
handleErrorCode(nebula::cpp2::ErrorCode::E_INVALID_PARM);
onFinished();
return;
} else if (col.type.get_type() == nebula::cpp2::PropertyType::GEOGRAPHY && fields.size() > 1) {
// TODO(jie): Support joint index for geography
LOG(ERROR) << "Only support to create index on a single geography column currently";
LOG(INFO) << "Only support to create index on a single geography column currently";
handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED);
onFinished();
return;
Expand All @@ -173,7 +177,7 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
std::vector<kvstore::KV> data;
auto tagIndexRet = autoIncrementIdInSpace(space);
if (!nebula::ok(tagIndexRet)) {
LOG(ERROR) << "Create tag index failed : Get tag index ID failed";
LOG(INFO) << "Create tag index failed : Get tag index ID failed";
handleErrorCode(nebula::error(tagIndexRet));
onFinished();
return;
Expand Down
8 changes: 8 additions & 0 deletions src/meta/processors/index/CreateTagIndexProcessor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,14 @@
namespace nebula {
namespace meta {

/**
* @brief Create tag index on given tag fields. This processor has similar logic with the
* CreateEdgeIndexProcessor. It only check if the tag index could be built
* and then create a tag index item to save all the tag index meta.
* After tag index created, any vertex inserted satisfying the tag and fields
* will trigger corresponding index built.
*
*/
class CreateTagIndexProcessor : public BaseProcessor<cpp2::ExecResp> {
public:
static CreateTagIndexProcessor* instance(kvstore::KVStore* kvstore) {
Expand Down
14 changes: 7 additions & 7 deletions src/meta/processors/index/DropEdgeIndexProcessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ void DropEdgeIndexProcessor::process(const cpp2::DropEdgeIndexReq& req) {
if (req.get_if_exists()) {
retCode = nebula::cpp2::ErrorCode::SUCCEEDED;
} else {
LOG(ERROR) << "Drop Edge Index Failed, index name " << indexName
<< " not exists in Space: " << spaceID;
LOG(INFO) << "Drop Edge Index Failed, index name " << indexName
<< " not exists in Space: " << spaceID;
}
} else {
LOG(ERROR) << "Drop Edge Index Failed, index name " << indexName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Drop Edge Index Failed, index name " << indexName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
}
handleErrorCode(retCode);
onFinished();
Expand All @@ -45,16 +45,16 @@ void DropEdgeIndexProcessor::process(const cpp2::DropEdgeIndexReq& req) {
if (retCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) {
retCode = nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND;
}
LOG(ERROR) << "Drop Edge Index Failed: SpaceID " << spaceID << " Index Name: " << indexName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
LOG(INFO) << "Drop Edge Index Failed: SpaceID " << spaceID << " Index Name: " << indexName
<< " error: " << apache::thrift::util::enumNameSafe(retCode);
handleErrorCode(retCode);
onFinished();
return;
}

auto item = MetaKeyUtils::parseIndex(nebula::value(indexItemRet));
if (item.get_schema_id().getType() != nebula::cpp2::SchemaID::Type::edge_type) {
LOG(ERROR) << "Drop Edge Index Failed: Index Name " << indexName << " is not EdgeIndex";
LOG(INFO) << "Drop Edge Index Failed: Index Name " << indexName << " is not EdgeIndex";
resp_.code_ref() = nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND;
onFinished();
return;
Expand Down
Loading

0 comments on commit e89491a

Please sign in to comment.