CurveBS: the volume creation should fail when the CurveBS cluster cap…

…acity reaches a certain watermark

Signed-off-by: Zhao ZeYu <[email protected]>

src/mds/nameserver2/chunk_allocator.h

@@ -25,6 +25,7 @@
 
 #include <stdint.h>
 #include <vector>
+#include <map>
 #include <memory>
 #include "src/mds/common/mds_define.h"
 #include "src/mds/nameserver2/idgenerator/chunk_id_generator.h"
@@ -42,6 +43,9 @@ class ChunkSegmentAllocator {
     virtual bool AllocateChunkSegment(FileType type,
         SegmentSizeType segmentSize, ChunkSizeType chunkSize,
         offset_t offset, PageFileSegment *segment) = 0;
+    virtual void GetRemainingSpaceInLogicalPool(
+        const std::vector<PoolIdType>& logicalPools,
+        std::map<PoolIdType, double>* remianingSpace) = 0;
 };
 
 
@@ -65,6 +69,13 @@ class ChunkSegmentAllocatorImpl: public ChunkSegmentAllocator {
         SegmentSizeType segmentSize, ChunkSizeType chunkSize,
         offset_t offset, PageFileSegment *segment) override;
 
+    void GetRemainingSpaceInLogicalPool(
+        const std::vector<PoolIdType>& logicalPools,
+        std::map<PoolIdType, double>* remianingSpace) {
+            return topologyChunkAllocator_->GetRemainingSpaceInLogicalPool(
+                            logicalPools, remianingSpace);
+        }
+
  private:
     std::shared_ptr<TopologyChunkAllocator> topologyChunkAllocator_;
     std::shared_ptr<ChunkIDGenerator> chunkIDGenerator_;

src/mds/nameserver2/curvefs.cpp

@@ -39,6 +39,8 @@
 using curve::common::TimeUtility;
 using curve::mds::topology::LogicalPool;
 using curve::mds::topology::LogicalPoolIdType;
+using curve::mds::topology::PhysicalPool;
+using curve::mds::topology::PhysicalPoolIdType;
 using curve::mds::topology::CopySetIdType;
 using curve::mds::topology::ChunkServer;
 using curve::mds::topology::ChunkServerStatus;
@@ -280,6 +282,22 @@ StatusCode CurveFS::CreateFile(const std::string & fileName,
         }
     }
 
+    std::vector<PoolIdType> logicalPools =
+                topology_->GetLogicalPoolInCluster();
+    std::map<PoolIdType, double> remianingSpace;
+    uint64_t allRemianingSpace = 0;
+    chunkSegAllocator_->GetRemainingSpaceInLogicalPool(logicalPools,
+                                &remianingSpace);
+
+    for (auto it = remianingSpace.begin(); it != remianingSpace.end(); it++) {
+        allRemianingSpace +=it->second;
+    }
+    if (allRemianingSpace < length) {
+        LOG(ERROR) << "CreateFile file length > LeftSize, fileName = "
+                    << fileName << ", length = " << length;
+        return StatusCode::kFileLengthNotSupported;
+    }
+
     auto ret = CheckStripeParam(stripeUnit, stripeCount);
     if (ret != StatusCode::kOK) {
         return ret;

src/mds/topology/topology_chunk_allocator.cpp

@@ -166,7 +166,30 @@ bool TopologyChunkAllocatorImpl::ChooseSingleLogicalPool(
 
     std::map<PoolIdType, double> poolWeightMap;
     std::vector<PoolIdType> poolToChoose;
-    for (auto pid : logicalPools) {
+    std::map<PoolIdType, double> poolsEnough;
+    GetRemainingSpaceInLogicalPool(logicalPools, &poolsEnough);
+    for (auto pool : poolsEnough) {
+        // choose logical pool according to its weight
+        if (ChoosePoolPolicy::kWeight == policy_) {
+            // record capacity remaining as the weight of this logicalpool
+            poolWeightMap.emplace(pool.first, pool.second);
+        } else {
+            poolToChoose.push_back(pool.first);
+        }
+    }
+    if (ChoosePoolPolicy::kWeight == policy_) {
+        return AllocateChunkPolicy::ChooseSingleLogicalPoolByWeight(
+            poolWeightMap, poolOut);
+    } else {
+        return AllocateChunkPolicy::ChooseSingleLogicalPoolRandom(
+            poolToChoose, poolOut);
+    }
+}
+
+void TopologyChunkAllocatorImpl::GetRemainingSpaceInLogicalPool(
+        const std::vector<PoolIdType>& logicalPools,
+        std::map<PoolIdType, double>* enoughSpacePools) {
+        for (auto pid : logicalPools) {
         LogicalPool lPool;
         if (!topology_->GetLogicalPool(pid, &lPool)) {
             continue;
@@ -196,25 +219,11 @@ bool TopologyChunkAllocatorImpl::ChooseSingleLogicalPool(
                   << ", diskAlloc:" << alloc
                   << ", diskRemainning:" << diskRemainning
                   << "}";
-        // choose logical pool according to its weight
-        if (ChoosePoolPolicy::kWeight == policy_) {
-            // record capacity remaining as the weight of this logicalpool
-            poolWeightMap.emplace(pid, diskRemainning);
-        } else {
-            if (diskRemainning > 0) {
-                poolToChoose.push_back(pid);
-            }
+        if (diskRemainning > 0) {
+            (*enoughSpacePools)[pid] = diskRemainning;
         }
     }
-    if (ChoosePoolPolicy::kWeight == policy_) {
-        return AllocateChunkPolicy::ChooseSingleLogicalPoolByWeight(
-            poolWeightMap, poolOut);
-    } else {
-        return AllocateChunkPolicy::ChooseSingleLogicalPoolRandom(
-            poolToChoose, poolOut);
-    }
 }
-
 bool AllocateChunkPolicy::AllocateChunkRandomInSingleLogicalPool(
     std::vector<CopySetIdType> copySetIds,
     PoolIdType logicalPoolId,

src/mds/topology/topology_chunk_allocator.h

@@ -59,6 +59,9 @@ class TopologyChunkAllocator {
         uint32_t chunkNumer,
         ChunkSizeType chunkSize,
         std::vector<CopysetIdInfo> *infos) = 0;
+    virtual void GetRemainingSpaceInLogicalPool(
+        const std::vector<PoolIdType>& logicalPools,
+        std::map<PoolIdType, double>* remianingSpace) = 0;
 };
 
 class TopologyChunkAllocatorImpl : public TopologyChunkAllocator {
@@ -109,6 +112,9 @@ class TopologyChunkAllocatorImpl : public TopologyChunkAllocator {
         uint32_t chunkNumber,
         ChunkSizeType chunkSize,
         std::vector<CopysetIdInfo> *infos) override;
+    void GetRemainingSpaceInLogicalPool(
+        const std::vector<PoolIdType>& logicalPools,
+        std::map<PoolIdType, double>* remianingSpace) override;
 
  private:
     /**

test/mds/nameserver2/curvefs_test.cpp

@@ -26,7 +26,7 @@
 #include "src/mds/nameserver2/namespace_storage.h"
 #include "src/common/timeutility.h"
 #include "src/mds/common/mds_define.h"
-
+#include "src/mds/topology/topology_item.h"
 
 #include "test/mds/nameserver2/mock/mock_namespace_storage.h"
 #include "test/mds/nameserver2/mock/mock_inode_id_generator.h"
@@ -90,8 +90,8 @@ class CurveFSTest: public ::testing::Test {
         fileInfo.set_filetype(FileType::INODE_DIRECTORY);
         fileInfo.set_owner(authOptions_.rootOwner);
         EXPECT_CALL(*storage_, GetFile(_, _, _))
-            .Times(1)
-            .WillOnce(DoAll(SetArgPointee<2>(fileInfo),
+            .Times(AtLeast(1))
+            .WillRepeatedly(DoAll(SetArgPointee<2>(fileInfo),
                 Return(StoreStatus::OK)));
 
         curvefs_->Init(storage_, inodeIdGenerator_, mockChunkAllocator_,
@@ -131,6 +131,13 @@ class CurveFSTest: public ::testing::Test {
 
 TEST_F(CurveFSTest, testCreateFile1) {
     // test parm error
+    std::map<PoolIdType, double> spacePools;
+    spacePools.insert(std::pair<PoolIdType, double>(1,
+            kMaxFileLength - kMiniFileLength));
+    EXPECT_CALL(*mockChunkAllocator_,
+        GetRemainingSpaceInLogicalPool(_, _))
+        .Times(AtLeast(1))
+        .WillRepeatedly(DoAll(SetArgPointee<1>(spacePools), Return()));
     ASSERT_EQ(curvefs_->CreateFile("/file1", "owner1", FileType::INODE_PAGEFILE,
                     kMiniFileLength - 1, 0, 0),
                     StatusCode::kFileLengthNotSupported);
@@ -143,6 +150,10 @@ TEST_F(CurveFSTest, testCreateFile1) {
                                    kMiniFileLength + 1, 0, 0),
               StatusCode::kFileLengthNotSupported);
 
+    ASSERT_EQ(curvefs_->CreateFile("/flie1", "owner1", FileType::INODE_PAGEFILE,
+                kMaxFileLength - kMiniFileLength + DefaultSegmentSize, 0, 0),
+              StatusCode::kFileLengthNotSupported);
+
     ASSERT_EQ(curvefs_->CreateFile("/", "", FileType::INODE_DIRECTORY, 0, 0, 0),
               StatusCode::kFileExists);
 
@@ -271,6 +282,13 @@ TEST_F(CurveFSTest, testCreateFile1) {
 TEST_F(CurveFSTest, testCreateStripeFile) {
     {
         // test create ok
+        std::map<PoolIdType, double> spacePools;
+        spacePools.insert(std::pair<PoolIdType, double>(1, kMaxFileLength));
+        spacePools.insert(std::pair<PoolIdType, double>(2, kMaxFileLength));
+        EXPECT_CALL(*mockChunkAllocator_,
+            GetRemainingSpaceInLogicalPool(_, _))
+         .Times(AtLeast(1))
+         .WillRepeatedly(DoAll(SetArgPointee<1>(spacePools), Return()));
         EXPECT_CALL(*storage_, GetFile(_, _, _))
         .Times(AtLeast(1))
         .WillOnce(Return(StoreStatus::KeyNotExist));

test/mds/nameserver2/fakes.h

@@ -41,6 +41,7 @@ using ::curve::common::SNAPSHOTFILEINFOKEYPREFIX;
 
 const uint64_t FACK_INODE_INITIALIZE = 0;
 const uint64_t FACK_CHUNKID_INITIALIZE = 0;
+const uint64_t FACK_FILE_INTTIALIZE = 10737418240;
 
 namespace curve {
 namespace mds {
@@ -97,6 +98,14 @@ class FackTopologyChunkAllocator: public TopologyChunkAllocator {
         }
         return true;
     }
+    void GetRemainingSpaceInLogicalPool(
+        const std::vector<PoolIdType>& logicalPools,
+        std::map<PoolIdType, double>* enoughSpacePools) override {
+            for (auto i = logicalPools.begin(); i != logicalPools.end(); i++){
+                enoughSpacePools->insert(std::pair<PoolIdType,
+                    double>(*i, 10*FACK_FILE_INTTIALIZE));
+            }
+        }
 };
 
 class FakeNameServerStorage : public NameServerStorage {

test/mds/nameserver2/mock/mock_chunk_allocate.h

@@ -24,6 +24,8 @@
 #define  TEST_MDS_NAMESERVER2_MOCK_MOCK_CHUNK_ALLOCATE_H_
 
 #include <gmock/gmock.h>
+#include <vector>
+#include <map>
 #include "src/mds/nameserver2/chunk_allocator.h"
 
 namespace curve {
@@ -36,6 +38,9 @@ class MockChunkAllocator: public ChunkSegmentAllocator {
 
     MOCK_METHOD5(AllocateChunkSegment, bool(FileType, SegmentSizeType,
       ChunkSizeType, offset_t, PageFileSegment*));
+    MOCK_METHOD2(GetRemainingSpaceInLogicalPool,
+    void(const std::vector <PoolIdType>&,
+    std::map<PoolIdType, double>*));
 };
 }  // namespace mds
 }  // namespace curve

test/mds/nameserver2/mock/mock_topology_chunk_allocator.h

@@ -26,6 +26,7 @@
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include <vector>
+#include <map>
 #include "src/mds/topology/topology_chunk_allocator.h"
 
 using ::curve::mds::topology::TopologyChunkAllocator;
@@ -45,6 +46,9 @@ class  MockTopologyChunkAllocator: public TopologyChunkAllocator {
     MOCK_METHOD4(AllocateChunkRoundRobinInSingleLogicalPool,
         bool(FileType, uint32_t,
             ChunkSizeType chunkSize, std::vector<CopysetIdInfo> *));
+    MOCK_METHOD2(GetRemainingSpaceInLogicalPool,
+        void(const std::vector <PoolIdType>&,
+        std::map<PoolIdType, double>*));
 };
 
 }  // namespace mds

test/mds/nameserver2/namespace_service_test.cpp

@@ -32,6 +32,7 @@
 #include "src/common/string_util.h"
 #include "test/mds/nameserver2/fakes.h"
 #include "test/mds/nameserver2/mock/mock_clean_manager.h"
+#include "test/mds/nameserver2/mock/mock_chunk_allocate.h"
 #include "src/mds/nameserver2/clean_manager.h"
 #include "src/mds/nameserver2/clean_core.h"
 #include "src/mds/nameserver2/clean_task_manager.h"
@@ -51,6 +52,7 @@ using ::testing::AtLeast;
 using ::testing::SetArgPointee;
 using ::testing::DoAll;
 using ::testing::Invoke;
+using ::testing::Matcher;
 
 namespace curve {
 namespace mds {
@@ -128,6 +130,7 @@ class NameSpaceServiceTest : public ::testing::Test {
         ASSERT_EQ(curveFSOptions.maxFileLength, kCurveFS.GetMaxFileLength());
         DefaultSegmentSize = kCurveFS.GetDefaultSegmentSize();
         kMiniFileLength = kCurveFS.GetMinFileLength();
+        kMaxFileLength = kCurveFS.GetMaxFileLength();
         kCurveFS.Run();
 
         std::this_thread::sleep_for(std::chrono::microseconds(
@@ -236,14 +239,15 @@ class NameSpaceServiceTest : public ::testing::Test {
     std::shared_ptr<CleanTaskManager> cleanTaskManager_;
     std::shared_ptr<CleanManager> cleanManager_;
     std::shared_ptr<MockTopology> topology_;
-    std::shared_ptr<AllocStatistic> allocStatistic_;
+    std::shared_ptr<MockAllocStatistic> allocStatistic_;
     std::shared_ptr<FileRecordManager> fileRecordManager_;
     std::shared_ptr<CurveFSService_Stub> stub_;
     struct FileRecordOptions fileRecordOptions;
     struct RootAuthOption authOptions;
     struct CurveFSOption curveFSOptions;
     uint64_t DefaultSegmentSize;
     uint64_t kMiniFileLength;
+    uint64_t kMaxFileLength;
 };
 
 TEST_F(NameSpaceServiceTest, test1) {
@@ -267,6 +271,10 @@ TEST_F(NameSpaceServiceTest, test1) {
 
     // test CreateFile
     // create /file1(owner1) , /file2(owner2), /dir/file3(owner3)
+    std::vector<PoolIdType> logicalPools{1, 2, 3};
+    EXPECT_CALL(*topology_, GetLogicalPoolInCluster(_))
+        .Times(AtLeast(1))
+        .WillRepeatedly(Return(logicalPools));
     CreateFileRequest request;
     CreateFileResponse response;
     brpc::Controller cntl;
@@ -1152,6 +1160,10 @@ TEST_F(NameSpaceServiceTest, snapshottests) {
 
 
     // test create file
+    std::vector<PoolIdType> logicalPools{1, 2, 3};
+    EXPECT_CALL(*topology_, GetLogicalPoolInCluster(_))
+        .Times(AtLeast(1))
+        .WillRepeatedly(Return(logicalPools));
     CreateFileRequest request;
     CreateFileResponse response;
 
@@ -1404,6 +1416,10 @@ TEST_F(NameSpaceServiceTest, deletefiletests) {
     CurveFSService_Stub stub(&channel);
 
     // 先创建文件/file1，目录/dir1，文件/dir1/file2
+    std::vector<PoolIdType> logicalPools{1, 2, 3};
+    EXPECT_CALL(*topology_, GetLogicalPoolInCluster(_))
+        .Times(AtLeast(1))
+        .WillRepeatedly(Return(logicalPools));
     CreateFileRequest request;
     CreateFileResponse response;
 
@@ -2103,6 +2119,10 @@ TEST_F(NameSpaceServiceTest, testRecoverFile) {
     CurveFSService_Stub stub(&channel);
 
     // create file /file1，dir /dir1 and file /dir1/file2
+    std::vector<PoolIdType> logicalPools{1, 2, 3};
+    EXPECT_CALL(*topology_, GetLogicalPoolInCluster(_))
+        .Times(AtLeast(1))
+        .WillRepeatedly(Return(logicalPools));
     CreateFileRequest createRequest;
     CreateFileResponse createResponse;
     brpc::Controller cntl;
@@ -2718,6 +2738,10 @@ TEST_F(NameSpaceServiceTest, TestDeAllocateSegment) {
 
     // create file and allocate segment
     {
+        std::vector<PoolIdType> logicalPools{1, 2, 3};
+         EXPECT_CALL(*topology_, GetLogicalPoolInCluster(_))
+         .Times(AtLeast(1))
+         .WillRepeatedly(Return(logicalPools));
         CreateFileRequest createRequest;
         CreateFileResponse createResponse;
         createRequest.set_filename(filename);