curvefs/client: change icache and dcache (lru cache) to lru cache wit…

…h timeout

Signed-off-by: wanghai01 <[email protected]>

curvefs/conf/client.conf

@@ -86,6 +86,7 @@ fuseClient.iCacheLruSize=65536
 fuseClient.dCacheLruSize=1000000
 fuseClient.enableICacheMetrics=true
 fuseClient.enableDCacheMetrics=true
+fuseClient.lruTimeOutSec=60
 fuseClient.cto=true
 
 ### kvcache opt

curvefs/src/client/async_request_closure.cpp

@@ -101,7 +101,7 @@ void UpdateInodeAttrAndExtentClosure::Run() {
                    << ", inode: " << inode_->GetInodeId();
         inode_->MarkInodeError();
     }
-
+    inode_->ClearDirty();
     inode_->syncingVolumeExtentsMtx_.unlock();
     inode_->ReleaseSyncingInode();
 

curvefs/src/client/common/config.cpp

@@ -284,6 +284,8 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
                               &clientOption->enableICacheMetrics);
     conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
                               &clientOption->enableDCacheMetrics);
+    conf->GetValueFatalIfFail("fuseClient.lruTimeOutSec",
+                              &clientOption->lruTimeOutSec);
     conf->GetValueFatalIfFail("client.dummyServer.startPort",
                               &clientOption->dummyServerStartPort);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",

curvefs/src/client/common/config.h

@@ -199,6 +199,7 @@ struct FuseClientOption {
     uint64_t dCacheLruSize;
     bool enableICacheMetrics;
     bool enableDCacheMetrics;
+    uint32_t lruTimeOutSec;
     uint32_t dummyServerStartPort;
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;

curvefs/src/client/dentry_cache_manager.h

@@ -39,7 +39,7 @@
 #include "src/common/concurrent/name_lock.h"
 
 using ::curvefs::metaserver::Dentry;
-using ::curve::common::LRUCache;
+using ::curve::common::TimedLRUCache;
 using ::curve::common::CacheMetrics;
 
 namespace curvefs {
@@ -59,7 +59,8 @@ class DentryCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) = 0;
+    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+        uint32_t cacheTimeOutSec) = 0;
 
     virtual void InsertOrReplaceCache(const Dentry& dentry) = 0;
 
@@ -93,14 +94,15 @@ class DentryCacheManagerImpl : public DentryCacheManager {
       : metaClient_(metaClient),
         dCache_(nullptr) {}
 
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) override {
+    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+        uint32_t cacheTimeOutSec) override {
         if (enableCacheMetrics) {
             dCache_ = std::make_shared<
-                LRUCache<std::string, Dentry>>(cacheSize,
+                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize,
                     std::make_shared<CacheMetrics>("dcache"));
         } else {
             dCache_ = std::make_shared<
-                LRUCache<std::string, Dentry>>(cacheSize);
+                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize);
         }
         return CURVEFS_ERROR::OK;
     }
@@ -129,7 +131,7 @@ class DentryCacheManagerImpl : public DentryCacheManager {
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
     // key is parentId + name
-    std::shared_ptr<LRUCache<std::string, Dentry>> dCache_;
+    std::shared_ptr<TimedLRUCache<std::string, Dentry>> dCache_;
     curve::common::GenericNameLock<Mutex> nameLock_;
 };
 

curvefs/src/client/fuse_client.cpp

@@ -123,12 +123,14 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
 
     CURVEFS_ERROR ret3 =
         inodeManager_->Init(option.iCacheLruSize, option.enableICacheMetrics,
-                            option.flushPeriodSec, option.refreshDataOption);
+                            option.flushPeriodSec, option.refreshDataOption,
+                            option.lruTimeOutSec);
     if (ret3 != CURVEFS_ERROR::OK) {
         return ret3;
     }
     ret3 =
-        dentryManager_->Init(option.dCacheLruSize, option.enableDCacheMetrics);
+        dentryManager_->Init(option.dCacheLruSize, option.enableDCacheMetrics,
+            option.lruTimeOutSec);
     if (ret3 != CURVEFS_ERROR::OK) {
         return ret3;
     }

curvefs/src/client/inode_cache_manager.cpp

@@ -108,7 +108,7 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
     // get inode from cache
     bool ok = iCache_->Get(inodeId, &out);
-    if (ok && NeedUseCahce(inodeId, out->IsDirty())) {
+    if (ok && NeedUseCache(inodeId, out, false)) {
         curve::common::UniqueLock lgGuard = out->GetUniqueLock();
         if (out->GetType() == FsFileType::TYPE_FILE) {
             return CURVEFS_ERROR::OK;
@@ -141,7 +141,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
     // 1. find in icache
     std::shared_ptr<InodeWrapper> inodeWrapper;
     bool ok = iCache_->Get(inodeId, &inodeWrapper);
-    if (ok && NeedUseCahce(inodeId, inodeWrapper->IsDirty())) {
+    if (ok && NeedUseCache(inodeId, inodeWrapper, true)) {
         inodeWrapper->GetInodeAttr(out);
         return CURVEFS_ERROR::OK;
     }
@@ -178,7 +178,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
             InodeAttr tmpAttr;
             inodeWrapper->GetInodeAttr(&tmpAttr);
             attrs->emplace_back(std::move(tmpAttr));
@@ -215,7 +215,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
             InodeAttr tmpAttr;
             inodeWrapper->GetInodeAttr(&tmpAttr);
             attrs->emplace(*iter, std::move(tmpAttr));
@@ -272,7 +272,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
             xattr->emplace_back(inodeWrapper->GetXattr());
             iter = inodeIds->erase(iter);
         } else {
@@ -511,8 +511,32 @@ bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
     return iter != openedInodes_.end();
 }
 
-bool InodeCacheManagerImpl::NeedUseCahce(uint64_t inodeId, bool IsDirty) {
-    if (!FLAGS_enableCto || OpenInodeCached(inodeId) || IsDirty) {
+bool InodeCacheManagerImpl::NeedUseCache(uint64_t inodeId,
+    const std::shared_ptr<InodeWrapper> &inodeWrapper,
+    bool onlyAttr) {
+    auto lock = inodeWrapper->GetUniqueLock();
+    if (onlyAttr) {
+        if (inodeWrapper->IsDirty()) {
+            return true;
+        }
+    } else {
+        if (IsDirtyInode(inodeWrapper.get(), false)) {
+            return true;
+        }
+    }
+
+    if (((FLAGS_enableCto && OpenInodeCached(inodeId)) || !FLAGS_enableCto)
+        && !IsTimeOut(inodeWrapper)) {
+        return true;
+    }
+    return false;
+}
+
+bool InodeCacheManagerImpl::IsTimeOut(
+    const std::shared_ptr<InodeWrapper> &inodeWrapper) const {
+    uint32_t time = inodeWrapper->GetCachedTime();
+    if (cacheTimeOutSec_ > 0 &&
+        TimeUtility::GetTimeofDaySec() - time >= cacheTimeOutSec_) {
         return true;
     }
     return false;

curvefs/src/client/inode_cache_manager.h

@@ -128,7 +128,8 @@ class InodeCacheManager {
 
     virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
                                uint32_t flushPeriodSec,
-                               RefreshDataOption option) = 0;
+                               RefreshDataOption option,
+                               uint32_t cacheTimeOutSec) = 0;
 
     virtual void Run() = 0;
 
@@ -186,17 +187,20 @@ class InodeCacheManagerImpl : public InodeCacheManager,
       : metaClient_(std::make_shared<MetaServerClientImpl>()),
         iCache_(nullptr),
         iAttrCache_(nullptr),
-        isStop_(true) {}
+        isStop_(true),
+        cacheTimeOutSec_(0) {}
 
     explicit InodeCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
       : metaClient_(metaClient),
         iCache_(nullptr),
-        iAttrCache_(nullptr) {}
+        iAttrCache_(nullptr),
+        cacheTimeOutSec_(0) {}
 
     CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
                        uint32_t flushPeriodSec,
-                       RefreshDataOption option) override {
+                       RefreshDataOption option,
+                       uint32_t cacheTimeOutSec) override {
         if (enableCacheMetrics) {
             iCache_ = std::make_shared<
                 LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0,
@@ -208,6 +212,7 @@ class InodeCacheManagerImpl : public InodeCacheManager,
         maxCacheSize_ = cacheSize;
         option_ = option;
         flushPeriodSec_ = flushPeriodSec;
+        cacheTimeOutSec_ = cacheTimeOutSec;
         iAttrCache_ = std::make_shared<InodeAttrCache>();
         s3ChunkInfoMetric_ = std::make_shared<S3ChunkInfoMetric>();
         return CURVEFS_ERROR::OK;
@@ -272,7 +277,11 @@ class InodeCacheManagerImpl : public InodeCacheManager,
 
     void RemoveOpenedInode(uint64_t inodeId) override;
 
-    bool NeedUseCahce(uint64_t inodeId, bool IsDirty);
+    bool NeedUseCache(uint64_t inodeId,
+        const std::shared_ptr<InodeWrapper> &inodeWrapper,
+        bool onlyAttr);
+
+    bool IsTimeOut(const std::shared_ptr<InodeWrapper> &inodeWrapper) const;
 
  private:
     virtual void FlushInodeBackground();
@@ -283,7 +292,8 @@ class InodeCacheManagerImpl : public InodeCacheManager,
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    std::shared_ptr<LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>> iCache_;
+    std::shared_ptr<LRUCache<uint64_t,
+        std::shared_ptr<InodeWrapper>>> iCache_;
     std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric_;
 
     std::shared_ptr<InodeAttrCache> iAttrCache_;
@@ -306,6 +316,8 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     Thread flushThread_;
     InterruptibleSleeper sleeper_;
     Atomic<bool> isStop_;
+    // cache timeout seconds, 0 means never timeout
+    uint32_t cacheTimeOutSec_;
 };
 
 class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {

curvefs/src/client/inode_wrapper.cpp

@@ -107,6 +107,7 @@ class UpdateInodeAsyncDone : public MetaServerClientDone {
                        << ", inodeid: " << inodeWrapper_->GetInodeId();
             inodeWrapper_->MarkInodeError();
         }
+        inodeWrapper_->ClearDirty();
         inodeWrapper_->ReleaseSyncingInode();
 
         if (parent_ != nullptr) {
@@ -226,7 +227,6 @@ void InodeWrapper::AsyncFlushAttr(MetaServerClientDone* done,
         metaClient_->UpdateInodeWithOutNlinkAsync(
             inode_.fsid(), inode_.inodeid(), dirtyAttr_,
             new UpdateInodeAsyncDone(shared_from_this(), done));
-        dirty_ = false;
         dirtyAttr_.Clear();
         return;
     }
@@ -301,7 +301,7 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
     UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize() - before);
     ClearS3ChunkInfoAdd();
     UpdateMaxS3ChunkInfoSize();
-    lastRefreshTime_ = ::curve::common::TimeUtility::GetTimeofDaySec();
+    lastRefreshTime_ = TimeUtility::GetTimeofDaySec();
     return CURVEFS_ERROR::OK;
 }
 
@@ -489,7 +489,6 @@ void InodeWrapper::AsyncFlushAttrAndExtents(MetaServerClientDone *done,
             new UpdateInodeAttrAndExtentClosure{shared_from_this(), done},
             std::move(indices));
 
-        dirty_ = false;
         dirtyAttr_.Clear();
         return;
     }
@@ -543,6 +542,7 @@ class UpdateInodeAsyncS3Done : public MetaServerClientDone {
             inodeWrapper_->MarkInodeError();
         }
         VLOG(9) << "inode " << inodeWrapper_->GetInodeId() << " async success.";
+        inodeWrapper_->ClearDirty();
         inodeWrapper_->ReleaseSyncingInode();
         inodeWrapper_->ReleaseSyncingS3ChunkInfo();
 
@@ -570,7 +570,6 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
             inode_.fsid(), inode_.inodeid(), dirtyAttr_,
             new UpdateInodeAsyncS3Done{shared_from_this(), done},
             std::move(indices));
-        dirty_ = false;
         dirtyAttr_.Clear();
         ClearS3ChunkInfoAdd();
         return;

curvefs/src/client/inode_wrapper.h

@@ -67,6 +67,7 @@ using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;
 using metric::S3ChunkInfoMetric;
 using common::NlinkChange;
+using curve::common::TimeUtility;
 
 std::ostream &operator<<(std::ostream &os, const struct stat &attr);
 void AppendS3ChunkInfoToMap(uint64_t chunkIndex, const S3ChunkInfo &info,
@@ -86,11 +87,12 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
           baseMaxDataSize_(maxDataSize),
           maxDataSize_(maxDataSize),
           refreshDataInterval_(refreshDataInterval),
-          lastRefreshTime_(::curve::common::TimeUtility::GetTimeofDaySec()),
+          lastRefreshTime_(TimeUtility::GetTimeofDaySec()),
           s3ChunkInfoAddSize_(0),
           metaClient_(std::move(metaClient)),
           s3ChunkInfoMetric_(std::move(s3ChunkInfoMetric)),
-          dirty_(false) {
+          dirty_(false),
+          time_(TimeUtility::GetTimeofDaySec()) {
         UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize());
         g_alive_inode_count << 1;
     }
@@ -339,6 +341,10 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         return false;
     }
 
+    uint32_t GetCachedTime() const {
+        return time_;
+    }
+
  private:
     CURVEFS_ERROR SyncS3ChunkInfo(bool internal = false);
 
@@ -413,6 +419,9 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
 
     mutable ::curve::common::Mutex syncingVolumeExtentsMtx_;
     ExtentCache extentCache_;
+
+    // timestamp when put in cache
+    uint64_t time_;
 };
 
 }  // namespace client

curvefs/test/client/mock_dentry_cache_mamager.h

@@ -24,10 +24,9 @@
 #define CURVEFS_TEST_CLIENT_MOCK_DENTRY_CACHE_MAMAGER_H_
 
 #include <gmock/gmock.h>
-
+#include <cstdint>
 #include <string>
 #include <list>
-
 #include "curvefs/src/client/dentry_cache_manager.h"
 
 namespace curvefs {
@@ -38,8 +37,8 @@ class MockDentryCacheManager : public DentryCacheManager {
     MockDentryCacheManager() {}
     ~MockDentryCacheManager() {}
 
-    MOCK_METHOD2(Init, CURVEFS_ERROR(
-        uint64_t cacheSize, bool enableCacheMetrics));
+    MOCK_METHOD3(Init, CURVEFS_ERROR(
+        uint64_t cacheSize, bool enableCacheMetrics, uint32_t cacheTimeOutSec));
 
     MOCK_METHOD1(InsertOrReplaceCache, void(const Dentry& dentry));
 

curvefs/test/client/mock_inode_cache_manager.h

@@ -40,10 +40,11 @@ class MockInodeCacheManager : public InodeCacheManager {
     MockInodeCacheManager() {}
     ~MockInodeCacheManager() {}
 
-    MOCK_METHOD4(Init, CURVEFS_ERROR(uint64_t cacheSize,
+    MOCK_METHOD5(Init, CURVEFS_ERROR(uint64_t cacheSize,
                                      bool enableCacheMetrics,
                                      uint32_t flushPeriodSec,
-                                     RefreshDataOption option));
+                                     RefreshDataOption option,
+                                     uint32_t cacheTimeOutSec));
 
     MOCK_METHOD0(Run, void());