Add "RocksDB_MaxOpenFiles" to options we can pass to Agent and Hub. (A…

…zure#3441)

Expose the MaxOpenFiles setting in RocksDb to the user via the `RocksDB_MaxOpenFiles` environment variable.

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/Constants.cs

@@ -95,6 +95,8 @@ public static class Constants
 
         public const string StorageMaxTotalWalSize = "RocksDB_MaxTotalWalSize";
 
+        public const string StorageMaxOpenFiles = "RocksDB_MaxOpenFiles";
+
         public const string StorageLogLevel = "Storage_LogLevel";
 
         public const string WorkloadApiVersion = "2019-01-30";

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Program.cs

@@ -139,6 +139,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                 TimeSpan idleTimeout = TimeSpan.FromSeconds(idleTimeoutSecs);
                 experimentalFeatures = ExperimentalFeatures.Create(configuration.GetSection("experimentalFeatures"), logger);
                 Option<ulong> storageTotalMaxWalSize = GetConfigIfExists<ulong>(Constants.StorageMaxTotalWalSize, configuration, logger);
+                Option<int> storageMaxOpenFiles = GetConfigIfExists<int>(Constants.StorageMaxOpenFiles, configuration, logger);
                 Option<StorageLogLevel> storageLogLevel = GetConfigIfExists<StorageLogLevel>(Constants.StorageLogLevel, configuration, logger);
                 string iothubHostname;
                 string deviceId;
@@ -152,7 +153,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                         IotHubConnectionStringBuilder connectionStringParser = IotHubConnectionStringBuilder.Create(deviceConnectionString);
                         deviceId = connectionStringParser.DeviceId;
                         iothubHostname = connectionStringParser.HostName;
-                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageLogLevel));
+                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxOpenFiles, storageLogLevel));
                         builder.RegisterModule(new DockerModule(deviceConnectionString, edgeDeviceHostName, dockerUri, dockerAuthConfig, upstreamProtocol, proxy, productInfo, closeOnIdleTimeout, idleTimeout, useServerHeartbeat, backupConfigFilePath));
                         break;
 
@@ -165,7 +166,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                         string moduleGenerationId = configuration.GetValue<string>(Constants.EdgeletModuleGenerationIdVariableName);
                         apiVersion = configuration.GetValue<string>(Constants.EdgeletApiVersionVariableName);
                         TimeSpan performanceMetricsUpdateFrequency = configuration.GetTimeSpan("PerformanceMetricsUpdateFrequency", TimeSpan.FromMinutes(5));
-                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageLogLevel));
+                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxOpenFiles, storageLogLevel));
                         builder.RegisterModule(new EdgeletModule(iothubHostname, edgeDeviceHostName, deviceId, new Uri(managementUri), new Uri(workloadUri), apiVersion, dockerAuthConfig, upstreamProtocol, proxy, productInfo, closeOnIdleTimeout, idleTimeout, performanceMetricsUpdateFrequency, useServerHeartbeat, backupConfigFilePath));
 
                         IEnumerable<X509Certificate2> trustBundle = await CertificateHelper.GetTrustBundleFromEdgelet(new Uri(workloadUri), apiVersion, Constants.WorkloadApiVersion, moduleId, moduleGenerationId);
@@ -203,7 +204,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                             configuration.GetValue<string>(K8sConstants.EdgeK8sObjectOwnerUidKey));
                         bool runAsNonRoot = configuration.GetValue<bool>(K8sConstants.RunAsNonRootKey);
 
-                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageLogLevel));
+                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxOpenFiles, storageLogLevel));
                         builder.RegisterModule(
                             new KubernetesModule(
                                 iothubHostname,

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs

@@ -33,10 +33,11 @@ public class AgentModule : Module
         readonly bool useBackupAndRestore;
         readonly Option<string> storageBackupPath;
         readonly Option<ulong> storageTotalMaxWalSize;
+        readonly Option<int> storageMaxOpenFiles;
         readonly Option<StorageLogLevel> storageLogLevel;
 
-        public AgentModule(int maxRestartCount, TimeSpan intensiveCareTime, int coolOffTimeUnitInSeconds, bool usePersistentStorage, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageTotalMaxWalSize, Option<StorageLogLevel> storageLogLevel)
-            : this(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.None<Uri>(), Option.None<string>(), Constants.EdgeAgentModuleIdentityName, Option.None<string>(), useBackupAndRestore, storageBackupPath, storageTotalMaxWalSize, storageLogLevel)
+        public AgentModule(int maxRestartCount, TimeSpan intensiveCareTime, int coolOffTimeUnitInSeconds, bool usePersistentStorage, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageTotalMaxWalSize, Option<int> storageMaxOpenFiles, Option<StorageLogLevel> storageLogLevel)
+            : this(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.None<Uri>(), Option.None<string>(), Constants.EdgeAgentModuleIdentityName, Option.None<string>(), useBackupAndRestore, storageBackupPath, storageTotalMaxWalSize, storageMaxOpenFiles, storageLogLevel)
         {
         }
 
@@ -53,6 +54,7 @@ public AgentModule(
             bool useBackupAndRestore,
             Option<string> storageBackupPath,
             Option<ulong> storageTotalMaxWalSize,
+            Option<int> storageMaxOpenFiles,
             Option<StorageLogLevel> storageLogLevel)
         {
             this.maxRestartCount = maxRestartCount;
@@ -67,6 +69,7 @@ public AgentModule(
             this.useBackupAndRestore = useBackupAndRestore;
             this.storageBackupPath = storageBackupPath;
             this.storageTotalMaxWalSize = storageTotalMaxWalSize;
+            this.storageMaxOpenFiles = storageMaxOpenFiles;
             this.storageLogLevel = storageLogLevel;
         }
 
@@ -144,7 +147,7 @@ protected override void Load(ContainerBuilder builder)
 
             // IRocksDbOptionsProvider
             // For EdgeAgent, we don't need high performance from RocksDb, so always turn off optimizeForPerformance
-            builder.Register(c => new RocksDbOptionsProvider(c.Resolve<ISystemEnvironment>(), false, this.storageTotalMaxWalSize, this.storageLogLevel))
+            builder.Register(c => new RocksDbOptionsProvider(c.Resolve<ISystemEnvironment>(), false, this.storageTotalMaxWalSize, this.storageMaxOpenFiles, this.storageLogLevel))
                 .As<IRocksDbOptionsProvider>()
                 .SingleInstance();
 

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/Constants.cs

@@ -30,6 +30,7 @@ public static class ConfigKey
             public const string EdgeHubClientCertAuthEnabled = "ClientCertAuthEnabled";
             public const string SslProtocols = "SslProtocols";
             public const string StorageMaxTotalWalSize = "RocksDB_MaxTotalWalSize";
+            public const string StorageMaxOpenFiles = "RocksDB_MaxOpenFiles";
             public const string StorageLogLevel = "Storage_LogLevel";
         }
     }

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/DependencyManager.cs

@@ -80,7 +80,7 @@ public void Register(ContainerBuilder builder)
                 });
 
             bool optimizeForPerformance = this.configuration.GetValue("OptimizeForPerformance", true);
-            (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<StorageLogLevel> storageLogLevel) storeAndForward =
+            (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<int> storageMaxOpenFiles, Option<StorageLogLevel> storageLogLevel) storeAndForward =
                 this.GetStoreAndForwardConfiguration();
 
             IConfiguration configuration = this.configuration.GetSection("experimentalFeatures");
@@ -107,7 +107,7 @@ void RegisterAmqpModule(ContainerBuilder builder)
             builder.RegisterModule(new AmqpModule(amqpSettings["scheme"], amqpSettings.GetValue<ushort>("port"), this.serverCertificate, this.iotHubHostname, clientCertAuthEnabled, this.sslProtocols));
         }
 
-        void RegisterMqttModule(ContainerBuilder builder, (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<StorageLogLevel> storageLogLevel) storeAndForward, bool optimizeForPerformance)
+        void RegisterMqttModule(ContainerBuilder builder, (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<int> storageMaxOpenFiles, Option<StorageLogLevel> storageLogLevel) storeAndForward, bool optimizeForPerformance)
         {
             var topics = new MessageAddressConversionConfiguration(
                 this.configuration.GetSection(Constants.TopicNameConversionSectionName + ":InboundTemplates").Get<List<string>>(),
@@ -121,7 +121,7 @@ void RegisterMqttModule(ContainerBuilder builder, (bool isEnabled, bool usePersi
 
         void RegisterRoutingModule(
             ContainerBuilder builder,
-            (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<StorageLogLevel> storageLogLevel) storeAndForward,
+            (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<int> storageMaxOpenFiles, Option<StorageLogLevel> storageLogLevel) storeAndForward,
             ExperimentalFeatures experimentalFeatures)
         {
             var routes = this.configuration.GetSection("routes").Get<Dictionary<string, string>>();
@@ -191,7 +191,7 @@ void RegisterRoutingModule(
         void RegisterCommonModule(
             ContainerBuilder builder,
             bool optimizeForPerformance,
-            (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<StorageLogLevel> storageLogLevel) storeAndForward,
+            (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<int> storageMaxOpenFiles, Option<StorageLogLevel> storageLogLevel) storeAndForward,
             MetricsConfig metricsConfig)
         {
             bool cacheTokens = this.configuration.GetValue("CacheTokens", false);
@@ -234,6 +234,7 @@ void RegisterCommonModule(
                     storeAndForward.useBackupAndRestore,
                     storeAndForward.storageBackupPath,
                     storeAndForward.storageMaxTotalWalSize,
+                    storeAndForward.storageMaxOpenFiles,
                     storeAndForward.storageLogLevel));
         }
 
@@ -244,7 +245,7 @@ static string GetProductInfo()
             return productInfo;
         }
 
-        (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<StorageLogLevel> storageLogLevel) GetStoreAndForwardConfiguration()
+        (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageMaxTotalWalSize, Option<int> storageMaxOpenFiles, Option<StorageLogLevel> storageLogLevel) GetStoreAndForwardConfiguration()
         {
             int defaultTtl = -1;
             bool usePersistentStorage = this.configuration.GetValue<bool>("usePersistentStorage");
@@ -254,6 +255,7 @@ static string GetProductInfo()
             string storagePath = GetOrCreateDirectoryPath(this.configuration.GetValue<string>("StorageFolder"), Constants.EdgeHubStorageFolder);
             bool storeAndForwardEnabled = this.configuration.GetValue<bool>("storeAndForwardEnabled");
             Option<ulong> storageMaxTotalWalSize = this.GetConfigIfExists<ulong>(Constants.ConfigKey.StorageMaxTotalWalSize, this.configuration);
+            Option<int> storageMaxOpenFiles = this.GetConfigIfExists<int>(Constants.ConfigKey.StorageMaxOpenFiles, this.configuration);
             Option<StorageLogLevel> storageLogLevel = this.GetConfigIfExists<StorageLogLevel>(Constants.ConfigKey.StorageLogLevel, this.configuration);
 
             if (storeAndForwardEnabled)
@@ -270,7 +272,7 @@ static string GetProductInfo()
             }
 
             var storeAndForwardConfiguration = new StoreAndForwardConfiguration(timeToLiveSecs);
-            return (storeAndForwardEnabled, usePersistentStorage, storeAndForwardConfiguration, storagePath, useBackupAndRestore, storageBackupPath, storageMaxTotalWalSize, storageLogLevel);
+            return (storeAndForwardEnabled, usePersistentStorage, storeAndForwardConfiguration, storagePath, useBackupAndRestore, storageBackupPath, storageMaxTotalWalSize, storageMaxOpenFiles, storageLogLevel);
         }
 
         // TODO: Move this function to a common location that can be shared between EdgeHub and EdgeAgent

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/modules/CommonModule.cs

@@ -45,6 +45,7 @@ public class CommonModule : Module
         readonly bool useBackupAndRestore;
         readonly Option<string> storageBackupPath;
         readonly Option<ulong> storageMaxTotalWalSize;
+        readonly Option<int> storageMaxOpenFiles;
         readonly Option<StorageLogLevel> storageLogLevel;
 
         public CommonModule(
@@ -69,6 +70,7 @@ public CommonModule(
             bool useBackupAndRestore,
             Option<string> storageBackupPath,
             Option<ulong> storageMaxTotalWalSize,
+            Option<int> storageMaxOpenFiles,
             Option<StorageLogLevel> storageLogLevel)
         {
             this.productInfo = productInfo;
@@ -92,6 +94,7 @@ public CommonModule(
             this.useBackupAndRestore = useBackupAndRestore;
             this.storageBackupPath = storageBackupPath;
             this.storageMaxTotalWalSize = storageMaxTotalWalSize;
+            this.storageMaxOpenFiles = storageMaxOpenFiles;
             this.storageLogLevel = storageLogLevel;
         }
 
@@ -144,7 +147,7 @@ protected override void Load(ContainerBuilder builder)
                 .SingleInstance();
 
             // DataBase options
-            builder.Register(c => new RocksDbOptionsProvider(c.Resolve<ISystemEnvironment>(), this.optimizeForPerformance, this.storageMaxTotalWalSize, this.storageLogLevel))
+            builder.Register(c => new RocksDbOptionsProvider(c.Resolve<ISystemEnvironment>(), this.optimizeForPerformance, this.storageMaxTotalWalSize, this.storageMaxOpenFiles, this.storageLogLevel))
                 .As<IRocksDbOptionsProvider>()
                 .SingleInstance();
 
@@ -232,9 +235,9 @@ protected override void Load(ContainerBuilder builder)
             // Task<IStoreProvider>
             builder.Register(async c =>
                 {
-                var dbStoreProvider = await c.Resolve<Task<IDbStoreProvider>>();
-                IStoreProvider storeProvider = new StoreProvider(dbStoreProvider);
-                return storeProvider;
+                    var dbStoreProvider = await c.Resolve<Task<IDbStoreProvider>>();
+                    IStoreProvider storeProvider = new StoreProvider(dbStoreProvider);
+                    return storeProvider;
                 })
                 .As<Task<IStoreProvider>>()
                 .SingleInstance();

edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/DependencyManager.cs

@@ -168,6 +168,7 @@ public void Register(ContainerBuilder builder)
                     enableNonPersistentStorageBackup,
                     backupFolder,
                     Option.None<ulong>(),
+                    Option.None<int>(),
                     Option.None<StorageLogLevel>()));
 
             builder.RegisterModule(

edge-util/src/Microsoft.Azure.Devices.Edge.Storage.RocksDb/RocksDbOptionsProvider.cs

@@ -44,19 +44,26 @@ public class RocksDbOptionsProvider : IRocksDbOptionsProvider
         // Once write-ahead logs exceed this size, we will start forcing the flush of
         // column families whose memtables are backed by the oldest live WAL file
         const ulong DefaultMaxTotalWalSize = 512 * 1024 * 1024;
+
+        // Set a large default is to allow RocksDb to keep open,
+        // but not the default unlimited setting (unlimted = -1)
+        const int DefaultMaxOpenFiles = 5000;
+
         const ulong DefaultKeepLogFileNum = 1;
         const StorageLogLevel DefaultLogLevel = StorageLogLevel.NONE;
 
         readonly ISystemEnvironment env;
         readonly bool optimizeForPerformance;
         readonly ulong maxTotalWalSize;
+        readonly int maxOpenFiles;
         readonly StorageLogLevel logLevel;
 
-        public RocksDbOptionsProvider(ISystemEnvironment env, bool optimizeForPerformance, Option<ulong> maxTotalWalsize, Option<StorageLogLevel> logLevel)
+        public RocksDbOptionsProvider(ISystemEnvironment env, bool optimizeForPerformance, Option<ulong> maxTotalWalsize, Option<int> maxOpenFiles, Option<StorageLogLevel> logLevel)
         {
             this.env = Preconditions.CheckNotNull(env);
             this.optimizeForPerformance = optimizeForPerformance;
             this.maxTotalWalSize = maxTotalWalsize.GetOrElse(DefaultMaxTotalWalSize);
+            this.maxOpenFiles = maxOpenFiles.GetOrElse(DefaultMaxOpenFiles);
             this.logLevel = logLevel.GetOrElse(DefaultLogLevel);
         }
 
@@ -67,6 +74,7 @@ public DbOptions GetDbOptions()
                 .SetCreateMissingColumnFamilies();
 
             options.SetMaxTotalWalSize(this.maxTotalWalSize)
+                .SetMaxOpenFiles(this.maxOpenFiles)
                 .SetKeepLogFileNum(DefaultKeepLogFileNum)
                 .SetInfoLogLevel((int)this.logLevel);