Merge branch 'master' into changeLogo

.github/workflows/go.yml

@@ -27,19 +27,20 @@ jobs:
       - name: Install Dependency
         run: sudo apt-get update && sudo apt-get -y install librocksdb-dev
 
-      - name: Clear cache
-        run: go clean -modcache
-
-      - name: Cache Go modules
-        uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
       - name: Check Format
-        run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then exit 1; fi
+        run: |
+          unformatted=$(gofmt -s -l .)
+          if [ "$unformatted" != "" ]; then
+            echo "the following files are not formatted:"
+            for file in $unformatted; do
+              echo "$file:"
+              gofmt -s -d "$file"
+              echo "------"
+            done
+            exit 1
+          fi
+          
+      
 
       - name: Lint
         uses: golangci/[email protected]

config/cluster_options.go

@@ -2,16 +2,81 @@ package config
 
 import "time"
 
+// Config holds configuration options for a distributed database.
 type Config struct {
-	ReplicationFactor   int
-	ShardingStrategy    string
-	SchedulingStrategy  string
-	LogDataStorage      string
-	LogDataStoragePath  string
-	SnapshotStorage     string
+	// ReplicationFactor specifies the number of replicas for each piece of data,
+	// impacting redundancy and availability.
+	ReplicationFactor int
+
+	// ShardingStrategy defines the strategy for data sharding, determining how data
+	// is partitioned and distributed across nodes in the cluster.
+	ShardingStrategy string
+
+	// SchedulingStrategy specifies the task scheduling strategy, affecting data balancing
+	// and load distribution.
+	SchedulingStrategy string
+
+	// LogDataStorage specifies the storage type for log data, which could be disk,
+	// network storage, etc.
+	LogDataStorage string
+
+	// LogDataStoragePath is the path for storing log data.
+	LogDataStoragePath string
+
+	// SnapshotStorage specifies the storage type for snapshot data, used for backup
+	// and restoration.
+	SnapshotStorage string
+
+	// SnapshotStoragePath is the path for storing snapshot data.
 	SnapshotStoragePath string
-	LogDataStorageSize  int64
-	HeartbeatInterval   time.Duration
-	MetaNodes           []string
-	StoreNodes          []string
+
+	// LogDataStorageSize specifies the maximum capacity for log data storage.
+	LogDataStorageSize int64
+
+	// HeartbeatInterval defines the interval for heartbeats, used to maintain communication
+	// and state synchronization among nodes in the cluster.
+	HeartbeatInterval time.Duration
+
+	// MetaNodes contains the addresses of metadata nodes, used for managing the cluster's
+	// metadata information.
+	MetaNodes []string
+
+	// StoreNodes contains the addresses of storage nodes, used for storing and
+	// accessing actual data.
+	StoreNodes []string
+}
+
+// RegionConfig encapsulates configuration and boundary information for a specific region
+// within a distributed Raft-based database.
+type RegionConfig struct {
+	// Options contains a set of configuration options specific to the behavior of the Raft region.
+	Options Options
+
+	// Config holds additional configuration settings related to the operation of the Raft region.
+	Config Config
+
+	// Id represents the unique identifier for the Raft region.
+	Id int64
+
+	// Start specifies the starting boundary key of the Raft region.
+	Start []byte
+
+	// End specifies the ending boundary key of the Raft region.
+	End []byte
+}
+
+// StoreConfig encapsulates configuration and identification information for a store
+// within a distributed system.
+type StoreConfig struct {
+	// Options contains a set of configuration options specific to the behavior of the store.
+	Options Options
+
+	// Config holds additional configuration settings related to the operation of the store.
+	Config Config
+
+	// Id represents the unique identifier for the store.
+	Id int64
+
+	// Addr specifies the network address at which the store can be accessed.
+	Addr string
 }

config/options.go

@@ -1,19 +1,75 @@
 package config
 
-import "os"
+import (
+	"github.com/ByteStorage/FlyDB/lib/wal"
+	"os"
+)
 
+// Options is a comprehensive configuration struct that
+// encapsulates various settings for configuring the behavior of a database.
 type Options struct {
-	DirPath      string // Database data directory
-	DataFileSize int64  // Size of data files
-	SyncWrite    bool   // Whether to persist data on every write
-	IndexType    IndexerType
-	FIOType      FIOType
+	// DirPath specifies the path to the directory where the database will store its data files.
+	DirPath string
+
+	// DataFileSize defines the maximum size of each data file in the database.
+	DataFileSize int64
+
+	// SyncWrite determines whether the database should ensure data persistence with
+	// every write operation.
+	SyncWrite bool
+
+	// IndexType selects the type of indexing mechanism to be used for efficient data retrieval.
+	IndexType IndexerType
+
+	// FIOType indicates the type of file I/O optimization to be applied by the database.
+	FIOType FIOType
+}
+
+// ColumnOptions are configurations for database column families
+type ColumnOptions struct {
+	// DbMemoryOptions contains configuration settings
+	// for managing database memory usage and caching.
+	DbMemoryOptions DbMemoryOptions
+
+	// WalOptions contains configuration settings for the Write-Ahead Logging (WAL) mechanism.
+	WalOptions wal.Options
+}
+
+// DbMemoryOptions is related to configuration of database memory tables
+type DbMemoryOptions struct {
+	// Option contains a set of database configuration options
+	// to influence memory management behavior.
+	Option Options
+
+	// LogNum specifies the number of logs to keep in memory
+	// for efficient access and performance.
+	LogNum uint32
+
+	// FileSize defines the maximum size of data files to be kept in memory.
+	FileSize int64
+
+	// SaveTime determines the interval at which data should be
+	// saved from memory to disk to ensure durability.
+	SaveTime int64
+
+	// MemSize sets the limit on the amount of memory to be used for caching purposes.
+	MemSize int64
+
+	// TotalMemSize defines the overall memory capacity allocated for database operations.
+	TotalMemSize int64
+
+	// ColumnName identifies the specific database column to which these memory options apply.
+	ColumnName string
+
+	// Wal is a reference to the Write-Ahead Logging (WAL) mechanism that ensures data durability.
+	Wal *wal.Wal
 }
 
 // IteratorOptions is the configuration for index iteration.
 type IteratorOptions struct {
 	// Prefix specifies the prefix value for keys to iterate over. Default is empty.
 	Prefix []byte
+
 	// Reverse indicates whether to iterate in reverse order.
 	// Default is false for forward iteration.
 	Reverse bool
@@ -23,6 +79,7 @@ type IteratorOptions struct {
 type WriteBatchOptions struct {
 	// MaxBatchNum is the maximum number of data entries in a batch.
 	MaxBatchNum uint
+
 	// SyncWrites indicates whether to sync (persist) the data on batch commit.
 	SyncWrites bool
 }
@@ -46,6 +103,9 @@ const (
 
 	// SkipList
 	SkipList
+
+	// ARTWithBloom index With Bloom Filter
+	ARTWithBloom
 )
 
 const (
@@ -69,3 +129,14 @@ var DefaultWriteBatchOptions = WriteBatchOptions{
 	MaxBatchNum: 10000,
 	SyncWrites:  true,
 }
+
+var DefaultDbMemoryOptions = DbMemoryOptions{
+	Option:       DefaultOptions,
+	LogNum:       1000,
+	FileSize:     256 * 1024 * 1024, // 256MB
+	SaveTime:     100 * 1000,
+	MemSize:      256 * 1024 * 1024,      // 256MB
+	TotalMemSize: 1 * 1024 * 1024 * 1024, // 2GB
+	ColumnName:   "default",
+	Wal:          nil,
+}