diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index 46774343..60f5a79e 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -12,5 +12,5 @@ jobs: - uses: actions/first-interaction@v1 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - issue-message: "Message that will be displayed on users' first issue" - pr-message: "Message that will be displayed on users' first pull request" + issue-message: "Thank you very much for your issue and we will discuss it." + pr-message: "Thank you very much for your contribution, we will promptly review your code if there are no errors and pass ci. We will merge your pull request into the master branch." diff --git a/cluster/region/region_test.go b/cluster/region/region_test.go index c0201cf8..d432fb5a 100644 --- a/cluster/region/region_test.go +++ b/cluster/region/region_test.go @@ -25,8 +25,11 @@ func ReturnNewDB() *engine.DB { func destroyRegion(r Region) { // Close the region's database db := r.(*TestRegionStruct).db - _ = db.Close() - err := os.RemoveAll(dirpath) + err := db.Close() + if err != nil { + return + } + err = os.RemoveAll(dirpath) if err != nil { return } diff --git a/config/options.go b/config/options.go index e9090409..005b8399 100644 --- a/config/options.go +++ b/config/options.go @@ -9,6 +9,7 @@ type Options struct { IndexType IndexerType FIOType FIOType Addr string // Addr DB Server Listen + IsCli bool // Is Cli } // IteratorOptions is the configuration for index iteration. @@ -57,6 +58,7 @@ var DefaultOptions = Options{ IndexType: ART, FIOType: MmapIOType, Addr: DefaultAddr, + IsCli: false, } var DefaultIteratorOptions = IteratorOptions{ diff --git a/engine/batch_test.go b/engine/batch_test.go index 2be58d30..2e58682a 100644 --- a/engine/batch_test.go +++ b/engine/batch_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "os" "testing" + "time" ) func TestDB_WriteBatch(t *testing.T) { @@ -14,7 +15,7 @@ func TestDB_WriteBatch(t *testing.T) { dir, _ := os.MkdirTemp("", "flydb-batch-1") opts.DirPath = dir db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -51,7 +52,7 @@ func TestDB_WriteBatchRestart(t *testing.T) { dir, _ := os.MkdirTemp("", "flydb-batch-2") opts.DirPath = dir db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -77,6 +78,8 @@ func TestDB_WriteBatchRestart(t *testing.T) { assert.Nil(t, err) db2, err := NewDB(opts) + time.Sleep(time.Millisecond * 100) + defer db2.Clean() assert.Nil(t, err) _, err = db2.Get(randkv.GetTestKey(1)) @@ -91,6 +94,7 @@ func TestDB_WriteBatch1(t *testing.T) { dir := "/tmp/batch-3" opts.DirPath = dir db, err := NewDB(opts) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) diff --git a/engine/benchmark/benchmark_test.go b/engine/benchmark/benchmark_test.go index 349fed5d..a3cce793 100644 --- a/engine/benchmark/benchmark_test.go +++ b/engine/benchmark/benchmark_test.go @@ -39,6 +39,7 @@ func init() { opts.DirPath = filepath.Join("benchmark", "flydbtest") FlyDB, err = flydb.NewFlyDB(opts) + defer FlyDB.Clean() if err != nil { panic(err) } diff --git a/engine/db.go b/engine/db.go index 1f0904cf..b46f7f9c 100644 --- a/engine/db.go +++ b/engine/db.go @@ -10,16 +10,19 @@ import ( "github.com/ByteStorage/FlyDB/lib/const" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" "io" "net" "os" + "os/signal" "path/filepath" "sort" "strconv" "strings" "sync" + "syscall" ) // DB represents a FlyDB database instance, @@ -102,10 +105,6 @@ func NewDB(options config.Options) (*DB, error) { if err := db.loadIndexFromDataFiles(); err != nil { return nil, err } - - // start grpc server - db.startGrpcServer() - return db, nil } @@ -141,8 +140,6 @@ func (db *DB) Close() error { return err } } - // close grpc server - db.server.Stop() return nil } @@ -263,8 +260,8 @@ func (db *DB) setActiveDataFile() error { // Get Read data according to the key func (db *DB) Get(key []byte) ([]byte, error) { zap.L().Info("get", zap.ByteString("key", key)) - db.lock.Lock() - defer db.lock.Unlock() + db.lock.RLock() + defer db.lock.RUnlock() // Determine the validity of the key if len(key) == 0 { @@ -556,7 +553,8 @@ func (db *DB) loadIndexFromDataFiles() error { return nil } -func (db *DB) startGrpcServer() { +// StartGrpcServer starts the grpc server +func (db *DB) StartGrpcServer() { listener, err := net.Listen("tcp", db.options.Addr) if err != nil { panic("tcp listen error: " + err.Error()) @@ -571,5 +569,41 @@ func (db *DB) startGrpcServer() { panic("db server start error: " + err.Error()) } }() + //wait for server start + for { + conn, err := grpc.Dial(db.options.Addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + continue + } + err = conn.Close() + if err != nil { + continue + } + break + } + if db.options.IsCli { + // graceful shutdown + sig := make(chan os.Signal) + signal.Notify(sig, syscall.SIGINT, syscall.SIGKILL) + + <-sig + } +} +func (db *DB) StopGrpcServer() { + if db.server != nil { + db.server.Stop() + } +} + +// Clean the DB data directory after the test is complete +func (db *DB) Clean() { + if db != nil { + db.StopGrpcServer() + _ = db.Close() + err := os.RemoveAll(db.options.DirPath) + if err != nil { + panic(err) + } + } } diff --git a/engine/db_test.go b/engine/db_test.go index 1cd4af17..daf21739 100644 --- a/engine/db_test.go +++ b/engine/db_test.go @@ -9,27 +9,15 @@ import ( "os" "sync" "testing" + "time" ) -// Destroy the DB data directory after the test is complete -func destroyDB(db *DB) { - if db != nil { - if db.activeFile != nil { - _ = db.Close() - } - err := os.RemoveAll(db.options.DirPath) - if err != nil { - panic(err) - } - } -} - func TestNewFlyDB(t *testing.T) { opts := config.DefaultOptions dir, _ := os.MkdirTemp("", "flydb") opts.DirPath = dir db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) } @@ -40,7 +28,7 @@ func TestDB_Put(t *testing.T) { opts.DirPath = dir opts.DataFileSize = 64 * 1024 * 1024 db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -82,6 +70,7 @@ func TestDB_Put(t *testing.T) { // Restart the database db2, err := NewDB(opts) + defer db2.Clean() assert.Nil(t, err) assert.NotNil(t, db2) val4 := randkv.RandomValue(128) @@ -98,7 +87,7 @@ func TestDB_ConcurrentPut(t *testing.T) { opts.DirPath = dir opts.DataFileSize = 64 * 1024 * 1024 db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -177,6 +166,7 @@ func TestDB_ConcurrentPut(t *testing.T) { // Restart the database db2, err := NewDB(opts) + defer db2.Clean() val1, err := db2.Get(randkv.GetTestKey(1)) assert.Nil(t, err) @@ -214,7 +204,7 @@ func TestDB_Get(t *testing.T) { opts.DirPath = dir opts.DataFileSize = 64 * 1024 * 1024 db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -263,6 +253,7 @@ func TestDB_Get(t *testing.T) { // Restart the database db2, err := NewDB(opts) + defer db2.Clean() val6, err := db2.Get(randkv.GetTestKey(11)) assert.Nil(t, err) assert.NotNil(t, val6) @@ -284,7 +275,7 @@ func TestDB_Delete(t *testing.T) { opts.DirPath = dir opts.DataFileSize = 64 * 1024 * 1024 db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -322,6 +313,8 @@ func TestDB_Delete(t *testing.T) { // Restart the database db2, err := NewDB(opts) + time.Sleep(time.Millisecond * 100) + defer db2.Clean() _, err = db2.Get(randkv.GetTestKey(11)) assert.Equal(t, _const.ErrKeyNotFound, err) @@ -336,7 +329,7 @@ func TestDB_GetListKeys(t *testing.T) { opts.DirPath = dir opts.DataFileSize = 64 * 1024 * 1024 db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -370,7 +363,7 @@ func TestDB_Fold(t *testing.T) { opts.DirPath = dir opts.DataFileSize = 64 * 1024 * 1024 db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -413,7 +406,7 @@ func TestDB_Sync(t *testing.T) { opts.DirPath = dir opts.DataFileSize = 64 * 1024 * 1024 db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) diff --git a/engine/iterator_test.go b/engine/iterator_test.go index f342fcf3..af4d55c3 100644 --- a/engine/iterator_test.go +++ b/engine/iterator_test.go @@ -13,6 +13,7 @@ func TestDB_NewIterator(t *testing.T) { dir, _ := os.MkdirTemp("", "flydb-iterator-1") opt.DirPath = dir db, err := NewDB(opt) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -26,6 +27,7 @@ func TestDB_Iterator_One_Value(t *testing.T) { dir, _ := os.MkdirTemp("", "flydb-iterator-2") opt.DirPath = dir db, err := NewDB(opt) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -45,6 +47,7 @@ func TestDB_Iterator_Multi_Value(t *testing.T) { dir, _ := os.MkdirTemp("", "flydb-iterator-3") opt.DirPath = dir db, err := NewDB(opt) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) diff --git a/engine/merge_test.go b/engine/merge_test.go index 023935b2..5429835c 100644 --- a/engine/merge_test.go +++ b/engine/merge_test.go @@ -14,7 +14,7 @@ func TestDB_Merge(t *testing.T) { dir, _ := os.MkdirTemp("", "flydb-merge-1") opts.DirPath = dir db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -29,7 +29,7 @@ func TestDB_Merge2(t *testing.T) { opts.DataFileSize = 32 * 1024 * 1024 opts.DirPath = dir db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) @@ -68,7 +68,7 @@ func TestDB_Merge3(t *testing.T) { opts.DataFileSize = 32 * 1024 * 1024 opts.DirPath = dir db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) diff --git a/engine/put_get_test.go b/engine/put_get_test.go index 046ffc9a..bb04dadd 100644 --- a/engine/put_get_test.go +++ b/engine/put_get_test.go @@ -16,7 +16,7 @@ func TestPutAndGet(t *testing.T) { opts.DirPath = dir opts.DataFileSize = 64 * 1024 * 1024 db, err := NewDB(opts) - defer destroyDB(db) + defer db.Clean() assert.Nil(t, err) assert.NotNil(t, db) diff --git a/protocol/http/http_server_test.go b/protocol/http/http_server_test.go index 5f746ab2..3714706a 100644 --- a/protocol/http/http_server_test.go +++ b/protocol/http/http_server_test.go @@ -5,10 +5,12 @@ import ( "encoding/json" "github.com/ByteStorage/FlyDB/config" "github.com/ByteStorage/FlyDB/engine" + "github.com/stretchr/testify/assert" "io" "net/http" "net/http/httptest" "testing" + "time" ) func newHttpHandler() (*HttpHandler, error) { @@ -24,21 +26,15 @@ func newHttpHandler() (*HttpHandler, error) { func TestNewHTTPServer(t *testing.T) { server, err := newHttpHandler() - defer func(server *HttpHandler) { - err := server.Close() - if err != nil { - - } - }(server) - if err != nil { - t.Error(err) - } + time.Sleep(time.Millisecond * 100) + defer server.Clean() + assert.Nil(t, err) } // 测试Put方法 func TestPut(t *testing.T) { handler, _ := newHttpHandler() - defer handler.Close() + defer handler.Clean() // 创建一个测试用的http server server := httptest.NewServer(http.HandlerFunc(handler.PutHandler)) defer server.Close() @@ -92,7 +88,7 @@ func TestPut(t *testing.T) { func TestDel(t *testing.T) { handler, _ := newHttpHandler() - defer handler.Close() + defer handler.Clean() // 创建一个测试用的http server server := httptest.NewServer(http.HandlerFunc(handler.DelHandler)) defer server.Close() @@ -140,7 +136,7 @@ func TestDel(t *testing.T) { func TestGet(t *testing.T) { handler, _ := newHttpHandler() - defer handler.Close() + defer handler.Clean() // 创建一个测试用的http server server := httptest.NewServer(http.HandlerFunc(handler.GetHandler)) defer server.Close() @@ -187,7 +183,7 @@ func TestGet(t *testing.T) { func TestPost(t *testing.T) { handler, _ := newHttpHandler() - defer handler.Close() + defer handler.Clean() // 创建一个测试用的 HTTP 服务器 server := httptest.NewServer(http.HandlerFunc(handler.PostHandler)) defer server.Close() @@ -242,7 +238,7 @@ func TestPost(t *testing.T) { func TestGetListKeysHandler(t *testing.T) { handler, _ := newHttpHandler() - defer handler.Close() + defer handler.DB.Clean() // 创建一个测试用的http server server := httptest.NewServer(http.HandlerFunc(handler.GetListKeysHandler)) defer server.Close() diff --git a/structure/list.go b/structure/list.go new file mode 100644 index 00000000..225da391 --- /dev/null +++ b/structure/list.go @@ -0,0 +1,562 @@ +package structure + +import ( + "bytes" + "encoding/binary" + "errors" + + "github.com/ByteStorage/FlyDB/config" + "github.com/ByteStorage/FlyDB/engine" + _const "github.com/ByteStorage/FlyDB/lib/const" +) + +// Due to the complexity of each operation is at least O(n) +// So we can directly use slice to implement the list at the bottom level +// If the implementation of the db is improved later, we need to switch to a linked list + +// ListStructure is a structure that stores list data + +// NewListStructure returns a new ListStructure +// It will return a nil ListStructure if the database cannot be opened +// or the database cannot be created +// The database will be created if it does not exist + +// LPush adds a value to the left of the list corresponding to the key +// If the key does not exist, it will create the key + +// LPushs adds one or more values to the left of the list corresponding to the key +// If the key does not exist, it will create the key + +// RPush adds a value to the right of the list corresponding to the key +// If the key does not exist, it will create the key + +// RPushs adds one or more values to the right of the list corresponding to the key +// If the key does not exist, it will create the key + +// Due to the complexity of each operation is at least O(n) +// So we can directly use slice to implement the list at the bottom level +// If the implementation of the db is improved later, we need to switch to a linked list +type ListStructure struct { + db *engine.DB +} + +// NewListStructure returns a new ListStructure +// It will return a nil ListStructure if the database cannot be opened +// or the database cannot be created +// The database will be created if it does not exist +func NewListStructure(options config.Options) (*ListStructure, error) { + db, err := engine.NewDB(options) + if err != nil { + return nil, err + } + return &ListStructure{db: db}, nil +} + +// LPush adds a value to the left of the list corresponding to the key +// If the key does not exist, it will create the key +func (l *ListStructure) LPush(key []byte, value []byte) error { + // Check if value is empty + if value == nil { + return ErrInvalidValue + } + + // Get the list + lstArr, err := l.getListFromDB(key, true) + if err != nil { + return err + } + + // Use new slice space, add data at the head, and append all original data afterwards + tmpArr := make([][]byte, 1+len(lstArr)) + tmpArr[0] = value + copy(tmpArr[1:], lstArr) + lstArr = tmpArr + + // Store to db + return l.setListToDB(key, lstArr) +} + +// LPushs adds one or more values to the left of the list corresponding to the key +// If the key does not exist, it will create the key +func (l *ListStructure) LPushs(key []byte, values ...[]byte) error { + // Check if values are valid + if len(values) == 0 { + return ErrInvalidValue + } + for _, v := range values { + if len(v) == 0 { + return ErrInvalidValue + } + } + + // Get the list + lstArr, err := l.getListFromDB(key, true) + if err != nil { + return err + } + + // Use new slice space, add data at the head, and append all original data afterwards + tmpArr := make([][]byte, len(values)+len(lstArr)) + copy(tmpArr[:len(values)], values) + copy(tmpArr[len(values):], lstArr) + lstArr = tmpArr + + // Store to db + return l.setListToDB(key, lstArr) +} + +// RPush adds a value to the right of the list corresponding to the key +// If the key does not exist, it will create the key +func (l *ListStructure) RPush(key []byte, value []byte) error { + // Check if value is empty + if value == nil { + return ErrInvalidValue + } + + // Get the list + lstArr, err := l.getListFromDB(key, true) + if err != nil { + return err + } + + // Append the new data to the end + lstArr = append(lstArr, value) + + // Store to db + return l.setListToDB(key, lstArr) +} + +// RPushs appends one or more values to the right side of a list associated with a key. +// If the key does not exist, it will be created. +func (l *ListStructure) RPushs(key []byte, values ...[]byte) error { + // Check if values are valid + if len(values) == 0 { + return ErrInvalidValue + } + for _, v := range values { + if len(v) == 0 { + return ErrInvalidValue + } + } + + // Get the list + lstArr, err := l.getListFromDB(key, true) + if err != nil { + return err + } + + // Append new values to the end + lstArr = append(lstArr, values...) + + // Store in the database + return l.setListToDB(key, lstArr) +} + +// LPop returns and removes the leftmost value of a list associated with a key. +// If the key does not exist, an error is returned. +// If the list is empty, an error is returned. +func (l *ListStructure) LPop(key []byte) ([]byte, error) { + // Get the list + lstArr, err := l.getListFromDB(key, false) + if err != nil { + return nil, err + } + + // Return error if the list is empty + if len(lstArr) == 0 { + return nil, ErrListEmpty + } + + leftData := lstArr[0] + lstArr = lstArr[1:] + + // Store in the database + return leftData, l.setListToDB(key, lstArr) +} + +// RPop returns and removes the rightmost value of a list associated with a key. +// If the key does not exist, an error is returned. +// If the list is empty, an error is returned. +func (l *ListStructure) RPop(key []byte) ([]byte, error) { + // Get the list + lstArr, err := l.getListFromDB(key, false) + if err != nil { + return nil, err + } + + // Return error if the list is empty + if len(lstArr) == 0 { + return nil, ErrListEmpty + } + + rightData := lstArr[len(lstArr)-1] + lstArr = lstArr[:len(lstArr)-1] + + // Store in the database + return rightData, l.setListToDB(key, lstArr) +} + +// LRange returns a range of elements from a list associated with a key. +// The range is inclusive, including both the start and stop indices. +// If the key does not exist, an error is returned. +// If the list is empty, an error is returned. +// Negative indices can be used, where -1 represents the last element of the list, +// -2 represents the second last element, and so on. +func (l *ListStructure) LRange(key []byte, start int, stop int) ([][]byte, error) { + // Get the list + lstArr, err := l.getListFromDB(key, false) + if err != nil { + return nil, err + } + + // Return error if the list is empty + if len(lstArr) == 0 { + return nil, ErrListEmpty + } + + lstLen := len(lstArr) + + // Calculate the correct indices + start = (start%lstLen + lstLen) % lstLen + stop = (stop%lstLen + lstLen) % lstLen + + // Return empty if the range length is less than 1 + if stop-start+1 < 1 { + return nil, nil + } + + return lstArr[start : stop+1], nil +} + +// LLen returns the size of a list associated with a key. +// If the key does not exist, an error is returned. +func (l *ListStructure) LLen(key []byte) (int, error) { + // Get the list + lstArr, err := l.getListFromDB(key, false) + if err != nil { + return 0, err + } + + return len(lstArr), nil +} + +// LRem removes elements from a list associated with a key based on the count and value parameters. +// The count can have the following values: +// count > 0: Remove count occurrences of the value from the beginning of the list. +// count < 0: Remove count occurrences of the value from the end of the list. +// count = 0: Remove all occurrences of the value from the list. +// If the key does not exist, an error is returned. +func (l *ListStructure) LRem(key []byte, count int, value []byte) error { + // Get the list + lstArr, err := l.getListFromDB(key, false) + if err != nil { + return err + } + + // Store whether an element is removed + isRemoved := make([]bool, len(lstArr)) + + num := 0 + // Process different counts to calculate the isRemoved array + if count > 0 { + for i := 0; i < len(lstArr) && num < count; i++ { + if bytes.Equal(lstArr[i], value) { + isRemoved[i] = true + num++ + } + } + } else if count < 0 { + for i := len(lstArr) - 1; i >= 0 && num < -count; i-- { + if bytes.Equal(lstArr[i], value) { + isRemoved[i] = true + num++ + } + } + } else { + for i := 0; i < len(lstArr); i++ { + if bytes.Equal(lstArr[i], value) { + isRemoved[i] = true + num++ + } + } + } + + // Create a new slice to store the list after removal + tmpList := make([][]byte, 0, len(lstArr)-num) + for i := 0; i < len(lstArr); i++ { + if !isRemoved[i] { + tmpList = append(tmpList, lstArr[i]) + } + } + lstArr = tmpList + + // Store in the database + return l.setListToDB(key, lstArr) +} + +// LSet sets the value of an element in a list associated with a key based on the index. +// If the index is out of range, an error is returned. +// If the list is empty, an error is returned. +func (l *ListStructure) LSet(key []byte, index int, value []byte) error { + // Check if the value is valid + if len(value) == 0 { + return ErrInvalidValue + } + + // Get the list + lstArr, err := l.getListFromDB(key, false) + if err != nil { + return err + } + + // Return error if the list is empty + if len(lstArr) == 0 { + return ErrListEmpty + } + + // Check if the index is out of range + if index < 0 || index >= len(lstArr) { + return ErrIndexOutOfRange + } + + lstArr[index] = value + + // Store in the database + return l.setListToDB(key, lstArr) +} + +// LTrim retains a range of elements in a list associated with a key. +// The range is inclusive, including both the start and stop indices. +// If the key does not exist, an error is returned. +// If the list is empty, an error is returned. +// Negative indices can be used, where -1 represents the last element of the list, +// -2 represents the second last element, and so on. +func (l *ListStructure) LTrim(key []byte, start int, stop int) error { + // Get the list + lstArr, err := l.getListFromDB(key, false) + if err != nil { + return err + } + + if len(lstArr) == 0 { + return ErrListEmpty + } + + lstLen := len(lstArr) + + // Calculate the correct indices + start = (start%lstLen + lstLen) % lstLen + stop = (stop%lstLen + lstLen) % lstLen + + if stop-start+1 < 1 { // Empty the list if the range length is less than 1 + lstArr = make([][]byte, 0) + } else { + lstArr = lstArr[start : stop+1] + } + + // Store in the database + return l.setListToDB(key, lstArr) +} + +// LIndex returns the value of an element in a list associated with a key based on the index. +// If the key does not exist, an error is returned. +// If the list is empty, an error is returned. +// Negative indices can be used, where -1 represents the last element of the list, +// -2 represents the second last element, and so on. +func (l *ListStructure) LIndex(key []byte, index int) ([]byte, error) { + // Get the list + lstArr, err := l.getListFromDB(key, false) + if err != nil { + return nil, err + } + + // Return error if the list is empty + if len(lstArr) == 0 { + return nil, ErrListEmpty + } + + lstLen := len(lstArr) + + // Calculate the correct index + index = (index%lstLen + lstLen) % lstLen + + return lstArr[index], nil +} + +// RPOPLPUSH removes the last element from one list and pushes it to another list. +// If the source list is empty, an error is returned. +// If the destination list is empty, it is created. +// Atomicity is not guaranteed. +func (l *ListStructure) RPOPLPUSH(source []byte, destination []byte) error { + // Get the source list + lstArr1, err := l.getListFromDB(source, false) + if err != nil { + return err + } + + // Get the destination list + lstArr2, err := l.getListFromDB(destination, true) + if err != nil { + return err + } + + // Return error if the source list is empty + if len(lstArr1) == 0 { + return ErrListEmpty + } + + // Insert lstArr1's last element at the beginning of lstArr2 + tmpLst := make([][]byte, len(lstArr2)+1) + tmpLst[0] = lstArr1[len(lstArr1)-1] + copy(tmpLst[1:], lstArr2) + lstArr2 = tmpLst + + // Truncate lstArr1's last element + lstArr1 = lstArr1[:len(lstArr1)-1] + + // Store in the database + err = l.setListToDB(source, lstArr1) + if err != nil { + return err + } + return l.setListToDB(destination, lstArr2) +} + +var ( + // ErrListEmpty is returned if the list is empty. + ErrListEmpty = errors.New("Wrong operation: list is empty") + // ErrIndexOutOfRange is returned if the index out of range. + ErrIndexOutOfRange = errors.New("Wrong operation: index out of range") +) + +// getListFromDB retrieves data from the database. When isKeyCanNotExist is true, it returns an empty slice if the key doesn't exist instead of an error. +func (l *ListStructure) getListFromDB(key []byte, isKeyCanNotExist bool) ([][]byte, error) { + if isKeyCanNotExist { + // Get data corresponding to the key from the database + dbData, err := l.db.Get(key) + + // Since the key might not exist, we need to handle ErrKeyNotFound separately as it is a valid case + if err != nil && err != _const.ErrKeyNotFound { + return nil, err + } + + // Deserialize the data into a list + lstArr, err := l.decodeList(dbData) + if err != nil { + if len(dbData) != 0 { + return nil, err + } else { + lstArr = make([][]byte, 0) + } + } + return lstArr, nil + } else { + // Get data corresponding to the key from the database + dbData, err := l.db.Get(key) + if err != nil { + return nil, err + } + + // Deserialize the data into a list + lstArr, err := l.decodeList(dbData) + if err != nil { + return nil, err + } + return lstArr, nil + } +} + +// setListToDB stores the data into the database. +func (l *ListStructure) setListToDB(key []byte, lstArr [][]byte) error { + // Serialize into binary array + encValue, err := l.encodeList(lstArr) + if err != nil { + return err + } + // Store in the database + return l.db.Put(key, encValue) +} + +// encodeList encodes the value +// format: [type][len1][value1][len2][value2]... +// len: variable number of bytes +// value: len bytes +func (l *ListStructure) encodeList(data [][]byte) ([]byte, error) { + dataLen := len(data) + + // Calculate the required buffer space in advance + bufMaxLen := 1 + for i := 0; i < dataLen; i++ { + bufMaxLen += len(data[i]) + binary.MaxVarintLen64 + } + buf := make([]byte, bufMaxLen) + + buf[0] = List + + bufIndex := 1 + + for i := 0; i < dataLen; i++ { + bufIndex += binary.PutVarint(buf[bufIndex:], int64(len(data[i]))) + bufIndex += copy(buf[bufIndex:], data[i]) + } + + return buf[:bufIndex], nil +} + +// decodeList decodes the value +// format: [type][len1][value1][len2][value2]... +// len: variable number of bytes +// value: len bytes +func (l *ListStructure) decodeList(value []byte) ([][]byte, error) { + // Check the length of the value + if len(value) < 1 { + return nil, ErrInvalidValue + } + + // Check the type of the value + if value[0] != List { + return nil, ErrInvalidType + } + + valueLen := len(value) + + nowIndex := 1 + + lstLen := 0 + + // Calculate the length of the list + for nowIndex < valueLen { + length, lenOfLen := binary.Varint(value[nowIndex:]) + nowIndex += lenOfLen + int(length) + lstLen++ + } + + result := make([][]byte, 0, lstLen) + nowIndex = 1 + for nowIndex < valueLen { + // Read the data length + length, lenOfLen := binary.Varint(value[nowIndex:]) + + // Check the number of bytes read + if lenOfLen <= 0 { + return nil, ErrInvalidValue + } + + // Jump to the start of the data + nowIndex += lenOfLen + + // Check if the next operation will go out of bounds + if nowIndex+int(length) > valueLen { + return nil, ErrInvalidValue + } + + // Add the data to the result + result = append(result, make([]byte, length)) + copy(result[len(result)-1], value[nowIndex:nowIndex+int(length)]) + + // Jump to the next data length + nowIndex += int(length) + } + + return result, nil +} diff --git a/structure/list_test.go b/structure/list_test.go new file mode 100644 index 00000000..82743ebc --- /dev/null +++ b/structure/list_test.go @@ -0,0 +1,325 @@ +package structure + +import ( + "os" + "testing" + + "github.com/ByteStorage/FlyDB/config" + _const "github.com/ByteStorage/FlyDB/lib/const" + "github.com/ByteStorage/FlyDB/lib/randkv" + "github.com/stretchr/testify/assert" +) + +var listErr error + +func initList() *ListStructure { + opts := config.DefaultOptions + dir, _ := os.MkdirTemp("", "TestListStructure") + opts.DirPath = dir + list, _ := NewListStructure(opts) + return list +} + +func TestListStructure_LPush(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test LPush function when the key exists + listErr = list.LPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + + // Test LPush function when the key does not exist + listErr = list.LPush(randkv.GetTestKey(2), randkv.RandomValue(100)) + assert.Nil(t, listErr) +} + +func TestListStructure_LPushs(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test LPushs function when the key exists + listErr = list.LPushs(randkv.GetTestKey(1), randkv.RandomValue(100), randkv.RandomValue(100)) + assert.Nil(t, listErr) + + // Test LPushs function when the key does not exist + listErr = list.LPushs(randkv.GetTestKey(2), randkv.RandomValue(100), randkv.RandomValue(100)) + assert.Nil(t, listErr) +} + +func TestListStructure_RPush(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test RPush function when the key exists + listErr = list.RPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + + // Test RPush function when the key does not exist + listErr = list.RPush(randkv.GetTestKey(2), randkv.RandomValue(100)) + assert.Nil(t, listErr) +} + +func TestListStructure_RPushs(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test RPushs function when the key exists + listErr = list.RPushs(randkv.GetTestKey(1), randkv.RandomValue(100), randkv.RandomValue(100)) + assert.Nil(t, listErr) + + // Test RPushs function when the key does not exist + listErr = list.RPushs(randkv.GetTestKey(2), randkv.RandomValue(100), randkv.RandomValue(100)) + assert.Nil(t, listErr) +} + +func TestListStructure_LPop(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test LPop function when the key exists + listErr = list.LPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + value, err := list.LPop(randkv.GetTestKey(1)) + assert.Nil(t, err) + assert.NotNil(t, value) + + // Test LPop function when the key does not exist + _, err = list.LPop(randkv.GetTestKey(2)) + assert.Equal(t, err, _const.ErrKeyNotFound) + + // Test LPop function when the list is empty + err = list.LPush(randkv.GetTestKey(3), randkv.RandomValue(100)) + assert.Nil(t, err) + _, err = list.LPop(randkv.GetTestKey(3)) + assert.Nil(t, err) + _, err = list.LPop(randkv.GetTestKey(3)) + assert.Equal(t, err, ErrListEmpty) +} + +func TestListStructure_RPop(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test RPop function when the key exists + listErr = list.RPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + value, err := list.RPop(randkv.GetTestKey(1)) + assert.Nil(t, err) + assert.NotNil(t, value) + + // Test RPop function when the key does not exist + _, err = list.RPop(randkv.GetTestKey(2)) + assert.Equal(t, err, _const.ErrKeyNotFound) + + // Test RPop function when the list is empty + err = list.RPush(randkv.GetTestKey(3), randkv.RandomValue(100)) + assert.Nil(t, err) + _, err = list.RPop(randkv.GetTestKey(3)) + assert.Nil(t, err) + _, err = list.RPop(randkv.GetTestKey(3)) + assert.Equal(t, err, ErrListEmpty) +} + +func TestListStructure_LRange(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test LRange function when the key exists + listErr = list.LPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + values, err := list.LRange(randkv.GetTestKey(1), 0, 1) + assert.Nil(t, err) + assert.NotNil(t, values) + + // Test LRange function when the key does not exist + _, err = list.LRange(randkv.GetTestKey(2), 0, 1) + assert.Equal(t, err, _const.ErrKeyNotFound) + + // Test LRange function when the list is empty + err = list.LPush(randkv.GetTestKey(3), randkv.RandomValue(100)) + assert.Nil(t, err) + _, err = list.LPop(randkv.GetTestKey(3)) + assert.Nil(t, err) + _, err = list.LRange(randkv.GetTestKey(3), 0, 1) + assert.Equal(t, err, ErrListEmpty) +} + +func TestListStructure_LLen(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test LLen function when the key exists + listErr = list.LPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + len, err := list.LLen(randkv.GetTestKey(1)) + assert.Nil(t, err) + assert.Equal(t, len, 1) + + // Test LLen function when the key does not exist + _, err = list.LLen(randkv.GetTestKey(2)) + assert.Equal(t, err, _const.ErrKeyNotFound) +} + +func TestListStructure_LRem(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test LRem function when the key exists + listErr = list.LPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + listErr = list.LRem(randkv.GetTestKey(1), 1, randkv.RandomValue(100)) + assert.Nil(t, listErr) + + // Test LRem function when the key does not exist + listErr = list.LRem(randkv.GetTestKey(2), 1, randkv.RandomValue(100)) + assert.Equal(t, listErr, _const.ErrKeyNotFound) +} + +func TestListStructure_LSet(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test LSet function when the key exists + listErr = list.LPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + listErr = list.LSet(randkv.GetTestKey(1), 0, randkv.RandomValue(200)) + assert.Nil(t, listErr) + + // Test LSet function when the key does not exist + listErr = list.LSet(randkv.GetTestKey(2), 0, randkv.RandomValue(100)) + assert.Equal(t, listErr, _const.ErrKeyNotFound) + + // Test LSet function when the list is empty + listErr = list.LPush(randkv.GetTestKey(3), randkv.RandomValue(100)) + assert.Nil(t, listErr) + _, listErr = list.LPop(randkv.GetTestKey(3)) + assert.Nil(t, listErr) + listErr = list.LSet(randkv.GetTestKey(3), 0, randkv.RandomValue(100)) + assert.Equal(t, listErr, ErrListEmpty) +} + +func TestListStructure_LTrim(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test LTrim function when the key exists + listErr = list.LPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + listErr = list.LTrim(randkv.GetTestKey(1), 0, 1) + assert.Nil(t, listErr) + + // Test LTrim function when the key does not exist + listErr = list.LTrim(randkv.GetTestKey(2), 0, 1) + assert.Equal(t, listErr, _const.ErrKeyNotFound) + + // Test LTrim function when the list is empty + listErr = list.LPush(randkv.GetTestKey(3), randkv.RandomValue(100)) + assert.Nil(t, listErr) + _, listErr = list.LPop(randkv.GetTestKey(3)) + assert.Nil(t, listErr) + listErr = list.LTrim(randkv.GetTestKey(3), 0, 1) + assert.Equal(t, listErr, ErrListEmpty) +} + +func TestListStructure_LIndex(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test LIndex function when the key exists + listErr = list.LPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + value, err := list.LIndex(randkv.GetTestKey(1), 0) + assert.Nil(t, err) + assert.NotNil(t, value) + + // Test LIndex function when the key does not exist + _, err = list.LIndex(randkv.GetTestKey(2), 0) + assert.Equal(t, err, _const.ErrKeyNotFound) + + // Test LIndex function when the list is empty + err = list.LPush(randkv.GetTestKey(3), randkv.RandomValue(100)) + assert.Nil(t, err) + _, err = list.LPop(randkv.GetTestKey(3)) + assert.Nil(t, err) + _, err = list.LIndex(randkv.GetTestKey(3), 0) + assert.Equal(t, err, ErrListEmpty) +} + +func TestListStructure_RPOPLPUSH(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Test RPOPLPUSH function when the source list exists + listErr = list.RPush(randkv.GetTestKey(1), randkv.RandomValue(100)) + assert.Nil(t, listErr) + listErr = list.RPOPLPUSH(randkv.GetTestKey(1), randkv.GetTestKey(2)) + assert.Nil(t, listErr) + + // Test RPOPLPUSH function when the source list does not exist + listErr = list.RPOPLPUSH(randkv.GetTestKey(3), randkv.GetTestKey(2)) + assert.Equal(t, listErr, _const.ErrKeyNotFound) + + // Test RPOPLPUSH function when the source list is empty + listErr = list.RPush(randkv.GetTestKey(4), randkv.RandomValue(100)) + assert.Nil(t, listErr) + _, listErr = list.RPop(randkv.GetTestKey(4)) + assert.Nil(t, listErr) + listErr = list.RPOPLPUSH(randkv.GetTestKey(4), randkv.GetTestKey(2)) + assert.Equal(t, listErr, ErrListEmpty) +} + +func TestListStructure_Integration(t *testing.T) { + list := initList() + defer list.db.Clean() + + // Create a key and use LPush to add some values + key := randkv.GetTestKey(1) + values := [][]byte{randkv.RandomValue(100), randkv.RandomValue(100), randkv.RandomValue(100)} + for _, value := range values { + listErr = list.RPush(key, value) + assert.Nil(t, listErr) + } + + // Use LLen to check the length of the list + tmplen, err := list.LLen(key) + assert.Nil(t, err) + assert.Equal(t, tmplen, len(values)) + + // Use LRange to get all values of the list and check if they are correct + rangeValues, err := list.LRange(key, 0, -1) + assert.Nil(t, err) + assert.Equal(t, values, rangeValues) + + // Use LRem to remove a value and check if it is properly removed + err = list.LRem(key, 1, values[0]) + assert.Nil(t, err) + rangeValues, err = list.LRange(key, 0, -1) + assert.Nil(t, err) + assert.NotContains(t, rangeValues, values[0]) + + // Use LSet to modify a value and check if it is properly modified + newValue := randkv.RandomValue(100) + err = list.LSet(key, 0, newValue) + assert.Nil(t, err) + rangeValues, err = list.LRange(key, 0, -1) + assert.Nil(t, err) + assert.Contains(t, rangeValues, newValue) + + // Use LTrim to trim the list and check if it is properly trimmed + err = list.LTrim(key, 0, 0) + assert.Nil(t, err) + rangeValues, err = list.LRange(key, 0, -1) + assert.Nil(t, err) + assert.Equal(t, len(rangeValues), 1) + + // Use RPOPLPUSH to move a value to another list and check if it is properly moved + destination := randkv.GetTestKey(2) + err = list.RPOPLPUSH(key, destination) + assert.Nil(t, err) + rangeValues, err = list.LRange(key, 0, -1) + assert.Equal(t, ErrListEmpty, err) + assert.Equal(t, len(rangeValues), 0) + rangeValues, err = list.LRange(destination, 0, -1) + assert.Nil(t, err) + assert.Equal(t, len(rangeValues), 1) +} diff --git a/structure/string_test.go b/structure/string_test.go index 9ab732c7..f1f02c06 100644 --- a/structure/string_test.go +++ b/structure/string_test.go @@ -22,6 +22,7 @@ func initdb() *StringStructure { func TestStringStructure_Get(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), randkv.RandomValue(100), 0) assert.Nil(t, err) @@ -44,6 +45,7 @@ func TestStringStructure_Get(t *testing.T) { func TestStringStructure_Del(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), randkv.RandomValue(100), 0) assert.Nil(t, err) @@ -57,6 +59,7 @@ func TestStringStructure_Del(t *testing.T) { func TestStringStructure_Type(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), randkv.RandomValue(100), 0) assert.Nil(t, err) @@ -70,6 +73,7 @@ func TestStringStructure_Type(t *testing.T) { func TestStringStructure_StrLen(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), randkv.RandomValue(100), 0) assert.Nil(t, err) @@ -81,6 +85,7 @@ func TestStringStructure_StrLen(t *testing.T) { func TestStringStructure_GetSet(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), randkv.RandomValue(100), 0) assert.Nil(t, err) @@ -94,6 +99,7 @@ func TestStringStructure_GetSet(t *testing.T) { func TestStringStructure_Append(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), randkv.RandomValue(10), 0) //assert.Nil(t, err) @@ -108,6 +114,7 @@ func TestStringStructure_Append(t *testing.T) { func TestStringStructure_Incr(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), []byte("1"), 0) assert.Nil(t, err) @@ -125,6 +132,7 @@ func TestStringStructure_Incr(t *testing.T) { func TestStringStructure_IncrBy(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), []byte("1"), 0) assert.Nil(t, err) @@ -142,6 +150,7 @@ func TestStringStructure_IncrBy(t *testing.T) { func TestStringStructure_IncrByFloat(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), []byte("1"), 0) assert.Nil(t, err) @@ -159,6 +168,7 @@ func TestStringStructure_IncrByFloat(t *testing.T) { func TestStringStructure_Decr(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), []byte("1"), 0) assert.Nil(t, err) @@ -176,6 +186,7 @@ func TestStringStructure_Decr(t *testing.T) { func TestStringStructure_DecrBy(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), []byte("1"), 0) assert.Nil(t, err) @@ -193,6 +204,7 @@ func TestStringStructure_DecrBy(t *testing.T) { func TestStringStructure_Exists(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), []byte("1"), 0) assert.Nil(t, err) @@ -208,6 +220,7 @@ func TestStringStructure_Exists(t *testing.T) { func TestStringStructure_Expire(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), []byte("1"), 0) assert.Nil(t, err) @@ -226,6 +239,7 @@ func TestStringStructure_Expire(t *testing.T) { func TestStringStructure_Persist(t *testing.T) { str := initdb() + defer str.db.Clean() err = str.Set(randkv.GetTestKey(1), []byte("1"), 0) assert.Nil(t, err) diff --git a/structure/types.go b/structure/types.go index ceefcd8a..fe118af1 100644 --- a/structure/types.go +++ b/structure/types.go @@ -8,7 +8,7 @@ const ( // Hash is a hash data structure Hash // List is a list data structure - List + List DataStructure = iota + 1 // Set is a set data structure Set // ZSet is a zset data structure