支持多数据库，批量添加

docs/api.md

@@ -2,6 +2,17 @@
 
 `gofound`启动之后，会监听一个TCP端口，接收来自客户端的搜索请求。处理http请求部分使用`gin`框架。
 
+## 多数据库支持
+
+从1.1版本开始，我们支持了多数据库，API接口中通过get参数来指定数据库。
+
+如果不指定，默认数据库为`default`。
+
+如：`api/index?database=db1` 其他post参数不变
+
+如果指定的数据库名没有存在，将会自动创建一个新的数据库。如果需要删除，直接删除改数据库目录，然后重启gofound即可。
+
+
 ## 增加/修改索引
 
 | 接口地址 | /api/index       |
@@ -45,6 +56,35 @@ curl -H "Content-Type:application/json" -X POST --data '{"id":88888,"text":"深
 }
 ```
 
+## 批量增加/修改索引
+
+
+| 接口地址 | /api/index/batch       |
+|------|------------------|
+| 请求方式 | POST             |
+| 请求类型 | application/json |
+
+参数与单个一致，只是需要用数组包裹多个json对象，例如：
+
+```json
+[{
+  "id": 88888,
+  "text": "深圳北站",
+  "document": {
+    "title": "阿森松岛所445",
+    "number": 223
+  }
+},{
+  "id": 22222,
+  "text": "北京东站",
+  "document": {
+    "title": "123123123",
+    "number": 123123
+  }
+}]
+```
+
+
 ## 删除索引
 
 | 接口地址 | /api/remove      |

main.go

@@ -50,20 +50,17 @@ func initTokenizer(dictionaryPath string) *words.Tokenizer {
 	return words.NewTokenizer(dictionaryPath)
 }
 
-func initEngine(args Args, tokenizer *words.Tokenizer) *searcher.Engine {
-	var engine = &searcher.Engine{
-		IndexPath: args.DataDir,
+func initContainer(args Args, tokenizer *words.Tokenizer) *searcher.Container {
+	container := &searcher.Container{
+		Dir:       args.DataDir,
+		Debug:     args.Debug,
 		Tokenizer: tokenizer,
 	}
-	option := engine.GetOptions()
-
-	go engine.InitOption(option)
-	engine.IsDebug = args.Debug
-
-	return engine
+	container.Init()
+	return container
 }
 
-func initGin(args Args, engine *searcher.Engine) {
+func initGin(args Args, container *searcher.Container) {
 	if args.Debug {
 		gin.SetMode(gin.DebugMode)
 	} else {
@@ -82,7 +79,7 @@ func initGin(args Args, engine *searcher.Engine) {
 	//注册api
 	api.Register(router)
 
-	api.SetEngine(engine)
+	api.SetContainer(container)
 
 	log.Println("API url： \t http://" + args.Addr + "/api")
 
@@ -106,11 +103,8 @@ func main() {
 	//初始化分词器
 	tokenizer := initTokenizer(*args.DictionaryPath)
 
-	//初始化引擎
-	engine := initEngine(args, tokenizer)
-	//保存索引到磁盘
-	defer engine.Close()
+	container := initContainer(args, tokenizer)
 
 	//初始化gin
-	initGin(args, engine)
+	initGin(args, container)
 }

router/api/api.go

@@ -9,10 +9,10 @@ import (
 	"runtime/debug"
 )
 
-var Engine *searcher.Engine
+var container *searcher.Container
 
-func SetEngine(e *searcher.Engine) {
-	Engine = e
+func SetContainer(c *searcher.Container) {
+	container = c
 }
 
 func query(c *gin.Context) {
@@ -25,8 +25,7 @@ func query(c *gin.Context) {
 	}
 
 	//调用搜索
-	//r := Engine.Search(request)
-	r := Engine.MultiSearch(request)
+	r := container.GetDataBase(c.Query("database")).MultiSearch(request)
 	c.JSON(200, result.Success(r))
 }
 
@@ -41,13 +40,6 @@ func status(c *gin.Context) {
 	var m runtime.MemStats
 	runtime.ReadMemStats(&m)
 
-	//索引状态
-	index := &map[string]any{
-		"size":  Engine.GetIndexSize(),
-		"shard": Engine.Option.Shard,
-		"queue": len(Engine.addDocumentWorkerChan),
-	}
-
 	memory := map[string]any{
 		"alloc":         m.Alloc,
 		"total":         m.TotalAlloc,
@@ -69,7 +61,6 @@ func status(c *gin.Context) {
 	r := gin.H{
 		"memory": memory,
 		"system": system,
-		"index":  index,
 		"status": "ok",
 	}
 	// 获取服务器状态
@@ -84,7 +75,7 @@ func addIndex(c *gin.Context) {
 		return
 	}
 
-	go Engine.IndexDocument(document)
+	go container.GetDataBase(c.Query("database")).IndexDocument(document)
 
 	c.JSON(200, result.Success(nil))
 }
@@ -97,8 +88,9 @@ func batchAddIndex(c *gin.Context) {
 		return
 	}
 
+	db := container.GetDataBase(c.Query("database"))
 	for _, doc := range documents {
-		go Engine.IndexDocument(doc)
+		go db.IndexDocument(doc)
 	}
 
 	c.JSON(200, result.Success(nil))
@@ -112,7 +104,7 @@ func dump(c *gin.Context) {
 
 func wordCut(c *gin.Context) {
 	q := c.Query("q")
-	r := Engine.Tokenizer.Cut(q)
+	r := container.Tokenizer.Cut(q)
 	c.JSON(200, result.Success(r))
 
 }
@@ -128,15 +120,20 @@ func removeIndex(c *gin.Context) {
 		c.JSON(200, result.Error(err.Error()))
 		return
 	}
+	db := container.GetDataBase(c.Query("database"))
 
-	err = Engine.RemoveIndex(removeIndexModel.Id)
+	err = db.RemoveIndex(removeIndexModel.Id)
 	if err != nil {
 		c.JSON(200, result.Error(err.Error()))
 		return
 	}
 	c.JSON(200, result.Success(nil))
 }
 
+func dbs(ctx *gin.Context) {
+	ctx.JSON(200, result.Success(container.GetDataBases()))
+}
+
 //Recover 处理异常
 func Recover(c *gin.Context) {
 	defer func() {
@@ -148,10 +145,13 @@ func Recover(c *gin.Context) {
 	}()
 	c.Next()
 }
+
 func Register(router *gin.Engine) {
 
 	router.GET("/api/", welcome)
 
+	router.GET("/api/dbs", dbs)
+
 	router.POST("/api/query", query)
 
 	router.GET("/api/status", status).POST("/api/status", status)

searcher/container.go

@@ -2,18 +2,21 @@ package searcher
 
 import (
 	"fmt"
+	"gofound/searcher/words"
 	"io/ioutil"
 	"log"
 	"os"
 )
 
 type Container struct {
-	Dir     string             //文件夹
-	engines map[string]*Engine //引擎
-	Debug   bool
+	Dir       string             //文件夹
+	engines   map[string]*Engine //引擎
+	Debug     bool               //调试
+	Tokenizer *words.Tokenizer   //分词器
 }
 
 func (c *Container) Init() error {
+
 	c.engines = make(map[string]*Engine)
 
 	//读取当前路径下的所有目录，就是数据库名称
@@ -32,7 +35,7 @@ func (c *Container) Init() error {
 	//初始化数据库
 	for _, dir := range dirs {
 		if dir.IsDir() {
-			c.engines[dir.Name()] = c.GetOrCreate(dir.Name())
+			c.engines[dir.Name()] = c.GetDataBase(dir.Name())
 		}
 	}
 	return nil
@@ -44,6 +47,7 @@ func (c *Container) NewEngine(name string) *Engine {
 	var engine = &Engine{
 		IndexPath:    fmt.Sprintf("%s%c%s", c.Dir, os.PathSeparator, name),
 		DatabaseName: name,
+		Tokenizer:    c.Tokenizer,
 	}
 	option := engine.GetOptions()
 
@@ -53,9 +57,15 @@ func (c *Container) NewEngine(name string) *Engine {
 	return engine
 }
 
-// GetOrCreate 获取或创建引擎
-func (c *Container) GetOrCreate(name string) *Engine {
-	log.Println("Get Engine:", name)
+// GetDataBase 获取或创建引擎
+func (c *Container) GetDataBase(name string) *Engine {
+
+	//默认数据库名为default
+	if name == "" {
+		name = "default"
+	}
+
+	log.Println("Get DataBase:", name)
 	engine, ok := c.engines[name]
 	if !ok {
 		//创建引擎
@@ -66,6 +76,7 @@ func (c *Container) GetOrCreate(name string) *Engine {
 	return engine
 }
 
-func (c *Container) GetEngines() map[string]*Engine {
+// GetDataBases 获取数据库列表
+func (c *Container) GetDataBases() map[string]*Engine {
 	return c.engines
 }

searcher/container_test.go

@@ -15,11 +15,11 @@ func TestContainer_Init(t *testing.T) {
 		panic(err)
 	}
 
-	test := c.GetOrCreate("test")
+	test := c.GetDataBase("test")
 
 	fmt.Println(test.GetIndexSize())
 
-	all := c.GetEngines()
+	all := c.GetDataBases()
 	for name, engine := range all {
 		fmt.Println(name)
 		fmt.Println(engine)

searcher/engine.go

@@ -38,7 +38,6 @@ type Option struct {
 	InvertedIndexName string //倒排索引
 	PositiveIndexName string //正排索引
 	DocIndexName      string //文档存储
-	Dictionary        string //词典路径
 	Shard             int    //分片数，默认为5
 }
 
@@ -142,7 +141,6 @@ func (e *Engine) GetOptions() *Option {
 		DocIndexName:      "docs",
 		InvertedIndexName: "inverted_index",
 		PositiveIndexName: "positive_index",
-		Dictionary:        "./data/dictionary.txt",
 		Shard:             5,
 	}
 }

searcher/model/doc.go

@@ -2,7 +2,6 @@ package model
 
 // IndexDoc 索引实体
 type IndexDoc struct {
-	Database
 	Id       uint32                 `json:"id,omitempty"`
 	Text     string                 `json:"text,omitempty"`
 	Document map[string]interface{} `json:"document,omitempty"`
@@ -20,6 +19,5 @@ type ResponseDoc struct {
 }
 
 type RemoveIndexModel struct {
-	Database
 	Id uint32 `json:"id,omitempty"`
 }