修改分词逻辑，重命名依赖包

linzedo · Dec 25, 2023 · 7a219b2 · 7a219b2
1 parent 669549a
commit 7a219b2
Show file tree

Hide file tree

Showing 33 changed files with 88 additions and 66 deletions.
diff --git a/core/initialize.go b/core/initialize.go
@@ -3,11 +3,11 @@ package core
 import (
 	"context"
 	"fmt"
-	"github.com/sea-team/gofound/global"
-	"github.com/sea-team/gofound/searcher"
-	"github.com/sea-team/gofound/searcher/words"
-	"github.com/sea-team/gofound/web/controller"
-	"github.com/sea-team/gofound/web/router"
+	"github.com/linzedo/gofound/global"
+	"github.com/linzedo/gofound/searcher"
+	"github.com/linzedo/gofound/searcher/words"
+	"github.com/linzedo/gofound/web/controller"
+	"github.com/linzedo/gofound/web/router"
 	"log"
 	"net/http"
 

diff --git a/core/parser.go b/core/parser.go
@@ -3,7 +3,7 @@ package core
 import (
 	"flag"
 	"fmt"
-	"github.com/sea-team/gofound/global"
+	"github.com/linzedo/gofound/global"
 	"os"
 	"runtime"
 

diff --git a/global/global.go b/global/global.go
@@ -1,7 +1,7 @@
 package global
 
 import (
-	"github.com/sea-team/gofound/searcher"
+	"github.com/linzedo/gofound/searcher"
 )
 
 var (

diff --git a/main.go b/main.go
@@ -1,10 +1,10 @@
 package main
 
 import (
-	"github.com/sea-team/gofound/core"
+	"github.com/linzedo/gofound/core"
 )
 
 func main() {
 	//初始化容器和参数解析
-	core.Initialize("")
+	core.Initialize("./config/config.yaml")
 }
diff --git a/sdk/base.go b/sdk/base.go
@@ -1,8 +1,8 @@
 package gofound
 
 import (
-	"github.com/sea-team/gofound/searcher/model"
-	"github.com/sea-team/gofound/searcher/system"
+	"github.com/linzedo/gofound/searcher/model"
+	"github.com/linzedo/gofound/searcher/system"
 	"runtime"
 )
 

diff --git a/sdk/client.go b/sdk/client.go
@@ -2,9 +2,9 @@ package gofound
 
 import (
 	"fmt"
-	"github.com/sea-team/gofound/core"
-	"github.com/sea-team/gofound/global"
-	"github.com/sea-team/gofound/searcher"
+	"github.com/linzedo/gofound/core"
+	"github.com/linzedo/gofound/global"
+	"github.com/linzedo/gofound/searcher"
 	"os"
 	"runtime"
 	"sync"

diff --git a/sdk/database.go b/sdk/database.go
@@ -1,7 +1,7 @@
 package gofound
 
 import (
-	"github.com/sea-team/gofound/searcher"
+	"github.com/linzedo/gofound/searcher"
 
 	"github.com/syndtr/goleveldb/leveldb/errors"
 )

diff --git a/sdk/index.go b/sdk/index.go
@@ -2,7 +2,7 @@ package gofound
 
 import (
 	"errors"
-	"github.com/sea-team/gofound/searcher/model"
+	"github.com/linzedo/gofound/searcher/model"
 )
 
 // AddIndex 添加索引

diff --git a/searcher/container.go b/searcher/container.go
@@ -3,7 +3,7 @@ package searcher
 import (
 	"errors"
 	"fmt"
-	"github.com/sea-team/gofound/searcher/words"
+	"github.com/linzedo/gofound/searcher/words"
 	"log"
 	"os"
 	"runtime"

diff --git a/searcher/engine.go b/searcher/engine.go
@@ -2,13 +2,13 @@ package searcher
 
 import (
 	"fmt"
-	"github.com/sea-team/gofound/searcher/arrays"
-	"github.com/sea-team/gofound/searcher/model"
-	"github.com/sea-team/gofound/searcher/pagination"
-	"github.com/sea-team/gofound/searcher/sorts"
-	"github.com/sea-team/gofound/searcher/storage"
-	"github.com/sea-team/gofound/searcher/utils"
-	"github.com/sea-team/gofound/searcher/words"
+	"github.com/linzedo/gofound/searcher/arrays"
+	"github.com/linzedo/gofound/searcher/model"
+	"github.com/linzedo/gofound/searcher/pagination"
+	"github.com/linzedo/gofound/searcher/sorts"
+	"github.com/linzedo/gofound/searcher/storage"
+	"github.com/linzedo/gofound/searcher/utils"
+	"github.com/linzedo/gofound/searcher/words"
 	"log"
 	"os"
 	"runtime"
@@ -352,9 +352,16 @@ func (e *Engine) addPositiveIndex(index *model.IndexDoc, keys []string) {
 func (e *Engine) MultiSearch(request *model.SearchRequest) (*model.SearchResult, error) {
 	//等待搜索初始化完成
 	e.Wait()
+	var (
+		words []string
+	)
 
 	//分词搜索
-	words := e.Tokenizer.Cut(request.Query)
+	if request.Query == "" {
+		words = []string{""}
+	} else {
+		words = e.Tokenizer.Cut(request.Query)[1:]
+	}
 
 	fastSort := &sorts.FastSort{
 		IsDebug: e.IsDebug,

diff --git a/searcher/sorts/fast.go b/searcher/sorts/fast.go
@@ -1,7 +1,7 @@
 package sorts
 
 import (
-	"github.com/sea-team/gofound/searcher/model"
+	"github.com/linzedo/gofound/searcher/model"
 	"sort"
 	"strings"
 	"sync"

diff --git a/searcher/sorts/sort.go b/searcher/sorts/sort.go
@@ -2,7 +2,7 @@ package sorts
 
 import (
 	"github.com/emirpasic/gods/trees/avltree"
-	"github.com/sea-team/gofound/searcher/utils"
+	"github.com/linzedo/gofound/searcher/utils"
 	"log"
 	"sync"
 )

diff --git a/searcher/utils/utils.go b/searcher/utils/utils.go
@@ -223,15 +223,27 @@ func DirSizeB(path string) int64 {
 }
 
 // RemovePunctuation 移除所有的标点符号
+var regPunctuation = regexp.MustCompile(`\p{P}+`)
+
 func RemovePunctuation(str string) string {
-	reg := regexp.MustCompile(`\p{P}+`)
-	return reg.ReplaceAllString(str, "")
+	//reg := regexp.MustCompile(`\p{P}+`)
+	return regPunctuation.ReplaceAllString(str, "")
+}
+
+func IsPunctuation(str string) bool {
+	return regPunctuation.MatchString(str)
 }
 
 // RemoveSpace 移除所有的空格
+var regSpace = regexp.MustCompile(`\s+`)
+
 func RemoveSpace(str string) string {
-	reg := regexp.MustCompile(`\s+`)
-	return reg.ReplaceAllString(str, "")
+	//reg := regexp.MustCompile(`\s+`)
+	return regSpace.ReplaceAllString(str, "")
+}
+
+func IsSpace(str string) bool {
+	return regSpace.MatchString(str)
 }
 
 // init 注册数据类型

diff --git a/searcher/words/tokenizer.go b/searcher/words/tokenizer.go
@@ -2,7 +2,7 @@ package words
 
 import (
 	"embed"
-	"github.com/sea-team/gofound/searcher/utils"
+	"github.com/linzedo/gofound/searcher/utils"
 	"strings"
 
 	"github.com/wangbin/jiebago"
@@ -45,12 +45,15 @@ func (t *Tokenizer) Cut(text string) []string {
 	var wordMap = make(map[string]struct{})
 
 	resultChan := t.seg.CutForSearch(text, true)
-	var wordsSlice []string
+	var wordsSlice = []string{""}
 	for {
 		w, ok := <-resultChan
 		if !ok {
 			break
 		}
+		if utils.IsSpace(w) || utils.IsPunctuation(w) {
+			continue
+		}
 		_, found := wordMap[w]
 		if !found {
 			//去除重复的词

diff --git a/tests/benchmark/array_test.go b/tests/benchmark/array_test.go
@@ -1,7 +1,7 @@
 package benchmark
 
 import (
-	"github.com/sea-team/gofound/searcher/arrays"
+	"github.com/linzedo/gofound/searcher/arrays"
 	"testing"
 )
 

diff --git a/tests/benchmark/skiplist_test.go b/tests/benchmark/skiplist_test.go
@@ -1,7 +1,7 @@
 package benchmark
 
 import (
-	"github.com/sea-team/gofound/searcher/arrays"
+	"github.com/linzedo/gofound/searcher/arrays"
 	"math/rand"
 	"testing"
 )

diff --git a/tests/index_test.go b/tests/index_test.go
@@ -3,10 +3,10 @@ package tests
 import (
 	"bufio"
 	"fmt"
-	"github.com/sea-team/gofound/searcher"
-	"github.com/sea-team/gofound/searcher/model"
-	"github.com/sea-team/gofound/searcher/utils"
-	"github.com/sea-team/gofound/searcher/words"
+	"github.com/linzedo/gofound/searcher"
+	"github.com/linzedo/gofound/searcher/model"
+	"github.com/linzedo/gofound/searcher/utils"
+	"github.com/linzedo/gofound/searcher/words"
 	"os"
 	"strings"
 	"testing"

diff --git a/tests/sort_test.go b/tests/sort_test.go
@@ -2,7 +2,7 @@ package tests
 
 import (
 	"fmt"
-	"github.com/sea-team/gofound/searcher/utils"
+	"github.com/linzedo/gofound/searcher/utils"
 	"math/rand"
 	"sort"
 	"testing"

diff --git a/web/admin/admin.go b/web/admin/admin.go
@@ -2,7 +2,7 @@ package admin
 
 import (
 	"github.com/gin-gonic/gin"
-	"github.com/sea-team/gofound/web/admin/assets"
+	"github.com/linzedo/gofound/web/admin/assets"
 	"net/http"
 	"net/url"
 	"os"

diff --git a/web/controller/base.go b/web/controller/base.go
@@ -1,7 +1,7 @@
 package controller
 
 import (
-	"github.com/sea-team/gofound/searcher/model"
+	"github.com/linzedo/gofound/searcher/model"
 
 	"github.com/gin-gonic/gin"
 )

diff --git a/web/controller/index.go b/web/controller/index.go
@@ -1,7 +1,7 @@
 package controller
 
 import (
-	"github.com/sea-team/gofound/searcher/model"
+	"github.com/linzedo/gofound/searcher/model"
 	"time"
 
 	"github.com/gin-gonic/gin"

diff --git a/web/controller/services.go b/web/controller/services.go
@@ -1,7 +1,7 @@
 package controller
 
 import (
-	service2 "github.com/sea-team/gofound/web/service"
+	service2 "github.com/linzedo/gofound/web/service"
 )
 
 var srv *Services

diff --git a/web/middleware/exception.go b/web/middleware/exception.go
@@ -1,7 +1,7 @@
 package middleware
 
 import (
-	"github.com/sea-team/gofound/web"
+	"github.com/linzedo/gofound/web"
 	"runtime/debug"
 
 	"github.com/gin-gonic/gin"

diff --git a/web/router/base.go b/web/router/base.go
@@ -1,7 +1,7 @@
 package router
 
 import (
-	"github.com/sea-team/gofound/web/controller"
+	"github.com/linzedo/gofound/web/controller"
 
 	"github.com/gin-gonic/gin"
 )

diff --git a/web/router/database.go b/web/router/database.go
@@ -1,7 +1,7 @@
 package router
 
 import (
-	"github.com/sea-team/gofound/web/controller"
+	"github.com/linzedo/gofound/web/controller"
 
 	"github.com/gin-gonic/gin"
 )

diff --git a/web/router/index.go b/web/router/index.go
@@ -1,7 +1,7 @@
 package router
 
 import (
-	"github.com/sea-team/gofound/web/controller"
+	"github.com/linzedo/gofound/web/controller"
 
 	"github.com/gin-gonic/gin"
 )

diff --git a/web/router/router.go b/web/router/router.go
@@ -1,9 +1,9 @@
 package router
 
 import (
-	"github.com/sea-team/gofound/global"
-	"github.com/sea-team/gofound/web/admin"
-	"github.com/sea-team/gofound/web/middleware"
+	"github.com/linzedo/gofound/global"
+	"github.com/linzedo/gofound/web/admin"
+	"github.com/linzedo/gofound/web/middleware"
 	"io"
 	"log"
 	"mime"

diff --git a/web/router/word.go b/web/router/word.go
@@ -1,7 +1,7 @@
 package router
 
 import (
-	"github.com/sea-team/gofound/web/controller"
+	"github.com/linzedo/gofound/web/controller"
 
 	"github.com/gin-gonic/gin"
 )

diff --git a/web/service/base.go b/web/service/base.go
@@ -1,10 +1,10 @@
 package service
 
 import (
-	"github.com/sea-team/gofound/global"
-	"github.com/sea-team/gofound/searcher"
-	"github.com/sea-team/gofound/searcher/model"
-	"github.com/sea-team/gofound/searcher/system"
+	"github.com/linzedo/gofound/global"
+	"github.com/linzedo/gofound/searcher"
+	"github.com/linzedo/gofound/searcher/model"
+	"github.com/linzedo/gofound/searcher/system"
 	"os"
 	"runtime"
 )

diff --git a/web/service/database.go b/web/service/database.go
@@ -1,8 +1,8 @@
 package service
 
 import (
-	"github.com/sea-team/gofound/global"
-	"github.com/sea-team/gofound/searcher"
+	"github.com/linzedo/gofound/global"
+	"github.com/linzedo/gofound/searcher"
 )
 
 type Database struct {

diff --git a/web/service/index.go b/web/service/index.go
@@ -1,9 +1,9 @@
 package service
 
 import (
-	"github.com/sea-team/gofound/global"
-	"github.com/sea-team/gofound/searcher"
-	"github.com/sea-team/gofound/searcher/model"
+	"github.com/linzedo/gofound/global"
+	"github.com/linzedo/gofound/searcher"
+	"github.com/linzedo/gofound/searcher/model"
 )
 
 type Index struct {