feat: index enhancement (close AlistGo#2632 pr AlistGo#2636)

* feat: index paths as setting * feat: clear index (AlistGo#2632) * feat: check indexMQ more frequently
nic562 · Dec 9, 2022 · 6f1aeb4 · 6f1aeb4
1 parent 1f7c1b4
commit 6f1aeb4
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 2 deletions.
diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go
@@ -132,6 +132,7 @@ func InitialSettings() []model.SettingItem {
 		// single settings
 		{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
 		{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.INDEX},
+		{Key: conf.IndexPaths, Value: "/", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
 		{Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
 		{Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE},
 	}

diff --git a/internal/conf/const.go b/internal/conf/const.go
@@ -44,6 +44,7 @@ const (
 
 	// index
 	SearchIndex = "search_index"
+	IndexPaths  = "index_paths"
 	IgnorePaths = "ignore_paths"
 
 	// aria2

diff --git a/internal/search/build.go b/internal/search/build.go
@@ -33,10 +33,17 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
 	Quit = make(chan struct{}, 1)
 	indexMQ := mq.NewInMemoryMQ[ObjWithParent]()
 	go func() {
-		ticker := time.NewTicker(5 * time.Second)
+		ticker := time.NewTicker(time.Second)
+		tickCount := 0
 		for {
 			select {
 			case <-ticker.C:
+				tickCount += 1
+				if indexMQ.Len() < 1000 && tickCount != 5 {
+					continue
+				} else if tickCount >= 5 {
+					tickCount = 0
+				}
 				log.Infof("index obj count: %d", objCount)
 				indexMQ.ConsumeAll(func(messages []mq.Message[ObjWithParent]) {
 					if len(messages) != 0 {
@@ -155,6 +162,10 @@ func Update(parent string, objs []model.Obj) {
 	if instance == nil || !instance.Config().AutoUpdate || Running.Load() {
 		return
 	}
+	indexPaths := GetIndexPaths()
+	if !isIndexPath(parent, indexPaths) {
+		return
+	}
 	ignorePaths, err := GetIgnorePaths()
 	if err != nil {
 		log.Errorf("update search index error while get ignore paths: %+v", err)

diff --git a/internal/search/util.go b/internal/search/util.go
@@ -35,6 +35,24 @@ func WriteProgress(progress *model.IndexProgress) {
 	}
 }
 
+func GetIndexPaths() []string {
+	indexPaths := make([]string, 0)
+	customIndexPaths := setting.GetStr(conf.IndexPaths)
+	if customIndexPaths != "" {
+		indexPaths = append(indexPaths, strings.Split(customIndexPaths, "\n")...)
+	}
+	return indexPaths
+}
+
+func isIndexPath(path string, indexPaths []string) bool {
+	for _, indexPaths := range indexPaths {
+		if strings.HasPrefix(path, indexPaths) {
+			return true
+		}
+	}
+	return false
+}
+
 func GetIgnorePaths() ([]string, error) {
 	storages, err := db.GetEnabledStorages()
 	if err != nil {

diff --git a/pkg/mq/mq.go b/pkg/mq/mq.go
@@ -18,6 +18,7 @@ type MQ[T any] interface {
 	Consume(BasicConsumer[T])
 	ConsumeAll(AllConsumer[T])
 	Clear()
+	Len() int
 }
 
 type inMemoryMQ[T any] struct {
@@ -54,3 +55,7 @@ func (mq *inMemoryMQ[T]) Clear() {
 	defer mq.Unlock()
 	mq.queue.Clear()
 }
+
+func (mq *inMemoryMQ[T]) Len() int {
+	return mq.queue.Len()
+}
diff --git a/server/handles/index.go b/server/handles/index.go
@@ -3,8 +3,10 @@ package handles
 import (
 	"context"
 
+	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/search"
 	"github.com/alist-org/alist/v3/server/common"
+	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
 )
@@ -26,6 +28,13 @@ func BuildIndex(c *gin.Context) {
 		common.ErrorStrResp(c, "index is running", 400)
 		return
 	}
+	indexPaths := search.GetIndexPaths()
+	indexPaths = append(indexPaths, req.Paths...)
+	indexPathsSet := mapset.NewSet[string]()
+	for _, indexPath := range indexPaths {
+		indexPathsSet.Add(indexPath)
+	}
+	indexPaths = indexPathsSet.ToSlice()
 	ignorePaths, err := search.GetIgnorePaths()
 	if err != nil {
 		common.ErrorResp(c, err, 500)
@@ -50,7 +59,7 @@ func BuildIndex(c *gin.Context) {
 				}
 			}
 		}
-		err = search.BuildIndex(context.Background(), req.Paths, ignorePaths, req.MaxDepth, true)
+		err = search.BuildIndex(context.Background(), indexPaths, ignorePaths, req.MaxDepth, true)
 		if err != nil {
 			log.Errorf("build index error: %+v", err)
 		}
@@ -67,6 +76,21 @@ func StopIndex(c *gin.Context) {
 	common.SuccessResp(c)
 }
 
+func ClearIndex(c *gin.Context) {
+	if search.Running.Load() {
+		common.ErrorStrResp(c, "index is running", 400)
+		return
+	}
+	search.Clear(c)
+	search.WriteProgress(&model.IndexProgress{
+		ObjCount:     0,
+		IsDone:       false,
+		LastDoneTime: nil,
+		Error:        "",
+	})
+	common.SuccessResp(c)
+}
+
 func GetProgress(c *gin.Context) {
 	progress, err := search.Progress()
 	if err != nil {

diff --git a/server/router.go b/server/router.go
@@ -111,6 +111,7 @@ func admin(g *gin.RouterGroup) {
 	index := g.Group("/index")
 	index.POST("/build", middlewares.SearchIndex, handles.BuildIndex)
 	index.POST("/stop", middlewares.SearchIndex, handles.StopIndex)
+	index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex)
 	index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)
 }