Language statistics bar for repositories (go-gitea#8037)

* Implementation for calculating language statistics

Impement saving code language statistics to database

Implement rendering langauge stats

Add primary laguage to show in repository list

Implement repository stats indexer queue

Add indexer test

Refactor to use queue module

* Do not timeout for queues

go.mod

@@ -85,6 +85,7 @@ require (
 	github.com/sergi/go-diff v1.0.0
 	github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
+	github.com/src-d/enry/v2 v2.1.0
 	github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect

go.sum

@@ -508,8 +508,13 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/src-d/enry v1.7.3 h1:jG2fmEaQaURh0qqU/sn82BRzVa6d4EVHJIw6gc98bak=
+github.com/src-d/enry/v2 v2.1.0 h1:z1L8t+B8bh3mmjPkJrgOTnVRpFGmTPJsplHX9wAn6BI=
+github.com/src-d/enry/v2 v2.1.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj0t028=
 github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
 github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
+github.com/src-d/go-oniguruma v1.1.0 h1:EG+Nm5n2JqWUaCjtM0NtutPxU7ZN5Tp50GWrrV8bTww=
+github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM=
 github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s=
 github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -530,6 +535,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
 github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/toqueteos/trie v1.0.0 h1:8i6pXxNUXNRAqP246iibb7w/pSFquNTQ+uNfriG7vlk=
+github.com/toqueteos/trie v1.0.0/go.mod h1:Ywk48QhEqhU1+DwhMkJ2x7eeGxDHiGkAdc9+0DYcbsM=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
 github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM=
 github.com/tstranex/u2f v1.0.0 h1:HhJkSzDDlVSVIVt7pDJwCHQj67k7A5EeBgPmeD+pVsQ=
@@ -747,6 +754,8 @@ gopkg.in/testfixtures.v2 v2.5.0 h1:N08B7l2GzFQenyYbzqthDnKAA+cmb17iAZhhFxr7JHw=
 gopkg.in/testfixtures.v2 v2.5.0/go.mod h1:vyAq+MYCgNpR29qitQdLZhdbLFf4mR/2MFJRFoQZZ2M=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/toqueteos/substring.v1 v1.0.2 h1:urLqCeMm6x/eTuQa1oZerNw8N1KNOIp5hD5kGL7lFsE=
+gopkg.in/toqueteos/substring.v1 v1.0.2/go.mod h1:Eb2Z1UYehlVK8LYW2WBVR2rwbujsz3aX8XDrM1vbNew=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=

integrations/testlogger.go

@@ -13,7 +13,6 @@ import (
 	"strings"
 	"sync"
 	"testing"
-	"time"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/queue"
@@ -101,7 +100,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() {
 	}
 	writerCloser.setT(&t)
 	return func() {
-		if err := queue.GetManager().FlushAll(context.Background(), 20*time.Second); err != nil {
+		if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
 			t.Errorf("Flushing queues failed with error %v", err)
 		}
 		_ = writerCloser.Close()

models/migrations/migrations.go

@@ -186,6 +186,8 @@ var migrations = []Migration{
 	NewMigration("Add some columns on review for migration", addReviewMigrateInfo),
 	// v126 -> v127
 	NewMigration("Fix topic repository count", fixTopicRepositoryCount),
+	// v127 -> v128
+	NewMigration("add repository code language statistics", addLanguageStats),
 }
 
 // Migrate database to current version

models/migrations/v127.go

@@ -0,0 +1,45 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func addLanguageStats(x *xorm.Engine) error {
+	// LanguageStat see models/repo_language_stats.go
+	type LanguageStat struct {
+		ID          int64 `xorm:"pk autoincr"`
+		RepoID      int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
+		CommitID    string
+		IsPrimary   bool
+		Language    string             `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"`
+		Percentage  float32            `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"`
+		Color       string             `xorm:"-"`
+		CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
+	}
+
+	type RepoIndexerType int
+
+	// RepoIndexerStatus see models/repo_stats_indexer.go
+	type RepoIndexerStatus struct {
+		ID          int64           `xorm:"pk autoincr"`
+		RepoID      int64           `xorm:"INDEX(s)"`
+		CommitSha   string          `xorm:"VARCHAR(40)"`
+		IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
+	}
+
+	if err := x.Sync2(new(LanguageStat)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+	if err := x.Sync2(new(RepoIndexerStatus)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+	return nil
+}

models/models.go

@@ -116,6 +116,7 @@ func init() {
 		new(OAuth2AuthorizationCode),
 		new(OAuth2Grant),
 		new(Task),
+		new(LanguageStat),
 	)
 
 	gonicNames := []string{"SSL", "UID"}

models/repo.go

@@ -175,8 +175,9 @@ type Repository struct {
 	*Mirror    `xorm:"-"`
 	Status     RepositoryStatus `xorm:"NOT NULL DEFAULT 0"`
 
-	RenderingMetas map[string]string `xorm:"-"`
-	Units          []*RepoUnit       `xorm:"-"`
+	RenderingMetas  map[string]string `xorm:"-"`
+	Units           []*RepoUnit       `xorm:"-"`
+	PrimaryLanguage *LanguageStat     `xorm:"-"`
 
 	IsFork                          bool               `xorm:"INDEX NOT NULL DEFAULT false"`
 	ForkID                          int64              `xorm:"INDEX"`
@@ -185,7 +186,8 @@ type Repository struct {
 	TemplateID                      int64              `xorm:"INDEX"`
 	TemplateRepo                    *Repository        `xorm:"-"`
 	Size                            int64              `xorm:"NOT NULL DEFAULT 0"`
-	IndexerStatus                   *RepoIndexerStatus `xorm:"-"`
+	CodeIndexerStatus               *RepoIndexerStatus `xorm:"-"`
+	StatsIndexerStatus              *RepoIndexerStatus `xorm:"-"`
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
 	CloseIssuesViaCommitInAnyBranch bool               `xorm:"NOT NULL DEFAULT false"`
 	Topics                          []string           `xorm:"TEXT JSON"`
@@ -1504,6 +1506,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
 		&Notification{RepoID: repoID},
 		&CommitStatus{RepoID: repoID},
 		&RepoIndexerStatus{RepoID: repoID},
+		&LanguageStat{RepoID: repoID},
 		&Comment{RefRepoID: repoID},
 		&Task{RepoID: repoID},
 	); err != nil {

models/repo_indexer.go

@@ -10,21 +10,32 @@ import (
 	"xorm.io/builder"
 )
 
+// RepoIndexerType specifies the repository indexer type
+type RepoIndexerType int
+
+const (
+	// RepoIndexerTypeCode code indexer
+	RepoIndexerTypeCode RepoIndexerType = iota // 0
+	// RepoIndexerTypeStats repository stats indexer
+	RepoIndexerTypeStats // 1
+)
+
 // RepoIndexerStatus status of a repo's entry in the repo indexer
 // For now, implicitly refers to default branch
 type RepoIndexerStatus struct {
-	ID        int64  `xorm:"pk autoincr"`
-	RepoID    int64  `xorm:"INDEX"`
-	CommitSha string `xorm:"VARCHAR(40)"`
+	ID          int64           `xorm:"pk autoincr"`
+	RepoID      int64           `xorm:"INDEX(s)"`
+	CommitSha   string          `xorm:"VARCHAR(40)"`
+	IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
 }
 
 // GetUnindexedRepos returns repos which do not have an indexer status
-func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) {
+func GetUnindexedRepos(indexerType RepoIndexerType, maxRepoID int64, page, pageSize int) ([]int64, error) {
 	ids := make([]int64, 0, 50)
 	cond := builder.Cond(builder.IsNull{
 		"repo_indexer_status.id",
 	})
-	sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id")
+	sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id AND repo_indexer_status.indexer_type = ?", indexerType)
 	if maxRepoID > 0 {
 		cond = builder.And(cond, builder.Lte{
 			"repository.id": maxRepoID,
@@ -43,40 +54,64 @@ func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) {
 	return ids, err
 }
 
-// GetIndexerStatus loads repo codes indxer status
-func (repo *Repository) GetIndexerStatus() error {
-	if repo.IndexerStatus != nil {
-		return nil
+// getIndexerStatus loads repo codes indxer status
+func (repo *Repository) getIndexerStatus(e Engine, indexerType RepoIndexerType) (*RepoIndexerStatus, error) {
+	switch indexerType {
+	case RepoIndexerTypeCode:
+		if repo.CodeIndexerStatus != nil {
+			return repo.CodeIndexerStatus, nil
+		}
+	case RepoIndexerTypeStats:
+		if repo.StatsIndexerStatus != nil {
+			return repo.StatsIndexerStatus, nil
+		}
 	}
-	status := &RepoIndexerStatus{RepoID: repo.ID}
-	has, err := x.Get(status)
+	status := &RepoIndexerStatus{RepoID: repo.ID, IndexerType: indexerType}
+	has, err := e.Get(status)
 	if err != nil {
-		return err
+		return nil, err
 	} else if !has {
 		status.CommitSha = ""
 	}
-	repo.IndexerStatus = status
-	return nil
+	switch indexerType {
+	case RepoIndexerTypeCode:
+		repo.CodeIndexerStatus = status
+	case RepoIndexerTypeStats:
+		repo.StatsIndexerStatus = status
+	}
+	return status, nil
 }
 
-// UpdateIndexerStatus updates indexer status
-func (repo *Repository) UpdateIndexerStatus(sha string) error {
-	if err := repo.GetIndexerStatus(); err != nil {
+// GetIndexerStatus loads repo codes indxer status
+func (repo *Repository) GetIndexerStatus(indexerType RepoIndexerType) (*RepoIndexerStatus, error) {
+	return repo.getIndexerStatus(x, indexerType)
+}
+
+// updateIndexerStatus updates indexer status
+func (repo *Repository) updateIndexerStatus(e Engine, indexerType RepoIndexerType, sha string) error {
+	status, err := repo.getIndexerStatus(e, indexerType)
+	if err != nil {
 		return fmt.Errorf("UpdateIndexerStatus: Unable to getIndexerStatus for repo: %s Error: %v", repo.FullName(), err)
 	}
-	if len(repo.IndexerStatus.CommitSha) == 0 {
-		repo.IndexerStatus.CommitSha = sha
-		_, err := x.Insert(repo.IndexerStatus)
+
+	if len(status.CommitSha) == 0 {
+		status.CommitSha = sha
+		_, err := e.Insert(status)
 		if err != nil {
 			return fmt.Errorf("UpdateIndexerStatus: Unable to insert repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err)
 		}
 		return nil
 	}
-	repo.IndexerStatus.CommitSha = sha
-	_, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha").
-		Update(repo.IndexerStatus)
+	status.CommitSha = sha
+	_, err = e.ID(status.ID).Cols("commit_sha").
+		Update(status)
 	if err != nil {
 		return fmt.Errorf("UpdateIndexerStatus: Unable to update repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err)
 	}
 	return nil
 }
+
+// UpdateIndexerStatus updates indexer status
+func (repo *Repository) UpdateIndexerStatus(indexerType RepoIndexerType, sha string) error {
+	return repo.updateIndexerStatus(x, indexerType, sha)
+}