Skip to content

Commit

Permalink
Move all push update operations to a queue (go-gitea#10133)
Browse files Browse the repository at this point in the history
* Fix test

* Add no queue for test only

* improve code

* Auto watch whatever branch operation

* Fix lint

* Rename noqueue to immediate

* Remove old PushUpdate function

* Fix tests

Co-authored-by: zeripath <[email protected]>
Co-authored-by: techknowlogick <[email protected]>
  • Loading branch information
3 people authored Sep 11, 2020
1 parent 910947f commit dd1a651
Show file tree
Hide file tree
Showing 16 changed files with 552 additions and 585 deletions.
5 changes: 4 additions & 1 deletion integrations/mssql.ini.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ REPO_INDEXER_PATH = integrations/indexers-mssql/repos.bleve
[queue.code_indexer]
TYPE = immediate

[queue.push_update]
TYPE = immediate

[repository]
ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mssql/gitea-repositories

Expand Down Expand Up @@ -89,4 +92,4 @@ LEVEL = Debug
[security]
INSTALL_LOCK = true
SECRET_KEY = 9pCviYTWSb
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ
5 changes: 4 additions & 1 deletion integrations/mysql.ini.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ REPO_INDEXER_PATH = integrations/indexers-mysql/repos.bleve
[queue.code_indexer]
TYPE = immediate

[queue.push_update]
TYPE = immediate

[repository]
ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mysql/gitea-repositories

Expand Down Expand Up @@ -109,4 +112,4 @@ LEVEL = Debug
[security]
INSTALL_LOCK = true
SECRET_KEY = 9pCviYTWSb
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ
4 changes: 4 additions & 0 deletions integrations/mysql8.ini.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ REPO_INDEXER_PATH = integrations/indexers-mysql8/repos.bleve
[queue.code_indexer]
TYPE = immediate

[queue.push_update]
TYPE = immediate

[repository]
ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mysql8/gitea-repositories

Expand Down Expand Up @@ -83,3 +86,4 @@ LEVEL = Debug
INSTALL_LOCK = true
SECRET_KEY = 9pCviYTWSb
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ

5 changes: 4 additions & 1 deletion integrations/pgsql.ini.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ REPO_INDEXER_PATH = integrations/indexers-pgsql/repos.bleve
[queue.code_indexer]
TYPE = immediate

[queue.push_update]
TYPE = immediate

[repository]
ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-pgsql/gitea-repositories

Expand Down Expand Up @@ -90,4 +93,4 @@ LEVEL = Debug
[security]
INSTALL_LOCK = true
SECRET_KEY = 9pCviYTWSb
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ
3 changes: 3 additions & 0 deletions integrations/sqlite.ini.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ REPO_INDEXER_PATH = integrations/indexers-sqlite/repos.bleve
[queue.code_indexer]
TYPE = immediate

[queue.push_update]
TYPE = immediate

[repository]
ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-sqlite/gitea-repositories

Expand Down
141 changes: 0 additions & 141 deletions modules/repofiles/action.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
package repofiles

import (
"encoding/json"
"fmt"
"html"
"regexp"
Expand All @@ -14,12 +13,9 @@ import (
"time"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/references"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
)

const (
Expand Down Expand Up @@ -220,140 +216,3 @@ func UpdateIssuesCommit(doer *models.User, repo *models.Repository, commits []*r
}
return nil
}

// CommitRepoActionOptions represent options of a new commit action.
type CommitRepoActionOptions struct {
PushUpdateOptions

RepoOwnerID int64
Commits *repository.PushCommits
}

// CommitRepoAction adds new commit action to the repository, and prepare
// corresponding webhooks.
func CommitRepoAction(optsList ...*CommitRepoActionOptions) error {
var pusher *models.User
var repo *models.Repository
actions := make([]*models.Action, len(optsList))

for i, opts := range optsList {
if pusher == nil || pusher.Name != opts.PusherName {
var err error
pusher, err = models.GetUserByName(opts.PusherName)
if err != nil {
return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err)
}
}

if repo == nil || repo.OwnerID != opts.RepoOwnerID || repo.Name != opts.RepoName {
var err error
if repo != nil {
// Change repository empty status and update last updated time.
if err := models.UpdateRepository(repo, false); err != nil {
return fmt.Errorf("UpdateRepository: %v", err)
}
}
repo, err = models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName)
if err != nil {
return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err)
}
}
refName := git.RefEndName(opts.RefFullName)

// Change default branch and empty status only if pushed ref is non-empty branch.
if repo.IsEmpty && opts.IsBranch() && !opts.IsDelRef() {
repo.DefaultBranch = refName
repo.IsEmpty = false
if refName != "master" {
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
return err
}
if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil {
if !git.IsErrUnsupportedVersion(err) {
gitRepo.Close()
return err
}
}
gitRepo.Close()
}
}

opType := models.ActionCommitRepo

// Check it's tag push or branch.
if opts.IsTag() {
opType = models.ActionPushTag
if opts.IsDelRef() {
opType = models.ActionDeleteTag
}
opts.Commits = &repository.PushCommits{}
} else if opts.IsDelRef() {
opType = models.ActionDeleteBranch
opts.Commits = &repository.PushCommits{}
} else {
// if not the first commit, set the compare URL.
if !opts.IsNewRef() {
opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID)
}

if err := UpdateIssuesCommit(pusher, repo, opts.Commits.Commits, refName); err != nil {
log.Error("updateIssuesCommit: %v", err)
}
}

if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum {
opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum]
}

data, err := json.Marshal(opts.Commits)
if err != nil {
return fmt.Errorf("Marshal: %v", err)
}

actions[i] = &models.Action{
ActUserID: pusher.ID,
ActUser: pusher,
OpType: opType,
Content: string(data),
RepoID: repo.ID,
Repo: repo,
RefName: refName,
IsPrivate: repo.IsPrivate,
}

var isHookEventPush = true
switch opType {
case models.ActionCommitRepo: // Push
if opts.IsNewBranch() {
notification.NotifyCreateRef(pusher, repo, "branch", opts.RefFullName)
}
case models.ActionDeleteBranch: // Delete Branch
notification.NotifyDeleteRef(pusher, repo, "branch", opts.RefFullName)

case models.ActionPushTag: // Create
notification.NotifyCreateRef(pusher, repo, "tag", opts.RefFullName)

case models.ActionDeleteTag: // Delete Tag
notification.NotifyDeleteRef(pusher, repo, "tag", opts.RefFullName)
default:
isHookEventPush = false
}

if isHookEventPush {
notification.NotifyPushCommits(pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits)
}
}

if repo != nil {
// Change repository empty status and update last updated time.
if err := models.UpdateRepository(repo, false); err != nil {
return fmt.Errorf("UpdateRepository: %v", err)
}
}

if err := models.NotifyWatchers(actions...); err != nil {
return fmt.Errorf("NotifyWatchers: %v", err)
}
return nil
}
121 changes: 0 additions & 121 deletions modules/repofiles/action_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,133 +8,12 @@ import (
"testing"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"

"github.com/stretchr/testify/assert"
)

func testCorrectRepoAction(t *testing.T, opts *CommitRepoActionOptions, actionBean *models.Action) {
models.AssertNotExistsBean(t, actionBean)
assert.NoError(t, CommitRepoAction(opts))
models.AssertExistsAndLoadBean(t, actionBean)
models.CheckConsistencyFor(t, &models.Action{})
}

func TestCommitRepoAction(t *testing.T) {
samples := []struct {
userID int64
repositoryID int64
commitRepoActionOptions CommitRepoActionOptions
action models.Action
}{
{
userID: 2,
repositoryID: 16,
commitRepoActionOptions: CommitRepoActionOptions{
PushUpdateOptions: PushUpdateOptions{
RefFullName: "refName",
OldCommitID: "oldCommitID",
NewCommitID: "newCommitID",
},
Commits: &repository.PushCommits{
Commits: []*repository.PushCommit{
{
Sha1: "69554a6",
CommitterEmail: "[email protected]",
CommitterName: "User2",
AuthorEmail: "[email protected]",
AuthorName: "User2",
Message: "not signed commit",
},
{
Sha1: "27566bd",
CommitterEmail: "[email protected]",
CommitterName: "User2",
AuthorEmail: "[email protected]",
AuthorName: "User2",
Message: "good signed commit (with not yet validated email)",
},
},
Len: 2,
},
},
action: models.Action{
OpType: models.ActionCommitRepo,
RefName: "refName",
},
},
{
userID: 2,
repositoryID: 1,
commitRepoActionOptions: CommitRepoActionOptions{
PushUpdateOptions: PushUpdateOptions{
RefFullName: git.TagPrefix + "v1.1",
OldCommitID: git.EmptySHA,
NewCommitID: "newCommitID",
},
Commits: &repository.PushCommits{},
},
action: models.Action{
OpType: models.ActionPushTag,
RefName: "v1.1",
},
},
{
userID: 2,
repositoryID: 1,
commitRepoActionOptions: CommitRepoActionOptions{
PushUpdateOptions: PushUpdateOptions{
RefFullName: git.TagPrefix + "v1.1",
OldCommitID: "oldCommitID",
NewCommitID: git.EmptySHA,
},
Commits: &repository.PushCommits{},
},
action: models.Action{
OpType: models.ActionDeleteTag,
RefName: "v1.1",
},
},
{
userID: 2,
repositoryID: 1,
commitRepoActionOptions: CommitRepoActionOptions{
PushUpdateOptions: PushUpdateOptions{
RefFullName: git.BranchPrefix + "feature/1",
OldCommitID: "oldCommitID",
NewCommitID: git.EmptySHA,
},
Commits: &repository.PushCommits{},
},
action: models.Action{
OpType: models.ActionDeleteBranch,
RefName: "feature/1",
},
},
}

for _, s := range samples {
models.PrepareTestEnv(t)

user := models.AssertExistsAndLoadBean(t, &models.User{ID: s.userID}).(*models.User)
repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: s.repositoryID, OwnerID: user.ID}).(*models.Repository)
repo.Owner = user

s.commitRepoActionOptions.PusherName = user.Name
s.commitRepoActionOptions.RepoOwnerID = user.ID
s.commitRepoActionOptions.RepoName = repo.Name

s.action.ActUserID = user.ID
s.action.RepoID = repo.ID
s.action.Repo = repo
s.action.IsPrivate = repo.IsPrivate

testCorrectRepoAction(t, &s.commitRepoActionOptions, &s.action)
}
}

func TestUpdateIssuesCommit(t *testing.T) {
assert.NoError(t, models.PrepareTestDatabase())
pushCommits := []*repository.PushCommit{
Expand Down
Loading

0 comments on commit dd1a651

Please sign in to comment.