fix(sonarqube): issue collector failed when issue status updated (apa…

…che#5544) (apache#5548) * fix(sonarqube): issue collector failed when issue status updated (apache#5544) cherry-pick * fix: fix golangci-lint error
ZhangNing10 · Jun 26, 2023 · 8b6a1ba · 8b6a1ba
1 parent 9d49a9f
commit 8b6a1ba
Show file tree

Hide file tree

Showing 11 changed files with 48 additions and 14 deletions.
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go b/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
@@ -66,15 +66,15 @@ func Test_topologicalSort(t *testing.T) {
 		{
 			name: "cycle error",
 			args: args{[]*plugin.SubTaskMeta{
-				&plugin.SubTaskMeta{
+				{
 					Name: "D",
-					Dependencies: []*plugin.SubTaskMeta{&plugin.SubTaskMeta{
+					Dependencies: []*plugin.SubTaskMeta{{
 						Name: "E",
 					}},
 				},
-				&plugin.SubTaskMeta{
+				{
 					Name: "E",
-					Dependencies: []*plugin.SubTaskMeta{&plugin.SubTaskMeta{
+					Dependencies: []*plugin.SubTaskMeta{{
 						Name: "D",
 					}},
 				},

diff --git a/backend/plugins/dora/impl/impl_test.go b/backend/plugins/dora/impl/impl_test.go
@@ -39,12 +39,12 @@ func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
 	doraOutputPlan := plugin.PipelinePlan{
 		plugin.PipelineStage{
 			{
-				Plugin:   "dora",
+				Plugin: "dora",
 				Subtasks: []string{
 					"generateDeploymentCommits",
 					"enrichPrevSuccessDeploymentCommits",
 				},
-				Options:  map[string]interface{}{"projectName": projectName},
+				Options: map[string]interface{}{"projectName": projectName},
 			},
 		},
 		plugin.PipelineStage{
@@ -56,7 +56,7 @@ func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
 		},
 		plugin.PipelineStage{
 			{
-				Plugin:  "dora",
+				Plugin: "dora",
 				Subtasks: []string{
 					"calculateChangeLeadTime",
 					"ConnectIncidentToDeployment",

diff --git a/backend/plugins/sonarqube/e2e/account_test.go b/backend/plugins/sonarqube/e2e/account_test.go
@@ -19,6 +19,7 @@ package e2e
 
 import (
 	"testing"
+	"time"
 
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
@@ -37,6 +38,7 @@ func TestSonarqubeAccountDataFlow(t *testing.T) {
 		Options: &tasks.SonarqubeOptions{
 			ConnectionId: 1,
 		},
+		TaskStartTime: time.Now(),
 	}
 
 	// import raw data table
@@ -51,6 +53,7 @@ func TestSonarqubeAccountDataFlow(t *testing.T) {
 		Options: &tasks.SonarqubeOptions{
 			ConnectionId: 2,
 		},
+		TaskStartTime: time.Now(),
 	}
 
 	dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData2)

diff --git a/backend/plugins/sonarqube/e2e/filemetrics_test.go b/backend/plugins/sonarqube/e2e/filemetrics_test.go
@@ -17,6 +17,7 @@ package e2e
 
 import (
 	"testing"
+	"time"
 
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
@@ -43,13 +44,15 @@ func TestSonarqubeFileMetricsDataFlow(t *testing.T) {
 			ConnectionId: 2,
 			ProjectKey:   "testDevLake",
 		},
+		TaskStartTime: time.Now(),
 	}
 	// Interfered data
 	taskData2 := &tasks.SonarqubeTaskData{
 		Options: &tasks.SonarqubeOptions{
 			ConnectionId: 1,
 			ProjectKey:   "testNone",
 		},
+		TaskStartTime: time.Now(),
 	}
 
 	// verify extraction

diff --git a/backend/plugins/sonarqube/e2e/hotspot_test.go b/backend/plugins/sonarqube/e2e/hotspot_test.go
@@ -17,6 +17,7 @@ package e2e
 
 import (
 	"testing"
+	"time"
 
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
@@ -41,13 +42,15 @@ func TestSonarqubeHotspotDataFlow(t *testing.T) {
 			ConnectionId: 1,
 			ProjectKey:   "f5a50c63-2e8f-4107-9014-853f6f467757",
 		},
+		TaskStartTime: time.Now(),
 	}
 	// Interfered data
 	taskData2 := &tasks.SonarqubeTaskData{
 		Options: &tasks.SonarqubeOptions{
 			ConnectionId: 2,
 			ProjectKey:   "testWarrenEtcd",
 		},
+		TaskStartTime: time.Now(),
 	}
 
 	// verify extraction

diff --git a/backend/plugins/sonarqube/e2e/issue_test.go b/backend/plugins/sonarqube/e2e/issue_test.go
@@ -17,6 +17,7 @@ package e2e
 
 import (
 	"testing"
+	"time"
 
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
@@ -41,13 +42,15 @@ func TestSonarqubeIssueDataFlow(t *testing.T) {
 			ConnectionId: 1,
 			ProjectKey:   "f5a50c63-2e8f-4107-9014-853f6f467757",
 		},
+		TaskStartTime: time.Now(),
 	}
 	// Interfered data
 	taskData2 := &tasks.SonarqubeTaskData{
 		Options: &tasks.SonarqubeOptions{
 			ConnectionId: 2,
 			ProjectKey:   "testWarrenEtcd",
 		},
+		TaskStartTime: time.Now(),
 	}
 
 	// verify extraction

diff --git a/backend/plugins/sonarqube/e2e/project_test.go b/backend/plugins/sonarqube/e2e/project_test.go
@@ -19,6 +19,7 @@ package e2e
 
 import (
 	"testing"
+	"time"
 
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
@@ -42,6 +43,7 @@ func TestSonarqubeProjectDataFlow(t *testing.T) {
 			ConnectionId: 2,
 			ProjectKey:   "e2c6d5e9-a321-4e8c-b322-03d9599ef962",
 		},
+		TaskStartTime: time.Now(),
 	}
 
 	dataflowTester.FlushTabler(&codequality.CqProject{})

diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
@@ -19,6 +19,7 @@ package impl
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/apache/incubator-devlake/core/dal"
 
@@ -127,8 +128,9 @@ func (p Sonarqube) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin
 		return nil, errors.Default.Wrap(err, "unable to get Sonarqube API client instance")
 	}
 	taskData := &tasks.SonarqubeTaskData{
-		Options:   op,
-		ApiClient: apiClient,
+		Options:       op,
+		ApiClient:     apiClient,
+		TaskStartTime: time.Now(),
 	}
 	// even we have project in _tool_sonaqube_projects, we still need to collect project to update LastAnalysisDate
 	var scope models.SonarqubeProject

diff --git a/backend/plugins/sonarqube/tasks/hotspots_collector.go b/backend/plugins/sonarqube/tasks/hotspots_collector.go
@@ -57,6 +57,22 @@ func CollectHotspots(taskCtx plugin.SubTaskContext) errors.Error {
 				Data []json.RawMessage `json:"hotspots"`
 			}
 			err := helper.UnmarshalResponse(res, &resData)
+
+			// check if sonar report updated during collecting
+			var issue struct {
+				UpdateDate *helper.Iso8601Time `json:"updateDate"`
+			}
+			for _, v := range resData.Data {
+				err = errors.Convert(json.Unmarshal(v, &issue))
+				if err != nil {
+					return nil, err
+				}
+				if issue.UpdateDate.ToTime().After(data.TaskStartTime) {
+					return nil, errors.Default.New(fmt.Sprintf(`Your data is affected by the latest analysis\n
+						Please recollect this project: %s`, data.Options.ProjectKey))
+				}
+			}
+
 			return resData.Data, err
 		},
 	})

diff --git a/backend/plugins/sonarqube/tasks/issues_collector.go b/backend/plugins/sonarqube/tasks/issues_collector.go
@@ -41,7 +41,6 @@ type SonarqubeIssueTimeIteratorNode struct {
 func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
 	logger := taskCtx.GetLogger()
 	logger.Info("collect issues")
-
 	iterator := helper.NewQueueIterator()
 	iterator.Push(
 		&SonarqubeIssueTimeIteratorNode{
@@ -154,19 +153,21 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
 			var resData struct {
 				Data []json.RawMessage `json:"issues"`
 			}
-			var issue struct {
-				UpdateDate *helper.Iso8601Time `json:"updateDate"`
-			}
 			err = helper.UnmarshalResponse(res, &resData)
 			if err != nil {
 				return nil, err
 			}
+
+			// check if sonar report updated during collecting
+			var issue struct {
+				UpdateDate *helper.Iso8601Time `json:"updateDate"`
+			}
 			for _, v := range resData.Data {
 				err = errors.Convert(json.Unmarshal(v, &issue))
 				if err != nil {
 					return nil, err
 				}
-				if issue.UpdateDate.ToTime().After(*data.LastAnalysisDate) {
+				if issue.UpdateDate.ToTime().After(data.TaskStartTime) {
 					return nil, errors.Default.New(fmt.Sprintf(`Your data is affected by the latest analysis\n
 						Please recollect this project: %s`, data.Options.ProjectKey))
 				}

diff --git a/backend/plugins/sonarqube/tasks/task_data.go b/backend/plugins/sonarqube/tasks/task_data.go
@@ -37,6 +37,7 @@ type SonarqubeTaskData struct {
 	Options          *SonarqubeOptions
 	ApiClient        *api.ApiAsyncClient
 	LastAnalysisDate *time.Time
+	TaskStartTime    time.Time
 }
 
 func DecodeAndValidateTaskOptions(options map[string]interface{}) (*SonarqubeOptions, errors.Error) {