Remove dupe events for details page (salesforce#66)

* Remove dupe events for details page

* Fix non-namespaces resources

* PR feedback

* PR Feedback

pkg/sloop/queries/respayloadquery.go

@@ -17,6 +17,7 @@ import (
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped"
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap"
 	"net/url"
+	"sort"
 	"time"
 )
 
@@ -34,7 +35,6 @@ func GetResPayload(params url.Values, t typed.Tables, startTime time.Time, endTi
 	var watchRes map[typed.WatchTableKey]*typed.KubeWatchResult
 	var previousKey *typed.WatchTableKey
 	var previousVal *typed.KubeWatchResult
-	var previousKeyFound bool
 
 	err := t.Db().View(func(txn badgerwrap.Txn) error {
 		var stats typed.RangeReadStats
@@ -59,7 +59,7 @@ func GetResPayload(params url.Values, t typed.Tables, startTime time.Time, endTi
 			var getErr error
 			previousVal, getErr = t.WatchTable().Get(txn, previousKey.String())
 			if getErr == nil {
-				previousKeyFound = true
+				watchRes[*previousKey] = previousVal
 			} else {
 				// we need to return error when getErr is not nil and its error is not keyNotFound
 				if getErr != badger.ErrKeyNotFound {
@@ -75,11 +75,11 @@ func GetResPayload(params url.Values, t typed.Tables, startTime time.Time, endTi
 		return []byte{}, err
 	}
 
-	payloadOutputList := getPayloadOutputList(watchRes, previousKeyFound, previousKey, previousVal)
+	payloadOutputList := getPayloadOutputList(watchRes)
 	glog.V(5).Infof("get the length of the resPayload is:%v", len(payloadOutputList))
-	if len(payloadOutputList) == 0 {
-		return []byte{}, nil
-	}
+
+	// Sort by time and remove entries with no payload change
+	payloadOutputList = removeDupePayloads(payloadOutputList)
 
 	var res ResPayLoadData
 	res.PayloadList = payloadOutputList
@@ -105,14 +105,13 @@ func getKeyComparator(params url.Values) *typed.WatchTableKey {
 	selectedName := params.Get(NameParam)
 	selectedKind := params.Get(KindParam)
 	if kubeextractor.IsClustersScopedResource(selectedKind) {
-		selectedNamespace = DefaultNamespace
+		selectedNamespace = ""
 	}
 	return typed.NewWatchTableKeyComparator(selectedKind, selectedNamespace, selectedName, time.Time{})
 }
 
 //todo: add unit tests
-func getPayloadOutputList(watchRes map[typed.WatchTableKey]*typed.KubeWatchResult, previousKeyFound bool,
-	previousKey *typed.WatchTableKey, previousVal *typed.KubeWatchResult) []PayloadOuput {
+func getPayloadOutputList(watchRes map[typed.WatchTableKey]*typed.KubeWatchResult) []PayloadOuput {
 
 	payloadOutputList := []PayloadOuput{}
 	for key, val := range watchRes {
@@ -124,14 +123,23 @@ func getPayloadOutputList(watchRes map[typed.WatchTableKey]*typed.KubeWatchResul
 		payloadOutputList = append(payloadOutputList, output)
 	}
 
-	// when previousKey is found, add it to the payload map as well
-	if previousKeyFound {
-		output := PayloadOuput{
-			PayLoadTime: previousKey.Timestamp.UnixNano(),
-			Payload:     previousVal.Payload,
-			PayloadKey:  previousKey.String(),
+	return payloadOutputList
+}
+
+func removeDupePayloads(payloads []PayloadOuput) []PayloadOuput {
+	sort.Slice(payloads, func(i, j int) bool {
+		return payloads[i].PayLoadTime < payloads[j].PayLoadTime
+	})
+
+	ret := []PayloadOuput{}
+
+	lastPayload := ""
+	for _, val := range payloads {
+		if val.Payload != lastPayload {
+			ret = append(ret, val)
 		}
-		payloadOutputList = append(payloadOutputList, output)
+		lastPayload = val.Payload
 	}
-	return payloadOutputList
+
+	return ret
 }

pkg/sloop/queries/respayloadquery_test.go

@@ -63,7 +63,7 @@ func Test_GetResPayload_False(t *testing.T) {
 	endTime := someTs.Add(60 * time.Minute)
 	tables := helper_get_resPayload(keys, t, somePTime)
 	res, err := GetResPayload(values, tables, starTime, endTime, someRequestId)
-	assert.Equal(t, string(res), "")
+	assert.Equal(t, "[]", string(res))
 	assert.Nil(t, err)
 }
 
@@ -88,7 +88,7 @@ func Test_GetResPayload_NotInTimeRange(t *testing.T) {
 	tables := helper_get_resPayload(keys, t, somePTime)
 	res, err := GetResPayload(values, tables, someTs.Add(2*time.Hour), someTs.Add(5*time.Hour), someRequestId)
 	assert.Nil(t, err)
-	assert.Equal(t, string(res), "")
+	assert.Equal(t, "[]", string(res))
 }
 
 func Test_GetResPayload_True_NoPreviousKeyFound(t *testing.T) {
@@ -146,12 +146,39 @@ func Test_GetResPayload_True_PreviousKeyFound(t *testing.T) {
   "payloadKey": "/watch/001546398000/someKind/someNamespace/someName/1546398245000000006",
   "payloadTime": 1546398245000000006,
   "payload": "{\n  \"metadata\": {\n    \"name\": \"someName\",\n    \"namespace\": \"someNamespace\",\n    \"uid\": \"6c2a9795-a282-11e9-ba2f-14187761de09\",\n    \"creationTimestamp\": \"2019-07-09T19:47:45Z\"\n  }\n}"
- },
- {
-  "payloadKey": "/watch/001546394400/someKind/someNamespace/someName/1546398245000000006",
-  "payloadTime": 1546398245000000006,
-  "payload": "{\n  \"metadata\": {\n    \"name\": \"someName\",\n    \"namespace\": \"someNamespace\",\n    \"uid\": \"6c2a9795-a282-11e9-ba2f-14187761de09\",\n    \"creationTimestamp\": \"2019-07-09T19:47:45Z\"\n  }\n}"
  }
 ]`
 	assertex.JsonEqual(t, expectedRes, string(res))
 }
+
+var somePayloadTs = time.Date(2019, 3, 1, 3, 4, 0, 0, time.UTC)
+
+func Test_removeDupePayloads_emptyWorks(t *testing.T) {
+	ret := removeDupePayloads([]PayloadOuput{})
+	assert.Equal(t, []PayloadOuput{}, ret)
+}
+
+func Test_removeDupePayloads_twoUnique_returnsTwo(t *testing.T) {
+	input := []PayloadOuput{
+		{PayLoadTime: somePayloadTs.Add(time.Minute).UnixNano(), Payload: "abc"},
+		{PayLoadTime: somePayloadTs.UnixNano(), Payload: "def"},
+	}
+	expected := []PayloadOuput{
+		{PayLoadTime: somePayloadTs.UnixNano(), Payload: "def"},
+		{PayLoadTime: somePayloadTs.Add(time.Minute).UnixNano(), Payload: "abc"},
+	}
+	ret := removeDupePayloads(input)
+	assert.Equal(t, expected, ret)
+}
+
+func Test_removeDupePayloads_twoTheSame_returnsFirstOne(t *testing.T) {
+	input := []PayloadOuput{
+		{PayLoadTime: somePayloadTs.Add(time.Minute).UnixNano(), Payload: "abc"},
+		{PayLoadTime: somePayloadTs.UnixNano(), Payload: "abc"},
+	}
+	expected := []PayloadOuput{
+		{PayLoadTime: somePayloadTs.UnixNano(), Payload: "abc"},
+	}
+	ret := removeDupePayloads(input)
+	assert.Equal(t, expected, ret)
+}