Skip to content

Commit

Permalink
Use StoreMetadata from trivy-db (aquasecurity#509)
Browse files Browse the repository at this point in the history 
* db_test: Remove cruft

Signed-off-by: Simarpreet Singh <[email protected]>

* db: Add StoreMetadata from trivy-db.

Signed-off-by: Simarpreet Singh <[email protected]>

* mod: Update trivy-db dependency

Signed-off-by: Simarpreet Singh <[email protected]>

* mod: Bump trivy-db version

Signed-off-by: Simarpreet Singh <[email protected]>

* db: Eliminate metadata.Store

Signed-off-by: Simarpreet Singh <[email protected]>

* db: Add a TODO to move things into trivy-db repo

Signed-off-by: Simarpreet Singh <[email protected]>
  • Loading branch information
@simar7
simar7 authored Jun 22, 2020
1 parent 11ae6b2 commit 2ac672a
Show file tree
Hide file tree
Showing 7 changed files with 164 additions and 53 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.13
require (
github.com/aquasecurity/fanal v0.0.0-20200528202907-79693bf4a058
github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b
github.com/aquasecurity/trivy-db v0.0.0-20200514134639-7e57e3e02470
github.com/aquasecurity/trivy-db v0.0.0-20200616161554-cd5b3da29bc8
github.com/caarlos0/env/v6 v6.0.0
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/cheggaaa/pb/v3 v3.0.3
Expand Down
7 changes: 3 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b h1:55Ul
github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b/go.mod h1:BpNTD9vHfrejKsED9rx04ldM1WIbeyXGYxUrqTVwxVQ=
github.com/aquasecurity/testdocker v0.0.0-20200426142840-5f05bce6f12a h1:hsw7PpiymXP64evn/K7gsj3hWzMqLrdoeE6JkqDocVg=
github.com/aquasecurity/testdocker v0.0.0-20200426142840-5f05bce6f12a/go.mod h1:psfu0MVaiTDLpNxCoNsTeILSKY2EICBwv345f3M+Ffs=
github.com/aquasecurity/trivy-db v0.0.0-20200514134639-7e57e3e02470 h1:6VE+g4AK2uivPqZtVk/QtcCBb2rUjAvKqDNexSgqMC0=
github.com/aquasecurity/trivy-db v0.0.0-20200514134639-7e57e3e02470/go.mod h1:F77bF2nRbcH4EIhhcNEP585MoAKdLpEP3dihF9V1Hbw=
github.com/aquasecurity/trivy-db v0.0.0-20200616161554-cd5b3da29bc8 h1:PvRcn3v8lpccqmEEzmJmXrm47ag47OCt8ui+9APi4hA=
github.com/aquasecurity/trivy-db v0.0.0-20200616161554-cd5b3da29bc8/go.mod h1:EiFA908RL0ACrbYo/9HfT7f9QcdC2bZoIO5XAAcvz9A=
github.com/aquasecurity/vuln-list-update v0.0.0-20191016075347-3d158c2bf9a2 h1:xbdUfr2KE4THsFx9CFWtWpU91lF+YhgP46moV94nYTA=
github.com/aquasecurity/vuln-list-update v0.0.0-20191016075347-3d158c2bf9a2/go.mod h1:6NhOP0CjZJL27bZZcaHECtzWdwDDm2g6yCY0QgXEGQQ=
github.com/araddon/dateparse v0.0.0-20190426192744-0d74ffceef83/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
Expand Down Expand Up @@ -473,7 +473,6 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand Down Expand Up @@ -513,6 +512,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand All @@ -538,7 +538,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down
11 changes: 8 additions & 3 deletions integration/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package integration
import (
"compress/gzip"
"context"
"encoding/json"
"flag"
"io"
"io/ioutil"
Expand All @@ -13,8 +14,6 @@ import (
"path/filepath"
"time"

dbFile "github.com/aquasecurity/trivy/pkg/db"

"github.com/aquasecurity/trivy-db/pkg/db"
"github.com/spf13/afero"
)
Expand Down Expand Up @@ -53,7 +52,9 @@ func gunzipDB() (string, error) {
return "", err
}

err = dbFile.NewMetadata(afero.NewOsFs(), tmpDir).Store(db.Metadata{
fs := afero.NewOsFs()
metadataFile := filepath.Join(dbDir, "metadata.json")
b, err := json.Marshal(db.Metadata{
Version: 1,
Type: 1,
NextUpdate: time.Time{},
Expand All @@ -62,6 +63,10 @@ func gunzipDB() (string, error) {
if err != nil {
return "", err
}
err = afero.WriteFile(fs, metadataFile, b, 0600)
if err != nil {
return "", err
}

return tmpDir, nil
}
Expand Down
9 changes: 6 additions & 3 deletions internal/app_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package internal

import (
"bytes"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
Expand All @@ -10,7 +11,6 @@ import (

"github.com/stretchr/testify/require"

dbFile "github.com/aquasecurity/trivy/pkg/db"
"github.com/spf13/afero"

"github.com/aquasecurity/trivy-db/pkg/db"
Expand Down Expand Up @@ -87,17 +87,20 @@ Vulnerability DB:
}

if tt.createDB {
m := dbFile.NewMetadata(afero.NewOsFs(), cacheDir)
fs := afero.NewOsFs()
err := os.MkdirAll(filepath.Join(cacheDir, "db"), os.ModePerm)
require.NoError(t, err)
metadataFile := filepath.Join(cacheDir, "db", "metadata.json")

err = m.Store(db.Metadata{
b, err := json.Marshal(db.Metadata{
Version: 42,
Type: 1,
NextUpdate: time.Unix(1584403020, 0),
UpdatedAt: time.Unix(1584402020, 0),
})
require.NoError(t, err)
err = afero.WriteFile(fs, metadataFile, b, 0600)
require.NoError(t, err)
}

fw := new(bytes.Buffer)
Expand Down
21 changes: 4 additions & 17 deletions pkg/db/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ type Operation interface {

type dbOperation interface {
GetMetadata() (metadata db.Metadata, err error)
StoreMetadata(metadata db.Metadata, dir string) (err error)
}

type Client struct {
Expand Down Expand Up @@ -181,17 +182,17 @@ func (c Client) UpdateMetadata(cacheDir string) error {

metadata, err := c.dbc.GetMetadata()
if err != nil {
return xerrors.Errorf("unable to get a metadata: %w", err)
return xerrors.Errorf("unable to get metadata: %w", err)
}

if err = c.metadata.Store(metadata); err != nil {
if err = c.dbc.StoreMetadata(metadata, filepath.Join(cacheDir, "db")); err != nil {
return xerrors.Errorf("failed to store metadata: %w", err)
}

return nil
}

type Metadata struct {
type Metadata struct { // TODO: Move all Metadata things to trivy-db repo
fs afero.Fs
filePath string
}
Expand All @@ -210,20 +211,6 @@ func MetadataPath(cacheDir string) string {
return filepath.Join(dbDir, metadataFile)
}

// StoreMetadata stores database metadata as a file
func (m Metadata) Store(metadata db.Metadata) error {
f, err := m.fs.Create(m.filePath)
if err != nil {
return xerrors.Errorf("unable to create a metadata file: %w", err)
}
defer f.Close()

if err = json.NewEncoder(f).Encode(metadata); err != nil {
return xerrors.Errorf("unable to encode metadata: %w", err)
}
return nil
}

// DeleteMetadata deletes the file of database metadata
func (m Metadata) Delete() error {
if err := m.fs.Remove(m.filePath); err != nil {
Expand Down
121 changes: 102 additions & 19 deletions pkg/db/db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package db

import (
"context"
"encoding/json"
"errors"
"io/ioutil"
"os"
"testing"
Expand Down Expand Up @@ -142,11 +144,13 @@ func TestClient_NeedsUpdate(t *testing.T) {
fs := afero.NewMemMapFs()
metadata := NewMetadata(fs, "/cache")
if tc.metadata != (db.Metadata{}) {
metadata.Store(tc.metadata)
b, err := json.Marshal(tc.metadata)
require.NoError(t, err)
err = afero.WriteFile(fs, metadata.filePath, b, 0600)
require.NoError(t, err)
}

client := Client{
//dbc: mockConfig,
clock: tc.clock,
metadata: metadata,
}
Expand All @@ -161,22 +165,15 @@ func TestClient_NeedsUpdate(t *testing.T) {
}

assert.Equal(t, tc.expected, needsUpdate)
//mockConfig.AssertExpectations(t)
})
}
}

func TestClient_Download(t *testing.T) {
type getMetadataOutput struct {
metadata db.Metadata
err error
}

testCases := []struct {
name string
light bool
downloadDB []github.DownloadDBExpectation
getMetadata dbOperationGetMetadataExpectation
expectedContent []byte
expectedError error
}{
Expand All @@ -191,15 +188,6 @@ func TestClient_Download(t *testing.T) {
},
},
},
getMetadata: dbOperationGetMetadataExpectation{
Returns: dbOperationGetMetadataReturns{
Metadata: db.Metadata{
Version: 1,
Type: db.TypeFull,
NextUpdate: time.Date(2019, 9, 1, 0, 0, 0, 0, time.UTC),
},
},
},
},
{
name: "DownloadDB returns an error",
Expand Down Expand Up @@ -235,7 +223,6 @@ func TestClient_Download(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
mockConfig := new(mockDbOperation)
mockConfig.ApplyGetMetadataExpectation(tc.getMetadata)

mockGitHubClient, err := github.NewMockClient(tc.downloadDB)
require.NoError(t, err, tc.name)
Expand Down Expand Up @@ -263,3 +250,99 @@ func TestClient_Download(t *testing.T) {
})
}
}

func TestClient_UpdateMetadata(t *testing.T) {
testCases := []struct {
name string
getMetadataExpectation dbOperationGetMetadataExpectation
storeMetadataExpectation dbOperationStoreMetadataExpectation
expectedError error
}{
{
name: "happy path",
getMetadataExpectation: dbOperationGetMetadataExpectation{
Returns: dbOperationGetMetadataReturns{
Metadata: db.Metadata{
Version: 1,
Type: 1,
NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC),
UpdatedAt: time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC),
},
Err: nil,
},
},
storeMetadataExpectation: dbOperationStoreMetadataExpectation{
Metadata: db.Metadata{
Version: 1,
Type: 1,
NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC),
UpdatedAt: time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC),
},
},
},
{
name: "sad path, get metadata fails",
getMetadataExpectation: dbOperationGetMetadataExpectation{
Returns: dbOperationGetMetadataReturns{
Err: errors.New("get metadata failed"),
},
},
expectedError: errors.New("unable to get metadata: get metadata failed"),
},
{
name: "sad path, store metadata fails",
getMetadataExpectation: dbOperationGetMetadataExpectation{
Returns: dbOperationGetMetadataReturns{
Metadata: db.Metadata{
Version: 1,
Type: 1,
NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC),
UpdatedAt: time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC),
},
Err: nil,
},
},
storeMetadataExpectation: dbOperationStoreMetadataExpectation{
Metadata: db.Metadata{
Version: 1,
Type: 1,
NextUpdate: time.Date(2020, 4, 30, 23, 59, 59, 0, time.UTC),
UpdatedAt: time.Date(2006, 4, 30, 23, 59, 59, 0, time.UTC),
},
Returns: dbOperationStoreMetadataReturns{
Err: errors.New("store metadata failed"),
},
},
expectedError: errors.New("failed to store metadata: store metadata failed"),
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
mockConfig := new(mockDbOperation)
err := log.InitLogger(false, true)
require.NoError(t, err, "failed to init logger")

mockConfig.ApplyGetMetadataExpectation(tc.getMetadataExpectation)
mockConfig.ApplyStoreMetadataExpectation(tc.storeMetadataExpectation)

fs := afero.NewMemMapFs()
metadata := NewMetadata(fs, "/cache")

dir, err := ioutil.TempDir("", "db")
require.NoError(t, err, tc.name)
defer os.RemoveAll(dir)

pb := indicator.NewProgressBar(true)
client := NewClient(mockConfig, nil, pb, nil, metadata)

err = client.UpdateMetadata(dir)
switch {
case tc.expectedError != nil:
assert.EqualError(t, err, tc.expectedError.Error(), tc.name)
default:
assert.NoError(t, err, tc.name)
}
})
}
}
Loading

0 comments on commit 2ac672a

Please sign in to comment.