Skip to content

Commit

Permalink
Ingestion for the vulnerabilities (guacsec#496)
Browse files Browse the repository at this point in the history 
* Generated code

Signed-off-by: Mihai Maruseac <[email protected]>

* Move resolvers to backends

Signed-off-by: Mihai Maruseac <[email protected]>

* Ingest builders

Signed-off-by: Mihai Maruseac <[email protected]>

* Explicitly write the neo4j query for retrieval

Signed-off-by: Mihai Maruseac <[email protected]>

* Generate mutations to ingest CVEs, GHSAs and OSVs

Signed-off-by: Mihai Maruseac <[email protected]>

* Generated code

Signed-off-by: Mihai Maruseac <[email protected]>

* Move resolvers to backends

Signed-off-by: Mihai Maruseac <[email protected]>

* Ensure both clients ingest the same data, remove duplicates

Signed-off-by: Mihai Maruseac <[email protected]>

* Handle mutations on the in-memory backend

Signed-off-by: Mihai Maruseac <[email protected]>

* Ingest data in neo4j backend

Signed-off-by: Mihai Maruseac <[email protected]>

* Extract queries outside of transaction

Signed-off-by: Mihai Maruseac <[email protected]>

* Format

Signed-off-by: Mihai Maruseac <[email protected]>

* Reorder the query examples

Signed-off-by: Mihai Maruseac <[email protected]>

* Use consistent naming

Signed-off-by: Mihai Maruseac <[email protected]>

* s/n/root

Signed-off-by: Mihai Maruseac <[email protected]>

---------

Signed-off-by: Mihai Maruseac <[email protected]>
  • Loading branch information
@mihaimaruseac
mihaimaruseac authored Feb 27, 2023
1 parent 027a150 commit 32395d4
Show file tree
Hide file tree
Showing 22 changed files with 852 additions and 149 deletions.
3 changes: 3 additions & 0 deletions pkg/assembler/backends/backends.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ type Backend interface {
IngestSource(ctx context.Context, source *model.SourceInputSpec) (*model.Source, error)
IngestArtifact(ctx context.Context, artifact *model.ArtifactInputSpec) (*model.Artifact, error)
IngestBuilder(ctx context.Context, builder *model.BuilderInputSpec) (*model.Builder, error)
IngestCve(ctx context.Context, cve *model.CVEInputSpec) (*model.Cve, error)
IngestGhsa(ctx context.Context, ghsa *model.GHSAInputSpec) (*model.Ghsa, error)
IngestOsv(ctx context.Context, osv *model.OSVInputSpec) (*model.Osv, error)
}

// BackendArgs interface allows each backend to specify the arguments needed to
Expand Down
100 changes: 70 additions & 30 deletions pkg/assembler/backends/neo4j/cve.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,6 @@ func (e *cveYearToCveID) IdentifiablePropertyNames() []string {
}

func (c *neo4jClient) Cve(ctx context.Context, cveSpec *model.CVESpec) ([]*model.Cve, error) {

// fields: [year cveId cveId.id]
fields := getPreloads(ctx)
cveIDImplRequired := false
Expand All @@ -162,29 +161,27 @@ func (c *neo4jClient) Cve(ctx context.Context, cveSpec *model.CVESpec) ([]*model
session := c.driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
defer session.Close()

result, err := session.ReadTransaction(
func(tx neo4j.Transaction) (interface{}, error) {

var sb strings.Builder
var firstMatch bool = true
queryValues := map[string]any{}

sb.WriteString("MATCH (n:Cve)-[:CveIsYear]->(cveYear:CveYear)-[:CveHasID]->(cveID:CveID)")
var sb strings.Builder
var firstMatch bool = true
queryValues := map[string]any{}

if cveSpec.Year != nil {
sb.WriteString("MATCH (root:Cve)-[:CveIsYear]->(cveYear:CveYear)-[:CveHasID]->(cveID:CveID)")

matchProperties(&sb, firstMatch, "cveYear", "year", "$cveYear")
firstMatch = false
if cveSpec.Year != nil {
matchProperties(&sb, firstMatch, "cveYear", "year", "$cveYear")
queryValues["cveYear"] = cveSpec.Year
firstMatch = false
}

queryValues["cveYear"] = cveSpec.Year
}
if cveSpec.CveID != nil {
matchProperties(&sb, firstMatch, "cveID", "id", "$cveID")
queryValues["cveID"] = strings.ToLower(*cveSpec.CveID)
}

if cveSpec.CveID != nil {
matchProperties(&sb, firstMatch, "cveID", "id", "$cveID")
queryValues["cveID"] = strings.ToLower(*cveSpec.CveID)
}
sb.WriteString(" RETURN cveYear.year, cveID.id")

sb.WriteString(" RETURN cveYear.year, cveID.id")
result, err := session.ReadTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
result, err := tx.Run(sb.String(), queryValues)
if err != nil {
return nil, err
Expand Down Expand Up @@ -223,21 +220,20 @@ func (c *neo4jClient) cveYear(ctx context.Context, cveSpec *model.CVESpec) ([]*m
session := c.driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
defer session.Close()

result, err := session.ReadTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
var sb strings.Builder
queryValues := map[string]any{}

var sb strings.Builder
queryValues := map[string]any{}
sb.WriteString("MATCH (n:Cve)-[:CveIsYear]->(cveYear:CveYear)")

sb.WriteString("MATCH (n:Cve)-[:CveIsYear]->(cveYear:CveYear)")

if cveSpec.Year != nil {
if cveSpec.Year != nil {
matchProperties(&sb, true, "cveYear", "year", "$cveYear")
queryValues["cveYear"] = cveSpec.Year
}

matchProperties(&sb, true, "cveYear", "year", "$cveYear")
queryValues["cveYear"] = cveSpec.Year
}
sb.WriteString(" RETURN cveYear.year")

sb.WriteString(" RETURN cveYear.year")
result, err := session.ReadTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
result, err := tx.Run(sb.String(), queryValues)
if err != nil {
return nil, err
Expand All @@ -263,3 +259,47 @@ func (c *neo4jClient) cveYear(ctx context.Context, cveSpec *model.CVESpec) ([]*m

return result.([]*model.Cve), nil
}

func (c *neo4jClient) IngestCve(ctx context.Context, cve *model.CVEInputSpec) (*model.Cve, error) {
session := c.driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
defer session.Close()

values := map[string]any{}
values["year"] = cve.Year
values["id"] = strings.ToLower(cve.CveID)

result, err := session.WriteTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
query := `MERGE (root:Cve)
MERGE (root) -[:CveIsYear]-> (cveYear:CveYear{year:$year})
MERGE (cveYear) -[:CveHasID]-> (cveID:CveID{id:$id})
RETURN cveYear.year, cveID.id`
result, err := tx.Run(query, values)
if err != nil {
return nil, err
}

// query returns a single record
record, err := result.Single()
if err != nil {
return nil, err
}

// TODO(mihaimaruseac): Extract this to a utility since it is repeated
idStr := record.Values[1].(string)
id := &model.CVEId{ID: idStr}

yearStr := record.Values[0].(string)
src := model.Cve{
Year: yearStr,
CveID: []*model.CVEId{id},
}

return &src, nil
})
if err != nil {
return nil, err
}

return result.(*model.Cve), nil
}
58 changes: 48 additions & 10 deletions pkg/assembler/backends/neo4j/ghsa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,20 +99,20 @@ func (c *neo4jClient) Ghsa(ctx context.Context, ghsaSpec *model.GHSASpec) ([]*mo
session := c.driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
defer session.Close()

result, err := session.ReadTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
var sb strings.Builder
queryValues := map[string]any{}

var sb strings.Builder
queryValues := map[string]any{}
sb.WriteString("MATCH (root:Ghsa)-[:GhsaHasID]->(ghsaID:GhsaID)")

sb.WriteString("MATCH (n:Ghsa)-[:GhsaHasID]->(ghsaID:GhsaID)")
if ghsaSpec.GhsaID != nil {
matchProperties(&sb, true, "ghsaID", "id", "$ghsaID")
queryValues["ghsaID"] = strings.ToLower(*ghsaSpec.GhsaID)
}

if ghsaSpec.GhsaID != nil {
matchProperties(&sb, true, "ghsaID", "id", "$ghsaID")
queryValues["ghsaID"] = strings.ToLower(*ghsaSpec.GhsaID)
}
sb.WriteString(" RETURN ghsaID.id")

sb.WriteString(" RETURN ghsaID.id")
result, err := session.ReadTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
result, err := tx.Run(sb.String(), queryValues)
if err != nil {
return nil, err
Expand Down Expand Up @@ -141,3 +141,41 @@ func (c *neo4jClient) Ghsa(ctx context.Context, ghsaSpec *model.GHSASpec) ([]*mo

return result.([]*model.Ghsa), nil
}

func (c *neo4jClient) IngestGhsa(ctx context.Context, ghsa *model.GHSAInputSpec) (*model.Ghsa, error) {
session := c.driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
defer session.Close()

values := map[string]any{}
values["id"] = strings.ToLower(ghsa.GhsaID)

result, err := session.WriteTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
query := `MERGE (root:Ghsa)
MERGE (root) -[:GhsaHasID]-> (ghsaID:GhsaID{id:$id})
RETURN ghsaID.id`
result, err := tx.Run(query, values)
if err != nil {
return nil, err
}

// query returns a single record
record, err := result.Single()
if err != nil {
return nil, err
}

id := record.Values[0].(string)
ghsaID := &model.GHSAId{ID: id}
ghsa := &model.Ghsa{
GhsaID: []*model.GHSAId{ghsaID},
}

return ghsa, nil
})
if err != nil {
return nil, err
}

return result.(*model.Ghsa), nil
}
18 changes: 5 additions & 13 deletions pkg/assembler/backends/neo4j/ingest_testdata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,35 +88,27 @@ func registerAllGHSA(client *neo4jClient) error {
if err != nil {
return err
}
err = client.registerGhsa("GHSA-h45f-rjvw-2rv2")
if err != nil {
return err
}
err = client.registerGhsa("GHSA-h45f-rjvw-2rv2")
if err != nil {
return err
}
return nil
}

func registerAllOSV(client *neo4jClient) error {
err := client.registerOSV("CVE-2019-3456")
err := client.registerOSV("CVE-2019-13110")
if err != nil {
return err
}
err = client.registerOSV("CVE-2014-53356")
err = client.registerOSV("CVE-2014-8139")
if err != nil {
return err
}
err = client.registerOSV("CVE-2014-4432")
err = client.registerOSV("CVE-2014-8140")
if err != nil {
return err
}
err = client.registerOSV("CVE-2022-9876")
err = client.registerOSV("CVE-2022-26499")
if err != nil {
return err
}
err = client.registerOSV("CVE-2014-4432")
err = client.registerOSV("GHSA-h45f-rjvw-2rv2")
if err != nil {
return err
}
Expand Down
58 changes: 48 additions & 10 deletions pkg/assembler/backends/neo4j/osv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,20 +99,20 @@ func (c *neo4jClient) Osv(ctx context.Context, osvSpec *model.OSVSpec) ([]*model
session := c.driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
defer session.Close()

result, err := session.ReadTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
var sb strings.Builder
queryValues := map[string]any{}

var sb strings.Builder
queryValues := map[string]any{}
sb.WriteString("MATCH (root:Osv)-[:OsvHasID]->(osvID:OsvID)")

sb.WriteString("MATCH (n:Osv)-[:OsvHasID]->(osvID:OsvID)")
if osvSpec.OsvID != nil {
matchProperties(&sb, true, "osvID", "id", "$osvID")
queryValues["osvID"] = strings.ToLower(*osvSpec.OsvID)
}

if osvSpec.OsvID != nil {
matchProperties(&sb, true, "osvID", "id", "$osvID")
queryValues["osvID"] = strings.ToLower(*osvSpec.OsvID)
}
sb.WriteString(" RETURN osvID.id")

sb.WriteString(" RETURN osvID.id")
result, err := session.ReadTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
result, err := tx.Run(sb.String(), queryValues)
if err != nil {
return nil, err
Expand Down Expand Up @@ -141,3 +141,41 @@ func (c *neo4jClient) Osv(ctx context.Context, osvSpec *model.OSVSpec) ([]*model

return result.([]*model.Osv), nil
}

func (c *neo4jClient) IngestOsv(ctx context.Context, osv *model.OSVInputSpec) (*model.Osv, error) {
session := c.driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
defer session.Close()

values := map[string]any{}
values["id"] = strings.ToLower(osv.OsvID)

result, err := session.WriteTransaction(
func(tx neo4j.Transaction) (interface{}, error) {
query := `MERGE (root:Osv)
MERGE (root) -[:OsvHasID]-> (osvID:OsvID{id:$id})
RETURN osvID.id`
result, err := tx.Run(query, values)
if err != nil {
return nil, err
}

// query returns a single record
record, err := result.Single()
if err != nil {
return nil, err
}

id := record.Values[0].(string)
osvID := &model.OSVId{ID: id}
osv := &model.Osv{
OsvID: []*model.OSVId{osvID},
}

return osv, nil
})
if err != nil {
return nil, err
}

return result.(*model.Osv), nil
}
10 changes: 8 additions & 2 deletions pkg/assembler/backends/testing/cve.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,20 @@ func registerAllCVE(client *demoClient) {

// Ingest CVE

func (c *demoClient) registerCVE(year, id string) {
func (c *demoClient) registerCVE(year, id string) *model.Cve {
idLower := strings.ToLower(id)
for i, s := range c.cve {
if s.Year == year {
c.cve[i] = registerCveID(s, idLower)
return
return c.cve[i]
}
}

newCve := &model.Cve{Year: year}
newCve = registerCveID(newCve, idLower)
c.cve = append(c.cve, newCve)

return newCve
}

func registerCveID(c *model.Cve, id string) *model.Cve {
Expand Down Expand Up @@ -90,3 +92,7 @@ func filterCVEID(cve *model.Cve, cveSpec *model.CVESpec) (*model.Cve, error) {
CveID: cveID,
}, nil
}

func (c *demoClient) IngestCve(ctx context.Context, cve *model.CVEInputSpec) (*model.Cve, error) {
return c.registerCVE(cve.Year, cve.CveID), nil
}
12 changes: 8 additions & 4 deletions pkg/assembler/backends/testing/ghsa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,22 @@ func registerAllGHSA(client *demoClient) {
client.registerGhsa("GHSA-h45f-rjvw-2rv2")
client.registerGhsa("GHSA-xrw3-wqph-3fxg")
client.registerGhsa("GHSA-8v4j-7jgf-5rg9")
client.registerGhsa("GHSA-h45f-rjvw-2rv2")
client.registerGhsa("GHSA-h45f-rjvw-2rv2")
}

// Ingest GHSA

func (c *demoClient) registerGhsa(id string) {
func (c *demoClient) registerGhsa(id string) *model.Ghsa {
idLower := strings.ToLower(id)
for i, g := range c.ghsa {
c.ghsa[i] = registerGhsaID(g, idLower)
return
return c.ghsa[i]
}

newGhsa := &model.Ghsa{}
newGhsa = registerGhsaID(newGhsa, idLower)
c.ghsa = append(c.ghsa, newGhsa)

return newGhsa
}

func registerGhsaID(g *model.Ghsa, id string) *model.Ghsa {
Expand Down Expand Up @@ -85,3 +85,7 @@ func filterGHSAID(ghsa *model.Ghsa, ghsaSpec *model.GHSASpec) (*model.Ghsa, erro
GhsaID: ghsaID,
}, nil
}

func (c *demoClient) IngestGhsa(ctx context.Context, ghsa *model.GHSAInputSpec) (*model.Ghsa, error) {
return c.registerGhsa(ghsa.GhsaID), nil
}
Loading

0 comments on commit 32395d4

Please sign in to comment.