Skip to content

Commit

Permalink
Merge pull request #265 from mmulholla/pyxisMultiPages
Browse files Browse the repository at this point in the history 
allow for pyxis responses spanning more than one page
  • Loading branch information
@mmulholla
mmulholla authored Jun 7, 2022
2 parents 524421e + 24b4249 commit d73fc97
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 70 deletions.
193 changes: 125 additions & 68 deletions pkg/chartverifier/pyxis/pyxis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,17 @@ import (
"io/ioutil"
"net/http"
"strings"

"github.com/redhat-certification/chart-verifier/pkg/chartverifier/utils"
)

var pyxisBaseUrl = "https://catalog.redhat.com/api/containers/v1/repositories"

type RepositoriesBody struct {
PyxisRepositories []PyxisRepository `json:"data"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Total int `json:"total"`
}

type PyxisRepository struct {
Expand All @@ -39,18 +44,17 @@ type PyxisRepository struct {

type RegistriesBody struct {
PyxisRegistries []PyxisRegistry `json:"data"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Total int `json:"total"`
}

type PyxisRegistry struct {
Id string `json:"_id"`
ParsedData ImageData `json:"parsed_data"`
ImageId string `json:"image_id"`
Repositories []RegistryRepository `json:"repositories"`
}

type ImageData struct {
Digest string `json:"docker_image_digest"`
}

type RegistryRepository struct {
Registry string `json:"registry"`
Repository string `json:"repository"`
Expand All @@ -73,34 +77,59 @@ func GetImageRegistries(repository string) ([]string, error) {
var err error
var registries []string

req, _ := http.NewRequest("GET", pyxisBaseUrl, nil)
queryString := req.URL.Query()
queryString.Add("filter", fmt.Sprintf("repository==%s", repository))
req.URL.RawQuery = queryString.Encode()
req.Header.Set("X-API-KEY", "RedHatChartVerifier")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
err = errors.New(fmt.Sprintf("Error getting repository %s : %v\n", repository, err))
} else {
if resp.StatusCode == 200 {
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
var repositoriesBody RepositoriesBody
json.Unmarshal(body, &repositoriesBody)

if len(repositoriesBody.PyxisRepositories) > 0 {
for _, repo := range repositoriesBody.PyxisRepositories {
registries = append(registries, repo.Registry)
read := 0
total := 0
nextPage := 0
allDataRead := false

for !allDataRead {

utils.LogInfo(fmt.Sprintf("Look for repository %s at %s, page %d", repository, pyxisBaseUrl, nextPage))
req, _ := http.NewRequest("GET", pyxisBaseUrl, nil)
queryString := req.URL.Query()
queryString.Add("filter", fmt.Sprintf("repository==%s", repository))
queryString.Add("page_size", "100")
queryString.Add("page", fmt.Sprintf("%d", nextPage))
req.URL.RawQuery = queryString.Encode()
req.Header.Set("X-API-KEY", "RedHatChartVerifier")
client := &http.Client{}
resp, reqErr := client.Do(req)
if reqErr != nil {
err = errors.New(fmt.Sprintf("Error getting repository %s : %v\n", repository, err))
} else {
if resp.StatusCode == 200 {
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
var repositoriesBody RepositoriesBody
json.Unmarshal(body, &repositoriesBody)

if total == 0 {
total = repositoriesBody.Total
}
read += repositoriesBody.PageSize
if read >= total {
allDataRead = true
} else {
nextPage += 1
}
utils.LogInfo(fmt.Sprintf("page: %d, page_size: %d, total: %d", repositoriesBody.Page, repositoriesBody.PageSize, total))
if len(repositoriesBody.PyxisRepositories) > 0 {
for _, repo := range repositoriesBody.PyxisRepositories {
registries = append(registries, repo.Registry)
utils.LogInfo(fmt.Sprintf("Found repository in registry: %s", repo.Registry))
fmt.Println(fmt.Sprintf("Found repository in registry: %s", repo.Registry))
}
} else {
err = errors.New(fmt.Sprintf("Respository not found: %s", repository))
}
} else {
err = errors.New(fmt.Sprintf("Respository not found: %s", repository))
err = errors.New(fmt.Sprintf("Bad response code from Pyxis: %d : %s", resp.StatusCode, req.URL))
}
} else {
err = errors.New(fmt.Sprintf("Bad response code from Pyxis: %d : %s", resp.StatusCode, req.URL))
}
}

if err != nil {
utils.LogError(err.Error())
}
return registries, err
}

Expand All @@ -115,59 +144,84 @@ func IsImageInRegistry(imageRef ImageReference) (bool, error) {
Loops:
for _, registry := range imageRef.Registries {

requestUrl := fmt.Sprintf("%s/registry/%s/repository/%s/images", pyxisBaseUrl, registry, imageRef.Repository)
req, _ := http.NewRequest("GET", requestUrl, nil)
queryString := req.URL.Query()
queryString.Add("filter", fmt.Sprintf("repositories=em=(repository==%s;registry==%s)", imageRef.Repository, registry))
req.URL.RawQuery = queryString.Encode()
req.Header.Set("X-API-KEY", "RedHatChartVerifier")
client := &http.Client{}
resp, reqErr := client.Do(req)
read := 0
total := 0
nextPage := 0
allDataRead := false

if reqErr == nil {
if resp.StatusCode == 200 {
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
var registriesBody RegistriesBody
json.Unmarshal(body, &registriesBody)

if len(registriesBody.PyxisRegistries) > 0 {
found = false
for _, reg := range registriesBody.PyxisRegistries {
if len(imageRef.Sha) > 0 {
if imageRef.Sha == reg.ParsedData.Digest {
found = true
err = nil
continue Loops
requestUrl := fmt.Sprintf("%s/registry/%s/repository/%s/images", pyxisBaseUrl, registry, imageRef.Repository)
utils.LogInfo(fmt.Sprintf("Search url: %s, tag: %s, sha: %s ", requestUrl, imageRef.Tag, imageRef.Sha))

for !allDataRead && err == nil && !found {

req, _ := http.NewRequest("GET", requestUrl, nil)
queryString := req.URL.Query()
queryString.Add("filter", fmt.Sprintf("repositories=em=(repository==%s;registry==%s)", imageRef.Repository, registry))
queryString.Add("page_size", "100")
queryString.Add("page", fmt.Sprintf("%d", nextPage))
req.URL.RawQuery = queryString.Encode()
req.Header.Set("X-API-KEY", "RedHatChartVerifier")
client := &http.Client{}
resp, reqErr := client.Do(req)

if reqErr == nil {
if resp.StatusCode == 200 {
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
var registriesBody RegistriesBody
json.Unmarshal(body, &registriesBody)

if total == 0 {
total = registriesBody.Total
}
read += registriesBody.PageSize
if read >= total {
allDataRead = true
} else {
nextPage += 1
}
utils.LogInfo(fmt.Sprintf("page: %d, page_size: %d, total: %d", registriesBody.Page, registriesBody.PageSize, registriesBody.Total))

if len(registriesBody.PyxisRegistries) > 0 {
found = false
for _, reg := range registriesBody.PyxisRegistries {
if len(imageRef.Sha) > 0 {
if imageRef.Sha == reg.ImageId {
utils.LogInfo(fmt.Sprintf("sha found: %s", imageRef.Sha))
found = true
err = nil
continue Loops
} else {
shas = append(shas, reg.ImageId)
}
} else {
shas = append(shas, reg.ParsedData.Digest)
}
} else {
for _, repo := range reg.Repositories {
if repo.Repository == imageRef.Repository && repo.Registry == registry {
if len(imageRef.Sha) == 0 {
for _, tag := range repo.Tags {
if tag.Name == imageRef.Tag {
found = true
err = nil
continue Loops
} else {
tags = append(tags, tag.Name)
for _, repo := range reg.Repositories {
if repo.Repository == imageRef.Repository && repo.Registry == registry {
if len(imageRef.Sha) == 0 {
for _, tag := range repo.Tags {
if tag.Name == imageRef.Tag {
utils.LogInfo(fmt.Sprintf("tag found: %s", imageRef.Tag))
found = true
err = nil
continue Loops
} else {
tags = append(tags, tag.Name)
}
}
}
}
}
}
}
} else {
err = errors.New(fmt.Sprintf("No images found for Registry/Repository: %s/%s", registry, imageRef.Repository))
}
} else {
err = errors.New(fmt.Sprintf("No images found for Registry/Repository: %s/%s", registry, imageRef.Repository))
err = errors.New(fmt.Sprintf("Bad response code %d from pyxis request : %s", resp.StatusCode, requestUrl))
}
} else {
err = errors.New(fmt.Sprintf("Bad response code %d from pyxis request : %s", resp.StatusCode, requestUrl))
err = reqErr
}
} else {
err = reqErr
}
}
if !found {
Expand All @@ -179,5 +233,8 @@ Loops:
}
}
}
if err != nil {
utils.LogError(err.Error())
}
return found, err
}
7 changes: 5 additions & 2 deletions pkg/chartverifier/pyxis/pyxis_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ func Test_getImageRegistries(t *testing.T) {
{description: "Test rhel6.7 respository", repository: "rhel6.7", registry: "registry.access.redhat.com", message: ""},
{description: "Test rhel8/nginx-116 respository", repository: "rhel8/nginx-116", registry: "registry.access.redhat.com", message: ""},
{description: "Test ibm/nginx respository", repository: "ibm/nginx", registry: "non_registry", message: ""},
{description: "Test turbonomic/zookeeper respository", repository: "turbonomic/zookeeper", registry: "registry.connect.redhat.com", message: ""}}
{description: "Test turbonomic/zookeeper respository", repository: "turbonomic/zookeeper", registry: "registry.connect.redhat.com", message: ""},
{description: "Test cpopen/ibmcloud-object-storage-driver respository", repository: "cpopen/ibmcloud-object-storage-driver", registry: "icr.io", message: ""}}

for _, tc := range PassTestCases {
t.Run(tc.description, func(t *testing.T) {
Expand Down Expand Up @@ -75,8 +76,10 @@ func Test_checkImageInRegistry(t *testing.T) {
{description: "Test turbonomic/zookeeper respository and version found.", imageRef: ImageReference{Repository: "turbonomic/zookeeper", Registries: []string{"registry.connect.redhat.com"}, Tag: "8.1.2", Sha: ""}, message: ""},
{description: "Test cpopen/ibmcloud-object-storage-driver respository and sha found.", imageRef: ImageReference{Repository: "cpopen/ibmcloud-object-storage-driver", Registries: []string{"icr.io"}, Tag: "", Sha: "sha256:fc17bb3e89d00b3eb0f50b3ea83aa75c52e43d8e56cf2e0f17475e934eeeeb5f"}, message: ""},
{description: "Test cpopen/ibmcloud-object-storage-plugin respository and sha found.", imageRef: ImageReference{Repository: "cpopen/ibmcloud-object-storage-plugin", Registries: []string{"icr.io"}, Tag: "", Sha: "sha256:cf654987c38d048bc9e654f3928e9ce9a2a4fd47ce0283bb5f339c1b99298e6e"}, message: ""},
{description: "Test postgresql-10-rhel7 respository and tag found", imageRef: ImageReference{Repository: "rhscl/postgresql-10-rhel7", Registries: []string{"registry.access.redhat.com"}, Tag: "latest", Sha: ""}, message: ""},
{description: "Test cpopen/ibmcloud-object-storage-plugin respository sha found.", imageRef: ImageReference{Repository: "cpopen/ibmcloud-object-storage-plugin", Registries: []string{"icr.io"}, Tag: "", Sha: "sha256:7c00bc76f91d456164f98375cd8932a0ec500c9dca1728368f3c1ccdbfd96e91"}, message: ""},
{description: "Test cpopen/ibmcloud-object-storage-driver respository sha found.", imageRef: ImageReference{Repository: "cpopen/ibmcloud-object-storage-driver", Registries: []string{"icr.io"}, Tag: "", Sha: "sha256:667667c5907d0ad145e8518ca0f8cf013ca778d6738b028d1cd08103b1b64667"}, message: ""},
}

for _, tc := range PassTestCases {
t.Run(tc.description, func(t *testing.T) {
found, err := IsImageInRegistry(tc.imageRef)
Expand Down

0 comments on commit d73fc97

Please sign in to comment.