From 59d5d5ffac0d19617b31e490b7786e1c2acb94aa Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger Date: Mon, 30 Jul 2018 14:06:17 -0700 Subject: [PATCH] Cleaning up unused code (#352) * removing pkg/repo Fixes https://github.com/gomods/athens/issues/335 * removing the gomod parser --- cmd/olympus/actions/app.go | 4 - cmd/olympus/actions/package_downloader.go | 79 --------- cmd/proxy/actions/app_proxy.go | 1 - cmd/proxy/actions/fetch.go | 70 -------- pkg/gomod/bytes/parser.go | 21 --- pkg/gomod/bytes/parser_test.go | 33 ---- pkg/gomod/file/parser.go | 26 --- pkg/gomod/file/parser_test.go | 43 ----- pkg/gomod/file/testdata/go.0.mod | 5 - pkg/gomod/file/testdata/go.1.mod | 5 - pkg/gomod/file/testdata/go.2.mod | 1 - pkg/gomod/file/testdata/go.3.mod | 1 - pkg/gomod/file/testdata/go.4.mod | 8 - pkg/gomod/parser.go | 51 ------ pkg/gomod/parser_test.go | 56 ------ pkg/gomod/zip/parser.go | 47 ------ pkg/gomod/zip/parser_test.go | 68 -------- pkg/gomod/zip/testdata/Gopkg.toml | 5 - pkg/gomod/zip/testdata/go.0.mod | 5 - pkg/gomod/zip/testdata/go.1.mod | 1 - pkg/gomod/zip/testdata/go.2.mod | 1 - pkg/gomod/zip/testdata/go.3.mod | 1 - pkg/gomod/zip/testdata/go.4.mod | 8 - pkg/repo/WARNING.md | 3 - pkg/repo/fetcher.go | 7 - pkg/repo/github/commitinfo.go | 66 -------- pkg/repo/github/github.go | 197 ---------------------- pkg/repo/github/github_test.go | 50 ------ pkg/repo/goget.go | 169 ------------------- pkg/repo/goget_test.go | 56 ------ 30 files changed, 1088 deletions(-) delete mode 100644 cmd/olympus/actions/package_downloader.go delete mode 100644 cmd/proxy/actions/fetch.go delete mode 100644 pkg/gomod/bytes/parser.go delete mode 100644 pkg/gomod/bytes/parser_test.go delete mode 100644 pkg/gomod/file/parser.go delete mode 100644 pkg/gomod/file/parser_test.go delete mode 100644 pkg/gomod/file/testdata/go.0.mod delete mode 100644 pkg/gomod/file/testdata/go.1.mod delete mode 100644 pkg/gomod/file/testdata/go.2.mod delete mode 100644 pkg/gomod/file/testdata/go.3.mod delete mode 100644 pkg/gomod/file/testdata/go.4.mod delete mode 100644 pkg/gomod/parser.go delete mode 100644 pkg/gomod/parser_test.go delete mode 100644 pkg/gomod/zip/parser.go delete mode 100644 pkg/gomod/zip/parser_test.go delete mode 100644 pkg/gomod/zip/testdata/Gopkg.toml delete mode 100644 pkg/gomod/zip/testdata/go.0.mod delete mode 100644 pkg/gomod/zip/testdata/go.1.mod delete mode 100644 pkg/gomod/zip/testdata/go.2.mod delete mode 100644 pkg/gomod/zip/testdata/go.3.mod delete mode 100644 pkg/gomod/zip/testdata/go.4.mod delete mode 100644 pkg/repo/WARNING.md delete mode 100644 pkg/repo/fetcher.go delete mode 100644 pkg/repo/github/commitinfo.go delete mode 100644 pkg/repo/github/github.go delete mode 100644 pkg/repo/github/github_test.go delete mode 100644 pkg/repo/goget.go delete mode 100644 pkg/repo/goget_test.go diff --git a/cmd/olympus/actions/app.go b/cmd/olympus/actions/app.go index 0618f3182..bf3627a64 100644 --- a/cmd/olympus/actions/app.go +++ b/cmd/olympus/actions/app.go @@ -144,9 +144,5 @@ func getWorker(store storage.Backend, eLog eventlog.Eventlog) (worker.Worker, er MaxFails: env.WorkerMaxFails(), } - if err := w.RegisterWithOptions(DownloadHandlerName, opts, GetPackageDownloaderJob(store, eLog, w)); err != nil { - return nil, err - } - return w, w.RegisterWithOptions(PushNotificationHandlerName, opts, GetProcessPushNotificationJob(store, eLog)) } diff --git a/cmd/olympus/actions/package_downloader.go b/cmd/olympus/actions/package_downloader.go deleted file mode 100644 index b1a265c1a..000000000 --- a/cmd/olympus/actions/package_downloader.go +++ /dev/null @@ -1,79 +0,0 @@ -package actions - -import ( - "context" - "errors" - "io/ioutil" - "path/filepath" - "time" - - "github.com/gobuffalo/buffalo/worker" - "github.com/gomods/athens/pkg/eventlog" - "github.com/gomods/athens/pkg/repo" - "github.com/gomods/athens/pkg/storage" - "github.com/spf13/afero" -) - -// GetPackageDownloaderJob porcesses queue of cache misses and downloads sources from VCS -func GetPackageDownloaderJob(s storage.Backend, e eventlog.Eventlog, w worker.Worker) worker.Handler { - return func(args worker.Args) error { - module, version, err := parsePackageDownloaderJobArgs(args) - if err != nil { - return err - } - - // download package - fs := afero.NewOsFs() - f, err := repo.NewGenericFetcher(fs, module, version) - if err != nil { - return err - } - - dirName, err := f.Fetch() - if err != nil { - return err - } - - modPath := filepath.Join(dirName, version+".mod") - modBytes, err := ioutil.ReadFile(modPath) - if err != nil { - return err - } - - zipPath := filepath.Join(dirName, version+".zip") - zipFile, err := fs.Open(zipPath) - if err != nil { - return err - } - defer zipFile.Close() - - infoPath := filepath.Join(dirName, version+".info") - infoBytes, err := ioutil.ReadFile(infoPath) - if err != nil { - return err - } - - // save it - if err := s.Save(context.Background(), module, version, modBytes, zipFile, infoBytes); err != nil { - return err - } - - // update log - _, err = e.Append(eventlog.Event{Module: module, Version: version, Time: time.Now(), Op: eventlog.OpAdd}) - return err - } -} - -func parsePackageDownloaderJobArgs(args worker.Args) (string, string, error) { - module, ok := args[workerModuleKey].(string) - if !ok { - return "", "", errors.New("module name not specified") - } - - version, ok := args[workerVersionKey].(string) - if !ok { - return "", "", errors.New("version not specified") - } - - return module, version, nil -} diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index 0b805900b..25dfbd59d 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -25,6 +25,5 @@ func addProxyRoutes( app.GET(download.PathVersionModule, download.VersionModuleHandler(dp, lggr, proxy)) app.GET(download.PathVersionZip, download.VersionZipHandler(dp, lggr, proxy)) - app.POST("/admin/fetch/{module:[a-zA-Z./]+}/{owner}/{repo}/{ref}/{version}", fetchHandler(storage)) return nil } diff --git a/cmd/proxy/actions/fetch.go b/cmd/proxy/actions/fetch.go deleted file mode 100644 index 529012c8c..000000000 --- a/cmd/proxy/actions/fetch.go +++ /dev/null @@ -1,70 +0,0 @@ -package actions - -import ( - "fmt" - "net/http" - "os" - "path/filepath" - - "github.com/bketelsen/buffet" - "github.com/gobuffalo/buffalo" - parser "github.com/gomods/athens/pkg/gomod/file" - "github.com/gomods/athens/pkg/module" - "github.com/gomods/athens/pkg/repo/github" - "github.com/gomods/athens/pkg/storage" - "github.com/pkg/errors" - "github.com/spf13/afero" -) - -// /admin/fetch/{module:[a-zA-Z./]+}/{owner}/{repo}/{ref}/{version} -func fetchHandler(store storage.Saver) func(c buffalo.Context) error { - return func(c buffalo.Context) error { - sp := buffet.SpanFromContext(c) - sp.SetOperationName("fetchHandler") - owner := c.Param("owner") - repo := c.Param("repo") - ref := c.Param("ref") - version := c.Param("version") - fs := afero.NewOsFs() - - git, err := github.NewGitFetcher(fs, owner, repo, ref) - if err != nil { - return err - } - - path, err := git.Fetch() - if err != nil { - os.Remove(path) - return err - } - - modFilePath := filepath.Join(path, "go.mod") - modBytes, err := afero.ReadFile(fs, modFilePath) - if err != nil { - return fmt.Errorf("couldn't find go.mod file (%s)", err) - } - - gomodParser := parser.NewFileParser(fs, modFilePath) - moduleName, err := gomodParser.ModuleName() - if err != nil { - return fmt.Errorf("couldn't parse go.mod file (%s)", err) - } - - zipBytes := module.MakeZip(fs, path, moduleName, version) - - infoFilePath := filepath.Join(path, version+".info") - infoBytes, err := afero.ReadFile(fs, infoFilePath) - if err != nil { - return fmt.Errorf("coudln't find .info file (%s)", err) - } - - saveErr := store.Save(c, moduleName, version, modBytes, zipBytes, infoBytes) - if storage.IsVersionAlreadyExistsErr(saveErr) { - return c.Render(http.StatusConflict, proxy.JSON(saveErr.Error())) - } else if err != nil { - return errors.WithStack(err) - } - - return c.Render(http.StatusOK, proxy.JSON(err.Error())) - } -} diff --git a/pkg/gomod/bytes/parser.go b/pkg/gomod/bytes/parser.go deleted file mode 100644 index 0cd27b4c9..000000000 --- a/pkg/gomod/bytes/parser.go +++ /dev/null @@ -1,21 +0,0 @@ -package bytes - -import ( - "bytes" - - "github.com/gomods/athens/pkg/gomod" -) - -// NewContentParser creates shorthand for parsing module name out of file content -func NewContentParser(content []byte) parser.GomodParser { - return contentParser{content: content} -} - -type contentParser struct { - content []byte -} - -func (p contentParser) ModuleName() (string, error) { - readCloser := bytes.NewReader(p.content) - return parser.Parse(readCloser) -} diff --git a/pkg/gomod/bytes/parser_test.go b/pkg/gomod/bytes/parser_test.go deleted file mode 100644 index 4d771f6d2..000000000 --- a/pkg/gomod/bytes/parser_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package bytes - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - parser "github.com/gomods/athens/pkg/gomod" -) - -func TestContentParser_ModuleName(t *testing.T) { - a := assert.New(t) - - var testCases = []struct { - content []byte - expected string - expectedErr error - }{ - {[]byte(`module "my/thing"`), "my/thing", nil}, - {[]byte(`module "my/thing2`), "", parser.ErrNotFound}, - {[]byte(`foobar`), "", parser.ErrNotFound}, - } - - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - cp := NewContentParser(tc.content) - actual, actualErr := cp.ModuleName() - - a.Equal(tc.expected, actual) - a.Equal(tc.expectedErr, actualErr) - }) - } -} diff --git a/pkg/gomod/file/parser.go b/pkg/gomod/file/parser.go deleted file mode 100644 index 3680d2eab..000000000 --- a/pkg/gomod/file/parser.go +++ /dev/null @@ -1,26 +0,0 @@ -package file - -import ( - "github.com/gomods/athens/pkg/gomod" - "github.com/spf13/afero" -) - -// NewFileParser creates shorthand for parsing module name out of file specified by filepath -func NewFileParser(fs afero.Fs, filepath string) parser.GomodParser { - return fileParser{filepath: filepath, fs: fs} -} - -type fileParser struct { - filepath string - fs afero.Fs -} - -func (p fileParser) ModuleName() (string, error) { - file, err := p.fs.Open(p.filepath) - if err != nil { - return "", err - } - defer file.Close() - - return parser.Parse(file) -} diff --git a/pkg/gomod/file/parser_test.go b/pkg/gomod/file/parser_test.go deleted file mode 100644 index a1584f034..000000000 --- a/pkg/gomod/file/parser_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package file - -import ( - "os" - "testing" - - "github.com/gomods/athens/pkg/gomod" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" -) - -func TestFileParser_ModuleName(t *testing.T) { - a := assert.New(t) - - var testCases = []struct { - file string - expected string - expectedErr error - }{ - {"testdata/go.0.mod", "my/thing", nil}, - {"testdata/go.1.mod", "my/thing2", nil}, - {"testdata/go.2.mod", "", parser.ErrNotFound}, - {"testdata/go.3.mod", "", parser.ErrNotFound}, - {"testdata/go.4.mod", "", parser.ErrNotFound}, - } - fs := afero.NewOsFs() - for _, tc := range testCases { - t.Run(tc.file, func(t *testing.T) { - fp := NewFileParser(fs, tc.file) - actual, actualErr := fp.ModuleName() - - a.Equal(tc.expected, actual) - a.Equal(tc.expectedErr, actualErr) - }) - } -} - -func TestFileParser_ModuleName_FileNotFound(t *testing.T) { - fs := afero.NewOsFs() - fp := NewFileParser(fs, "/not/exist") - _, err := fp.ModuleName() - assert.True(t, os.IsNotExist(err)) -} diff --git a/pkg/gomod/file/testdata/go.0.mod b/pkg/gomod/file/testdata/go.0.mod deleted file mode 100644 index 0e9376d1e..000000000 --- a/pkg/gomod/file/testdata/go.0.mod +++ /dev/null @@ -1,5 +0,0 @@ -module "my/thing" -require "other/thing" v1.0.2 -require "new/thing" v2.3.4 -exclude "old/thing" v1.2.3 -replace "bad/thing" v1.4.5 => "good/thing" v1.4.5 diff --git a/pkg/gomod/file/testdata/go.1.mod b/pkg/gomod/file/testdata/go.1.mod deleted file mode 100644 index d9bb9b722..000000000 --- a/pkg/gomod/file/testdata/go.1.mod +++ /dev/null @@ -1,5 +0,0 @@ -require "other/thing" v1.0.2 -require "new/thing" v2.3.4 -exclude "old/thing" v1.2.3 -replace "bad/thing" v1.4.5 => "good/thing" v1.4.5 -module "my/thing2" diff --git a/pkg/gomod/file/testdata/go.2.mod b/pkg/gomod/file/testdata/go.2.mod deleted file mode 100644 index 0d7780104..000000000 --- a/pkg/gomod/file/testdata/go.2.mod +++ /dev/null @@ -1 +0,0 @@ -module "my/thing \ No newline at end of file diff --git a/pkg/gomod/file/testdata/go.3.mod b/pkg/gomod/file/testdata/go.3.mod deleted file mode 100644 index 0d7780104..000000000 --- a/pkg/gomod/file/testdata/go.3.mod +++ /dev/null @@ -1 +0,0 @@ -module "my/thing \ No newline at end of file diff --git a/pkg/gomod/file/testdata/go.4.mod b/pkg/gomod/file/testdata/go.4.mod deleted file mode 100644 index dd3d047c0..000000000 --- a/pkg/gomod/file/testdata/go.4.mod +++ /dev/null @@ -1,8 +0,0 @@ -[[constraint]] - name = "github.com/user/project" - version = "1.0.0" - -[[constraint]] - name = "github.com/user/project2" - branch = "dev" - source = "github.com/myfork/project2" \ No newline at end of file diff --git a/pkg/gomod/parser.go b/pkg/gomod/parser.go deleted file mode 100644 index 2b8c611d4..000000000 --- a/pkg/gomod/parser.go +++ /dev/null @@ -1,51 +0,0 @@ -package parser - -import ( - "bufio" - "errors" - "io" - "regexp" -) - -const ( - moduleRegexp = `module "([\w\.@:%_\+-.~#?&]+/.+)"` -) - -var ( - re = regexp.MustCompile(moduleRegexp) - // ErrNotFound describe an error when can't find module name - ErrNotFound = errors.New("name not found") -) - -// GomodParser retrieves module name from parsable source -type GomodParser interface { - ModuleName() (string, error) -} - -// Parse parses file passed as a reader and returns module name out of it. -func Parse(reader io.Reader) (string, error) { - scanner := bufio.NewScanner(reader) - - for scanner.Scan() { - line := scanner.Text() - if name, found := checkVersion(line, re); found { - return name, nil - } - } - - if err := scanner.Err(); err != nil { - return "", err - } - - return "", ErrNotFound -} - -func checkVersion(line string, expression *regexp.Regexp) (string, bool) { - matches := expression.FindAllStringSubmatch(line, 1) - - if len(matches) == 0 || len(matches[0]) < 2 { - return "", false - } - - return matches[0][1], true -} diff --git a/pkg/gomod/parser_test.go b/pkg/gomod/parser_test.go deleted file mode 100644 index bcd1ecc52..000000000 --- a/pkg/gomod/parser_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package parser - -import ( - "bytes" - "errors" - "io" - "testing" - - "github.com/stretchr/testify/assert" -) - -var errReaderError = errors.New("errReader error") - -type errReader struct{} - -// Read implements io.Reader but returns error (not EOF) -func (e *errReader) Read(p []byte) (n int, err error) { return 0, errReaderError } - -func TestParse(t *testing.T) { - a := assert.New(t) - - var testCases = []struct { - reader io.Reader - expected string - expectedErr error - }{ - {newBuf(`module "my/thing"`), "my/thing", nil}, - {newBuf("module \"my/thing2\"\nmodule \"my/thing1\""), "my/thing2", nil}, - {newBuf(`module "github.com/gomods/athens"`), "github.com/gomods/athens", nil}, - {newBuf(`module "github.com.athens/gomods"`), "github.com.athens/gomods", nil}, - {newBuf(``), "", ErrNotFound}, - {newBuf(`module "my/thing2`), "", ErrNotFound}, - {newBuf(`module my/thing3`), "", ErrNotFound}, - {newBuf(`module github.com/gomods/athens`), "", ErrNotFound}, - {newBuf(`module "github.com?gomods"`), "", ErrNotFound}, - {newBuf(`module "github.com.athens"`), "", ErrNotFound}, - {newBuf(`module "github.com.athens"`), "", ErrNotFound}, - {newBuf(`module "github.com&athens"`), "", ErrNotFound}, - {newBuf(`module "?github%com&athens"`), "", ErrNotFound}, - {newBuf(`foobar`), "", ErrNotFound}, - {new(errReader), "", errReaderError}, - } - - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - actual, actualErr := Parse(tc.reader) - - a.Equal(tc.expected, actual) - a.Equal(tc.expectedErr, actualErr) - }) - } -} - -func newBuf(content string) io.Reader { - return bytes.NewBuffer([]byte(content)) -} diff --git a/pkg/gomod/zip/parser.go b/pkg/gomod/zip/parser.go deleted file mode 100644 index 9d99d7b46..000000000 --- a/pkg/gomod/zip/parser.go +++ /dev/null @@ -1,47 +0,0 @@ -package zip - -import ( - "archive/zip" - "fmt" - - "github.com/gomods/athens/pkg/gomod" -) - -const ( - gomodFilename = "go.mod" -) - -// NewZipParser creates shorthand for parsing module name out of zip package containing go.mod file -func NewZipParser(reader zip.ReadCloser) parser.GomodParser { - return zipParser{zipReader: reader} -} - -type zipParser struct { - zipReader zip.ReadCloser -} - -func (p zipParser) ModuleName() (string, error) { - defer p.zipReader.Close() - - var file *zip.File - for _, f := range p.zipReader.File { - if f.Name != gomodFilename { - continue - } - - file = f - break - } - - if file == nil { - return "", fmt.Errorf("go.mod not found") - } - - fileReader, err := file.Open() - if err != nil { - return "", err - } - defer fileReader.Close() - - return parser.Parse(fileReader) -} diff --git a/pkg/gomod/zip/parser_test.go b/pkg/gomod/zip/parser_test.go deleted file mode 100644 index f44c061cf..000000000 --- a/pkg/gomod/zip/parser_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package zip - -import ( - "archive/zip" - "errors" - "io" - "testing" - - "github.com/gomods/athens/pkg/gomod" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestZipParser_ModuleName(t *testing.T) { - a := assert.New(t) - fs := afero.NewOsFs() - var testCases = []struct { - file string - dstFileName string - expected string - expectedErr error - }{ - {"testdata/go.0.mod", "go.mod", "my/thing", nil}, - {"testdata/go.1.mod", "go.mod", "my/thing2", nil}, - {"testdata/go.2.mod", "go.mod", "", parser.ErrNotFound}, - {"testdata/go.3.mod", "go.mod", "", parser.ErrNotFound}, - {"testdata/go.4.mod", "go.mod", "", parser.ErrNotFound}, - {"testdata/Gopkg.toml", "Gopkg.toml", "", errors.New("go.mod not found")}, - } - - for _, tc := range testCases { - t.Run(tc.file, func(t *testing.T) { - zipfile := zipTestMod(t, fs, tc.file, tc.dstFileName) - - reader, err := zip.OpenReader(zipfile) - a.NoError(err) - defer reader.Close() - fp := NewZipParser(*reader) - actual, actualErr := fp.ModuleName() - - a.Equal(tc.expected, actual) - a.Equal(tc.expectedErr, actualErr) - }) - } -} - -func zipTestMod(t *testing.T, fs afero.Fs, src string, dstFileName string) (target string) { - r := require.New(t) - zipfile, err := afero.TempFile(fs, "", "") - r.NoError(err, "an error occurred while creating temporary file") - defer zipfile.Close() - - archive := zip.NewWriter(zipfile) - defer archive.Close() - - srcfile, err := fs.Open(src) - r.NoError(err, "an error occurred while opening fixture file") - defer srcfile.Close() - - f, err := archive.Create(dstFileName) - r.NoError(err, "an error occurred while creating file in archive") - - _, err = io.Copy(f, srcfile) - r.NoError(err, "an error occurred while coping data to archive") - - return zipfile.Name() -} diff --git a/pkg/gomod/zip/testdata/Gopkg.toml b/pkg/gomod/zip/testdata/Gopkg.toml deleted file mode 100644 index 0e9376d1e..000000000 --- a/pkg/gomod/zip/testdata/Gopkg.toml +++ /dev/null @@ -1,5 +0,0 @@ -module "my/thing" -require "other/thing" v1.0.2 -require "new/thing" v2.3.4 -exclude "old/thing" v1.2.3 -replace "bad/thing" v1.4.5 => "good/thing" v1.4.5 diff --git a/pkg/gomod/zip/testdata/go.0.mod b/pkg/gomod/zip/testdata/go.0.mod deleted file mode 100644 index 0e9376d1e..000000000 --- a/pkg/gomod/zip/testdata/go.0.mod +++ /dev/null @@ -1,5 +0,0 @@ -module "my/thing" -require "other/thing" v1.0.2 -require "new/thing" v2.3.4 -exclude "old/thing" v1.2.3 -replace "bad/thing" v1.4.5 => "good/thing" v1.4.5 diff --git a/pkg/gomod/zip/testdata/go.1.mod b/pkg/gomod/zip/testdata/go.1.mod deleted file mode 100644 index c9e10f1dd..000000000 --- a/pkg/gomod/zip/testdata/go.1.mod +++ /dev/null @@ -1 +0,0 @@ -module "my/thing2" \ No newline at end of file diff --git a/pkg/gomod/zip/testdata/go.2.mod b/pkg/gomod/zip/testdata/go.2.mod deleted file mode 100644 index 0d7780104..000000000 --- a/pkg/gomod/zip/testdata/go.2.mod +++ /dev/null @@ -1 +0,0 @@ -module "my/thing \ No newline at end of file diff --git a/pkg/gomod/zip/testdata/go.3.mod b/pkg/gomod/zip/testdata/go.3.mod deleted file mode 100644 index 0d7780104..000000000 --- a/pkg/gomod/zip/testdata/go.3.mod +++ /dev/null @@ -1 +0,0 @@ -module "my/thing \ No newline at end of file diff --git a/pkg/gomod/zip/testdata/go.4.mod b/pkg/gomod/zip/testdata/go.4.mod deleted file mode 100644 index dd3d047c0..000000000 --- a/pkg/gomod/zip/testdata/go.4.mod +++ /dev/null @@ -1,8 +0,0 @@ -[[constraint]] - name = "github.com/user/project" - version = "1.0.0" - -[[constraint]] - name = "github.com/user/project2" - branch = "dev" - source = "github.com/myfork/project2" \ No newline at end of file diff --git a/pkg/repo/WARNING.md b/pkg/repo/WARNING.md deleted file mode 100644 index 04c6a0fe3..000000000 --- a/pkg/repo/WARNING.md +++ /dev/null @@ -1,3 +0,0 @@ -This package is going to be deleted after https://github.com/gomods/athens/pull/291, -https://github.com/gomods/athens/pull/319, https://github.com/gomods/athens/pull/316, and -https://github.com/gomods/athens/pull/314 are merged diff --git a/pkg/repo/fetcher.go b/pkg/repo/fetcher.go deleted file mode 100644 index f26833c75..000000000 --- a/pkg/repo/fetcher.go +++ /dev/null @@ -1,7 +0,0 @@ -package repo - -// Fetcher is interface used for fetching sources from VCS or Services -type Fetcher interface { - Fetch() (string, error) - Clear() error -} diff --git a/pkg/repo/github/commitinfo.go b/pkg/repo/github/commitinfo.go deleted file mode 100644 index 3a06be2d3..000000000 --- a/pkg/repo/github/commitinfo.go +++ /dev/null @@ -1,66 +0,0 @@ -package github - -import ( - "encoding/json" - "fmt" - "net/http" - "time" - - "github.com/gomods/athens/pkg/config/env" -) - -const ( - fetchCommitURI string = "https://api.github.com/repos/%s/%s/commits/%s" -) - -// CommitInfo represents information store in go.info -type CommitInfo struct { - Version string - Name string - Short string - Time time.Time -} - -func getCommitInfo(owner, repo, tag string) (CommitInfo, error) { - commit := CommitResponse{} - commitInfo := CommitInfo{Version: tag} - - uri := fmt.Sprintf(fetchCommitURI, owner, repo, tag) - - client := http.Client{Timeout: env.Timeout()} - resp, err := client.Get(uri) - if err != nil { - return commitInfo, err - } - defer resp.Body.Close() - - decoder := json.NewDecoder(resp.Body) - if err := decoder.Decode(&commit); err != nil { - return commitInfo, err - } - - commitInfo.Name = commit.Sha - commitInfo.Short = commit.Sha - if commit.Sha != "" && len(commit.Sha) > 12 { - commitInfo.Short = commit.Sha[:12] - } - commitInfo.Time = commit.Commit.Author.Date - - return commitInfo, nil -} - -// CommitResponse represents commit information from response message of git API -type CommitResponse struct { - Sha string - Commit Commit -} - -// Commit represents commit information from response message of git API -type Commit struct { - Author Author -} - -// Author represents author information from response message of git API -type Author struct { - Date time.Time -} diff --git a/pkg/repo/github/github.go b/pkg/repo/github/github.go deleted file mode 100644 index 7f072ad4c..000000000 --- a/pkg/repo/github/github.go +++ /dev/null @@ -1,197 +0,0 @@ -// Package github is working version only -// We need to (re-)consider -// - multiple go.mod files in one repo e.g (owner/repo/cli/go.mod owner/repo/web/go.mod) -// - v2 (or any other vX) folder -// - something else which I cannot think of right now because I had too many cakes today -package github - -import ( - "archive/tar" - "compress/gzip" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - - "github.com/gomods/athens/pkg/config/env" - "github.com/gomods/athens/pkg/gomod/file" - "github.com/gomods/athens/pkg/module" - "github.com/gomods/athens/pkg/repo" - "github.com/spf13/afero" -) - -const ( - fetchRepoURI string = "https://api.github.com/repos/%s/%s/tarball/%s" -) - -type gitFetcher struct { - fs afero.Fs - owner string - repoName string - tag string - dirName string -} - -// NewGitFetcher creates a new Fetcher for repositories hosted on github -func NewGitFetcher(fs afero.Fs, owner string, repoName string, tag string) (repo.Fetcher, error) { - if owner == "" || repoName == "" { - return nil, errors.New("invalid repository identifier") - } - - gf := &gitFetcher{ - owner: owner, - repoName: repoName, - tag: tag, - fs: fs, - } - return gf, nil -} - -// Fetches a tarball of a repo and untars it into a temp dir which is used later in the workflow. -func (g gitFetcher) Fetch() (string, error) { - uri := fmt.Sprintf(fetchRepoURI, g.owner, g.repoName, g.tag) - - client := http.Client{Timeout: env.Timeout()} - resp, err := client.Get(uri) - - if err != nil { - return "", err - } - defer resp.Body.Close() - - tmpDir, err := afero.TempDir(g.fs, "", "") - if err != nil { - return "", err - } - g.dirName, err = untar(g.fs, resp.Body, tmpDir) - if err != nil { - g.fs.RemoveAll(tmpDir) - return "", err - } - - // Get module name from go.mod - gomodPath := filepath.Join(g.dirName, "go.mod") - parser := file.NewFileParser(g.fs, gomodPath) - - moduleName, err := parser.ModuleName() - if err != nil { - g.fs.RemoveAll(tmpDir) - return g.dirName, err - } - - // Generate zip - if err := g.generateZip(moduleName); err != nil { - g.fs.RemoveAll(tmpDir) - return g.dirName, err - } - - // Rename go.mod - verModPath := filepath.Join(g.dirName, g.tag+".mod") - if err := g.fs.Rename(gomodPath, verModPath); err != nil { - g.fs.RemoveAll(tmpDir) - return g.dirName, err - } - - // Generate info - if err := g.generateInfo(); err != nil { - g.fs.RemoveAll(tmpDir) - return g.dirName, err - } - - return g.dirName, nil -} - -// Clear removes all downloaded data -func (g *gitFetcher) Clear() error { - if g.dirName == "" { - return nil - } - - return g.fs.RemoveAll(g.dirName) -} - -func (g *gitFetcher) generateZip(moduleName string) error { - zipReader := module.MakeZip(g.fs, g.dirName, moduleName, g.tag) - - zipPath := filepath.Join(g.dirName, g.tag+".zip") - f, err := g.fs.OpenFile(zipPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) - if err != nil { - return err - } - _, err = io.Copy(f, zipReader) - return err -} - -func (g *gitFetcher) generateInfo() error { - info, err := getCommitInfo(g.owner, g.repoName, g.tag) - if err != nil { - return err - } - - infoContent, err := json.Marshal(info) - if err != nil { - return err - } - - goinfoPath := filepath.Join(g.dirName, g.tag+".info") - return afero.WriteFile(g.fs, goinfoPath, infoContent, os.ModePerm) -} - -func untar(fs afero.Fs, content io.Reader, tmpDir string) (string, error) { - gzr, err := gzip.NewReader(content) - if err != nil { - return "", err - } - defer gzr.Close() - - tr := tar.NewReader(gzr) - var dirName string - - for { - hdr, err := tr.Next() - - if err != nil { - if err == io.EOF { - break - } - - return "", err - } - - if hdr == nil { - continue - } - - target := filepath.Join(tmpDir, hdr.Name) - - switch hdr.Typeflag { - - case tar.TypeDir: - if dirName == "" { - dirName = target - } - - if _, err := fs.Stat(target); err != nil { - if err := fs.MkdirAll(target, 0755); err != nil { - return "", err - } - } - - case tar.TypeReg: - f, err := fs.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode)) - if err != nil { - return "", err - } - defer f.Close() - - if _, err := io.Copy(f, tr); err != nil { - return "", err - } - } - } - - return dirName, nil -} diff --git a/pkg/repo/github/github_test.go b/pkg/repo/github/github_test.go deleted file mode 100644 index 1541aa375..000000000 --- a/pkg/repo/github/github_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package github - -import ( - "path/filepath" - "testing" - - "github.com/spf13/afero" -) - -func Test_Download(t *testing.T) { - owner := "bketelsen" - repo := "captainhook" - version := "v0.1.8" - - memFs := afero.NewMemMapFs() - - fetcher, err := NewGitFetcher(memFs, owner, repo, version) - if err != nil { - t.Error(err) - } - - path, err := fetcher.Fetch() - if err != nil { - t.Error(err) - } - if path == "" { - t.Error("path null") - } - - t.Log(path) - if _, err := memFs.Stat(filepath.Join(path, version+".mod")); err != nil { - t.Error(err) - t.Fail() - } - - if _, err := memFs.Stat(filepath.Join(path, version+".zip")); err != nil { - t.Error(err) - t.Fail() - } - - if _, err := memFs.Stat(filepath.Join(path, version+".info")); err != nil { - t.Error(err) - t.Fail() - } - - err = fetcher.Clear() - if err != nil { - t.Error(err) - } -} diff --git a/pkg/repo/goget.go b/pkg/repo/goget.go deleted file mode 100644 index e33e33211..000000000 --- a/pkg/repo/goget.go +++ /dev/null @@ -1,169 +0,0 @@ -package repo - -import ( - "bytes" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/spf13/afero" -) - -const ( - tmpRepoDir = "%s-%s" // owner-repo-ref -) - -var ( - // ErrLimitExceeded signals that github.com refused to serve the request due to exceeded quota - ErrLimitExceeded = errors.New("github limit exceeded") -) - -type genericFetcher struct { - fs afero.Fs - repoURI string - version string - dirName string -} - -// NewGenericFetcher creates fetcher which uses go get tool to fetch sources -// returns path of directory containing vx.y.z.(zip|info|mod) -func NewGenericFetcher(fs afero.Fs, repoURI, version string) (Fetcher, error) { - if !isVgoInstalled() { - return nil, errors.New("vgo not installed") - } - - if repoURI == "" { - return nil, errors.New("invalid repository identifier") - } - - return &genericFetcher{ - fs: fs, - repoURI: repoURI, - version: version, - }, nil -} - -// Fetch downloads the sources and returns path where it can be found -func (g *genericFetcher) Fetch() (string, error) { - escapedURI := strings.Replace(g.repoURI, "/", "-", -1) - repoDirName := fmt.Sprintf(tmpRepoDir, escapedURI, g.version) - - gopath, repoRoot, err := setupTmp(g.fs, repoDirName) - if err != nil { - return "", err - } - g.dirName = repoRoot - - prepareStructure(g.fs, repoRoot) - - dirName, err := getSources(g.fs, gopath, repoRoot, g.repoURI, g.version) - - return dirName, err -} - -// Clear removes all downloaded data -func (g *genericFetcher) Clear() error { - if g.dirName == "" { - return nil - } - - return g.fs.RemoveAll(g.dirName) -} - -func isVgoInstalled() bool { - cmd := exec.Command("vgo") - err := cmd.Run() // this will fail even if vgo is installed - - if err != nil { - _, ok := err.(*exec.ExitError) // if vgo is installed it will return ExitError, otherwise normal error - return ok == true - } - - return false -} - -func setupTmp(fs afero.Fs, repoDirName string) (string, string, error) { - gopathDir, err := afero.TempDir(fs, "", "") - if err != nil { - return "", "", err - } - - path := filepath.Join(gopathDir, "src", repoDirName) - - return gopathDir, path, fs.MkdirAll(path, os.ModeDir|os.ModePerm) -} - -// Hacky thing makes vgo not to complain -func prepareStructure(fs afero.Fs, repoRoot string) error { - // vgo expects go.mod file present with module statement or .go file with import comment - gomodPath := filepath.Join(repoRoot, "go.mod") - gomodContent := []byte("module \"mod\"") - if err := afero.WriteFile(fs, gomodPath, gomodContent, 0666); err != nil { - return err - } - - sourcePath := filepath.Join(repoRoot, "mod.go") - sourceContent := []byte(`package mod // import "mod"`) - return afero.WriteFile(fs, sourcePath, sourceContent, 0666) -} - -func getSources(fs afero.Fs, gopath, repoRoot, repoURI, version string) (string, error) { - version = strings.TrimPrefix(version, "@") - if !strings.HasPrefix(version, "v") { - version = "v" + version - } - uri := strings.TrimSuffix(repoURI, "/") - - fullURI := fmt.Sprintf("%s@%s", uri, version) - - gopathEnv := fmt.Sprintf("GOPATH=%s", gopath) - cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache")) - disableCgo := "CGO_ENABLED=0" - - cmd := exec.Command("vgo", "get", fullURI) - // PATH is needed for vgo to recognize vcs binaries - // this breaks windows. - cmd.Env = []string{"PATH=" + os.Getenv("PATH"), gopathEnv, cacheEnv, disableCgo} - cmd.Dir = repoRoot - - packagePath := filepath.Join(gopath, "src", "mod", "cache", "download", repoURI, "@v") - - o, err := cmd.CombinedOutput() - if err != nil { - switch { - case isLimitHit(o): - // github quota exceeded - return packagePath, ErrLimitExceeded - case checkFiles(fs, packagePath, version) == nil: - // some compilation error - return packagePath, nil - default: - return packagePath, err - } - } - - return packagePath, err -} - -func checkFiles(fs afero.Fs, path, version string) error { - if _, err := fs.Stat(filepath.Join(path, version+".mod")); err != nil { - return errors.New("go.mod not found") - } - - if _, err := fs.Stat(filepath.Join(path, version+".zip")); err != nil { - return errors.New("zip package not found") - } - - if _, err := fs.Stat(filepath.Join(path, version+".info")); err != nil { - return errors.New("info file not found") - } - - return nil -} - -func isLimitHit(o []byte) bool { - return bytes.Contains(o, []byte("403 response from api.github.com")) -} diff --git a/pkg/repo/goget_test.go b/pkg/repo/goget_test.go deleted file mode 100644 index bba3d6f62..000000000 --- a/pkg/repo/goget_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package repo - -import ( - "path/filepath" - "testing" - - "github.com/spf13/afero" -) - -func Test_Download(t *testing.T) { - version := "v0.1.8" - gitURI := "github.com/bketelsen/captainhook" - fs := afero.NewOsFs() - fetcher, err := NewGenericFetcher(fs, gitURI, version) - if err != nil { - t.Error(err) - t.Fail() - } - - path, err := fetcher.Fetch() - - if err == ErrLimitExceeded { - t.Log("Skipped due to exceeded github quota") - t.Skip() - } - - if err != nil { - t.Error(err) - t.Fail() - } - if path == "" { - t.Error("path null") - t.Fail() - } - - if _, err := fs.Stat(filepath.Join(path, version+".mod")); err != nil { - t.Error(err) - t.Fail() - } - - if _, err := fs.Stat(filepath.Join(path, version+".zip")); err != nil { - t.Error(err) - t.Fail() - } - - if _, err := fs.Stat(filepath.Join(path, version+".info")); err != nil { - t.Error(err) - t.Fail() - } - - t.Log(path) - err = fetcher.Clear() - if err != nil { - t.Error(err) - } -}