Skip to content

Commit

Permalink
Backfill the cache on a miss (gomods#342)
Browse files Browse the repository at this point in the history
* add middlewares to fill the cache and to populate contexts with module and versions

* Carolyn tries to fix Aaron's code

* #DOINSTUFF

* updated env var

* fix all the things

* magic patch for olympus

* Add latest handler

* Remove deprecated return param

* go fmt all the things
  • Loading branch information
carolynvs authored and arschles committed Jul 27, 2018
1 parent f138bbb commit fd7c132
Show file tree
Hide file tree
Showing 14 changed files with 132 additions and 84 deletions.
8 changes: 4 additions & 4 deletions cmd/olympus/actions/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,12 @@ func App(config *AppConfig) (*buffalo.App, error) {
app.POST("/cachemiss", cachemissHandler(w))
app.POST("/push", pushNotificationHandler(w))

dp := goget.New()
// Download Protocol
dp := download.New(goget.New(), config.Storage)
app.GET(download.PathList, download.ListHandler(dp, lggr, renderEng))
app.GET(download.PathVersionInfo, download.VersionInfoHandler(config.Storage, renderEng))
app.GET(download.PathVersionModule, download.VersionModuleHandler(config.Storage, renderEng))
app.GET(download.PathVersionZip, download.VersionZipHandler(config.Storage, renderEng, lggr))
app.GET(download.PathVersionInfo, download.VersionInfoHandler(dp, lggr, renderEng))
app.GET(download.PathVersionModule, download.VersionModuleHandler(dp, lggr, renderEng))
app.GET(download.PathVersionZip, download.VersionZipHandler(dp, lggr, renderEng))

app.ServeFiles("/", assetsBox) // serve files from the public directory
}
Expand Down
5 changes: 4 additions & 1 deletion cmd/proxy/actions/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package actions

import (
"fmt"
"github.com/spf13/afero"

"github.com/gobuffalo/buffalo"
"github.com/gobuffalo/buffalo/middleware"
Expand Down Expand Up @@ -80,6 +81,8 @@ func App() (*buffalo.App, error) {
return nil, err
}

fetcher := module.NewGoGetFetcher(env.GoBinPath(), afero.NewOsFs())

worker, err := getWorker(store, mf)
if err != nil {
return nil, err
Expand Down Expand Up @@ -126,7 +129,7 @@ func App() (*buffalo.App, error) {
}
app.Use(T.Middleware())

if err := addProxyRoutes(app, store, mf, lggr); err != nil {
if err := addProxyRoutes(app, store, fetcher, mf, lggr); err != nil {
err = fmt.Errorf("error adding proxy routes (%s)", err)
return nil, err
}
Expand Down
8 changes: 5 additions & 3 deletions cmd/proxy/actions/app_proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
func addProxyRoutes(
app *buffalo.App,
storage storage.Backend,
fetcher module.Fetcher,
mf *module.Filter,
lggr *log.Logger,
) error {
Expand All @@ -20,9 +21,10 @@ func addProxyRoutes(
dp := download.New(goget.New(), storage)
// Download Protocol
app.GET(download.PathList, download.ListHandler(dp, lggr, proxy))
app.GET(download.PathVersionInfo, cacheMissHandler(download.VersionInfoHandler(storage, proxy), app.Worker, mf, lggr))
app.GET(download.PathVersionModule, cacheMissHandler(download.VersionModuleHandler(storage, proxy), app.Worker, mf, lggr))
app.GET(download.PathVersionZip, cacheMissHandler(download.VersionZipHandler(storage, proxy, lggr), app.Worker, mf, lggr))
app.GET(download.PathLatest, download.LatestHandler(dp, lggr, proxy))
app.GET(download.PathVersionInfo, download.VersionInfoHandler(dp, lggr, proxy))
app.GET(download.PathVersionModule, download.VersionModuleHandler(dp, lggr, proxy))
app.GET(download.PathVersionZip, download.VersionZipHandler(dp, lggr, proxy))

app.POST("/admin/fetch/{module:[a-zA-Z./]+}/{owner}/{repo}/{ref}/{version}", fetchHandler(storage))
return nil
Expand Down
6 changes: 3 additions & 3 deletions pkg/config/env/go.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ func GoPath() (string, error) {
return env, nil
}

// GoBinPath returns the path to Go's executable binary
// this binary must have Go Modules enabled.
// GoBinPath returns the path to the go binary to use, defined by
// GO_BINARY_PATH. This value can be a name on the PATH, or the full path
func GoBinPath() string {
return envy.Get("GO_BIN_PATH", "vgo")
return envy.Get("GO_BINARY_PATH", "go")
}
22 changes: 22 additions & 0 deletions pkg/download/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,25 @@
// can share the same download protocol
// implementation.
package download

import (
"github.com/gobuffalo/buffalo"
"github.com/gomods/athens/pkg/log"
"github.com/gomods/athens/pkg/paths"
"github.com/gomods/athens/pkg/storage"
"github.com/pkg/errors"
)

func getModuleVersion(c buffalo.Context, lggr *log.Logger, dp Protocol) (string, string, *storage.Version, error) {
params, err := paths.GetAllParams(c)
if err != nil {
lggr.SystemErr(err)
panic(err)
}

versionInfo, err := dp.Version(c, params.Module, params.Version)
if err != nil {
return "", "", nil, errors.WithStack(err)
}
return params.Module, params.Version, versionInfo, nil
}
6 changes: 3 additions & 3 deletions pkg/download/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (p *protocol) List(ctx context.Context, mod string) ([]string, error) {
func (p *protocol) Info(ctx context.Context, mod, ver string) ([]byte, error) {
const op errors.Op = "protocol.Info"
v, err := p.s.Get(mod, ver)
if errors.ErrNotFound(err) {
if storage.IsNotFoundError(err) {
v, err = p.fillCache(ctx, mod, ver)
}
if err != nil {
Expand Down Expand Up @@ -80,7 +80,7 @@ func (p *protocol) Latest(ctx context.Context, mod string) (*storage.RevInfo, er
func (p *protocol) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
const op errors.Op = "protocol.GoMod"
v, err := p.s.Get(mod, ver)
if errors.ErrNotFound(err) {
if storage.IsNotFoundError(err) {
v, err = p.fillCache(ctx, mod, ver)
}
if err != nil {
Expand All @@ -93,7 +93,7 @@ func (p *protocol) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
const op errors.Op = "protocol.Zip"
v, err := p.s.Get(mod, ver)
if errors.ErrNotFound(err) {
if storage.IsNotFoundError(err) {
v, err = p.fillCache(ctx, mod, ver)
}
if err != nil {
Expand Down
6 changes: 3 additions & 3 deletions pkg/download/goget/goget.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,15 @@ func (gg *goget) List(ctx context.Context, mod string) ([]string, error) {
type listResp struct {
Path string
Version string
Versions []string `json:"omitempty"`
Versions []string `json:",omitempty"`
Time time.Time
}

func (gg *goget) Info(ctx context.Context, mod string, ver string) ([]byte, error) {
const op errors.Op = "goget.Info"
v, err := gg.Version(ctx, mod, ver)
if err != nil {
return nil, errors.E(op)
return nil, errors.E(op, err)
}
v.Zip.Close()

Expand Down Expand Up @@ -139,7 +139,7 @@ func (gg *goget) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error

func (gg *goget) Version(ctx context.Context, mod, ver string) (*storage.Version, error) {
const op errors.Op = "goget.Version"
fetcher, _ := module.NewGoGetFetcher(gg.goBinPath, gg.fs) // TODO: remove err from func call
fetcher := module.NewGoGetFetcher(gg.goBinPath, gg.fs)
ref, err := fetcher.Fetch(mod, ver)
if err != nil {
return nil, errors.E(op, err)
Expand Down
36 changes: 36 additions & 0 deletions pkg/download/latest.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package download

import (
"net/http"

"github.com/bketelsen/buffet"
"github.com/gobuffalo/buffalo"
"github.com/gobuffalo/buffalo/render"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/log"
"github.com/gomods/athens/pkg/paths"
)

// PathLatest URL.
const PathLatest = "/{module:.+}/@latest"

// LatestHandler implements GET baseURL/module/@latest
func LatestHandler(dp Protocol, lggr *log.Logger, eng *render.Engine) func(c buffalo.Context) error {
return func(c buffalo.Context) error {
sp := buffet.SpanFromContext(c)
sp.SetOperationName("latestHandler")
mod, err := paths.GetModule(c)
if err != nil {
lggr.SystemErr(err)
return c.Render(500, nil)
}

info, err := dp.Latest(c, mod)
if err != nil {
lggr.SystemErr(err)
return c.Render(errors.Kind(err), eng.JSON(errors.KindText(err)))
}

return c.Render(http.StatusOK, eng.JSON(info))
}
}
26 changes: 12 additions & 14 deletions pkg/download/version_info.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,36 @@ package download

import (
"encoding/json"
"fmt"
"net/http"

"github.com/bketelsen/buffet"
"github.com/gobuffalo/buffalo"
"github.com/gobuffalo/buffalo/render"
"github.com/gomods/athens/pkg/paths"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/log"
"github.com/gomods/athens/pkg/storage"
)

// PathVersionInfo URL.
const PathVersionInfo = "/{module:.+}/@v/{version}.info"

// VersionInfoHandler implements GET baseURL/module/@v/version.info
func VersionInfoHandler(getter storage.Getter, eng *render.Engine) func(c buffalo.Context) error {
func VersionInfoHandler(dp Protocol, lggr *log.Logger, eng *render.Engine) buffalo.Handler {
return func(c buffalo.Context) error {
const op errors.Op = "download.versionInfoHandler"

sp := buffet.SpanFromContext(c)
sp.SetOperationName("versionInfoHandler")
params, err := paths.GetAllParams(c)

mod, ver, verInfo, err := getModuleVersion(c, lggr, dp)
if err != nil {
return err
}
version, err := getter.Get(params.Module, params.Version)
if storage.IsNotFoundError(err) {
msg := fmt.Sprintf("%s@%s not found", params.Module, params.Version)
return c.Render(http.StatusNotFound, eng.JSON(msg))
} else if err != nil {
return err
err := errors.E(op, errors.M(mod), errors.V(ver), err)
lggr.SystemErr(err)
c.Render(http.StatusInternalServerError, nil)
}

var revInfo storage.RevInfo
err = json.Unmarshal(version.Info, &revInfo)
if err != nil {
if err := json.Unmarshal(verInfo.Info, &revInfo); err != nil {
return err
}
return c.Render(http.StatusOK, eng.JSON(revInfo))
Expand Down
26 changes: 11 additions & 15 deletions pkg/download/version_module.go
Original file line number Diff line number Diff line change
@@ -1,39 +1,35 @@
package download

import (
"fmt"
"net/http"

"github.com/bketelsen/buffet"
"github.com/gobuffalo/buffalo"
"github.com/gobuffalo/buffalo/render"
"github.com/gomods/athens/pkg/paths"
"github.com/gomods/athens/pkg/storage"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/log"
)

// PathVersionModule URL.
const PathVersionModule = "/{module:.+}/@v/{version}.mod"

// VersionModuleHandler implements GET baseURL/module/@v/version.mod
func VersionModuleHandler(getter storage.Getter, eng *render.Engine) func(c buffalo.Context) error {
func VersionModuleHandler(dp Protocol, lggr *log.Logger, eng *render.Engine) buffalo.Handler {
return func(c buffalo.Context) error {
const op errors.Op = "download.VersionModuleHandler"

sp := buffet.SpanFromContext(c)
sp.SetOperationName("versionModuleHandler")
params, err := paths.GetAllParams(c)
if err != nil {
return err
}

version, err := getter.Get(params.Module, params.Version)
if storage.IsNotFoundError(err) {
msg := fmt.Sprintf("%s@%s not found", params.Module, params.Version)
return c.Render(http.StatusNotFound, eng.JSON(msg))
} else if err != nil {
return err
mod, ver, verInfo, err := getModuleVersion(c, lggr, dp)
if err != nil {
err := errors.E(op, errors.M(mod), errors.V(ver), err)
lggr.SystemErr(err)
c.Render(http.StatusInternalServerError, nil)
}

c.Response().WriteHeader(http.StatusOK)
_, err = c.Response().Write(version.Mod)
_, err = c.Response().Write(verInfo.Mod)
return err
}
}
42 changes: 13 additions & 29 deletions pkg/download/version_zip.go
Original file line number Diff line number Diff line change
@@ -1,65 +1,49 @@
package download

import (
"fmt"
"io"
"net/http"

"github.com/bketelsen/buffet"
"github.com/gobuffalo/buffalo"
"github.com/gobuffalo/buffalo/render"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/log"
"github.com/gomods/athens/pkg/paths"
"github.com/gomods/athens/pkg/storage"
"github.com/sirupsen/logrus"
)

// PathVersionZip URL.
const PathVersionZip = "/{module:.+}/@v/{version}.zip"

// VersionZipHandler implements GET baseURL/module/@v/version.zip
func VersionZipHandler(getter storage.Getter, eng *render.Engine, lggr *log.Logger) func(c buffalo.Context) error {
func VersionZipHandler(dp Protocol, lggr *log.Logger, eng *render.Engine) buffalo.Handler {
const op errors.Op = "download.VersionZipHandler"

return func(c buffalo.Context) error {
sp := buffet.SpanFromContext(c)
sp.SetOperationName("versionZipHandler")
params, err := paths.GetAllParams(c)
if err != nil {
lggr.SystemErr(errors.E(op, err))
c.Render(http.StatusInternalServerError, nil) // 500 because handler should not be called in the first place.
return nil
}

version, err := getter.Get(params.Module, params.Version)
lggr.Println("===getModuleVersion")
mod, ver, verInfo, err := getModuleVersion(c, lggr, dp)
if err != nil {
lvl := logrus.ErrorLevel
status := http.StatusInternalServerError
msg := http.StatusText(status)
// TODO: move this function to pkg/errors
if storage.IsNotFoundError(err) {
lvl = logrus.DebugLevel
msg = fmt.Sprintf("%v not found", config.FmtModVer(params.Module, params.Version))
status = http.StatusNotFound
}
lggr.Println("===carolyn found an error", err)

lggr.SystemErr(errors.E(op, errors.M(params.Module), errors.V(params.Version), lvl, err))
return c.Render(status, eng.JSON(msg))
err := errors.E(op, errors.M(mod), errors.V(ver), err)
lggr.SystemErr(err)
return c.Render(http.StatusInternalServerError, nil)
}
defer version.Zip.Close()

lggr.Println("===Copy")
status := http.StatusOK

_, err = io.Copy(c.Response(), version.Zip)
_, err = io.Copy(c.Response(), verInfo.Zip)
if err != nil {
lggr.Println("===Copy errored out", err)
status = http.StatusInternalServerError
lggr.SystemErr(errors.E(op, errors.M(params.Module), errors.V(params.Version), err))
lggr.SystemErr(errors.E(op, errors.M(mod), errors.V(ver), err))
}

c.Response().WriteHeader(status)
lggr.Println("===writeheader")

c.Response().WriteHeader(status)
return nil
}
}
Loading

0 comments on commit fd7c132

Please sign in to comment.