Parse bundle download URL and allow optional insecure TLS for it (Tyk…

…Technologies#2876) Bundle fetcher now parses the bundle base URL and joins the bundle ID to its path component instead of simple string concatenation. Resulting URL is then rendered from the parsed structure. This ensures correct URL will be constructed if the base contains e.g. query string parameters. This also adds a "bundle_insecure_skip_verify" config option to enable downloading from e.g. test sources that might not have a trusted certificate. Disabled by default. Addresses TykTechnologies#2874
backwardn · Feb 20, 2020 · 0473476 · 0473476
1 parent f308337
commit 0473476
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 17 deletions.
diff --git a/cli/linter/schema.json b/cli/linter/schema.json
@@ -278,6 +278,9 @@
     "enable_bundle_downloader": {
       "type": "boolean"
     },
+    "bundle_insecure_skip_verify": {
+      "type": "boolean"
+    },
     "enable_custom_domains": {
       "type": "boolean"
     },

diff --git a/config/config.go b/config/config.go
@@ -412,14 +412,15 @@ type Config struct {
 	CacheStorage             StorageOptionsConf    `json:"cache_storage"`
 
 	// Middleware/Plugin Configuration
-	EnableBundleDownloader  bool            `bson:"enable_bundle_downloader" json:"enable_bundle_downloader"`
-	BundleBaseURL           string          `bson:"bundle_base_url" json:"bundle_base_url"`
-	EnableJSVM              bool            `json:"enable_jsvm"`
-	JSVMTimeout             int             `json:"jsvm_timeout"`
-	DisableVirtualPathBlobs bool            `json:"disable_virtual_path_blobs"`
-	TykJSPath               string          `json:"tyk_js_path"`
-	MiddlewarePath          string          `json:"middleware_path"`
-	CoProcessOptions        CoProcessConfig `json:"coprocess_options"`
+	EnableBundleDownloader   bool            `bson:"enable_bundle_downloader" json:"enable_bundle_downloader"`
+	BundleBaseURL            string          `bson:"bundle_base_url" json:"bundle_base_url"`
+	BundleInsecureSkipVerify bool            `bson:"bundle_insecure_skip_verify" json:"bundle_insecure_skip_verify"`
+	EnableJSVM               bool            `json:"enable_jsvm"`
+	JSVMTimeout              int             `json:"jsvm_timeout"`
+	DisableVirtualPathBlobs  bool            `json:"disable_virtual_path_blobs"`
+	TykJSPath                string          `json:"tyk_js_path"`
+	MiddlewarePath           string          `json:"middleware_path"`
+	CoProcessOptions         CoProcessConfig `json:"coprocess_options"`
 
 	// Monitoring, Logging & Profiling
 	LogLevel                string         `json:"log_level"`

diff --git a/gateway/coprocess_bundle.go b/gateway/coprocess_bundle.go
@@ -1,6 +1,8 @@
 package gateway
 
 import (
+	"path"
+
 	"github.com/sirupsen/logrus"
 
 	"github.com/TykTechnologies/goverify"
@@ -10,6 +12,7 @@ import (
 	"archive/zip"
 	"bytes"
 	"crypto/md5"
+	"crypto/tls"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -124,12 +127,23 @@ type BundleGetter interface {
 
 // HTTPBundleGetter is a simple HTTP BundleGetter.
 type HTTPBundleGetter struct {
-	URL string
+	URL                string
+	InsecureSkipVerify bool
+}
+
+// MockBundleGetter is a BundleGetter for testing.
+type MockBundleGetter struct {
+	URL                string
+	InsecureSkipVerify bool
 }
 
 // Get performs an HTTP GET request.
 func (g *HTTPBundleGetter) Get() ([]byte, error) {
-	resp, err := http.Get(g.URL)
+	tr := &(*http.DefaultTransport.(*http.Transport))
+	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: g.InsecureSkipVerify}
+	client := &http.Client{Transport: tr}
+
+	resp, err := client.Get(g.URL)
 	if err != nil {
 		return nil, err
 	}
@@ -143,6 +157,14 @@ func (g *HTTPBundleGetter) Get() ([]byte, error) {
 	return ioutil.ReadAll(resp.Body)
 }
 
+// Get mocks an HTTP(S) GET request.
+func (g *MockBundleGetter) Get() ([]byte, error) {
+	if g.InsecureSkipVerify {
+		return []byte("bundle-insecure"), nil
+	}
+	return []byte("bundle"), nil
+}
+
 // BundleSaver is an interface used by bundle saver structures.
 type BundleSaver interface {
 	Save(*Bundle, string, *APISpec) error
@@ -198,22 +220,32 @@ func fetchBundle(spec *APISpec) (bundle Bundle, err error) {
 		return bundle, err
 	}
 
-	bundleURL := config.Global().BundleBaseURL + spec.CustomMiddlewareBundle
-
-	var getter BundleGetter
-
-	u, err := url.Parse(bundleURL)
+	u, err := url.Parse(config.Global().BundleBaseURL)
 	if err != nil {
 		return bundle, err
 	}
+
+	u.Path = path.Join(u.Path, spec.CustomMiddlewareBundle)
+
+	bundleURL := u.String()
+
+	var getter BundleGetter
+
 	switch u.Scheme {
 	case "http":
 		getter = &HTTPBundleGetter{
-			URL: bundleURL,
+			URL:                bundleURL,
+			InsecureSkipVerify: false,
 		}
 	case "https":
 		getter = &HTTPBundleGetter{
-			URL: bundleURL,
+			URL:                bundleURL,
+			InsecureSkipVerify: config.Global().BundleInsecureSkipVerify,
+		}
+	case "mock":
+		getter = &MockBundleGetter{
+			URL:                bundleURL,
+			InsecureSkipVerify: config.Global().BundleInsecureSkipVerify,
 		}
 	default:
 		err = errors.New("Unknown URL scheme")

diff --git a/gateway/coprocess_bundle_test.go b/gateway/coprocess_bundle_test.go
@@ -80,6 +80,55 @@ func TestBundleLoader(t *testing.T) {
 	})
 }
 
+func TestBundleFetcher(t *testing.T) {
+	bundleID := "testbundle"
+	defer ResetTestConfig()
+
+	t.Run("Simple bundle base URL", func(t *testing.T) {
+		globalConf := config.Global()
+		globalConf.BundleBaseURL = "mock://somepath"
+		globalConf.BundleInsecureSkipVerify = false
+		config.SetGlobal(globalConf)
+		specs := BuildAndLoadAPI(func(spec *APISpec) {
+			spec.CustomMiddlewareBundle = bundleID
+		})
+		spec := specs[0]
+		bundle, err := fetchBundle(spec)
+		if err != nil {
+			t.Fatalf("Couldn't fetch bundle: %s", err.Error())
+		}
+
+		if string(bundle.Data) != "bundle" {
+			t.Errorf("Wrong bundle data: %s", bundle.Data)
+		}
+		if bundle.Name != bundleID {
+			t.Errorf("Wrong bundle name: %s", bundle.Name)
+		}
+	})
+
+	t.Run("Bundle base URL with querystring", func(t *testing.T) {
+		globalConf := config.Global()
+		globalConf.BundleBaseURL = "mock://somepath?api_key=supersecret"
+		globalConf.BundleInsecureSkipVerify = true
+		config.SetGlobal(globalConf)
+		specs := BuildAndLoadAPI(func(spec *APISpec) {
+			spec.CustomMiddlewareBundle = bundleID
+		})
+		spec := specs[0]
+		bundle, err := fetchBundle(spec)
+		if err != nil {
+			t.Fatalf("Couldn't fetch bundle: %s", err.Error())
+		}
+
+		if string(bundle.Data) != "bundle-insecure" {
+			t.Errorf("Wrong bundle data: %s", bundle.Data)
+		}
+		if bundle.Name != bundleID {
+			t.Errorf("Wrong bundle name: %s", bundle.Name)
+		}
+	})
+}
+
 var overrideResponsePython = map[string]string{
 	"manifest.json": `
 		{