Go 1.5 migration (TykTechnologies#3454)

Go 1.4 introduced TLS 1.3 by default, which we can't do since a lot of our clients depend on 1.2.
In order to disable 1.3 Go added new `MaxVersion` option to `tls.Config` structure.
For example one of 1.3 breaking "features" is the inability to explicitly set TLS ciphers (in 1.3 they should be picked automatically), and a lot of enterprises depend on it.

In order to make it non-breaking change I added new max SSL version attributes to all needed sections (everywhere where there is min version), and it is set to 1.2 by default.

New config fields:
`http_server_options.max_version`
`proxy_ssl_max_version`
Api definition fields:
`proxy.transport.ssl_max_version`

Additionally, fixed a bunch of tests (not related to 1.5 change from what I see).

All the vendoring removed as well!

As part of QA we need to ensure that all outbound and inbound Gateway calls use TLS 1.2, which includes:
- [ ] Uptime tests
- [ ] JSVM HTTP calls
- [ ] Batch API
- [ ] Reverse proxy
- [ ] Gateway -> Dashboard communication
- [ ] Gateway -> MDCB communication
- [ ] UDG GraphQL and JSON data sources

## Types of changes
<!-- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)
- [ ] Refactoring or add test (improvements in base code or adds test coverage to functionality)

## Checklist
<!-- Go over all the following points, and put an `x` in all the boxes that apply -->
<!-- If you're unsure about any of these, don't hesitate to ask; we're here to help! -->
- [x] Make sure you are requesting to **pull a topic/feature/bugfix branch** (right side). If pulling from your own
      fork, don't request your `master`!
- [x] Make sure you are making a pull request against the **`master` branch** (left side). Also, you should start
      *your branch* off *our latest `master`*.
- [x] My change requires a change to the documentation.
  - [ ] If you've changed APIs, describe what needs to be updated in the documentation.
  - [ ] If new config option added, ensure that it can be set via ENV variable
- [ ] I have updated the documentation accordingly.
- [x] Modules and vendor dependencies have been updated; run `go mod tidy && go mod vendor`
- [ ] When updating library version must provide reason/explanation for this update.
- [x] I have added tests to cover my changes.
- [x] All new and existing tests passed.
- [x] Check your code additions will not fail linting checks:
  - [x] `go fmt -s`
  - [x] `go vet`

.travis.yml

@@ -19,7 +19,7 @@ addons:
 
 matrix:
   include:
-    - go: 1.12.x
+    - go: 1.15.x
 
 services:
   - redis-server

apidef/api_definitions.go

@@ -520,6 +520,7 @@ type ProxyConfig struct {
 		SSLInsecureSkipVerify   bool     `bson:"ssl_insecure_skip_verify" json:"ssl_insecure_skip_verify"`
 		SSLCipherSuites         []string `bson:"ssl_ciphers" json:"ssl_ciphers"`
 		SSLMinVersion           uint16   `bson:"ssl_min_version" json:"ssl_min_version"`
+		SSLMaxVersion           uint16   `bson:"ssl_max_version" json:"ssl_max_version"`
 		SSLForceCommonNameCheck bool     `json:"ssl_force_common_name_check"`
 		ProxyURL                string   `bson:"proxy_url" json:"proxy_url"`
 	} `bson:"transport" json:"transport"`

apidef/schema.go

@@ -245,9 +245,9 @@ const Schema = `{
                 }
             }
         },
-		"auth_configs":{
-			"type": ["object", "null"]
-		},
+        "auth_configs":{
+            "type": ["object", "null"]
+        },
         "definition": {
             "type": ["object", "null"],
             "id": "http://jsonschema.net/definition",
@@ -320,6 +320,9 @@ const Schema = `{
                         "ssl_min_version": {
                             "type": "number"
                         },
+                        "ssl_max_version": {
+                            "type": "number"
+                        },
                         "proxy_url": {
                             "type": "string"
                         },
@@ -431,88 +434,88 @@ const Schema = `{
         ]
         },
         "graphql": {
-			"type": ["object", "null"],
-			"properties": {
-				"enabled": {
-					"type": "boolean"
-				},
-				"execution_mode": {
-					"type": "string",
-					"enum": [
-						"proxyOnly",
-						"executionEngine",
-						""
-					]
-				},
-				"schema": {
-					"type": "string"
-				},
-				"last_schema_update": {
-					"type": "string",
-					"format": "date-time"
-				},
-				"type_field_configurations": {
-					"type": ["array", "null"],
-					"properties": {
-						"type_name": {
-							"type": "string"
-						},
-						"field_name": {
-							"type": "string"
-						},
-						"mapping": {
-							"type": ["object", "null"],
-							"properties": {
-								"disabled": {
-									"type": "boolean"
-								},
-								"path": {
-									"type": "string"
-								}
-							},
-							"required": [
-								"disabled"
-							]
-						},
-						"data_source": {
-							"type": ["object", "null"],
-							"properties": {
-								"kind": {
-									"type": "boolean"
-								},
-								"data_source_config": {
-									"type": ["object", "null"]
-								}
-							},
-							"required": [
-								"kind"
-							]
-						}
-					},
-					"required": [
-						"type_name",
-						"field_name"
-					]
-				},
-				"playground": {
-					"type": ["object", "null"],
-					"properties": {
-						"enabled": {
-							"type": "boolean"
-						},
-						"path": {
-							"type": "string"
-						}
-					},
-					"required": [
-						"enabled"
-					]
-				}
-			},
-			"required": [
-				"enabled"
-			]
-		}
+            "type": ["object", "null"],
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                },
+                "execution_mode": {
+                    "type": "string",
+                    "enum": [
+                        "proxyOnly",
+                        "executionEngine",
+                        ""
+                    ]
+                },
+                "schema": {
+                    "type": "string"
+                },
+                "last_schema_update": {
+                    "type": "string",
+                    "format": "date-time"
+                },
+                "type_field_configurations": {
+                    "type": ["array", "null"],
+                    "properties": {
+                        "type_name": {
+                            "type": "string"
+                        },
+                        "field_name": {
+                            "type": "string"
+                        },
+                        "mapping": {
+                            "type": ["object", "null"],
+                            "properties": {
+                                "disabled": {
+                                    "type": "boolean"
+                                },
+                                "path": {
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "disabled"
+                            ]
+                        },
+                        "data_source": {
+                            "type": ["object", "null"],
+                            "properties": {
+                                "kind": {
+                                    "type": "boolean"
+                                },
+                                "data_source_config": {
+                                    "type": ["object", "null"]
+                                }
+                            },
+                            "required": [
+                                "kind"
+                            ]
+                        }
+                    },
+                    "required": [
+                        "type_name",
+                        "field_name"
+                    ]
+                },
+                "playground": {
+                    "type": ["object", "null"],
+                    "properties": {
+                        "enabled": {
+                            "type": "boolean"
+                        },
+                        "path": {
+                            "type": "string"
+                        }
+                    },
+                    "required": [
+                        "enabled"
+                    ]
+                }
+            },
+            "required": [
+                "enabled"
+            ]
+        }
     },
     "required": [
         "name",

bin/integration_build.sh

@@ -49,7 +49,7 @@ cp tyk.conf.example ${bdir}/tyk.conf
 cp -R coprocess ${bdir}
 
 echo "Building Tyk binaries"
-go build -tags 'goplugin' -mod=vendor
+go build -tags 'goplugin'
 mv tyk ${bdir}
 
 echo "Making tarball"

certs/manager.go

@@ -345,22 +345,18 @@ func (c *CertificateManager) List(certIDs []string, mode CertificateType) (out [
 			continue
 		}
 
-		if isSHA256(id) {
-			var val string
-			val, err = c.storage.GetKey("raw-" + id)
-			if err != nil {
-				c.logger.Warn("Can't retrieve certificate from Redis:", id, err)
-				out = append(out, nil)
-				continue
-			}
-			rawCert = []byte(val)
-		} else {
+		var val string
+		val, err = c.storage.GetKey("raw-" + id)
+		if err != nil {
+			// Try read from file
 			rawCert, err = ioutil.ReadFile(id)
 			if err != nil {
-				c.logger.Error("Error while reading certificate from file:", id, err)
+				c.logger.Warn("Can't retrieve certificate:", id, err)
 				out = append(out, nil)
 				continue
 			}
+		} else {
+			rawCert = []byte(val)
 		}
 
 		cert, err = ParsePEMCertificate(rawCert, c.secret)
@@ -476,6 +472,7 @@ func (c *CertificateManager) ListAllIds(prefix string) (out []string) {
 		}
 
 		keys := c.storage.GetKeys("raw-" + prefix + "*")
+
 		for _, key := range keys {
 			if prefix != "" {
 				c.storage.AppendToSet(indexKey, key)

certs/manager_test.go

@@ -235,7 +235,7 @@ func TestCertificateStorage(t *testing.T) {
 	t.Run("File certificates", func(t *testing.T) {
 		certs := m.List([]string{certPath, "wrong"}, CertificatePublic)
 		if len(certs) != 2 {
-			t.Fatal("Should contain 1 cert", len(certs))
+			t.Fatal("Should contain 2 cert", len(certs))
 		}
 
 		if certs[1] != nil {

cli/linter/schema.json

@@ -506,6 +506,9 @@
         "min_version": {
           "type": "integer"
         },
+        "max_version": {
+          "type": "integer"
+        },
         "override_defaults": {
           "type": "boolean"
         },
@@ -784,6 +787,9 @@
     "proxy_ssl_min_version": {
       "type": "integer"
     },
+    "proxy_ssl_max_version": {
+      "type": "integer"
+    },
     "proxy_ssl_ciphers": {
       "type": [
         "array",

config/config.go

@@ -190,6 +190,7 @@ type HttpServerOptionsConfig struct {
 	SSLCertificates        []string   `json:"ssl_certificates"`
 	ServerName             string     `json:"server_name"`
 	MinVersion             uint16     `json:"min_version"`
+	MaxVersion             uint16     `json:"max_version"`
 	FlushInterval          int        `json:"flush_interval"`
 	SkipURLCleaning        bool       `json:"skip_url_cleaning"`
 	SkipTargetPathEscaping bool       `json:"skip_target_path_escaping"`
@@ -386,6 +387,7 @@ type Config struct {
 	ProxySSLInsecureSkipVerify    bool                 `json:"proxy_ssl_insecure_skip_verify"`
 	ProxyEnableHttp2              bool                 `json:"proxy_enable_http2"`
 	ProxySSLMinVersion            uint16               `json:"proxy_ssl_min_version"`
+	ProxySSLMaxVersion            uint16               `json:"proxy_ssl_max_version"`
 	ProxySSLCipherSuites          []string             `json:"proxy_ssl_ciphers"`
 	ProxyDefaultTimeout           float64              `json:"proxy_default_timeout"`
 	ProxySSLDisableRenegotiation  bool                 `json:"proxy_ssl_disable_renegotiation"`

gateway/api_loader.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -121,6 +122,14 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
 		spec.Proxy.TargetURL = strings.Replace(spec.Proxy.TargetURL, "h2c://", "http://", 1)
 	}
 
+	if spec.Proxy.Transport.SSLMaxVersion > 0 {
+		spec.Proxy.Transport.SSLMaxVersion = tls.VersionTLS12
+	}
+
+	if spec.Proxy.Transport.SSLMinVersion > spec.Proxy.Transport.SSLMaxVersion {
+		spec.Proxy.Transport.SSLMaxVersion = spec.Proxy.Transport.SSLMinVersion
+	}
+
 	if len(spec.TagHeaders) > 0 {
 		// Ensure all headers marked for tagging are lowercase
 		lowerCaseHeaders := make([]string, len(spec.TagHeaders))

gateway/batch_requests.go

@@ -48,6 +48,15 @@ func (b *BatchRequestHandler) doRequest(req *http.Request, relURL string) BatchR
 	}
 
 	tr.TLSClientConfig.InsecureSkipVerify = config.Global().ProxySSLInsecureSkipVerify
+
+	if config.Global().ProxySSLMaxVersion > 0 {
+		tr.TLSClientConfig.MaxVersion = config.Global().ProxySSLMaxVersion
+	}
+
+	if b.API != nil && b.API.Proxy.Transport.SSLMaxVersion > 0 {
+		tr.TLSClientConfig.MaxVersion = b.API.Proxy.Transport.SSLMaxVersion
+	}
+
 	tr.DialTLS = customDialTLSCheck(b.API, tr.TLSClientConfig)
 
 	tr.Proxy = proxyFromAPI(b.API)