build: Update most things for Go 1.11 modules

We're still using vendoring for now until we get _all_ of our tooling
updated, so the main idea here is to force use of the vendor directory
when running tests and building for development so we can quickly find
situations where we forget to run "go mod vendor".

We also setting GO111MODULE=off for installation of tools. Right now this
is the best way to install a tool in GOBIN without also interfering with
go.mod and go.sum, until a better pattern for managing tool dependencies
is devised by the Go team.

Finally, we run "go mod download" before launching "gox" in the main
build process, to prime the local module cache once so that the concurrent
"go build" processes won't race to populate it redundantly. This means
that we'll be producing final builds from the module cache rather than
from vendor as with everything else -- there's currently no way to tell
gox to use -mod=vendor -- but that should be fine in practice since
our go.sum file will ensure that we get the exact sources we expect in
the module cache before building.

.travis.yml

@@ -11,7 +11,7 @@ go:
 # add TF_ETCDV3_TEST=1 to run etcdv3 tests
 # if added, TF_ETCDV3_ENDPOINTS must be set to a comma-separated list of (insecure) etcd endpoints against which to test
 env:
-  - CONSUL_VERSION=0.7.5 GOMAXPROCS=4
+  - CONSUL_VERSION=0.7.5 GOMAXPROCS=4 GO111MODULE=on
 
 # Fetch consul for the backend and provider tests
 before_install:

Makefile

@@ -6,10 +6,10 @@ WEBSITE_REPO=github.com/hashicorp/terraform-website
 default: test
 
 tools:
-	go get -u github.com/kardianos/govendor
-	go get -u golang.org/x/tools/cmd/stringer
-	go get -u golang.org/x/tools/cmd/cover
-	go get -u github.com/golang/mock/mockgen
+	GO111MODULE=off go get -u github.com/kardianos/govendor
+	GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer
+	GO111MODULE=off go get -u golang.org/x/tools/cmd/cover
+	GO111MODULE=off go get -u github.com/golang/mock/mockgen
 
 # bin generates the releaseable binaries for Terraform
 bin: fmtcheck generate
@@ -18,10 +18,10 @@ bin: fmtcheck generate
 # dev creates binaries for testing Terraform locally. These are put
 # into ./bin/ as well as $GOPATH/bin
 dev: fmtcheck generate
-	@TF_DEV=1 sh -c "'$(CURDIR)/scripts/build.sh'"
+	go install -mod=vendor .
 
 quickdev: generate
-	@TF_DEV=1 sh -c "'$(CURDIR)/scripts/build.sh'"
+	go install -mod=vendor .
 
 # Shorthand for building and installing just one plugin for local testing.
 # Run as (for example): make plugin-dev PLUGIN=provider-aws
@@ -33,34 +33,34 @@ plugin-dev: generate
 # we run this one package at a time here because running the entire suite in
 # one command creates memory usage issues when running in Travis-CI.
 test: fmtcheck generate
-	go list $(TEST) | xargs -t -n4 go test $(TESTARGS) -timeout=2m -parallel=4
+	go list -mod=vendor $(TEST) | xargs -t -n4 go test $(TESTARGS) -mod=vendor -timeout=2m -parallel=4
 
 # testacc runs acceptance tests
 testacc: fmtcheck generate
 	@if [ "$(TEST)" = "./..." ]; then \
 		echo "ERROR: Set TEST to a specific package. For example,"; \
-		echo "  make testacc TEST=./builtin/providers/aws"; \
+		echo "  make testacc TEST=./builtin/providers/test"; \
 		exit 1; \
 	fi
-	TF_ACC=1 go test $(TEST) -v $(TESTARGS) -timeout 120m
+	TF_ACC=1 go test $(TEST) -v $(TESTARGS) -mod=vendor -timeout 120m
 
 # e2etest runs the end-to-end tests against a generated Terraform binary
 # The TF_ACC here allows network access, but does not require any special
 # credentials since the e2etests use local-only providers such as "null".
 e2etest: generate
-	TF_ACC=1 go test -v ./command/e2etest
+	TF_ACC=1 go test -mod=vendor -v ./command/e2etest
 
 test-compile: fmtcheck generate
 	@if [ "$(TEST)" = "./..." ]; then \
 		echo "ERROR: Set TEST to a specific package. For example,"; \
-		echo "  make test-compile TEST=./builtin/providers/aws"; \
+		echo "  make test-compile TEST=./builtin/providers/test"; \
 		exit 1; \
 	fi
 	go test -c $(TEST) $(TESTARGS)
 
 # testrace runs the race checker
 testrace: fmtcheck generate
-	TF_ACC= go test -race $(TEST) $(TESTARGS)
+	TF_ACC= go test -mod=vendor -race $(TEST) $(TESTARGS)
 
 cover:
 	@go tool cover 2>/dev/null; if [ $$? -eq 3 ]; then \
@@ -75,10 +75,13 @@ cover:
 # "make protobuf".
 generate:
 	@which stringer > /dev/null; if [ $$? -ne 0 ]; then \
-	  go get -u golang.org/x/tools/cmd/stringer; \
+	  GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer; \
 	fi
-	go generate ./...
-	@go fmt command/internal_plugin_list.go > /dev/null
+	# We turn off modules for "go generate" because our downstream generate
+	# commands are not all ready to deal with Go modules yet, and this
+	# avoids downloading all of the deps that are in the vendor dir anyway.
+	GO111MODULE=off go generate ./...
+	GO111MODULE=off go fmt command/internal_plugin_list.go > /dev/null
 
 # We separate the protobuf generation because most development tasks on
 # Terraform do not involve changing protobuf files and protoc is not a

scripts/build.sh

@@ -47,6 +47,10 @@ if [[ -n "${TF_RELEASE}" ]]; then
     LD_FLAGS="-X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X github.com/hashicorp/terraform/version.Prerelease= -s -w"
 fi
 
+# Ensure all remote modules are downloaded and cached before build so that
+# the concurrent builds launched by gox won't race to redundantly download them.
+go mod download
+
 # Build!
 echo "==> Building..."
 gox \