chore: portable and simpler toolchain install (argoproj#7920)

Signed-off-by: Jesse Suen <[email protected]>

hack/generate-proto.sh

@@ -9,24 +9,16 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd)
+PATH="${PROJECT_ROOT}/dist:${PATH}"
+
 # output tool versions
 protoc --version
 swagger version
 jq --version
 
-PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd)
-CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${PROJECT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ../code-generator)}
-PATH="${PROJECT_ROOT}/dist:${PATH}"
-MOD_ROOT=${GOPATH}/pkg/mod
-
-. ${PROJECT_ROOT}/hack/versions.sh
-
 export GO111MODULE=off
 
-# protobuf tooling required to build .proto files from go annotations from k8s-like api types
-go build -o dist/go-to-protobuf ./vendor/k8s.io/code-generator/cmd/go-to-protobuf
-go build -o dist/protoc-gen-gogo ./vendor/k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo
-
 # Generate pkg/apis/<group>/<apiversion>/(generated.proto,generated.pb.go)
 # NOTE: any dependencies of our types to the k8s.io apimachinery types should be added to the
 # --apimachinery-packages= option so that go-to-protobuf can locate the types, but prefixed with a
@@ -56,26 +48,20 @@ ${PROJECT_ROOT}/dist/go-to-protobuf \
 # server/*/<service>.pb.go from .proto files. golang/protobuf and gogo/protobuf can be used
 # interchangeably. The difference in the options are:
 # 1. protoc-gen-go - official golang/protobuf
-#go build -o dist/protoc-gen-go ./vendor/github.com/golang/protobuf/protoc-gen-go
 #GOPROTOBINARY=go
 # 2. protoc-gen-gofast - fork of golang golang/protobuf. Faster code generation
-#go build -o dist/protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast
 #GOPROTOBINARY=gofast
 # 3. protoc-gen-gogofast - faster code generation and gogo extensions and flexibility in controlling
 # the generated go code (e.g. customizing field names, nullable fields)
-go build -o dist/protoc-gen-gogofast ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast
 GOPROTOBINARY=gogofast
 
-# protoc-gen-grpc-gateway is used to build <service>.pb.gw.go files from from .proto files
-go build -o dist/protoc-gen-grpc-gateway ./vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
-# protoc-gen-swagger is used to build swagger.json
-go build -o dist/protoc-gen-swagger ./vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
-
 # Generate server/<service>/(<service>.pb.go|<service>.pb.gw.go)
+MOD_ROOT=${GOPATH}/pkg/mod
+grpc_gateway_version=$(go list -m github.com/grpc-ecosystem/grpc-gateway | awk '{print $NF}' | head -1)
+GOOGLE_PROTO_API_PATH=${MOD_ROOT}/github.com/grpc-ecosystem/grpc-gateway@${grpc_gateway_version}/third_party/googleapis
+GOGO_PROTOBUF_PATH=${PROJECT_ROOT}/vendor/github.com/gogo/protobuf
 PROTO_FILES=$(find $PROJECT_ROOT \( -name "*.proto" -and -path '*/server/*' -or -path '*/reposerver/*' -and -name "*.proto" -or -path '*/cmpserver/*' -and -name "*.proto" \) | sort)
 for i in ${PROTO_FILES}; do
- GOOGLE_PROTO_API_PATH=${MOD_ROOT}/github.com/grpc-ecosystem/grpc-gateway@${grpc_gateway_version}/third_party/googleapis
- GOGO_PROTOBUF_PATH=${PROJECT_ROOT}/vendor/github.com/gogo/protobuf
  protoc \
  -I${PROJECT_ROOT} \
  -I/usr/local/include \

hack/install.sh

@@ -9,6 +9,7 @@ mkdir -p $DOWNLOADS
 ARCHITECTURE=""
 case $(uname -m) in
  x86_64) ARCHITECTURE="amd64" ;;
+ arm64) ARCHITECTURE="arm64" ;;
  arm|armv7l|armv8l|aarch64) dpkg --print-architecture | grep -q "arm64" && ARCHITECTURE="arm64" || ARCHITECTURE="arm" ;;
 esac
 

hack/installers/compare-chksum.sh

@@ -23,4 +23,4 @@ if ! grep -q "${TARGET_FILE}" ${CHKSUM_FILE}; then
  exit 1
 fi
 
-sha256sum -c ${CHKSUM_FILE}
+shasum -a 256 -c ${CHKSUM_FILE}

hack/installers/install-codegen-go-tools.sh

@@ -1,9 +1,52 @@
 #!/bin/bash
 set -eux -o pipefail
 
-GO111MODULE=on go get github.com/gogo/protobuf/[email protected]
-GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/[email protected]
-GO111MODULE=on go get github.com/golang/protobuf/[email protected]
-GO111MODULE=on go get github.com/grpc-ecosystem/grpc-gateway/[email protected]
-GO111MODULE=on go get github.com/grpc-ecosystem/grpc-gateway/[email protected]
-GO111MODULE=on go get golang.org/x/tools/cmd/[email protected]
+SRCROOT="$( CDPATH='' cd -- "$(dirname "$0")/../.." && pwd -P )"
+
+# This script installs all our golang-based codegen utility CLIs necessary for codegen.
+# Some dependencies are vendored in go.mod (ones which are actually imported in our codebase).
+# Other dependencies are only used as a CLI and do not need vendoring in go.mod (doing so adds
+# unecessary dependencies to go.mod). We want to maintain a single source of truth for versioning
+# our binaries (either go.mod or go install <pkg>@<version>), so we use two techniques to install
+# our CLIs:
+# 1. For CLIs which are NOT vendored in go.mod, we can run `go install` with an explicit version
+# 2. For packages which we *do* vendor in go.mod, we can run `go install` from the vendor directory
+go_mod_install() {
+ go install -mod=vendor ./vendor/$1
+}
+
+# All binaries are compiled into the argo-cd/dist directory, which is added to the PATH during codegen
+export GOBIN="${SRCROOT}/dist"
+mkdir -p $GOBIN
+
+# protoc-gen-go* is used to generate <service>.pb.go from .proto files
+#go_mod_install github.com/golang/protobuf/protoc-gen-go
+#go_mod_install github.com/gogo/protobuf/protoc-gen-gogo
+go_mod_install github.com/gogo/protobuf/protoc-gen-gogofast
+
+# protoc-gen-grpc-gateway is used to build <service>.pb.gw.go files from from .proto files
+go_mod_install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
+
+# # protoc-gen-swagger is used to build swagger.json
+go_mod_install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
+
+# k8s tools to codegen .proto files, client libraries, and helpers from types.go
+go_mod_install k8s.io/code-generator/cmd/go-to-protobuf
+go_mod_install k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo
+go_mod_install k8s.io/code-generator/cmd/client-gen
+go_mod_install k8s.io/code-generator/cmd/deepcopy-gen
+go_mod_install k8s.io/code-generator/cmd/defaulter-gen
+go_mod_install k8s.io/code-generator/cmd/informer-gen
+go_mod_install k8s.io/code-generator/cmd/lister-gen
+
+# We still install openapi-gen from go.mod since upstream does not utilize release tags
+go_mod_install k8s.io/kube-openapi/cmd/openapi-gen
+
+# controller-gen is run by ./hack/gen-crd-spec to generate the CRDs
+go install sigs.k8s.io/controller-tools/cmd/[email protected]
+
+# swagger cli is used to generate swagger docs
+go install github.com/go-swagger/go-swagger/cmd/[email protected]
+
+# goimports is used to auto-format generated code
+go install golang.org/x/tools/cmd/[email protected]

hack/installers/install-codegen-tools.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -eux -o pipefail
 
-KUSTOMIZE_VERSION=4.2.0 "$(dirname $0)/../install.sh" helm2-linux jq-linux kustomize-linux protoc-linux swagger-linux
+KUSTOMIZE_VERSION=4.2.0 "$(dirname $0)/../install.sh" helm2-linux jq-linux kustomize-linux protoc-linux

hack/tool-versions.sh

@@ -16,4 +16,3 @@ kubectl_version=1.17.8
 kubectx_version=0.6.3
 kustomize4_version=4.2.0
 protoc_version=3.7.1
-swagger_version=0.19.0

hack/tools.go

@@ -1,19 +1,28 @@
+//go:build tools
 // +build tools
 
 package tools
 
 import (
+ // gogo/protobuf is vendored because the generated *.pb.go code imports it.
+ // Also, we need the gogo/protobuf/gogoproto/gogo.proto file
  _ "github.com/gogo/protobuf/protoc-gen-gogofast"
+
+ // grpc-ecosystem/grpc-gateway is vendored because the generated *.pb.gw.go code imports it.
+ // Also, we need the .proto files under grpc-gateway/third_party/googleapis
  _ "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway"
  _ "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger"
+
+ // k8s.io/code-generator is vendored to get generate-groups.sh, and k8s codegen utilities
+ _ "k8s.io/code-generator"
  _ "k8s.io/code-generator/cmd/client-gen"
  _ "k8s.io/code-generator/cmd/deepcopy-gen"
  _ "k8s.io/code-generator/cmd/defaulter-gen"
- _ "k8s.io/code-generator/cmd/informer-gen"
- _ "k8s.io/code-generator/cmd/lister-gen"
  _ "k8s.io/code-generator/cmd/go-to-protobuf"
  _ "k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo"
- _ "k8s.io/code-generator/pkg/util"
- _ "k8s.io/code-generator/third_party/forked/golang/reflect"
+ _ "k8s.io/code-generator/cmd/informer-gen"
+ _ "k8s.io/code-generator/cmd/lister-gen"
+
+ // openapi-gen is vendored because upstream does not have tagged releases
  _ "k8s.io/kube-openapi/cmd/openapi-gen"
 )

hack/update-codegen.sh

@@ -19,25 +19,14 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
-. ${SCRIPT_ROOT}/hack/versions.sh
-CODEGEN_PKG=$GOPATH/pkg/mod/k8s.io/code-generator@${kube_version}
-TARGET_SCRIPT=/tmp/generate-groups.sh
-
-(
- cd $CODEGEN_PKG
- go install ./cmd/{defaulter-gen,client-gen,lister-gen,informer-gen,deepcopy-gen}
-)
-
-export GO111MODULE=off
+PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd)
 
-sed -e '/go install/d' ${CODEGEN_PKG}/generate-groups.sh > ${TARGET_SCRIPT}
-
-export GO111MODULE=on
+TARGET_SCRIPT=/tmp/generate-groups.sh
+sed -e '/go install/d' ${PROJECT_ROOT}/vendor/k8s.io/code-generator/generate-groups.sh > ${TARGET_SCRIPT}
 
 [ -e ./v2 ] || ln -s . v2
-bash -x ${TARGET_SCRIPT} "deepcopy,client,informer,lister" \
+GOBIN=${PROJECT_ROOT}/dist bash -x ${TARGET_SCRIPT} "deepcopy,client,informer,lister" \
  github.com/argoproj/argo-cd/v2/pkg/client github.com/argoproj/argo-cd/v2/pkg/apis \
  "application:v1alpha1" \
- --go-header-file ${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt
+ --go-header-file ${PROJECT_ROOT}/hack/custom-boilerplate.go.txt
 [ -e ./v2 ] && rm -rf v2

hack/update-openapi.sh

@@ -6,14 +6,9 @@ set -o nounset
 set -o pipefail
 
 PROJECT_ROOT=$(cd $(dirname "$0")/.. ; pwd)
-CODEGEN_PKG=${PROJECT_ROOT}/vendor/k8s.io/kube-openapi
+PATH="${PROJECT_ROOT}/dist:${PATH}"
 VERSION="v1alpha1"
 
-export GO111MODULE=off
-go build -o dist/openapi-gen ${CODEGEN_PKG}/cmd/openapi-gen
-
-export GO111MODULE=on
-
 [ -e ./v2 ] || ln -s . v2
 ./dist/openapi-gen \
  --go-header-file ${PROJECT_ROOT}/hack/custom-boilerplate.go.txt \

test/container/Dockerfile

@@ -38,8 +38,7 @@ ADD ./hack/tool-versions.sh .
 ADD ./hack/install.sh .
 ADD ./hack/installers installers
 
-RUN ./install.sh dep-linux && \
- ./install.sh ksonnet-linux && \
+RUN ./install.sh ksonnet-linux && \
  ./install.sh helm2-linux && \
  ./install.sh helm-linux && \
  ./install.sh kubectl-linux && \