New Makefile (spiffe#1311)

* New Makefile

- toolchain is automatically installed/updated (no setup step)
- explicit protobuf generation
- explicit plugin/service/hostservice stub generation
- explicit mock generation
- deprecates build.sh
- cleans up and simplifies developer Docker image management
- moves travis specific steps into .travis/ scripts

.dockerignore

@@ -1,3 +1,6 @@
 # Ignore build tools and cache
-.build-*
+.build*
 .cache
+bin/
+releases/
+artifacts/

.gitignore

@@ -5,6 +5,7 @@
 .tmp
 *.swp
 *.log
+/bin
 /vendor
 /test_results
 /artifacts

.golangci.yml

@@ -1,6 +1,6 @@
 run:
  # timeout for analysis, e.g. 30s, 5m, default is 1m
- deadline: 5m
+ deadline: 10m
 
  skip-dirs:
  - testdata$

.travis.yml

@@ -19,49 +19,39 @@ cache:
  - $HOME/go/bin
 
 stages:
- - setup
- - lint
- - test
+ - lint and test
  - build release
  - publish images
  - nightly integration tests
 
 jobs:
  include:
- - stage: setup
+ - stage: lint and test
+ name: lint
  script:
- - ./build.sh setup
- - ./build.sh utils
- os: macos
- - stage: setup
- script:
- - ./build.sh setup
- - ./build.sh utils
- os: linux
- dist: xenial
- - stage: lint
- script:
- - ./build.sh protobuf_verify
- - shellcheck build.sh
  - make lint
+ - make generate-check
+ - shellcheck script/*.sh
+ - shellcheck .travis/*.sh
  # linting is OS agnostic but protobuf_verify needs utils to lets be explicit
  # about its runtime environment
  os: linux 
  dist: xenial
- - stage: test
+ - stage: lint and test
+ name: linux tests
  script:
- - ./build.sh race-test
+ - .travis/run-unit-tests.sh
  os: macos
- - stage: test
+ - stage: lint and test
+ name: macos tests
  script:
- - ./build.sh race-test
+ - .travis/run-unit-tests.sh
  os: linux
  dist: xenial
  - stage: build release
  script:
- - ./build.sh binaries
- - ./build.sh artifact
- - ./build.sh release
+ - make artifact
+ - .travis/build-release.sh
  os: linux
  dist: xenial
  deploy:
@@ -94,22 +84,17 @@ jobs:
  # Decrypt credentials needed to log into gcr registry
  - openssl aes-256-cbc -K $encrypted_b48f9e852489_key -iv $encrypted_b48f9e852489_iv -in .travis/spire-travis-ci.json.enc -out .travis/spire-travis-ci.json -d
  script:
- # Build spire images
- - make spire-images
- # Run integration tests
- - ./build.sh integration
- # Publish images
+ - make images
+ - make integration
  - .travis/publish-images.sh
 
  - stage: nightly integration tests
  if: type = cron
  os: linux
  dist: xenial
  script:
- # Build spire images
- - make spire-images
- # Run integration tests
- - ./build.sh integration
+ - make images
+ - make integration
 
 notifications:
  email:

.travis/build-release.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+TAG="$(git describe --abbrev=0 2>/dev/null || true)"
+ALWAYS="$(git describe --always || true)"
+if [ "$TAG" == "$ALWAYS" ]; then
+ make -C "${DIR}/.." TAG="${TAG}" OUTDIR=./releases artifact
+fi

.travis/publish-images.sh

@@ -3,7 +3,6 @@
 set -e
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-REPODIR=${DIR}/..
 
 echo "Preparing to build and push images..."
 echo "Travis Branch : ${TRAVIS_BRANCH}"
@@ -14,28 +13,28 @@ echo "Travis Commit : ${TRAVIS_COMMIT}"
 # file is NOT stored plaintext and is decrypted by Travis CI before this script
 # is run.
 echo "Logging into gcr.io..."
-cat "${DIR}/spire-travis-ci.json" | docker login -u _json_key --password-stdin https://gcr.io
+docker login -u _json_key --password-stdin https://gcr.io < "${DIR}/spire-travis-ci.json"
 
 # Tag and push latest build by Git hash
-docker tag spire-server gcr.io/spiffe-io/spire-server:${TRAVIS_COMMIT}
-docker push gcr.io/spiffe-io/spire-server:${TRAVIS_COMMIT}
-docker tag spire-agent gcr.io/spiffe-io/spire-agent:${TRAVIS_COMMIT}
-docker push gcr.io/spiffe-io/spire-agent:${TRAVIS_COMMIT}
-docker tag k8s-workload-registrar gcr.io/spiffe-io/k8s-workload-registrar:${TRAVIS_COMMIT}
-docker push gcr.io/spiffe-io/k8s-workload-registrar:${TRAVIS_COMMIT}
-docker tag oidc-discovery-provider gcr.io/spiffe-io/oidc-discovery-provider:${TRAVIS_COMMIT}
-docker push gcr.io/spiffe-io/oidc-discovery-provider:${TRAVIS_COMMIT}
+docker tag spire-server gcr.io/spiffe-io/spire-server:"${TRAVIS_COMMIT}"
+docker push gcr.io/spiffe-io/spire-server:"${TRAVIS_COMMIT}"
+docker tag spire-agent gcr.io/spiffe-io/spire-agent:"${TRAVIS_COMMIT}"
+docker push gcr.io/spiffe-io/spire-agent:"${TRAVIS_COMMIT}"
+docker tag k8s-workload-registrar gcr.io/spiffe-io/k8s-workload-registrar:"${TRAVIS_COMMIT}"
+docker push gcr.io/spiffe-io/k8s-workload-registrar:"${TRAVIS_COMMIT}"
+docker tag oidc-discovery-provider gcr.io/spiffe-io/oidc-discovery-provider:"${TRAVIS_COMMIT}"
+docker push gcr.io/spiffe-io/oidc-discovery-provider:"${TRAVIS_COMMIT}"
 
 if [ -n "${TRAVIS_TAG}" ]; then
  # This is a tagged build. Tag and push under the git tag.
- docker tag spire-server gcr.io/spiffe-io/spire-server:${TRAVIS_TAG}
- docker push gcr.io/spiffe-io/spire-server:${TRAVIS_TAG}
- docker tag spire-agent gcr.io/spiffe-io/spire-agent:${TRAVIS_TAG}
- docker push gcr.io/spiffe-io/spire-agent:${TRAVIS_TAG}
- docker tag k8s-workload-registrar gcr.io/spiffe-io/k8s-workload-registrar:${TRAVIS_TAG}
- docker push gcr.io/spiffe-io/k8s-workload-registrar:${TRAVIS_TAG}
- docker tag oidc-discovery-provider gcr.io/spiffe-io/oidc-discovery-provider:${TRAVIS_TAG}
- docker push gcr.io/spiffe-io/oidc-discovery-provider:${TRAVIS_TAG}
+ docker tag spire-server gcr.io/spiffe-io/spire-server:"${TRAVIS_TAG}"
+ docker push gcr.io/spiffe-io/spire-server:"${TRAVIS_TAG}"
+ docker tag spire-agent gcr.io/spiffe-io/spire-agent:"${TRAVIS_TAG}"
+ docker push gcr.io/spiffe-io/spire-agent:"${TRAVIS_TAG}"
+ docker tag k8s-workload-registrar gcr.io/spiffe-io/k8s-workload-registrar:"${TRAVIS_TAG}"
+ docker push gcr.io/spiffe-io/k8s-workload-registrar:"${TRAVIS_TAG}"
+ docker tag oidc-discovery-provider gcr.io/spiffe-io/oidc-discovery-provider:"${TRAVIS_TAG}"
+ docker push gcr.io/spiffe-io/oidc-discovery-provider:"${TRAVIS_TAG}"
 elif [ x"${TRAVIS_BRANCH}" = x"master" ]; then
  # This is an untagged build for master. Tag and push as unstable
  docker tag spire-server gcr.io/spiffe-io/spire-server:unstable

.travis/run-unit-tests.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+REPODIR=$(git rev-parse --show-toplevel)
+
+COVERPROFILE=
+if [ -n "${COVERALLS_TOKEN}" ]; then
+ COVERPROFILE=profile.cov
+ go get github.com/mattn/[email protected]
+fi
+
+make -C "${REPODIR}" COVERPROFILE="${COVERPROFILE}" test
+
+if [ -n "${COVERALLS_TOKEN}" ]; then
+ "$(go env GOPATH)"/bin/goveralls -service=travis-ci
+fi

CONTRIBUTING.md

@@ -25,57 +25,39 @@ Since go modules are used, this repository can live in any folder on your local
 
 A Makefile is provided for common actions.
 
-* `make all` - installs 3rd-party dependencies, build all binaries, and run all tests
 * `make` - builds all binaries
-* `make cmd/spire-agent` - builds one binary
-* `make test` - runs all tests
+* `make all` - builds all binaries, lints code, and runs all unit tests
+* `make bin/spire-server` - builds SPIRE server
+* `make bin/spire-agent` - builds SPIRE agent
+* `make images` - builds SPIRE docker images
+* `make test` - runs unit tests
 
-**Other Makefile targets**
+See `make help` for other targets
 
-* `vendor` - Make vendored copy of dependencies using go mod
-* `race-test` - run `go test -race`
-* `clean` - cleans `vendor` directory
-* `distclean` - removes caches in addition to `make clean`
-* `utils` - installs gRPC related development utilities
-* `protobuf` - regenerates the gRPC pb.go and README_pb.md files
-* `protobuf_verify` - checks that the checked-in generated code is up-to-date
-* `help` - shows makefile targets and description
+The Makefile takes care of installing the required toolchain as needed. The
+toolchain and other build related files are cached under the `.build` folder
+(ignored by git).
 
 ## Development in Docker
 
-You can either build Spire on your host or in a Ubuntu docker container. In both cases you will use
-the same Makefile commands.
+You can either build SPIRE on your host or in a Ubuntu docker container. In
+both cases you will use the same Makefile commands.
 
-To run in a docker container set the environment variable `SPIRE_DEV_HOST` to `docker` like so:
+To build SPIRE within a container, first build the development image:
 
 ```
-$ export SPIRE_DEV_HOST=docker
+$ make dev-image
 ```
 
-To set up the build container and run bash within it:
+Then launch a shell inside of development container:
 
 ```
-$ make container
-$ make cmd
+$ make dev-shell
 ```
 
-Because the docker container shares `$GOPATH/pkg/mod` you will not have to re-install the go dependencies every time you run the container.
-
-## CI
-
-The script `build.sh` manages the CI build process, implementing several unique steps and sanity
-checks. It is also used to bootstrap the Go environment in the Docker container.
-
-* `setup` - download and install necessary build tools into the directory `.build-<os>-<arch>`
-* `protobuf` - calls `make protobuf` and regenerates the gRPC pb.go and README_pb.md files
-* `protobuf_verify` - calls `make protobuf_verify` and checks that the checked-in generated code is up-to-date
-* `distclean` - calls `make distclean` and removes the directory `.build-<os>-<arch>`
-* `artifact` - generate a `.tgz` containing all of the SPIFFE binaries
-* `test` - when called from within a Travis-CI build, runs coverage tests in addition to the
- regular tests
-* `utils` - calls `make utils` and installs additional packages for the CI build
-* `eval $(build.sh env)` - configure GOPATH, GOROOT and PATH to use the private build tool directory
-
+Because the docker container shares the `.build` cache and `$GOPATH/pkg/mod`
+you will not have to re-install the toolchain or go dependencies every time you
+run the container.
 
 # Conventions
 
@@ -200,4 +182,4 @@ before sending a pull request. From the project root:
 ln -s .githooks/pre-commit .git/hooks/pre-commit
 ```
 # Reporting security vulnerabilities
-If you've found a vulnerability or a potential vulnerability in SPIRE please let us know at [email protected]. We'll send a confirmation email to acknowledge your report, and we'll send an additional email when we've identified the issue positively or negatively.
+If you've found a vulnerability or a potential vulnerability in SPIRE please let us know at [email protected]. We'll send a confirmation email to acknowledge your report, and we'll send an additional email when we've identified the issue positively or negatively.

Dockerfile

@@ -1,15 +1,44 @@
-FROM ubuntu:xenial
+# Build stage
+ARG goversion
+FROM golang:${goversion}-alpine as builder
+RUN apk add build-base git mercurial
+ADD go.mod /spire/go.mod
+ADD proto/spire/go.mod /spire/proto/spire/go.mod
+RUN cd /spire && go mod download
+ADD . /spire
+WORKDIR /spire
+RUN make build
 
-RUN apt-get update && apt-get -y install \
- curl unzip git build-essential ca-certificates
+# Common base
+FROM alpine AS spire-base
+RUN apk --no-cache add dumb-init
+RUN apk --no-cache add ca-certificates
+RUN mkdir -p /opt/spire/bin
 
-COPY build.sh /root/
-ENV BUILD_DIR=/root/build
-RUN /root/build.sh setup
+# SPIRE Server
+FROM spire-base AS spire-server
+COPY --from=builder /spire/bin/spire-server /opt/spire/bin/spire-server
+WORKDIR /opt/spire
+ENTRYPOINT ["/usr/bin/dumb-init", "/opt/spire/bin/spire-server", "run"]
+CMD []
 
-ENV GOPATH=/root/go
-ENV GOROOT=/root/build
-ENV GOBIN=$GOPATH/bin/linux_amd64
-ENV PATH=$GOROOT/bin:$GOBIN:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-RUN mkdir /root/go
-WORKDIR /root/spire
+# SPIRE Agent
+FROM spire-base AS spire-agent
+COPY --from=builder /spire/bin/spire-agent /opt/spire/bin/spire-agent
+WORKDIR /opt/spire
+ENTRYPOINT ["/usr/bin/dumb-init", "/opt/spire/bin/spire-agent", "run"]
+CMD []
+
+# K8S Workload Registrar
+FROM spire-base AS k8s-workload-registrar
+COPY --from=builder /spire/bin/k8s-workload-registrar /opt/spire/bin/k8s-workload-registrar
+WORKDIR /opt/spire
+ENTRYPOINT ["/usr/bin/dumb-init", "/opt/spire/bin/k8s-workload-registrar"]
+CMD []
+
+# OIDC Discovery Provider
+FROM spire-base AS oidc-discovery-provider
+COPY --from=builder /spire/bin/oidc-discovery-provider /opt/spire/bin/oidc-discovery-provider
+WORKDIR /opt/spire
+ENTRYPOINT ["/usr/bin/dumb-init", "/opt/spire/bin/oidc-discovery-provider"]
+CMD []

Dockerfile.dev

@@ -0,0 +1,6 @@
+FROM ubuntu:xenial
+
+RUN apt-get update && apt-get -y install \
+ curl unzip git build-essential ca-certificates
+
+WORKDIR /spire