Add Protolock. (istio#675)

* Add Protolock.

Move the protoc docker files to tools/protoc. Add docker files for
protolock image. Update Makefile precommit hook and prow presubmit
check to run protolock.

* Add combined image to be used on CircleCI

* Update proto.lock for policy_check_fail_open in mesh config.

* Updade circle config.

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 defaults: &defaults
   working_directory: /src/istio.io/api
   docker:
-    - image: gcr.io/istio-testing/protoc:2018-06-12
+    - image: gcr.io/istio-testing/api-build-tools:2018-10-24
   environment:
     GOPATH: /go
     OUT_PATH: /src
@@ -15,10 +15,11 @@ jobs:
       - run:
           command: |
             ./scripts/generate-protos.sh || die "could not generate *.pb.go"
+            make proto-commit || die "could not regenerate proto.lock"
             if [[ -n $(git status --porcelain) ]]; then
               git status
               git --no-pager diff
-              echo "Repo has unstaged changes. Re-run ./scripts/generate-protos.sh"
+              echo "Repo has unstaged changes. Re-run ./scripts/generate-protos.sh or make proto-commit"
               exit 1
             fi
 

Makefile

@@ -10,14 +10,17 @@ ifdef CIRCLECI
 repo_dir = .
 docker_gen = /usr/bin/protoc -I/protobuf -I$(repo_dir)
 out_path = $(OUT_PATH)
+docker_lock = protolock
 else
 gen_img := gcr.io/istio-testing/protoc:2018-06-12
+lock_img := gcr.io/istio-testing/protolock:2018-10-23
 pwd := $(shell pwd)
 mount_dir := /src
 repo_dir := istio.io/api
 repo_mount := $(mount_dir)/istio.io/api
 docker_gen := docker run --rm -v $(pwd):$(repo_mount) -w $(mount_dir) $(gen_img) -I$(repo_dir)
 out_path = .
+docker_lock = docker run --rm -v $(pwd):$(repo_mount) -w $(repo_mount) $(lock_img)
 endif
 
 ########################
@@ -99,12 +102,14 @@ generate-mcp-go: $(config_mcp_pb_gos) $(config_mcp_pb_doc)
 
 $(config_mcp_pb_gos) $(config_mcp_pb_doc): $(config_mcp_protos)
 	## Generate mcp/v1alpha1/*.pb.go + $(config_mcp_pb_doc)
+	@$(docker_lock) status
 	@$(docker_gen) $(gogofast_plugin) $(protoc_gen_docs_plugin)$(config_mcp_path) $^
 
 generate-mcp-python: $(config_mcp_pb_pythons)
 
 $(config_mcp_pb_pythons): $(config_mcp_protos)
 	## Generate python/istio_api/mcp/v1alpha1/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 clean-mcp:
@@ -125,12 +130,14 @@ generate-mesh-go: $(mesh_pb_gos) $(mesh_pb_doc)
 
 $(mesh_pb_gos) $(mesh_pb_doc): $(mesh_protos)
 	## Generate mesh/v1alpha1/*.pb.go + $(mesh_pb_doc)
+	@$(docker_lock) status
 	@$(docker_gen) $(gogofast_plugin) $(protoc_gen_docs_plugin)$(mesh_path) $^
 
 generate-mesh-python: $(mesh_pb_pythons)
 
 $(mesh_pb_pythons): $(mesh_protos)
 	## Generate python/istio_api/mesh/v1alpha1/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 clean-mesh:
@@ -173,18 +180,22 @@ generate-mixer-go: \
 
 $(mixer_v1_pb_gos) $(mixer_v1_pb_doc): $(mixer_v1_protos)
 	## Generate mixer/v1/*.pb.go + $(mixer_v1_pb_doc)
+	@$(docker_lock) status
 	@$(docker_gen) $(gogoslick_plugin) $(protoc_gen_docs_plugin)$(mixer_v1_path) $^
 
 $(mixer_config_client_pb_gos) $(mixer_config_client_pb_doc): $(mixer_config_client_protos)
 	## Generate mixer/v1/config/client/*.pb.go + $(mixer_config_client_pb_doc)
+	@$(docker_lock) status
 	@$(docker_gen) $(gogoslick_plugin) $(protoc_gen_docs_plugin)$(mixer_config_client_path) $^
 
 $(mixer_adapter_model_v1beta1_pb_gos) $(mixer_adapter_model_v1beta1_pb_doc) : $(mixer_adapter_model_v1beta1_protos)
 	## Generate mixer/adapter/model/v1beta1/*.pb.go + $(mixer_adapter_model_v1beta1_pb_doc)
+	@$(docker_lock) status
 	@$(docker_gen) $(gogoslick_plugin) $(protoc_gen_docs_plugin)$(mixer_adapter_model_v1beta1_path) $^
 
 $(policy_v1beta1_pb_gos) $(policy_v1beta1_pb_doc) : $(policy_v1beta1_protos)
 	## Generate policy/v1beta1/*.pb.go + $(policy_v1beta1_pb_doc)
+	@$(docker_lock) status
 	@$(docker_gen) $(gogoslick_plugin) $(protoc_gen_docs_plugin)$(policy_v1beta1_path) $^
 
 generate-mixer-python: \
@@ -195,18 +206,22 @@ generate-mixer-python: \
 
 $(mixer_v1_pb_pythons): $(mixer_v1_protos)
 	## Generate python/istio_api/mixer/v1/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 $(mixer_config_client_pb_pythons): $(mixer_config_client_protos)
 	## Generate python/istio_api/mixer/v1/config/client/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 $(mixer_adapter_model_v1beta1_pb_pythons): $(mixer_adapter_model_v1beta1_protos)
 	## Generate python/istio_api/mixer/adapter/model/v1beta1/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 $(policy_v1beta1_pb_pythons): $(policy_v1beta1_protos)
 	## Generate python/istio_api/policy/v1beta1/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 clean-mixer:
@@ -227,12 +242,14 @@ generate-routing-go: $(routing_v1alpha3_pb_gos) $(routing_v1alpha3_pb_doc)
 
 $(routing_v1alpha3_pb_gos) $(routing_v1alpha3_pb_doc): $(routing_v1alpha3_protos)
 	## Generate networking/v1alpha3/*.pb.go
+	@$(docker_lock) status
 	@$(docker_gen) $(gogofast_plugin) $(protoc_gen_docs_plugin)$(routing_v1alpha3_path) $^
 
 generate-routing-python: $(routing_v1alpha3_pb_pythons)
 
 $(routing_v1alpha3_pb_pythons): $(routing_v1alpha3_protos)
 	## Generate python/istio_api/networking/v1alpha3/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 clean-routing:
@@ -253,12 +270,14 @@ generate-rbac-go: $(rbac_v1alpha1_pb_gos) $(rbac_v1alpha1_pb_doc)
 
 $(rbac_v1alpha1_pb_gos) $(rbac_v1alpha1_pb_doc): $(rbac_v1alpha1_protos)
 	## Generate rbac/v1alpha1/*.pb.go
+	@$(docker_lock) status
 	@$(docker_gen) $(gogofast_plugin) $(protoc_gen_docs_plugin)$(rbac_v1alpha1_path) $^
 
 generate-rbac-python: $(rbac_v1alpha1_protos)
 
 $(rbac_v1alpha1_pb_pythons): $(rbac_v1alpha1_protos)
 	## Generate python/istio_api/rbac/v1alpha1/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 clean-rbac:
@@ -280,12 +299,14 @@ generate-authn-go: $(authn_v1alpha1_pb_gos) $(authn_v1alpha1_pb_doc)
 
 $(authn_v1alpha1_pb_gos) $(authn_v1alpha1_pb_doc): $(authn_v1alpha1_protos)
 	## Generate authentication/v1alpha1/*.pb.go
+	@$(docker_lock) status
 	@$(docker_gen) $(gogofast_plugin) $(protoc_gen_docs_plugin)$(authn_v1alpha1_path) $^
 
 generate-authn-python: $(authn_v1alpha1_pb_pythons)
 
 $(authn_v1alpha1_pb_pythons): $(authn_v1alpha1_protos)
 	## Generate python/istio_api/authentication/v1alpha1/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 clean-authn:
@@ -306,18 +327,26 @@ generate-envoy-go: $(envoy_pb_gos) $(envoy_pb_doc)
 # Envoy APIs is internal APIs, documents is not required.
 $(envoy_pb_gos): $(envoy_protos)
 	## Generate envoy/*/*.pb.go
+	@$(docker_lock) status
 	@$(docker_gen) $(gogofast_plugin) $^
 
 generate-envoy-python: $(envoy_pb_pythons)
 
 # Envoy APIs is internal APIs, documents is not required.
 $(envoy_pb_pythons): $(envoy_protos)
 	## Generate envoy/*/*_pb2.py
+	@$(docker_lock) status
 	@$(docker_gen) $(protoc_gen_python_plugin) $^
 
 clean-envoy:
 	rm -f $(envoy_pb_gos)
 
+#####################
+# Protolock
+#####################
+proto-commit:
+	@$(docker_lock) commit
+
 #####################
 # Cleanup
 #####################

README.md

@@ -12,6 +12,12 @@ When making changes to the protos in this repository, your changes **must** comp
 
 ## Updating
 
-After the [protobuf](https://github.com/google/protobuf) definitions are updated, the corresponding `*pb.go` and `_pb2.py` files must be generated by running `scripts/generate-protos.sh` and submitted as part of the same PR as the updated definitions.
+After the [protobuf](https://github.com/google/protobuf) definitions
+are updated, the corresponding `*pb.go` and `_pb2.py` files must be
+generated by running `scripts/generate-protos.sh` and submitted as
+part of the same PR as the updated definitions. Also `make
+proto-commit` must be run to update the proto.lock file with new
+changes.
+
 
 If releasing a new tagged version, please update python/istio-api/setup.py version to reflect.