run secret and role verifications before we create a slow build or ev…

…en a release

plugins/kubernetes/app/models/kubernetes/deploy_executor.rb

@@ -25,6 +25,7 @@ def stop!(_signal)
     end
 
     def execute!(*_commands)
+      verify_kubernetes_templates!
       build = find_or_create_build
       return false if stopped?
       release = create_release(build)
@@ -243,46 +244,9 @@ def ensure_build_is_successful(build)
 
     # create a release, storing all the configuration
     def create_release(build)
-      # find role configs to avoid N+1s
-      roles_configs = Kubernetes::DeployGroupRole.where(
-        project_id: @job.project_id,
-        deploy_group: @job.deploy.stage.deploy_groups.map(&:id)
-      )
-
-      # get all the roles that are configured for this sha
-      configured_roles = Kubernetes::Role.configured_for_project(@job.project, @job.commit)
-      if configured_roles.empty?
-        raise Samson::Hooks::UserError, "No kubernetes config files found at sha #{@job.commit}"
-      end
-
-      # build config for every cluster and role we want to deploy to
-      errors = []
-      group_config = @job.deploy.stage.deploy_groups.map do |group|
-        roles = configured_roles.map do |role|
-          role_config = roles_configs.detect do |dgr|
-            dgr.deploy_group_id == group.id && dgr.kubernetes_role_id == role.id
-          end
-
-          unless role_config
-            errors << "No config for role #{role.name} and group #{group.name} found, add it on the stage page."
-            next
-          end
-
-          {
-            id: role.id,
-            replicas: role_config.replicas,
-            cpu: role_config.cpu,
-            ram: role_config.ram
-          }
-        end
-        {id: group.id, roles: roles}
-      end
-
-      raise Samson::Hooks::UserError, errors.join("\n") if errors.any?
-
       release = Kubernetes::Release.create_release(
         deploy_id: @job.deploy.id,
-        deploy_groups: group_config,
+        deploy_groups: deploy_group_configs,
         build_id: build.try(:id),
         git_sha: @job.commit,
         git_ref: @reference,
@@ -294,10 +258,52 @@ def create_release(build)
         raise Samson::Hooks::UserError, "Failed to create release: #{release.errors.full_messages.inspect}"
       end
 
-      @output.puts("Created release #{release.id}\nConfig: #{group_config.to_json}")
+      @output.puts("Created release #{release.id}\nConfig: #{deploy_group_configs.to_json}")
       release
     end
 
+    def deploy_group_configs
+      @deploy_group_configs ||= begin
+        # find role configs to avoid N+1s
+        roles_configs = Kubernetes::DeployGroupRole.where(
+          project_id: @job.project_id,
+          deploy_group: @job.deploy.stage.deploy_groups.map(&:id)
+        )
+
+        # get all the roles that are configured for this sha
+        configured_roles = Kubernetes::Role.configured_for_project(@job.project, @job.commit)
+        if configured_roles.empty?
+          raise Samson::Hooks::UserError, "No kubernetes config files found at sha #{@job.commit}"
+        end
+
+        # build config for every cluster and role we want to deploy to
+        errors = []
+        group_configs = @job.deploy.stage.deploy_groups.map do |group|
+          roles = configured_roles.map do |role|
+            role_config = roles_configs.detect do |dgr|
+              dgr.deploy_group_id == group.id && dgr.kubernetes_role_id == role.id
+            end
+
+            unless role_config
+              errors << "No config for role #{role.name} and group #{group.name} found, add it on the stage page."
+              next
+            end
+
+            {
+              role: role,
+              replicas: role_config.replicas,
+              cpu: role_config.cpu,
+              ram: role_config.ram
+            }
+          end
+          {deploy_group: group, roles: roles}
+        end
+
+        raise Samson::Hooks::UserError, errors.join("\n") if errors.any?
+        group_configs
+      end
+    end
+
     def deploy(release_docs)
       release_docs.each do |release_doc|
         @output.puts "Creating for #{release_doc.deploy_group.name} role #{release_doc.kubernetes_role.name}"
@@ -329,5 +335,27 @@ def success
     def seconds_waiting
       (Time.now - @wait_start_time).to_i if @wait_start_time
     end
+
+    # verify with a temp release so we can verify everything before creating a real release
+    # and having to wait for docker build to finish
+    def verify_kubernetes_templates!
+      release = Kubernetes::Release.new(project: @job.project, git_sha: @job.commit)
+      deploy_group_configs.each do |config|
+        config.fetch(:roles).each do |role|
+          doc = Kubernetes::ReleaseDoc.new(
+            kubernetes_release: release,
+            deploy_group: config.fetch(:deploy_group),
+            kubernetes_role: role.fetch(:role)
+          )
+
+          # verifies that config files are readable
+          doc.deploy_template
+
+          # verifies that secrets are findable
+          template = Kubernetes::ResourceTemplate.new(doc)
+          template.set_secrets
+        end
+      end
+    end
   end
 end

plugins/kubernetes/app/models/kubernetes/release.rb

@@ -51,8 +51,8 @@ def create_release_docs(params)
       params.fetch(:deploy_groups).each do |dg|
         dg.fetch(:roles).to_a.each do |role|
           release_docs.create!(
-            deploy_group_id: dg.fetch(:id),
-            kubernetes_role_id: role.fetch(:id),
+            deploy_group: dg.fetch(:deploy_group),
+            kubernetes_role: role.fetch(:role),
             replica_target: role.fetch(:replicas),
             cpu: role.fetch(:cpu),
             ram: role.fetch(:ram)

plugins/kubernetes/app/models/kubernetes/resource_template.rb

@@ -18,10 +18,7 @@ def to_hash
         set_spec_template_metadata
         set_docker_image
         set_resource_usage
-        if needs_secret_sidecar?
-          set_secret_sidecar
-          expand_secret_annotations
-        end
+        set_secrets
         set_env
 
         hash = template
@@ -30,6 +27,12 @@ def to_hash
       end
     end
 
+    def set_secrets
+      return unless needs_secret_sidecar?
+      set_secret_sidecar
+      expand_secret_annotations
+    end
+
     private
 
     def template
@@ -38,7 +41,7 @@ def template
 
     # look up keys in all possible namespaces by specificity
     def expand_secret_annotations
-      resolver = Samson::Secrets::KeyResolver.new(@doc.build.project, [@doc.deploy_group])
+      resolver = Samson::Secrets::KeyResolver.new(project, [@doc.deploy_group])
       secret_annotations.each_value { |secret_key| resolver.expand!(secret_key) }
       resolver.verify!
     end
@@ -147,12 +150,16 @@ def set_resource_usage
 
     def set_docker_image
       if @doc.build
-        docker_path = @doc.build.docker_repo_digest || "#{@doc.build.project.docker_repo}:#{@doc.build.docker_ref}"
+        docker_path = @doc.build.docker_repo_digest || "#{project.docker_repo}:#{@doc.build.docker_ref}"
         # Assume first container is one we want to update docker image in
         container[:image] = docker_path
       end
     end
 
+    def project
+      @project ||= @doc.kubernetes_release.project
+    end
+
     # helpful env vars, also useful for log tagging
     def set_env
       env = (container[:env] ||= [])

plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb

@@ -120,6 +120,34 @@ def worker_is_unstable
       end
     end
 
+    describe "invalid configs" do
+      before { build.delete } # build needs to be created -> assertion fails
+      around { |test| refute_difference('Build.count') { refute_difference('Release.count', &test) } }
+
+      it "fails before building when roles are invalid" do
+        Kubernetes::ReleaseDoc.any_instance.stubs(raw_template: 'oops: "this is bad"')
+        e = assert_raises Samson::Hooks::UserError do
+          refute execute!
+        end
+        e.message.must_include "Error found when parsing kubernetes/resque_worker.yml"
+      end
+
+      it "fails before building when secrets are not configured in the backend" do
+        Kubernetes::ResourceTemplate.any_instance.stubs(:needs_secret_sidecar?).returns(true)
+        template = read_kubernetes_sample_file('kubernetes_deployment.yml')
+        assert template.sub!(
+          "      name: some-project-pod\n",
+          "      name: some-project-pod\n      annotations:\n        secret/foo: bar\n"
+        )
+        Kubernetes::ReleaseDoc.any_instance.stubs(raw_template: template)
+
+        e = assert_raises Samson::Hooks::UserError do
+          refute execute!
+        end
+        e.message.must_include "Failed to resolve secret keys:\n\tbar"
+      end
+    end
+
     describe "role settings" do
       it "uses configured role settings" do
         assert execute!
@@ -235,7 +263,7 @@ def worker_is_unstable
         # we need multiple different templates here
         # make the worker a job and keep the app server
         Kubernetes::ReleaseDoc.any_instance.unstub(:raw_template)
-        GitRepository.any_instance.expects(:file_content).with('kubernetes/resque_worker.yml', anything).returns({
+        GitRepository.any_instance.stubs(:file_content).with('kubernetes/resque_worker.yml', anything).returns({
           'kind' => 'Job',
           'spec' => {
             'template' => {

plugins/kubernetes/test/models/kubernetes/release_test.rb

@@ -148,10 +148,10 @@ def release_params
       project: project,
       deploy_groups: [
         {
-          id: deploy_group.id,
+          deploy_group: deploy_group,
           roles: [
             {
-              id: app_server.id,
+              role: app_server,
               replicas: 1,
               cpu: 1,
               ram: 50
@@ -165,7 +165,7 @@ def release_params
   def multiple_roles_release_params
     release_params.tap do |params|
       params[:deploy_groups].each do |dg|
-        dg[:roles].push(id: resque_worker.id, replicas: 2, cpu: 2, ram: 100)
+        dg[:roles].push(role: resque_worker, replicas: 2, cpu: 2, ram: 100)
       end
     end
   end