Skip to content

Commit

Permalink
show exactly which resource has issues and recommend manual deletion …
Browse files Browse the repository at this point in the history
…for stuck daemonsets
grosser committed Sep 11, 2018

Verified

This commit was signed with the committer’s verified signature.
grosser Michael Grosser
1 parent 535fd64 commit 3b869f1
Showing 2 changed files with 17 additions and 8 deletions.
18 changes: 13 additions & 5 deletions plugins/kubernetes/app/models/kubernetes/resource.rb
Original file line number Diff line number Diff line change
@@ -90,6 +90,10 @@ def desired_pod_count

private

def error_location
"#{name} #{namespace} #{@deploy_group.name}"
end

# when autoscaling we expect as many pods as we currently have
def replica_source
(@autoscaled && resource) || @template
@@ -100,7 +104,7 @@ def backoff_wait(backoff, reason)
yield
sleep wait
end
raise "Unable to #{reason} (#{name} #{namespace})"
raise "Unable to #{reason} (#{error_location})"
end

def request_delete
@@ -197,7 +201,7 @@ def request(method, *args)
rescue Kubeclient::HttpError
message = $!.message.to_s
if message.include?(" is invalid:") || message.include?(" no kind ")
raise Samson::Hooks::UserError, "Kubernetes error: #{message}"
raise Samson::Hooks::UserError, "Kubernetes error #{error_location}: #{message}"
else
raise
end
@@ -315,8 +319,8 @@ def desired_pod_count
if desired == 0
raise(
Samson::Hooks::UserError,
"Unable to find desired number of pods for daemonset #{name} on #{@deploy_group.name}\n" \
"delete it manually and make sure there is at least 1 node scheduleable."
"Unable to find desired number of pods for DaemonSet #{error_location}\n" \
"delete it manually and make sure there is at least 1 node schedulable."
)
end

@@ -360,7 +364,11 @@ def wait_for_termination_of_all_pods
expire_cache
return if no_pods_running?
end
raise Samson::Hooks::UserError, "Unable to terminate previous DaemonSet because it still has pods"
raise(
Samson::Hooks::UserError,
"Unable to terminate DaemonSet #{name} #{namespace} #{@deploy_group.name} because it still has pods.\n" \
"Delete it manually and redeploy again."
)
end
end

7 changes: 4 additions & 3 deletions plugins/kubernetes/test/models/kubernetes/resource_test.rb
Original file line number Diff line number Diff line change
@@ -116,7 +116,8 @@ def delete_resource!
assert_request(:get, url, to_return: {status: 404}) do
error = '{"message":"Foo.extensions \"app\" is invalid:"}'
assert_request(:post, base_url, to_return: {body: error, status: 400}) do
assert_raises(Samson::Hooks::UserError) { resource.deploy }.message.must_include "Kubernetes error: Foo"
e = assert_raises(Samson::Hooks::UserError) { resource.deploy }
e.message.must_include "Kubernetes error some-project pod1 Pod1: Foo"
end
end
end
@@ -220,7 +221,7 @@ def delete_resource!
resource.expects(:sleep).times(tries)

e = assert_raises(RuntimeError) { resource.delete }
e.message.must_equal "Unable to delete resource (some-project pod1)"
e.message.must_equal "Unable to delete resource (some-project pod1 Pod1)"
end
end
end
@@ -363,7 +364,7 @@ def daemonset_stub(scheduled, misscheduled)
e = assert_raises Samson::Hooks::UserError do
resource.deploy
end
e.message.must_include "Unable to terminate previous DaemonSet"
e.message.must_include "Unable to terminate DaemonSet some-project pod1 Pod1"
end
end

0 comments on commit 3b869f1

Please sign in to comment.