add exponential back off retry

mtnsat · Jun 18, 2014 · cccf4d8 · cccf4d8
1 parent 8211444
commit cccf4d8
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 12 deletions.
diff --git a/zerg/lib/zerg/version.rb b/zerg/lib/zerg/version.rb
@@ -22,5 +22,5 @@
 #++
 
 module Zerg
-  VERSION = "0.0.16"
+  VERSION = "0.0.17"
 end
diff --git a/zerg/zerg.gemspec b/zerg/zerg.gemspec
@@ -27,7 +27,7 @@ Gem::Specification.new do |s|
   s.add_dependency "thor"
   s.add_dependency "highline"
   s.add_dependency "zergrush_vagrant", ">= 0.0.4"
-  s.add_dependency "zergrush_cf", ">= 0.0.9"
+  s.add_dependency "zergrush_cf", ">= 0.0.10"
 
   s.files        = `git ls-files`.split("\n")
   s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact

diff --git a/zerg_plugins/zergrush_cf/lib/zergrush_cf/init.rb b/zerg_plugins/zergrush_cf/lib/zergrush_cf/init.rb
@@ -29,6 +29,7 @@
 require 'securerandom'
 require 'bunny'
 require 'time'
+require 'retries'
 require_relative 'renderer'
 
 class CloudFormation < ZergGemPlugin::Plugin "/driver"
@@ -85,6 +86,10 @@ def rush hive_location, task_name, task_hash, debug
             end
         end
 
+        Excon.defaults[:connect_timeout] = 600
+        Excon.defaults[:read_timeout] = 600
+        Excon.defaults[:write_timeout] = 600
+
         cf = Fog::AWS::CloudFormation.new(
             :aws_access_key_id => aws_key_id,
             :aws_secret_access_key => aws_secret,
@@ -112,10 +117,15 @@ def rush hive_location, task_name, task_hash, debug
             outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
         end
 
-        events = cf.describe_stack_events(stack_name).body['StackEvents']
-        while events == nil do
-            sleep 1 
+        events = nil
+        with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
             events = cf.describe_stack_events(stack_name).body['StackEvents']
+        }
+        while events == nil do
+            sleep 3
+            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+                events = cf.describe_stack_events(stack_name).body['StackEvents']
+            }
         end
 
         event_counter = 0
@@ -124,7 +134,9 @@ def rush hive_location, task_name, task_hash, debug
             logRabbitEvents(events.first(events.length - event_counter), rabbit_objects, eval_params(task_hash["vm"]["driver"]["driveroptions"][0]["rabbit"])) 
             event_counter = events.length
 
-            events = cf.describe_stack_events(stack_name).body['StackEvents']
+            with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+                events = cf.describe_stack_events(stack_name).body['StackEvents']
+            }
             outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
             if outputs_info.body["Stacks"][0]["StackStatus"] == "CREATE_COMPLETE"
                 logEvents(events.first(events.length - event_counter))
@@ -172,6 +184,10 @@ def clean hive_location, task_name, task_hash, debug
         # run fog cleanup on the stack.
         stack_name = "#{task_name}"
 
+        Excon.defaults[:connect_timeout] = 600
+        Excon.defaults[:read_timeout] = 600
+        Excon.defaults[:write_timeout] = 600
+
         cf = Fog::AWS::CloudFormation.new(
             :aws_access_key_id => aws_key_id,
             :aws_secret_access_key => aws_secret,
@@ -196,11 +212,16 @@ def clean hive_location, task_name, task_hash, debug
             end
         end
 
-        events = cf.describe_stack_events(stack_name).body['StackEvents']
+        events = nil
+        with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+            events = cf.describe_stack_events(stack_name).body['StackEvents']
+        }
         while events == nil do
-            sleep 1
+            sleep 3
             begin
-                events = cf.describe_stack_events(stack_name).body['StackEvents']
+                with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+                    events = cf.describe_stack_events(stack_name).body['StackEvents']
+                }
             rescue Fog::AWS::CloudFormation::NotFound
                 rabbit_objects[:connection].close unless rabbit_objects == nil
                 return 0
@@ -214,7 +235,9 @@ def clean hive_location, task_name, task_hash, debug
 
             event_counter = events.length
             begin
-                events = cf.describe_stack_events(stack_name).body['StackEvents']
+                with_retries(:max_tries => 10, :base_sleep_seconds => 3, :max_sleep_seconds => 20, :rescue => Fog::AWS::CloudFormation::Error) {
+                    events = cf.describe_stack_events(stack_name).body['StackEvents']
+                }
                 outputs_info = cf.describe_stacks({ 'StackName' => stack_name })
             rescue Fog::AWS::CloudFormation::NotFound
                 logEvents(events.first(events.length - event_counter))

diff --git a/zerg_plugins/zergrush_cf/lib/zergrush_cf/version.rb b/zerg_plugins/zergrush_cf/lib/zergrush_cf/version.rb
@@ -22,5 +22,5 @@
 #++
 
 module ZergrushCF
-  VERSION = "0.0.9"
+  VERSION = "0.0.10"
 end
diff --git a/zerg_plugins/zergrush_cf/zergrush_cf.gemspec b/zerg_plugins/zergrush_cf/zergrush_cf.gemspec
@@ -17,10 +17,11 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "bundler", ">= 1.0.0"
   s.add_development_dependency "rake"
-  s.add_development_dependency "zergrush", ">= 0.0.16"
+  s.add_development_dependency "zergrush", ">= 0.0.17"
 
   s.add_dependency "fog", ">=1.20.0"
   s.add_dependency "bunny", ">=1.2.1"
+  s.add_dependency "retries", ">=0.0.5"
 
   s.files        = `git ls-files`.split("\n")
   s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact