Add a configurable number of retries to the EC2 credential provider

Menno van der Sman · Menno van der Sman · commit 036c8e2e6cb3 · 2014-09-26T10:46:32.000+02:00
In cases where the EC2 provider is configured as the only valid provider
of credentials, intermittent failures are to be expected and can be
retried safely in order to avoid a credential error.
diff --git a/lib/aws/core/credential_providers.rb b/lib/aws/core/credential_providers.rb
@@ -354,6 +354,8 @@ class FailedRequestError < StandardError; end
         # @param [Hash] options
         # @option options [String] :ip_address ('169.254.169.254')
         # @option options [Integer] :port (80)
+        # @option options [Integer] :retries (0) Number of times to
+        #   retry retrieving credentials.
         # @option options [Float] :http_open_timeout (1)
         # @option options [Float] :http_read_timeout (1)
         # @option options [Object] :http_debug_output (nil) HTTP wire
@@ -362,6 +364,7 @@ class FailedRequestError < StandardError; end
         def initialize options = {}
           @ip_address = options[:ip_address] || '169.254.169.254'
           @port = options[:port] || 80
+          @retries = options[:retries] || 0
           @http_open_timeout = options[:http_open_timeout] || 1
           @http_read_timeout = options[:http_read_timeout] || 1
           @http_debug_output = options[:http_debug_output]
@@ -373,6 +376,9 @@ def initialize options = {}
         # @return [Integer] Defaults to port 80.
         attr_accessor :port
 
+        # @return [Integer] Defaults to 0
+        attr_accessor :retries
+
         # @return [Float]
         attr_accessor :http_open_timeout
 
@@ -404,6 +410,8 @@ def credentials
 
         # (see Provider#get_credentials)
         def get_credentials
+          retries_left = retries
+
           begin
 
             http = Net::HTTP.new(ip_address, port, nil)
@@ -432,7 +440,15 @@ def get_credentials
             credentials
 
           rescue *FAILURES => e
-            {}
+            if retries_left > 0
+              sleep_time = 2 ** (retries - retries_left)
+              Kernel.sleep(sleep_time)
+
+              retries_left -= 1
+              retry
+            else
+              {}
+            end
           end
         end
 
@@ -566,7 +582,7 @@ class AssumeRoleProvider
 
         include Provider
 
-        # @option options [AWS::STS] :sts (STS.new) An instance of {AWS::STS}. 
+        # @option options [AWS::STS] :sts (STS.new) An instance of {AWS::STS}.
         #   This is used to make the API call to assume role.
         # @option options [required, String] :role_arn
         # @option options [required, String] :role_session_name
diff --git a/spec/aws/core/credential_providers_spec.rb b/spec/aws/core/credential_providers_spec.rb
@@ -473,7 +473,7 @@ module CredentialProviders
 
         context 'timeouts' do
 
-          it 'has a short default timeouts' do
+          it 'has short default timeouts' do
             provider.http_open_timeout.should == 1
             provider.http_read_timeout.should == 1
           end
@@ -487,7 +487,7 @@ module CredentialProviders
             provider.http_read_timeout.should == 1.5
           end
 
-          it 'uses the timeouts when request credentials' do
+          it 'uses the timeouts when requesting credentials' do
 
             profiles = "profile-name\n"
             creds = <<-CREDS.strip
@@ -602,18 +602,44 @@ module CredentialProviders
           provider.credentials.should == creds2
         end
 
-        it 'is set when credentails is valid' do
+        it 'is set when credentials are valid' do
           provider.set?.should be_true
         end
 
-        it 'is not set when key_id or access_key is missing' do
+        it 'is not set when the request fails' do
           http = double('http').as_null_object
           http.should_receive(:start).and_raise(Errno::ECONNREFUSED)
           Net::HTTP.stub(:new).and_return(http)
 
           provider.set?.should be_false
         end
 
+        context 'retries' do
+
+          it 'does not retry by default' do
+            provider.retries.should == 0
+          end
+
+          it 'accepts a value for the number of retries' do
+            provider = EC2Provider.new({:retries => 2})
+            provider.retries.should == 2
+          end
+
+          it 'retries the request for the set amount of times' do
+            http = double('http').as_null_object
+            http.should_receive(:start).exactly(3).times.and_raise(Errno::ECONNREFUSED)
+            Net::HTTP.stub(:new).and_return(http)
+
+            Kernel.should_receive(:sleep).with(1)
+            Kernel.should_receive(:sleep).with(2)
+
+            provider = EC2Provider.new({:retries => 2})
+            provider.set?.should be_false
+            provider.retries.should == 2
+          end
+
+        end
+
       end
 
       describe SessionProvider do
