(PUP-10844) Warnings for unavailable servers

When processing server_list and trying to find a functional server,
throw warnings when it cannot connect to servers from server_list and
if there is no functional server in server_list, then throw the initial
error.

Author: Dorin-Pleava

lib/puppet/http/resolver/server_list.rb

@@ -54,7 +54,7 @@ def resolve(session, name, ssl_context: nil, canceled_handler: nil)
     end
 
     # Return the first simple service status endpoint we can connect to
-    @server_list_setting.value.each do |server|
+    @server_list_setting.value.each_with_index do |server, index|
       host = server[0]
       port = server[1] || @default_port
 
@@ -64,10 +64,21 @@ def resolve(session, name, ssl_context: nil, canceled_handler: nil)
         @resolved_url = service.url
         return Puppet::HTTP::Service.create_service(@client, session, name, @resolved_url.host, @resolved_url.port)
       rescue Puppet::HTTP::ResponseError => detail
-        Puppet.log_exception(detail, _("Puppet server %{host}:%{port} is unavailable: %{code} %{reason}") %
-                             { host: service.url.host, port: service.url.port, code: detail.response.code, reason: detail.response.reason })
+        if index < @server_list_setting.value.length - 1
+          Puppet.warning(_("Puppet server %{host}:%{port} is unavailable: %{code} %{reason}") %
+                              { host: service.url.host, port: service.url.port, code: detail.response.code, reason: detail.response.reason } +
+                              ' ' + _("Trying with next server from server_list."))
+        else
+          Puppet.log_exception(detail, _("Puppet server %{host}:%{port} is unavailable: %{code} %{reason}") %
+                               { host: service.url.host, port: service.url.port, code: detail.response.code, reason: detail.response.reason })
+        end
       rescue Puppet::HTTP::HTTPError => detail
-        Puppet.log_exception(detail, _("Unable to connect to server from server_list setting: %{detail}") % {detail: detail})
+        if index < @server_list_setting.value.length - 1
+          Puppet.warning(_("Unable to connect to server from server_list setting: %{detail}") % {detail: detail} +
+                             ' ' + _("Trying with next server from server_list."))
+        else
+          Puppet.log_exception(detail, _("Unable to connect to server from server_list setting: %{detail}") % {detail: detail})
+        end
       end
     end
 

spec/unit/configurer_spec.rb

@@ -1072,6 +1072,29 @@ def expects_neither_new_or_cached_catalog
       }.to raise_error(Puppet::Error, /Could not select a functional puppet server from server_list: 'myserver:123,someotherservername'/)
     end
 
+    it "should warn when servers in 'server_list' are unreachable" do
+      Puppet.settings[:server_list] = "mybadserver1:123,mybadserver2:123,mygoodserver"
+      Puppet[:usecacheonfailure] = false
+
+      stub_request(:get, 'https://mybadserver1:123/status/v1/simple/master').and_raise(Puppet::HTTP::HTTPError)
+      stub_request(:get, 'https://mybadserver2:123/status/v1/simple/master').and_raise(Puppet::HTTP::HTTPError)
+      stub_request(:get, 'https://mygoodserver:8140/status/v1/simple/master').to_return(status: 200)
+      
+      expect(Puppet).to receive(:warning).with(/^Unable to connect to server from server_list setting:.*Trying with next server from server_list.$/).twice
+      configurer.run
+    end
+
+    it "should warn when servers in 'server_list' respond with error" do
+      Puppet.settings[:server_list] = "mybadserver:123,someotherservername"
+      Puppet[:usecacheonfailure] = false
+
+      stub_request(:get, 'https://mybadserver:123/status/v1/simple/master').to_return(status: 400)
+      stub_request(:get, 'https://someotherservername:8140/status/v1/simple/master').to_return(status: 200)
+
+      expect(Puppet).to receive(:warning).with(/^Puppet server mybadserver:123 is unavailable: 400  Trying with next server from server_list.$/)
+      configurer.run
+    end
+
     it "should not error when usecacheonfailure is true and no servers in 'server_list' are reachable" do
       Puppet.settings[:server_list] = "myserver:123,someotherservername"
       Puppet[:usecacheonfailure] = true