Merge pull request #194 from BartoszBlizniak/main

(SUP-2952) (Issue #193) - Retry if provisioning target returns a 500

Author: david22swan

.fixtures.yml

@@ -1,7 +1,7 @@
 fixtures:
   repositories:
-    "facts": "git://github.com/puppetlabs/puppetlabs-facts.git"
-    "puppet_agent": "git://github.com/puppetlabs/puppetlabs-puppet_agent.git"
-    "puppet_conf": "git://github.com/puppetlabs/puppetlabs-puppet_conf.git"
+    "facts": "https://github.com/puppetlabs/puppetlabs-facts.git"
+    "puppet_agent": "https://github.com/puppetlabs/puppetlabs-puppet_agent.git"
+    "puppet_conf": "https://github.com/puppetlabs/puppetlabs-puppet_conf.git"
   symlinks:
     "provision": "#{source_dir}"

pdk.yaml

@@ -0,0 +1,2 @@
+---
+ignore: []

tasks/provision_service.json

@@ -23,6 +23,11 @@
     "vars": {
       "description": "The address of the provisioning service",
       "type": "Optional[String[1]]"
+    },
+    "retry_attempts": {
+      "description": "The number of times to retry the provisioning if it fails",
+      "type": "Optional[Integer[1]]",
+      "default": 5
     }
   },
   "files": [

tasks/provision_service.rb

@@ -28,7 +28,7 @@ def platform_to_cloud_request_parameters(platform, cloud, region, zone)
 end
 
 # curl -X POST https://facade-validation-6f3kfepqcq-ew.a.run.app/v1/provision --data @test_machines.json
-def invoke_cloud_request(params, uri, job_url, verb)
+def invoke_cloud_request(params, uri, job_url, verb, retry_attempts)
   headers =  {
     'Accept' => 'application/json',
     'Content-Type' => 'application/json',
@@ -62,6 +62,7 @@ def invoke_cloud_request(params, uri, job_url, verb)
   req_options = {
     use_ssl: uri.scheme == 'https',
     read_timeout: 60 * 5, # timeout reads after 5 minutes - that's longer than the backend service would keep the request open
+    max_retries: retry_attempts, # retry up to 5 times before throwing an error
   }
 
   response = Net::HTTP.start(uri.hostname, uri.port, req_options) do |http|
@@ -83,7 +84,7 @@ def invoke_cloud_request(params, uri, job_url, verb)
   # rubocop:enable Style/GuardClause
 end
 
-def provision(platform, inventory_location, vars)
+def provision(platform, inventory_location, vars, retry_attempts)
   # Call the provision service with the information necessary and write the inventory file locally
 
   if ENV['GITHUB_RUN_ID']
@@ -102,7 +103,7 @@ def provision(platform, inventory_location, vars)
   inventory_full_path = File.join(inventory_location, '/spec/fixtures/litmus_inventory.yaml')
 
   params = platform_to_cloud_request_parameters(platform, cloud, region, zone)
-  response = invoke_cloud_request(params, uri, job_url, 'post')
+  response = invoke_cloud_request(params, uri, job_url, 'post', retry_attempts)
   response_hash = YAML.safe_load(response)
 
   unless vars.nil?
@@ -138,7 +139,7 @@ def provision(platform, inventory_location, vars)
   }
 end
 
-def tear_down(platform, inventory_location, _vars)
+def tear_down(platform, inventory_location, _vars, retry_attempts)
   # remove all provisioned resources
   uri = URI.parse(ENV['SERVICE_URL'] || default_uri)
 
@@ -148,7 +149,7 @@ def tear_down(platform, inventory_location, _vars)
     inventory_hash = inventory_hash_from_inventory_file(inventory_full_path)
     facts = facts_from_node(inventory_hash, platform)
     job_id = facts['uuid']
-    response = invoke_cloud_request(job_id, uri, '', 'delete')
+    response = invoke_cloud_request(job_id, uri, '', 'delete', retry_attempts)
     response.to_json
   end
   # rubocop:enable Style/GuardClause
@@ -159,16 +160,17 @@ def tear_down(platform, inventory_location, _vars)
 action = params['action']
 vars = params['vars']
 node_name = params['node_name']
+retry_attempts = params['retry_attempts']
 inventory_location = sanitise_inventory_location(params['inventory'])
 
 begin
   case action
   when 'provision'
     raise 'specify a platform when provisioning' if platform.nil?
-    result = provision(platform, inventory_location, vars)
+    result = provision(platform, inventory_location, vars, retry_attempts)
   when 'tear_down'
     raise 'specify a node_name when tearing down' if node_name.nil?
-    result = tear_down(node_name, inventory_location, vars)
+    result = tear_down(node_name, inventory_location, vars, retry_attempts)
   else
     result = { _error: { kind: 'provision_service/argument_error', msg: "Unknown action '#{action}'" } }
   end