Add cartridge_app_config_upload_http_timeout variable (#402)

Closes #401

Before the patch, in the `upload_app_config` step,
the default timeouts were used for the clusterwide config patch
and for uploading the config via HTTP.

Now, the `cartridge_app_config_upload_http_timeout` variable is used
as a config upload timeout for HTTP mode.

Also, the `upload_app_config` step currently uses `twophase_upload_config_timeout`,
`twophase_netbox_call_timeout` and `twophase_apply_config_timeout` variables.

Author: hackallcode

CHANGELOG.md

@@ -15,6 +15,8 @@ README.md to use the newest tag with new release
 - Add `cartridge_log_dir_parent` to configure directory of logs
 - Add `cartridge_force_leader_control_instance` variable to choose a control
   instance among the leaders
+- Add `cartridge_app_config_upload_http_timeout` variable to configure timeout to
+  wait config upload in HTTP mode.
 
 ### Fixed
 
@@ -24,6 +26,7 @@ README.md to use the newest tag with new release
 - Fixed backup folder permissions
 - Handle empty values in `helpers.py`
 - Fixed templates of systemd units for TGZ packages
+- The twophase timeouts is used in the `upload_app_config` step
 
 ## [1.11.0] - 2021-07-30
 

defaults/main.yml

@@ -102,6 +102,7 @@ cartridge_app_config: null
 cartridge_app_config_path: null
 cartridge_app_config_upload_mode: null
 cartridge_app_config_upload_url: null
+cartridge_app_config_upload_http_timeout: 30
 cartridge_tdg_token: null
 
 cartridge_auth: null

doc/app_config.md

@@ -121,6 +121,9 @@ which is on the same machine as the control instance:
 cartridge_app_config_upload_url: 'http://10.0.0.102:8083/admin/config'
 ```
 
+If you have a large configuration or slow connection, you can increase
+the upload timeout with `cartridge_app_config_upload_http_timeout` variable (default is `30`).
+
 **Note** that directory uploading by HTTP mode is not yet supported.
 
 ### TDG mode

doc/steps.md

@@ -332,6 +332,7 @@ Input variables from config:
 - `cartridge_app_config_upload_url` - url of instance to upload config
   (`http://127.0.0.1:{control_instance.http_port}/admin/config` by default);
 - `cartridge_cluster_cookie` - cluster cookie for all cluster instances;
+- `cartridge_app_config_upload_http_timeout` - time in seconds to wait config upload in HTTP mode;
 - `cartridge_tdg_token` - token to upload config by HTTP in TDG.
 
 ## Step `configure_app_config`

doc/variables.md

@@ -158,6 +158,7 @@ For more details see [scenario documentation](/doc/scenario.md).
 - `cartridge_app_config_upload_mode` (`string`): mode of config uploading (`lua`, `http` or `tdg`);
 - `cartridge_app_config_upload_url` (`string`): url of instance to upload
   config (`http://127.0.0.1:{control_instance.http_port}/admin/config` by default);
+- `cartridge_app_config_upload_http_timeout` (`number`, default: `30`): time in seconds to wait config upload in HTTP mode;
 - `cartridge_tdg_token` (`string`): token to upload config by HTTP in TDG;
 - `cartridge_auth`: (`dict`): [authorization configuration](/doc/auth.md);
 - `cartridge_failover_params` (`dict`): [failover](/doc/failover.md) parameters;

library/cartridge_apply_app_config.py

@@ -16,6 +16,11 @@
     'upload_url': {'required': False, 'type': 'str'},
     'cluster_cookie': {'required': False, 'type': 'str'},
     'tdg_token': {'required': False, 'type': 'str'},
+    'http_timeout': {'required': False, 'type': 'int'},
+
+    'netbox_call_timeout': {'required': False, 'type': 'int'},
+    'upload_config_timeout': {'required': False, 'type': 'int'},
+    'apply_config_timeout': {'required': False, 'type': 'int'},
 }
 
 LUA_MODE = 'lua'
@@ -50,14 +55,14 @@ def patch_file_clusterwide(control_console, file_path):
     return changed
 
 
-def send_on_http(upload_url, headers, config_path):
+def send_on_http(upload_url, headers, config_path, http_timeout):
     headers.update({
         'Content-Length': os.stat(config_path).st_size
     })
 
     with open(config_path, 'rb') as data:
         try:
-            resp = open_url(upload_url, method='PUT', headers=headers, data=data)
+            resp = open_url(upload_url, method='PUT', headers=headers, data=data, timeout=http_timeout)
         except Exception as e:
             assert hasattr(e, 'code'), str(e)
             resp = e
@@ -77,8 +82,7 @@ def send_on_http(upload_url, headers, config_path):
     return True
 
 
-def apply_tdg_config(console_sock, path):
-    control_console = helpers.get_control_console(console_sock)
+def apply_tdg_config(control_console, path):
     _, err = control_console.eval_res_err('''
         return admin.upload_config_api(...)
     ''', path)
@@ -147,11 +151,7 @@ def prepare_config(config_path, output_format, upload_mode):
 ####################
 
 
-def get_tdg_upload_mode(console_sock):
-    if console_sock is None:
-        return 'http'
-
-    control_console = helpers.get_control_console(console_sock)
+def get_tdg_upload_mode(control_console):
     return control_console.eval_res_err('''
         if rawget(_G, 'admin') ~= nil and rawget(_G.admin, 'upload_config_api') ~= nil then
             return 'lua'
@@ -160,13 +160,10 @@ def get_tdg_upload_mode(console_sock):
     ''')[0]
 
 
-def get_tdg_http_headers(console_sock, tdg_token):
+def get_tdg_http_headers(control_console, tdg_token):
     if tdg_token is None:
         return {}
 
-    assert console_sock is not None, "Console socket is required for TDG mode"
-    control_console = helpers.get_control_console(console_sock)
-
     tdg_version = control_console.eval_res_err('''
         local ok, app_version = pcall(require, 'common.app_version')
         if not ok then
@@ -208,54 +205,51 @@ def prepare_upload_mode(upload_mode, remote_config_path):
     return upload_mode
 
 
-def get_output_format_and_apply_config_func_for_lua(console_sock):
-    assert console_sock is not None, "Console socket is required for Lua mode"
-
-    control_console = helpers.get_control_console(console_sock)
-
+def get_output_format_and_apply_config_func_for_lua(control_console):
     return FILE_OUTPUT_FORMAT, lambda path: patch_file_clusterwide(control_console, path)
 
 
-def get_output_format_and_apply_config_func_for_http(upload_url, cluster_cookie):
+def get_output_format_and_apply_config_func_for_http(upload_url, cluster_cookie, http_timeout):
     assert cluster_cookie is not None, 'Cluster cookie is required for HTTP mode'
     assert upload_url is not None, 'Upload URL is required for HTTP mode'
 
     headers = {'Authorization': basic_auth_header('admin', cluster_cookie)}
 
-    return FILE_OUTPUT_FORMAT, lambda path: send_on_http(upload_url, headers, path)
+    return FILE_OUTPUT_FORMAT, lambda path: send_on_http(upload_url, headers, path, http_timeout)
 
 
-def get_output_format_and_apply_config_func_for_tdg(console_sock, upload_url, tdg_token):
-    tdg_upload_mode = get_tdg_upload_mode(console_sock)
+def get_output_format_and_apply_config_func_for_tdg(control_console, upload_url, tdg_token, http_timeout):
+    tdg_upload_mode = get_tdg_upload_mode(control_console)
 
     if tdg_upload_mode == 'http':
         assert upload_url is not None, 'Upload URL is required for TDG mode'
 
-        headers = get_tdg_http_headers(console_sock, tdg_token)
+        headers = get_tdg_http_headers(control_console, tdg_token)
 
-        return ZIP_OUTPUT_FORMAT, lambda path: send_on_http(upload_url, headers, path)
+        return ZIP_OUTPUT_FORMAT, lambda path: send_on_http(upload_url, headers, path, http_timeout)
 
     elif tdg_upload_mode == 'lua':
-        return DIR_OUTPUT_FORMAT, lambda path: apply_tdg_config(console_sock, path)
+        return DIR_OUTPUT_FORMAT, lambda path: apply_tdg_config(control_console, path)
 
     raise AssertionError("Unknown TDG upload mode '%s'" % tdg_upload_mode)
 
 
 def get_output_format_and_apply_config_func(
     upload_mode,
-    console_sock=None,
+    control_console=None,
     upload_url=None,
     cluster_cookie=None,
     tdg_token=None,
+    http_timeout=None,
 ):
     if upload_mode == LUA_MODE:
-        return get_output_format_and_apply_config_func_for_lua(console_sock)
+        return get_output_format_and_apply_config_func_for_lua(control_console)
 
     elif upload_mode == HTTP_MODE:
-        return get_output_format_and_apply_config_func_for_http(upload_url, cluster_cookie)
+        return get_output_format_and_apply_config_func_for_http(upload_url, cluster_cookie, http_timeout)
 
     elif upload_mode == TDG_MODE:
-        return get_output_format_and_apply_config_func_for_tdg(console_sock, upload_url, tdg_token)
+        return get_output_format_and_apply_config_func_for_tdg(control_console, upload_url, tdg_token, http_timeout)
 
     raise AssertionError("Unknown upload mode '%s'" % upload_mode)
 
@@ -266,6 +260,10 @@ def get_output_format_and_apply_config_func(
 
 
 def apply_app_config(params):
+    console_sock = params['console_sock']
+    control_console = helpers.get_control_console(console_sock)
+    helpers.set_twophase_options_from_params(control_console, params)
+
     # We cannot use the fact 'dest' from the copy task,
     # because in the case of transferring a folder with one file,
     # 'dest' will contain the path to the file, not the path to the folder.
@@ -279,10 +277,11 @@ def apply_app_config(params):
 
     output_format, apply_config_func = get_output_format_and_apply_config_func(
         upload_mode,
-        console_sock=params['console_sock'],
+        control_console=control_console,
         upload_url=params['upload_url'],
         cluster_cookie=params['cluster_cookie'],
         tdg_token=params['tdg_token'],
+        http_timeout=params.get('http_timeout'),
     )
 
     temp_paths = [remote_config_path]

molecule/config_upload/converge.yml

@@ -27,6 +27,7 @@
         cartridge_app_config_path: './configs/config.yml'
         cartridge_app_config_upload_mode: null
         cartridge_app_config_upload_url: 'http://vm1:3301/admin/config'
+        cartridge_app_config_upload_http_timeout: 30
         expected_test_section: {config-file-key: yml}
 
     - name: 'Upload YAML config by Lua'
@@ -36,6 +37,7 @@
         cartridge_app_config_path: './configs/config.yaml'
         cartridge_app_config_upload_mode: null
         cartridge_app_config_upload_url: 'http://vm1:3302/admin/config'
+        cartridge_app_config_upload_http_timeout: 30
         expected_test_section: {config-file-key: yaml}
 
     # File config to Cartridge by HTTP
@@ -47,6 +49,7 @@
         cartridge_app_config_path: './configs/config.yml'
         cartridge_app_config_upload_mode: 'http'
         cartridge_app_config_upload_url: null
+        cartridge_app_config_upload_http_timeout: 30
         expected_test_section: {config-file-key: yml}
 
     - name: 'Upload YAML config by HTTP'
@@ -56,6 +59,7 @@
         cartridge_app_config_path: './configs/config.yaml'
         cartridge_app_config_upload_mode: 'http'
         cartridge_app_config_upload_url: null
+        cartridge_app_config_upload_http_timeout: 30
         expected_test_section: {config-file-key: yaml}
 
     ###############
@@ -72,6 +76,7 @@
         cartridge_app_config_path: './configs/config-yaml'
         cartridge_app_config_upload_mode: 'lua'
         cartridge_app_config_upload_url: null
+        cartridge_app_config_upload_http_timeout: 30
     - name: 'Check upload config failed'
       assert:
         fail_msg: 'Upload folder config by Lua should fail'
@@ -91,6 +96,7 @@
         cartridge_app_config_path: './configs/config-yml-json'
         cartridge_app_config_upload_mode: null
         cartridge_app_config_upload_url: null
+        cartridge_app_config_upload_http_timeout: 30
     - name: 'Check upload config failed'
       assert:
         fail_msg: 'Upload folder config by HTTP to Cartridge should fail'
@@ -110,6 +116,7 @@
         cartridge_app_config_path: './configs/config.yml'
         cartridge_app_config_upload_mode: 'tdg'
         cartridge_app_config_upload_url: null
+        cartridge_app_config_upload_http_timeout: 30
     - name: 'Check upload config failed'
       assert:
         fail_msg: 'Upload file config to Cartridge by TDG mode should fail'
@@ -129,9 +136,30 @@
         cartridge_app_config_path: './configs/config.zip'
         cartridge_app_config_upload_mode: 'tdg'
         cartridge_app_config_upload_url: null
+        cartridge_app_config_upload_http_timeout: 30
     - name: 'Check upload config failed'
       assert:
         fail_msg: 'Upload ZIP config to Cartridge by TDG mode should fail'
         success_msg: 'Upload ZIP config to Cartridge by TDG mode failed'
         that: apply_app_config_res.failed  # Here will be error from Cartridge
       run_once: true
+
+    # HTTP timeout for upload
+
+    - name: 'Upload YML config by HTTP'
+      ignore_errors: true
+      import_role:
+        name: ansible-cartridge
+      vars:
+        cartridge_app_config_path: './configs/config.yml'
+        cartridge_app_config_upload_mode: 'http'
+        cartridge_app_config_upload_url: 'http://vm1:3301/admin/config'
+        cartridge_app_config_upload_http_timeout: 0
+    - name: 'Check upload config failed'
+      assert:
+        fail_msg: 'urlopen should raise error'
+        success_msg: 'urlopen raises error'
+        that:
+          - apply_app_config_res.failed
+          - '"urlopen error" in apply_app_config_res.msg'
+      run_once: true

tasks/set_instance_facts.yml

@@ -104,6 +104,7 @@
       cartridge_app_config_path: '{{ cartridge_app_config_path }}'
       cartridge_app_config_upload_mode: '{{ cartridge_app_config_upload_mode }}'
       cartridge_app_config_upload_url: '{{ cartridge_app_config_upload_url }}'
+      cartridge_app_config_upload_http_timeout: '{{ cartridge_app_config_upload_http_timeout }}'
       cartridge_tdg_token: '{{ cartridge_tdg_token }}'
 
       cartridge_auth: '{{ cartridge_auth }}'

tasks/steps/upload_app_config.yml

@@ -28,6 +28,10 @@
         }}'
         cluster_cookie: '{{ cartridge_cluster_cookie }}'
         tdg_token: '{{ cartridge_tdg_token }}'
+        http_timeout: '{{ cartridge_app_config_upload_http_timeout }}'
+        netbox_call_timeout: '{{ twophase_netbox_call_timeout }}'
+        upload_config_timeout: '{{ twophase_upload_config_timeout }}'
+        apply_config_timeout: '{{ twophase_apply_config_timeout }}'
       register: apply_app_config_res
 
     - name: 'Add uploaded config to "temporary_files" fact'