Merge branch 'master' into issue/disallowDecryptKVForValuesMarkedSecret

Author: m4dcoder

.travis.yml

@@ -94,10 +94,8 @@ cache:
     #- .tox/
 
 before_install:
-  # Work around for apt Travis timeout issues, see https://github.com/travis-ci/travis-ci/issues/9112
-  #- sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60"
-  - pip install --upgrade "pip>=9.0,<9.1"
-  - sudo pip install --upgrade "virtualenv==15.1.0"
+  - pip install --upgrade "pip>=19.0,<20.0"
+  - sudo pip install --upgrade "virtualenv==16.6.0"
 
 install:
   - ./scripts/travis/install-requirements.sh

CHANGELOG.rst

@@ -17,6 +17,8 @@ Fixed
 
 * Fix orquesta st2kv to return empty string and null values. (bug fix) #4678
 * Allow tasks defined in the same task transition with ``fail`` to run for orquesta. (bug fix)
+* Fix workflow service to handle unexpected coordinator and database errors. (bug fix) #4704 #4705
+* Fix filter ``to_yaml_string`` to handle mongoengine base types for dict and list. (bug fix) #4700
 
 3.0.1 - May 24, 2019
 --------------------

Makefile

@@ -389,10 +389,10 @@ requirements: virtualenv .sdist-requirements install-runners
 	@echo
 	@echo "==================== requirements ===================="
 	@echo
-	# Make sure we use latest version of pip which is < 10.0.0
+	# Make sure we use latest version of pip which is 19
 	$(VIRTUALENV_DIR)/bin/pip --version
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=9.0,<9.1"
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "virtualenv==15.1.0" # Required for packs.install in dev envs.
+	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=19.0,<20.0"
+	$(VIRTUALENV_DIR)/bin/pip install --upgrade "virtualenv==16.6.0" # Required for packs.install in dev envs
 
 	# Generate all requirements to support current CI pipeline.
 	$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s st2*/in-requirements.txt contrib/runners/*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt

conf/st2.conf.sample

@@ -365,4 +365,12 @@ webui_base_url = https://localhost
 [workflow_engine]
 # Location of the logging configuration file.
 logging = /etc/st2/logging.workflowengine.conf
+# Max time to stop retrying.
+retry_stop_max_msec = 60000
+# Max jitter interval to smooth out retries.
+retry_max_jitter_msec = 1000
+# Interval inbetween retries.
+retry_wait_fixed_msec = 1000
+# Max seconds to allow workflow execution be idled before it is identified as orphaned and cancelled by the garbage collector.
+gc_max_idle_sec = 900
 

contrib/examples/actions/workflows/tests/orquesta-test-jinja-data-functions.yaml

@@ -5,6 +5,11 @@ description: A basic workflow testing data related functions.
 vars:
   - data_json_str_1: '{"foo": {"bar": "foobar"}}'
   - none: null
+  - data_list:
+    - a: 1
+      b: 2
+    - x: 3
+      y: 4
 
 tasks:
   task1:
@@ -30,3 +35,4 @@ output:
   - data_json_str_3: '{{ json_dump(ctx("data_json_obj_4")) }}'
   - data_none_str: '{{ use_none(ctx("none")) }}'
   - data_str: '{{ use_none("foobar") }}'
+  - data_list_str: '{{ to_yaml_string(ctx("data_list")) }}'

contrib/runners/orquesta_runner/in-requirements.txt

@@ -1 +1 @@
-git+https://github.com/StackStorm/orquesta.git@67fb92fb652073b37fe9678f9a9b9934ba8ac739#egg=orquesta
+git+https://github.com/StackStorm/orquesta.git@v1.0.0#egg=orquesta

contrib/runners/orquesta_runner/requirements.txt

@@ -1,2 +1,2 @@
 # Don't edit this file. It's generated automatically!
-git+https://github.com/StackStorm/orquesta.git@67fb92fb652073b37fe9678f9a9b9934ba8ac739#egg=orquesta
+git+https://github.com/StackStorm/orquesta.git@v1.0.0#egg=orquesta

contrib/runners/orquesta_runner/tests/unit/test_error_handling.py

@@ -34,7 +34,6 @@
 from st2common.persistence import execution as ex_db_access
 from st2common.persistence import liveaction as lv_db_access
 from st2common.persistence import workflow as wf_db_access
-from st2common.runners import base as runners
 from st2common.services import action as ac_svc
 from st2common.services import workflows as wf_svc
 from st2common.transport import liveaction as lv_ac_xport
@@ -76,7 +75,7 @@
     wf_ex_xport.WorkflowExecutionPublisher,
     'publish_state',
     mock.MagicMock(side_effect=mock_wf_ex_xport.MockWorkflowExecutionPublisher.publish_state))
-class OrquestaErrorHandlingTest(st2tests.ExecutionDbTestCase):
+class OrquestaErrorHandlingTest(st2tests.WorkflowTestCase):
     ensure_indexes = True
     ensure_indexes_models = [
         WorkflowExecutionDB,
@@ -100,13 +99,6 @@ def setUpClass(cls):
         for pack in PACKS:
             actions_registrar.register_from_pack(pack)
 
-    @classmethod
-    def get_runner_class(cls, runner_name):
-        return runners.get_runner(runner_name, runner_name).__class__
-
-    def sort_wf_runtime_errors(self, errors):
-        return sorted(errors, key=lambda x: x.get('task_id', None))
-
     def test_fail_inspection(self):
         expected_errors = [
             {
@@ -181,7 +173,7 @@ def test_fail_input_rendering(self):
         tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
         self.assertEqual(len(tk_ex_dbs), 0)
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -215,7 +207,7 @@ def test_fail_vars_rendering(self):
         tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
         self.assertEqual(len(tk_ex_dbs), 0)
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -251,7 +243,7 @@ def test_fail_start_task_action(self):
         tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
         self.assertEqual(len(tk_ex_dbs), 0)
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -288,7 +280,7 @@ def test_fail_start_task_input_expr_eval(self):
         tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
         self.assertEqual(len(tk_ex_dbs), 0)
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -326,7 +318,7 @@ def test_fail_start_task_input_value_type(self):
         # Assert workflow and task executions failed.
         wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         tk_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))[0]
         self.assertEqual(tk_ex_db.status, wf_statuses.FAILED)
@@ -376,7 +368,7 @@ def test_fail_next_task_action(self):
         self.assertEqual(tk_ex_db.status, wf_statuses.SUCCEEDED)
         wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -422,7 +414,7 @@ def test_fail_next_task_input_expr_eval(self):
         self.assertEqual(tk_ex_db.status, wf_statuses.SUCCEEDED)
         wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -471,7 +463,7 @@ def test_fail_next_task_input_value_type(self):
         # Assert workflow execution and task2 execution failed.
         wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(str(wf_ex_db.id))
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         tk2_ex_db = wf_db_access.TaskExecution.query(task_id='task2')[0]
         self.assertEqual(tk2_ex_db.status, wf_statuses.FAILED)
@@ -518,7 +510,7 @@ def test_fail_task_execution(self):
         # Assert workflow state and result.
         wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(str(wf_ex_db.id))
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -563,7 +555,7 @@ def test_fail_task_transition(self):
         self.assertEqual(tk_ex_db.status, wf_statuses.SUCCEEDED)
         wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -609,7 +601,7 @@ def test_fail_task_publish(self):
         self.assertEqual(tk_ex_db.status, wf_statuses.SUCCEEDED)
         wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -652,7 +644,7 @@ def test_fail_output_rendering(self):
         self.assertEqual(tk_ex_db.status, wf_statuses.SUCCEEDED)
         wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
         self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
         lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
         self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED)
@@ -770,7 +762,7 @@ def test_fail_manually(self):
             }
         ]
 
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)
 
     def test_fail_manually_with_recovery_failure(self):
         wf_file = 'fail-manually-with-recovery-failure.yaml'
@@ -833,4 +825,4 @@ def test_fail_manually_with_recovery_failure(self):
             }
         ]
 
-        self.assertListEqual(self.sort_wf_runtime_errors(wf_ex_db.errors), expected_errors)
+        self.assertListEqual(self.sort_workflow_errors(wf_ex_db.errors), expected_errors)

contrib/runners/orquesta_runner/tests/unit/test_functions_common.py

@@ -163,7 +163,8 @@ def test_data_functions_in_jinja(self):
             'data_query_1': ['foobar'],
             'data_pipe_str_1': '{"foo": {"bar": "foobar"}}',
             'data_none_str': data_funcs.NONE_MAGIC_VALUE,
-            'data_str': 'foobar'
+            'data_str': 'foobar',
+            'data_list_str': '- a: 1\n  b: 2\n- x: 3\n  y: 4\n'
         }
 
         self._execute_workflow(wf_name, expected_output)

fixed-requirements.txt

@@ -33,8 +33,8 @@ networkx==1.11
 python-keyczar==0.716
 cryptography==2.6.1
 retrying==1.3.3
-# Note: We use latest version of virtualenv which uses pip 9.0
-virtualenv==15.1.0
+# Note: We use latest version of virtualenv which uses pip 19
+virtualenv==16.6.0
 # NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc.
 sseclient-py==1.7
 python-editor==1.0.4