Skip to content

Commit dc85df7

Browse files
m4dcoderm4dcoder
authored
Merge pull request #196 from StackStorm/fix-variables-evaluation
Fix the yaql/jinja vars extraction to ignore methods of base ctx() dict
2 parents 419a4a9 + 387f56c commit dc85df7

12 files changed

+149
-55
lines changed

‎CHANGELOG.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ Fixed
3737
* When inspecting custom YAQL/Jinja function to see if there is a context arg, use getargspec
3838
for py2 and getfullargspec for py3. (bug fix)
3939
* Check syntax on with items task to ensure action is indented correctly. Fixes #184 (bug fix)
40+
* Fix variable inspection where ctx().get() method calls are identified as errors.
41+
Fixes StackStorm/st2#4866 (bug fix)
4042

4143
1.0.0
4244
-----

‎orquesta/expressions/base.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,6 @@ def evaluate(statement, data=None):
144144

145145

146146
def extract_vars(statement):
147-
148147
variables = []
149148

150149
if isinstance(statement, dict):

‎orquesta/expressions/jinja.py‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import functools
1616
import inspect
17+
import itertools
1718
import logging
1819
import re
1920
import six
@@ -53,16 +54,23 @@ class JinjaEvaluator(expr_base.Evaluator):
5354
_regex_pattern = '{{.*?}}'
5455
_regex_parser = re.compile(_regex_pattern)
5556

56-
_regex_dot_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*'
57-
_regex_ctx_pattern_1 = 'ctx\(\)\.%s' % _regex_dot_pattern
58-
_regex_ctx_pattern_2 = 'ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern)
59-
_regex_var_pattern = '.*?(%s|%s).*?' % (_regex_ctx_pattern_1, _regex_ctx_pattern_2)
60-
_regex_var_parser = re.compile(_regex_var_pattern)
61-
62-
_regex_dot_extract = '([a-zA-Z0-9_\-]*)'
63-
_regex_ctx_extract_1 = 'ctx\(\)\.%s' % _regex_dot_extract
64-
_regex_ctx_extract_2 = 'ctx\([\'|"]?%s(%s)' % (_regex_dot_extract, _regex_dot_pattern)
65-
_regex_var_extracts = ['%s\.?' % _regex_ctx_extract_1, '%s\.?' % _regex_ctx_extract_2]
57+
_regex_ctx_ref_pattern = r'[][a-zA-Z0-9_\'"\.()]*'
58+
# match any of:
59+
# word boundary ctx(*)
60+
# word boundary ctx()*
61+
# word boundary ctx().*
62+
# word boundary ctx(*)*
63+
# word boundary ctx(*).*
64+
_regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_ctx_ref_pattern)
65+
_regex_ctx_var_parser = re.compile(_regex_ctx_pattern)
66+
67+
_regex_var = r'[a-zA-Z0-9_-]+'
68+
_regex_var_extracts = [
69+
r'(?<=\bctx\(\)\.)({})\b(?!\()\.?'.format(_regex_var), # extract x in ctx().x
70+
r'(?:\bctx\(({})\))'.format(_regex_var), # extract x in ctx(x)
71+
r'(?:\bctx\(\'({})\'\))'.format(_regex_var), # extract x in ctx('x')
72+
r'(?:\bctx\("({})"\))'.format(_regex_var) # extract x in ctx("x")
73+
]
6674

6775
_block_delimiter = '{%}'
6876
_regex_block_pattern = '{%.*?%}'
@@ -234,9 +242,11 @@ def extract_vars(cls, text):
234242
if not isinstance(text, six.string_types):
235243
raise ValueError('Text to be evaluated is not typeof string.')
236244

237-
variables = []
245+
results = [
246+
cls._regex_ctx_var_parser.findall(expr.strip(cls._delimiter).strip())
247+
for expr in cls._regex_parser.findall(text)
248+
]
238249

239-
for expr in cls._regex_parser.findall(text):
240-
variables.extend(cls._regex_var_parser.findall(expr))
250+
variables = [v.strip() for v in itertools.chain.from_iterable(results)]
241251

242252
return sorted(list(set(variables)))

‎orquesta/expressions/yql.py‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# limitations under the License.
1414

1515
import inspect
16+
import itertools
1617
import logging
1718
import re
1819
import six
@@ -53,16 +54,23 @@ class YAQLEvaluator(expr_base.Evaluator):
5354
_regex_pattern = '<%.*?%>'
5455
_regex_parser = re.compile(_regex_pattern)
5556

56-
_regex_dot_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*'
57-
_regex_ctx_pattern_1 = 'ctx\(\)\.%s' % _regex_dot_pattern
58-
_regex_ctx_pattern_2 = 'ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern)
59-
_regex_var_pattern = '.*?(%s|%s).*?' % (_regex_ctx_pattern_1, _regex_ctx_pattern_2)
60-
_regex_var_parser = re.compile(_regex_var_pattern)
61-
62-
_regex_dot_extract = '([a-zA-Z0-9_\-]*)'
63-
_regex_ctx_extract_1 = 'ctx\(\)\.%s' % _regex_dot_extract
64-
_regex_ctx_extract_2 = 'ctx\([\'|"]?%s(%s)' % (_regex_dot_extract, _regex_dot_pattern)
65-
_regex_var_extracts = ['%s\.?' % _regex_ctx_extract_1, '%s\.?' % _regex_ctx_extract_2]
57+
_regex_ctx_ref_pattern = r'[][a-zA-Z0-9_\'"\.()]*'
58+
# match any of:
59+
# word boundary ctx(*)
60+
# word boundary ctx()*
61+
# word boundary ctx().*
62+
# word boundary ctx(*)*
63+
# word boundary ctx(*).*
64+
_regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_ctx_ref_pattern)
65+
_regex_ctx_var_parser = re.compile(_regex_ctx_pattern)
66+
67+
_regex_var = r'[a-zA-Z0-9_-]+'
68+
_regex_var_extracts = [
69+
r'(?<=\bctx\(\)\.)({})\b(?!\()\.?'.format(_regex_var), # extract x in ctx().x
70+
r'(?:\bctx\(({})\))'.format(_regex_var), # extract x in ctx(x)
71+
r'(?:\bctx\(\'({})\'\))'.format(_regex_var), # extract x in ctx('x')
72+
r'(?:\bctx\("({})"\))'.format(_regex_var) # extract x in ctx("x")
73+
]
6674

6775
_engine = yaql.language.factory.YaqlFactory().create()
6876
_root_ctx = yaql.create_context()
@@ -153,9 +161,11 @@ def extract_vars(cls, text):
153161
if not isinstance(text, six.string_types):
154162
raise ValueError('Text to be evaluated is not typeof string.')
155163

156-
variables = []
164+
results = [
165+
cls._regex_ctx_var_parser.findall(expr.strip(cls._delimiter).strip())
166+
for expr in cls._regex_parser.findall(text)
167+
]
157168

158-
for expr in cls._regex_parser.findall(text):
159-
variables.extend(cls._regex_var_parser.findall(expr))
169+
variables = [v.strip() for v in itertools.chain.from_iterable(results)]
160170

161171
return sorted(list(set(variables)))

‎orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@
1919
class JinjaFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest):
2020

2121
def test_empty_extraction(self):
22-
expr = '{{ just_text and _not_a_var }}'
22+
expr = (
23+
'{{ just_text and $not_a_var and notctx().bar and '
24+
'ctx(). and ctx().() and ctx().-foobar and ctx().foobar() }}'
25+
)
2326

2427
self.assertListEqual([], expr_base.extract_vars(expr))
2528

@@ -52,12 +55,14 @@ def test_single_functional_var_extraction(self):
5255
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
5356

5457
def test_multiple_vars_extraction(self):
55-
expr = '{{ ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().fu.bar[0] }}'
58+
expr = '{{ctx().fubar ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().foobaz.bar[0] }}'
5659

5760
expected_vars = [
5861
('jinja', expr, 'foo'),
5962
('jinja', expr, 'foobar'),
60-
('jinja', expr, 'fu')
63+
('jinja', expr, 'foobaz'),
64+
('jinja', expr, 'fu'),
65+
('jinja', expr, 'fubar')
6166
]
6267

6368
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))

‎orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,15 @@
1919
class JinjaFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest):
2020

2121
def test_empty_extraction(self):
22-
expr = '{{ just_text and _not_a_var }}'
22+
expr = (
23+
'{{ just_text and $not_a_var and '
24+
'notctx(foo) and notctx("bar") and notctx(\'fu\') '
25+
'ctx("foo\') and ctx(\'foo") and ctx(foo") and '
26+
'ctx("foo) and ctx(foo\') and ctx(\'foo) and '
27+
'ctx(-foo) and ctx("-bar") and ctx(\'-fu\') and '
28+
'ctx(foo.bar) and ctx("foo.bar") and ctx(\'foo.bar\') and '
29+
'ctx(foo()) and ctx("foo()") and ctx(\'foo()\') }}'
30+
)
2331

2432
self.assertListEqual([], expr_base.extract_vars(expr))
2533

@@ -52,12 +60,17 @@ def test_single_functional_var_extraction(self):
5260
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
5361

5462
def test_multiple_vars_extraction(self):
55-
expr = '{{ ctx("foobar") ctx("foo").get(bar) ctx("fu").bar ctx("fu").bar[0] }}'
63+
expr = (
64+
'{{ctx("fubar") ctx("foobar") ctx("foo").get(bar) '
65+
'ctx("fu").bar ctx("foobaz").bar[0] }}'
66+
)
5667

5768
expected_vars = [
5869
('jinja', expr, 'foo'),
5970
('jinja', expr, 'foobar'),
60-
('jinja', expr, 'fu')
71+
('jinja', expr, 'foobaz'),
72+
('jinja', expr, 'fu'),
73+
('jinja', expr, 'fubar')
6174
]
6275

6376
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
@@ -108,3 +121,21 @@ def test_vars_extraction_from_dict(self):
108121
]
109122

110123
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
124+
125+
def test_ignore_ctx_dict_funcs(self):
126+
expr = '{{ctx().keys() and ctx().values() and ctx().set("b", 3) }}'
127+
128+
expected_vars = []
129+
130+
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
131+
132+
def test_ignore_ctx_get_func_calls(self):
133+
expr = (
134+
'{{ctx().get(foo) and ctx().get(bar) and ctx().get("fu") and ctx().get(\'baz\') and '
135+
'ctx().get(foo, "bar") and ctx().get("fu", "bar") and ctx().get(\'bar\', \'foo\') and '
136+
'ctx().get("foo\') and ctx().get(\'foo") and ctx().get("foo) and ctx().get(foo") }}'
137+
)
138+
139+
expected_vars = []
140+
141+
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))

‎orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_dot_notation.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@
1919
class YAQLFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest):
2020

2121
def test_empty_extraction(self):
22-
expr = '<% just_text and $not_a_var %>'
22+
expr = (
23+
'<% just_text and $not_a_var and notctx().bar and '
24+
'ctx(). and ctx().() and ctx().-foobar and ctx().foobar() %>'
25+
)
2326

2427
self.assertListEqual([], expr_base.extract_vars(expr))
2528

@@ -52,12 +55,14 @@ def test_single_functional_var_extraction(self):
5255
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
5356

5457
def test_multiple_vars_extraction(self):
55-
expr = '<% ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().fu.bar[0] %>'
58+
expr = '<%ctx().fubar ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().foobaz.bar[0] %>'
5659

5760
expected_vars = [
5861
('yaql', expr, 'foo'),
5962
('yaql', expr, 'foobar'),
60-
('yaql', expr, 'fu')
63+
('yaql', expr, 'foobaz'),
64+
('yaql', expr, 'fu'),
65+
('yaql', expr, 'fubar')
6166
]
6267

6368
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))

‎orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,15 @@
1919
class YAQLFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest):
2020

2121
def test_empty_extraction(self):
22-
expr = '<% just_text and $not_a_var %>'
22+
expr = (
23+
'<% just_text and $not_a_var and '
24+
'notctx(foo) and notctx("bar") and notctx(\'fu\') '
25+
'ctx("foo\') and ctx(\'foo") and ctx(foo") and '
26+
'ctx("foo) and ctx(foo\') and ctx(\'foo) and '
27+
'ctx(-foo) and ctx("-bar") and ctx(\'-fu\') and '
28+
'ctx(foo.bar) and ctx("foo.bar") and ctx(\'foo.bar\') and '
29+
'ctx(foo()) and ctx("foo()") and ctx(\'foo()\') %>'
30+
)
2331

2432
self.assertListEqual([], expr_base.extract_vars(expr))
2533

@@ -52,12 +60,14 @@ def test_single_functional_var_extraction(self):
5260
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
5361

5462
def test_multiple_vars_extraction(self):
55-
expr = '<% ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(fu).bar[0] %>'
63+
expr = '<%ctx(fubar) ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(foobaz).bar[0] %>'
5664

5765
expected_vars = [
5866
('yaql', expr, 'foo'),
5967
('yaql', expr, 'foobar'),
60-
('yaql', expr, 'fu')
68+
('yaql', expr, 'foobaz'),
69+
('yaql', expr, 'fu'),
70+
('yaql', expr, 'fubar')
6171
]
6272

6373
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
@@ -108,3 +118,21 @@ def test_vars_extraction_from_dict(self):
108118
]
109119

110120
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
121+
122+
def test_ignore_ctx_dict_funcs(self):
123+
expr = '<%ctx().keys() and ctx().values() and ctx().set("b", 3) %>'
124+
125+
expected_vars = []
126+
127+
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))
128+
129+
def test_ignore_ctx_get_func_calls(self):
130+
expr = (
131+
'<%ctx().get(foo) and ctx().get(bar) and ctx().get("fu") and ctx().get(\'baz\') and '
132+
'ctx().get(foo, "bar") and ctx().get("fu", "bar") and ctx().get(\'bar\', \'foo\') and '
133+
'ctx().get("foo\') and ctx().get(\'foo") and ctx().get("foo) and ctx().get(foo") %>'
134+
)
135+
136+
expected_vars = []
137+
138+
self.assertListEqual(expected_vars, expr_base.extract_vars(expr))

‎orquesta/tests/unit/expressions/test_jinja_ctx_by_dot_notation.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ def setUpClass(cls):
2323
super(JinjaVariableExtractionTest, cls).setUpClass()
2424

2525
def test_empty_extraction(self):
26-
expr = '{{ just_text and _not_a_var }}'
26+
expr = '{{ just_text and $not_a_var and notctx().bar }}'
2727

2828
self.assertListEqual([], self.evaluator.extract_vars(expr))
2929

@@ -64,13 +64,14 @@ def test_single_functional_var_extraction(self):
6464
self.assertListEqual(expected_vars, self.evaluator.extract_vars(expr))
6565

6666
def test_multiple_vars_extraction(self):
67-
expr = '{{ ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().fu.bar[0] }}'
67+
expr = '{{ctx().fubar ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().foobaz.bar[0] }}'
6868

6969
expected_vars = [
7070
'ctx().foobar',
71+
'ctx().foobaz.bar[0]',
7172
'ctx().foo.get(bar)',
72-
'ctx().fu.bar',
73-
'ctx().fu.bar[0]'
73+
'ctx().fubar',
74+
'ctx().fu.bar'
7475
]
7576

7677
self.assertListEqual(

‎orquesta/tests/unit/expressions/test_jinja_ctx_by_function_arg.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ def setUpClass(cls):
2323
super(JinjaVariableExtractionTest, cls).setUpClass()
2424

2525
def test_empty_extraction(self):
26-
expr = '{{ just_text and _not_a_var }}'
26+
expr = '{{ just_text and $not_a_var and notctx(foo) and notctx("bar") and notctx(\'fu\') }}'
2727

2828
self.assertListEqual([], self.evaluator.extract_vars(expr))
2929

@@ -82,13 +82,14 @@ def test_single_functional_var_extraction(self):
8282
self.assertListEqual(expected_vars, self.evaluator.extract_vars(expr))
8383

8484
def test_multiple_vars_extraction(self):
85-
expr = '{{ ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(fu).bar[0] }}'
85+
expr = '{{ctx(fubar) ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(foobaz).bar[0] }}'
8686

8787
expected_vars = [
8888
'ctx(foobar)',
89+
'ctx(foobaz).bar[0]',
8990
'ctx(foo).get(bar)',
90-
'ctx(fu).bar',
91-
'ctx(fu).bar[0]'
91+
'ctx(fubar)',
92+
'ctx(fu).bar'
9293
]
9394

9495
self.assertListEqual(

0 commit comments

Comments
 (0)