Merge pull request #17 from Sebitosh/constants

airween · web-flow · commit 9a4d1d1d68de · 2025-01-15T21:50:46.000+01:00
Utility change: reusable constants in yaml schema
diff --git a/README.md b/README.md
@@ -454,7 +454,103 @@ The field `input.encoded_request` allows defining a whole request encoded in bas
               encoded_request: R0VUIC8gSFRUUC8xLjENCkhvc3Q6IGxvY2FsaG9zdA0KDQo=
 ```
 
-### output / additional checks
+### Constants
+The yaml schema has a mechanism to handle global and local constants.
+
+~~~yaml
+global:
+  default_constants:
+    one: 1
+    TWO: 2
+    two_in_list:
+      - 2
+    FOO_IN_DICT:
+      foo: attack
+...
+
+constants:
+  HEADERS_IN_DICTIONARY:
+    headers:
+      - name: test
+        value: test
+      - name: one
+        value: ~{one}~
+      - name: 2
+        value: ~{TWO}~
+  template_in_list:
+    - SecRule for TARGETS
+    - Template with constants
+  one: one
+~~~
+
+Global constants are defined under the `global.default_constants` field. They are accessible across files and are reset whenever a new `global` field is defined.
+
+Local constants are defined under a `constants` field at the root of a file. They are only accessible in the file they are defined in.
+
+#### Syntax
+Constants are defined as key-value pairs where:
+
+~~~yaml
+NAME: VALUE
+~~~
+
+The name is used for referencing the constant and the value is used for the substitution. Referencing a constant can be done inside the value of any other key in the API. References use the `~{...}~` separators like so:
+
+~~~yaml
+~{NAME}~
+~~~
+
+Variable names can be lower or upper case and are case sensitive.
+
+#### Properties
+
+Constants can be yaml scalars, lists, or dictionaries:
+
+~~~yaml
+scalar: 1
+list:
+  - 1
+dictionary:
+  1: 1
+~~~
+
+Constants can reference other constants in their values:
+
+~~~yaml
+headers:
+  - name: one
+    value: ~{one}~
+  - name: two
+    value: ~{TWO}~
+~~~
+
+Local constants with the same name as global constants have precedence in their local scope:
+~~~yaml
+global:
+  default_constants:
+    ONE: 1
+...
+constants:
+  ONE: one
+...
+key: ~{ONE}~  # substituted by 'one'
+~~~
+Values can contain multiple references, such as in templates:
+
+~~~yaml
+  - name: "Template with constants"
+    template: |
+      SecRule ~{target}~ "${OPERATOR}$ ${OPARG}$" \
+          "id:${CURRID}$,\
+          phase:${PHASE}$,\
+          deny,\
+          t:~{None}~,\
+          log,\
+          msg:'%{MATCHED_VAR_NAME} was caught in phase:${PHASE}$',\
+          ver:'~{VERSION}~'"
+~~~
+
+### Output / additional checks
 
 By default, the generator will produce checks for tests with `go-ftw`'s `expect_ids` field using the current rule id as parameter. If the associated rule matches and it's id put in the log, the test will pass.
 
diff --git a/feature_demo/config_tests/DEMO_000_GLOBAL.yaml b/feature_demo/config_tests/DEMO_000_GLOBAL.yaml
@@ -46,5 +46,26 @@ global:
           ver:'${VERSION}$'"
       
       ${DIRECTIVES}$
+  - name: "Template with constants"
+    template: |
+      SecRule ${TARGET}$ "${OPERATOR}$ ${OPARG}$" \
+          "id:${CURRID}$,\
+          phase:~{phase}~,\
+          deny,\
+          t:~{None}~,\
+          log,\
+          msg:'%{MATCHED_VAR_NAME} was caught in phase:~{phase}~',\
+          ver:'~{VERSION}~'"
   default_tests_phase_methods:
   - 2: post
+  default_constants:
+    one: local constants have precedence
+    TWO: 2
+    two_in_list:
+      - 2
+    FOO_IN_DICT:
+      foo: attack
+constants:
+  phase: ${PHASE}$
+  VERSION: ${VERSION}$
+  None: none
diff --git a/feature_demo/config_tests/DEMO_006_CONSTANTS.yaml b/feature_demo/config_tests/DEMO_006_CONSTANTS.yaml
@@ -0,0 +1,42 @@
+target: ARGS
+rulefile: DEMO_006_CONSTANTS.conf
+testfile: DEMO_006_CONSTANTS.yaml
+constants:
+  one: one
+  template_in_list:
+    - SecRule for TARGETS
+    - Template with constants
+  HEADERS_IN_DICTIONARY:
+    headers:
+      - name: test
+        value: test
+      - name: one
+        value: ~{one}~
+      - name: 2
+        value: ~{TWO}~
+templates: ~{template_in_list}~
+colkey:
+- - ''
+operator:
+- '@contains'
+oparg:
+- attack
+phase: ~{two_in_list}~
+testdata:
+  phase_methods:
+    2: post
+  targets:
+    - target: ''
+      test:
+        data:
+          foo: attack
+        input:
+          headers:
+            - name: one
+              value: ~{one}~
+            - name: 2
+              value: ~{TWO}~
+    - target: ''
+      test:
+        data: ~{FOO_IN_DICT}~
+        input: ~{HEADERS_IN_DICTIONARY}~
diff --git a/feature_demo/generated/rules/DEMO_006_CONSTANTS.conf b/feature_demo/generated/rules/DEMO_006_CONSTANTS.conf
@@ -0,0 +1,18 @@
+SecRule ARGS "@contains attack" \
+    "id:100011,\
+    phase:2,\
+    deny,\
+    t:none,\
+    log,\
+    msg:'%{MATCHED_VAR_NAME} was caught in phase:2',\
+    ver:'MRTS/0.1'"
+
+SecRule ARGS "@contains attack" \
+    "id:100012,\
+    phase:2,\
+    deny,\
+    t:none,\
+    log,\
+    msg:'%{MATCHED_VAR_NAME} was caught in phase:2',\
+    ver:'MRTS/0.1'"
+
diff --git a/feature_demo/generated/tests/DEMO_006_CONSTANTS_100011.yaml b/feature_demo/generated/tests/DEMO_006_CONSTANTS_100011.yaml
@@ -0,0 +1,56 @@
+---
+meta:
+  author: MRTS generate-rules.py
+  enabled: true
+  name: DEMO_006_CONSTANTS.yaml
+  description: Desc
+tests:
+- test_title: 100011-1
+  ruleid: 100011
+  test_id: 1
+  desc: 'Test case for rule 100011, #1'
+  stages:
+  - description: Send request
+    input:
+      dest_addr: 127.0.0.1
+      port: 80
+      protocol: http
+      method: POST
+      headers:
+        User-Agent: OWASP MRTS test agent
+        Host: localhost
+        Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+        one: one
+        2: 2
+      uri: /post
+      version: HTTP/1.1
+      data: foo=attack
+    output:
+      log:
+        expect_ids:
+        - 100011
+- test_title: 100011-2
+  ruleid: 100011
+  test_id: 2
+  desc: 'Test case for rule 100011, #2'
+  stages:
+  - description: Send request
+    input:
+      dest_addr: 127.0.0.1
+      port: 80
+      protocol: http
+      method: POST
+      headers:
+        User-Agent: OWASP MRTS test agent
+        Host: localhost
+        Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+        test: test
+        one: one
+        2: 2
+      uri: /post
+      version: HTTP/1.1
+      data: foo=attack
+    output:
+      log:
+        expect_ids:
+        - 100011
diff --git a/feature_demo/generated/tests/DEMO_006_CONSTANTS_100012.yaml b/feature_demo/generated/tests/DEMO_006_CONSTANTS_100012.yaml
@@ -0,0 +1,56 @@
+---
+meta:
+  author: MRTS generate-rules.py
+  enabled: true
+  name: DEMO_006_CONSTANTS.yaml
+  description: Desc
+tests:
+- test_title: 100012-1
+  ruleid: 100012
+  test_id: 1
+  desc: 'Test case for rule 100012, #1'
+  stages:
+  - description: Send request
+    input:
+      dest_addr: 127.0.0.1
+      port: 80
+      protocol: http
+      method: POST
+      headers:
+        User-Agent: OWASP MRTS test agent
+        Host: localhost
+        Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+        one: one
+        2: 2
+      uri: /post
+      version: HTTP/1.1
+      data: foo=attack
+    output:
+      log:
+        expect_ids:
+        - 100012
+- test_title: 100012-2
+  ruleid: 100012
+  test_id: 2
+  desc: 'Test case for rule 100012, #2'
+  stages:
+  - description: Send request
+    input:
+      dest_addr: 127.0.0.1
+      port: 80
+      protocol: http
+      method: POST
+      headers:
+        User-Agent: OWASP MRTS test agent
+        Host: localhost
+        Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+        test: test
+        one: one
+        2: 2
+      uri: /post
+      version: HTTP/1.1
+      data: foo=attack
+    output:
+      log:
+        expect_ids:
+        - 100012
diff --git a/mrts/generate-rules.py b/mrts/generate-rules.py
@@ -8,6 +8,7 @@
 import string
 import re
 import copy
+from ast import literal_eval
 
 NAME = "MRTS"
 VERSION = "0.1"
@@ -33,6 +34,7 @@ def __init__(self, flist, expdir, testdir):
             'phase'        : [1,2,3,4],
             'actions'      : [],
             'directives'   : [],
+            'constants'    : {},
             'phase_methods': {}
         }
         self.default_test_phase_methods = {
@@ -43,6 +45,8 @@ def __init__(self, flist, expdir, testdir):
             5: "post"
         }
 
+        self.default_constants = {}
+
         self.current_confdata = {}
         self.current_testdata = {}
 
@@ -54,6 +58,7 @@ def __init__(self, flist, expdir, testdir):
         self.testcontent      = {}
 
         self.re_tplvars  = re.compile(r"""\$\{[^ \n\t$,'"]*\}\$""")
+        self.re_constants = re.compile(r"""~\{([^ \n\t$,'"]*)\}~""")
 
         self.testdict         = {
             'header': {
@@ -115,6 +120,10 @@ def __init__(self, flist, expdir, testdir):
 
     def parseconf(self, c):
         """parsing a configuration file"""
+        # Before anything, load and replace constants for the current file.
+        if re.search(self.re_constants, str(c)) is not None:
+            c = self.parseconstants(c)
+
         # if there is a 'global' section, fill the possible global vars
         if 'global' in c:
             for k in c['global']:
@@ -156,6 +165,45 @@ def parseconf(self, c):
         else:
             pass
 
+    def parseconstants(self, c):
+        """Load and replace any used constants in current configuration"""
+        # per-file local
+        if 'constants' in c:
+            self.current_confdata['constants'] = c['constants']
+        # cross-file global
+        if 'global' in c:
+            if 'default_constants' in c['global']:
+                self.default_constants = c['global']['default_constants']
+            else:  # if no global constants, reset values defined under previous 'global' field
+                self.default_constants = {}
+        return self.swap_constants(c)
+
+    def swap_constants(self, c):
+        if isinstance(c, dict):
+            for key, val in c.items():
+                c[key] = self.swap_constants(val)
+        if isinstance(c, list):
+            for i in range(len(c)):
+                c[i] = self.swap_constants(c[i])
+        if isinstance(c, str):
+            matches = re.findall(self.re_constants, c)
+            for match in matches:
+                if match in self.current_confdata['constants']:
+                    og_type = type(self.current_confdata['constants'][match])
+                    c = c.replace(f"~{{{match}}}~", str(self.current_confdata['constants'][match]))
+                    if og_type in (list, dict):
+                        c = literal_eval(c)
+                    else:
+                        c = og_type(c)
+                elif match in self.default_constants:
+                    og_type = type(self.default_constants[match])
+                    c = c.replace(f"~{{{match}}}~", str(self.default_constants[match]))
+                    if og_type in (list, dict):
+                        c = literal_eval(c)
+                    else:
+                        c = og_type(c)
+        return c
+
     def genrulefromtemplate(self, tpl, current_confdata):
         """
             generate a rule from data based on the template
