fix: munge weird jinja2 quoting

Author: beckermr

conda_forge_tick/recipe_parser/_parser.py

@@ -447,6 +447,29 @@ def _build_jinja2_expr_tmp(jinja2_exprs):
     return "\n".join(exprs + tmpls)
 
 
+def _remove_quoted_jinja2_vars(lines):
+    """Remove any quoted jinja2 vars from the lines.
+
+    Sometimes people write
+
+        '{{ pin_compatible('x') }}'
+
+    which causes the parser to fail.
+
+    We remove all instances of "['\"]{{" and "}}['\"]" to be safe.
+    """
+    new_lines = []
+    for line in lines:
+        if (
+            ("'{{" in line and "}}'" in line) or ('"{{' in line and '}}"' in line)
+        ) and ("pin_" in line or "compiler" in line or "stdlib" in line):
+            new_lines.append(re.sub(r"['\"]{{", "{{", line))
+            new_lines[-1] = re.sub(r"}}['\"]", "}}", new_lines[-1])
+        else:
+            new_lines.append(line)
+    return new_lines
+
+
 class CondaMetaYAML:
     """Crude parsing of conda recipes.
 
@@ -498,9 +521,15 @@ def __init__(self, meta_yaml: str):
                     "with a conda build selector! (offending line: '%s')" % line,
                 )
 
-        # remove multiline jinja2 statements
+        # pre-munge odd syntax that we do not want
         lines = list(io.StringIO(meta_yaml).readlines())
+
+        # remove multiline jinja2 statements
         lines = _munge_multiline_jinja2(lines)
+
+        # get rid of quoted jinja2 vars
+        lines = _remove_quoted_jinja2_vars(lines)
+
         meta_yaml = "".join(lines)
 
         # get any variables set in the file by jinja2

tests/test_recipe_parser.py

@@ -1,4 +1,5 @@
 import io
+import os
 
 import pytest
 
@@ -1603,3 +1604,18 @@ def test_recipe_parses_strings_colons_quotes():
     s.seek(0)
     new_recipe = s.read()
     assert new_recipe == recipe_correct
+
+
+YAML_PATH = os.path.join(os.path.dirname(__file__), "test_yaml")
+
+
+@pytest.mark.parametrize("recipe", os.listdir(YAML_PATH))
+def test_recipe_parser_yaml_suite(recipe):
+    if recipe.endswith("_correct.yaml") and "duplicate_lines_cleanup" not in recipe:
+        with open(os.path.join(YAML_PATH, recipe)) as f:
+            recipe = f.read()
+            cm = CondaMetaYAML(recipe)
+            s = io.StringIO()
+            cm.dump(s)
+            s.seek(0)
+            assert s.read() == recipe

tests/test_version_migrator.py

@@ -57,6 +57,9 @@
         ("boostcpp", "1.74.0"),
         ("event_stream", "1.6.3"),
         ("21cmfast", "3.4.0"),
+        ("pyrsmq", "0.6.0"),
+        ("quart_trio", "0.11.1"),
+        ("reproc", "14.2.5"),
         # these contain sources that depend on conda build config variants
         pytest.param(
             "polars_mixed_selectors",

tests/test_yaml/version_giturl.yaml

@@ -10,7 +10,7 @@ source:
     git_tag: '{{version}}'
 
 build:
-  skip: True  # [py27 or py33 or win or win_64]
+  skip: true  # [py27 or py33 or win or win_64]
   number: 1
   script:
     - '{{PYTHON}} -m pip install --ignore-installed -vv .'
@@ -46,7 +46,7 @@ about:
   license_file: License.txt
   summary: 'NetworKit is a growing open-source toolkit for large-scale network analysis.'
   description: |
-           NetworKit is a growing open-source toolkit for large-scale network analysis. Its aim is to provide tools for the analysis of large networks in the size range from thousands to billions of edges. For this purpose, it implements efficient graph algorithms, many of them parallel to utilize multicore architectures. These are meant to compute standard measures of network analysis, such as degree sequences, clustering coefficients, and centrality measures. In this respect, NetworKit is comparable to packages such as NetworkX, albeit with a focus on parallelism and scalability. NetworKit is also a testbed for algorithm engineering and contains novel algorithms from recently published research (see list of Publications).
+    NetworKit is a growing open-source toolkit for large-scale network analysis. Its aim is to provide tools for the analysis of large networks in the size range from thousands to billions of edges. For this purpose, it implements efficient graph algorithms, many of them parallel to utilize multicore architectures. These are meant to compute standard measures of network analysis, such as degree sequences, clustering coefficients, and centrality measures. In this respect, NetworKit is comparable to packages such as NetworkX, albeit with a focus on parallelism and scalability. NetworKit is also a testbed for algorithm engineering and contains novel algorithms from recently published research (see list of Publications).
   doc_url: https://networkit.github.io/
   dev_url: https://github.com/networkit/networkit
 

tests/test_yaml/version_giturl_correct.yaml

@@ -10,7 +10,7 @@ source:
     git_tag: '{{version}}'
 
 build:
-  skip: True  # [py27 or py33 or win or win_64]
+  skip: true  # [py27 or py33 or win or win_64]
   number: 1
   script:
     - '{{PYTHON}} -m pip install --ignore-installed -vv .'
@@ -46,7 +46,7 @@ about:
   license_file: License.txt
   summary: 'NetworKit is a growing open-source toolkit for large-scale network analysis.'
   description: |
-           NetworKit is a growing open-source toolkit for large-scale network analysis. Its aim is to provide tools for the analysis of large networks in the size range from thousands to billions of edges. For this purpose, it implements efficient graph algorithms, many of them parallel to utilize multicore architectures. These are meant to compute standard measures of network analysis, such as degree sequences, clustering coefficients, and centrality measures. In this respect, NetworKit is comparable to packages such as NetworkX, albeit with a focus on parallelism and scalability. NetworKit is also a testbed for algorithm engineering and contains novel algorithms from recently published research (see list of Publications).
+    NetworKit is a growing open-source toolkit for large-scale network analysis. Its aim is to provide tools for the analysis of large networks in the size range from thousands to billions of edges. For this purpose, it implements efficient graph algorithms, many of them parallel to utilize multicore architectures. These are meant to compute standard measures of network analysis, such as degree sequences, clustering coefficients, and centrality measures. In this respect, NetworKit is comparable to packages such as NetworkX, albeit with a focus on parallelism and scalability. NetworKit is also a testbed for algorithm engineering and contains novel algorithms from recently published research (see list of Publications).
   doc_url: https://networkit.github.io/
   dev_url: https://github.com/networkit/networkit
 

tests/test_yaml/version_pyrsmq.yaml

@@ -0,0 +1,43 @@
+{% set name = "PyRSMQ" %}
+{% set version = "0.5.0" %}
+
+package:
+  name: "{{ name|lower }}"
+  version: "{{ version }}"
+
+source:
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
+  sha256: 68ba774ba544cd976622c9375c69e21b89a6fbb41221dbebb706733846d557a6
+
+build:
+  noarch: python
+  number: 0
+  script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed -vv "
+
+requirements:
+  host:
+    - pip
+    - python >=3.6
+    - setuptools
+  run:
+    - python >=3.6
+    - redis-py
+
+test:
+  imports:
+    - rsmq
+    - rsmq.cmd
+    - tests
+
+about:
+  home: https://mlasevich.github.io/PyRSMQ/
+  license: Apache-2.0
+  license_family: APACHE
+  license_file: LICENSE
+  summary: Python Implementation of Redis SMQ
+  dev_url: https://github.com/mlasevich/PyRSMQ
+
+extra:
+  recipe-maintainers:
+    - lincrosenbach
+    - kfasolin

tests/test_yaml/version_pyrsmq_correct.yaml

@@ -0,0 +1,43 @@
+{% set name = "PyRSMQ" %}
+{% set version = "0.6.0" %}
+
+package:
+  name: "{{ name|lower }}"
+  version: "{{ version }}"
+
+source:
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name|lower }}-{{ version }}.tar.gz
+  sha256: dd1f8467e541935489be018dbb0ba1df8b903eb855bf1725947ceee41df92fa4
+
+build:
+  noarch: python
+  number: 0
+  script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed -vv "
+
+requirements:
+  host:
+    - pip
+    - python >=3.6
+    - setuptools
+  run:
+    - python >=3.6
+    - redis-py
+
+test:
+  imports:
+    - rsmq
+    - rsmq.cmd
+    - tests
+
+about:
+  home: https://mlasevich.github.io/PyRSMQ/
+  license: Apache-2.0
+  license_family: APACHE
+  license_file: LICENSE
+  summary: Python Implementation of Redis SMQ
+  dev_url: https://github.com/mlasevich/PyRSMQ
+
+extra:
+  recipe-maintainers:
+    - lincrosenbach
+    - kfasolin

tests/test_yaml/version_quart_trio.yaml

@@ -0,0 +1,45 @@
+{% set name = "Quart-Trio" %}
+{% set version = "0.7.0" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
+  sha256: 933e3c18e232ece30ccbac7579fdc5f62f2f9c79c3273d6c341f5a1686791eb1
+
+build:
+  noarch: python
+  number: 0
+  script: {{ PYTHON }} -m pip install . -vv
+
+requirements:
+  host:
+    - python >=3.7.0
+    - pip
+  run:
+    - python >=3.7.0
+    - quart >=0.11.1
+    - trio >=0.10.0
+    - hypercorn >=0.6.0
+
+test:
+  imports:
+    - quart_trio
+
+about:
+  home: https://gitlab.com/pgjones/quart-trio
+  license: MIT
+  license_family: MIT
+  license_file: LICENSE
+  summary: Quart-Trio is an extension for Quart to support the Trio event loop
+  description: |
+    Quart-Trio is an extension for Quart to support the Trio event loop.
+    This is an alternative to using the asyncio event loop present in the Python
+    standard library and supported by default in Quart.
+
+extra:
+  recipe-maintainers:
+    - dhirschfeld
+    - scopatz

tests/test_yaml/version_quart_trio_correct.yaml

@@ -0,0 +1,45 @@
+{% set name = "Quart-Trio" %}
+{% set version = "0.11.1" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('-', '_').lower() }}-{{ version }}.tar.gz
+  sha256: 149c9c65c2faafdf455a4461b600e1983b71e593b6f8c8b91b592bbda36cea98
+
+build:
+  noarch: python
+  number: 0
+  script: {{ PYTHON }} -m pip install . -vv
+
+requirements:
+  host:
+    - python >=3.7.0
+    - pip
+  run:
+    - python >=3.7.0
+    - quart >=0.11.1
+    - trio >=0.10.0
+    - hypercorn >=0.6.0
+
+test:
+  imports:
+    - quart_trio
+
+about:
+  home: https://gitlab.com/pgjones/quart-trio
+  license: MIT
+  license_family: MIT
+  license_file: LICENSE
+  summary: Quart-Trio is an extension for Quart to support the Trio event loop
+  description: |
+    Quart-Trio is an extension for Quart to support the Trio event loop.
+    This is an alternative to using the asyncio event loop present in the Python
+    standard library and supported by default in Quart.
+
+extra:
+  recipe-maintainers:
+    - dhirschfeld
+    - scopatz