Support test_code to add pyright-config comment

challenges/advanced-forward/question.py

@@ -14,3 +14,6 @@ def copy(self) -> MyClass:
 
 inst = MyClass(x=1)
 assert_type(inst.copy(), MyClass)
+
+## End of test code ##
+# pyright: analyzeUnannotatedFunctions=false

docs/Contribute.md

@@ -42,6 +42,7 @@ Once you come up with an idea, go to the next steps.
    - Describe the challenge, make sure people understand what they need to accomplish (i.e. the `TODO:` part)
    - A comment `## End of your code ##`. This is mandatory, just copy and paste it.
    - Several test cases. Add a comment `# expect-type-error` after the lines where type errors should be thrown.
+   - (Optional) Add a comment `## End of test code ##`. Several [pyright-config](https://github.com/microsoft/pyright/blob/main/docs/configuration.md#type-check-diagnostics-settings) with the format `# pyright: <config_name>=<value>`
 
    `solution.py` contains the right solution, with everything else unchanged.
 

tests/assets/challenges/basic-foo-pyright-config/question.py

@@ -0,0 +1,14 @@
+"""A simple question, only for running tests.
+"""
+
+
+def foo():
+    pass
+
+
+## End of your code ##
+foo(1)
+foo(1, 2)  # expect-type-error
+
+## End of test code ##
+# pyright: reportGeneralTypeIssues=error

tests/conftest.py

@@ -10,6 +10,7 @@
 from flask.testing import FlaskClient
 
 from app import app
+from views.challenge import ChallengeManager
 
 CHALLENGES_DIR = Path(__file__).parent.parent / "challenges"
 ALL_QUESTIONS = list(CHALLENGES_DIR.glob("**/question.py"))
@@ -22,6 +23,11 @@ def assets_dir() -> Path:
     return Path(__file__).parent / "assets"
 
 
+@pytest.fixture()
+def mgr(assets_dir: Path):
+    return ChallengeManager(assets_dir / "challenges")
+
+
 @pytest.fixture()
 def test_client() -> FlaskClient:
     return app.test_client()

tests/test_challenge.py

@@ -1,6 +1,5 @@
 from pathlib import Path
 
-import pytest
 from views.challenge import ChallengeKey, ChallengeManager
 
 
@@ -11,23 +10,24 @@ def test_load_empty_dir(self, tmpdir):
     def test_defaults(self):
         assert ChallengeManager().challenge_count > 0
 
-    def test_load_tests_assets(self, assets_dir):
-        mgr = ChallengeManager(assets_dir / "challenges")
+    def test_load_tests_assets(self, mgr: ChallengeManager):
         assert mgr.challenge_count > 0
 
+    def test_partition_test_code(self, mgr: ChallengeManager):
+        pyright_config_test = mgr.get_challenge(
+            ChallengeKey.from_str("basic-foo-pyright-config")
+        ).test_code
+
+        _, pyright_basic_config = mgr._partition_test_code(pyright_config_test)
+        assert pyright_basic_config.endswith("pyright: reportGeneralTypeIssues=error\n")
 
-class TestChallengeWithHints:
-    @pytest.fixture()
-    def challenge_mgr(self, assets_dir):
-        return ChallengeManager(assets_dir / "challenges")
 
-    def test_misc(self, challenge_mgr):
-        c_foo = challenge_mgr.get_challenge(ChallengeKey.from_str("basic-foo"))
+class TestChallengeWithHints:
+    def test_misc(self, mgr: ChallengeManager):
+        c_foo = mgr.get_challenge(ChallengeKey.from_str("basic-foo"))
         assert c_foo.hints is None
 
         # Get the challenge with hints
-        c_foo_hints = challenge_mgr.get_challenge(
-            ChallengeKey.from_str("basic-foo-hints")
-        )
+        c_foo_hints = mgr.get_challenge(ChallengeKey.from_str("basic-foo-hints"))
         assert c_foo_hints.hints
         assert isinstance(c_foo_hints.hints, str)

tests/test_identical.py

@@ -8,17 +8,19 @@
 
 
 def test_identical(solution_file: Path):
-    level, challenge_name = solution_file.parent.name.split("-", maxsplit=1)
-    with solution_file.open() as f:
-        solution_code = f.read()
-    solution_test = Challenge(
-        name=challenge_name, level=Level(level), code=solution_code
-    ).test_code
-
-    question_file = solution_file.parent / "question.py"
-    with question_file.open() as f:
-        question_code = f.read()
-    question_test = Challenge(
-        name=challenge_name, level=Level(level), code=question_code
-    ).test_code
+    def get_test_code(path: Path):
+        TEST_SPLITTER = "\n## End of test code ##\n"
+        level, challenge_name = path.parent.name.split("-", maxsplit=1)
+
+        with solution_file.open() as f:
+            challenge_code = f.read()
+        challenge = Challenge(
+            name=challenge_name, level=Level(level), code=challenge_code
+        )
+
+        return challenge.test_code.partition(TEST_SPLITTER)[0]
+
+    solution_test = get_test_code(solution_file)
+    question_test = get_test_code(solution_file.parent / "question.py")
+
     assert solution_test.strip() == question_test.strip()

tests/test_questions.py

@@ -4,8 +4,6 @@
 
 from pathlib import Path
 
-import pytest
-
 from views.challenge import ChallengeManager
 
 

views/challenge.py

@@ -11,6 +11,29 @@
 from typing import ClassVar, Optional, TypeAlias
 
 ROOT_DIR = Path(__file__).parent.parent
+PYRIGHT_BASIC_CONFIG = """
+# pyright: analyzeUnannotatedFunctions=true
+# pyright: strictParameterNoneValue=true
+# pyright: disableBytesTypePromotions=false
+# pyright: strictListInference=false
+# pyright: strictDictionaryInference=false
+# pyright: strictSetInference=false
+# pyright: deprecateTypingAliases=false
+# pyright: enableExperimentalFeatures=false
+# pyright: reportMissingImports=error
+# pyright: reportUndefinedVariable=error
+# pyright: reportGeneralTypeIssues=error
+# pyright: reportOptionalSubscript=error
+# pyright: reportOptionalMemberAccess=error
+# pyright: reportOptionalCall=error
+# pyright: reportOptionalIterable=error
+# pyright: reportOptionalContextManager=error
+# pyright: reportOptionalOperand=error
+# pyright: reportTypedDictNotRequiredAccess=error
+# pyright: reportPrivateImportUsage=error
+# pyright: reportUnboundVariable=error
+# pyright: reportUnusedCoroutine=error
+"""
 
 
 class Level(StrEnum):
@@ -145,13 +168,30 @@ def _get_challenges_groupby_level(self) -> dict[Level, list[ChallengeName]]:
     # Pyright error messages look like:
     # `<filename>:<line_no>:<col_no> - <error|warning|information>: <message>`
     # Here we only capture the error messages and line numbers
-    PYRIGHT_MESSAGE_REGEX = r"^(?:.+?):(\d+):[\s\-\d]+(error:.+)$"
+    PYRIGHT_MESSAGE_REGEX = (
+        r"^(?:.+?):(?P<line_number>\d+):[\s\-\d]+(?P<message>error:.+)$"
+    )
+
+    @staticmethod
+    def _partition_test_code(test_code: str):
+        TEST_SPLITTER = "\n## End of test code ##\n"
+
+        # PYRIGHT_BASIC_CONFIG aim to limit user to modify the config
+        test_code, end_test_comment, pyright_config = test_code.partition(TEST_SPLITTER)
+        pyright_basic_config = PYRIGHT_BASIC_CONFIG
+
+        # Replace `## End of test code ##` with PYRIGHT_BASIC_CONFIG
+        if end_test_comment:
+            pyright_basic_config += pyright_config
+        return test_code, pyright_basic_config
 
     @classmethod
     def _type_check_with_pyright(
         cls, user_code: str, test_code: str
     ) -> TypeCheckResult:
-        code = f"{user_code}{test_code}"
+        test_code, pyright_basic_config = cls._partition_test_code(test_code)
+
+        code = f"{user_code}{test_code}{pyright_basic_config}"
         buffer = io.StringIO(code)
 
         # This produces a stream of TokenInfos, example:
@@ -187,37 +227,45 @@ def _type_check_with_pyright(
                 return TypeCheckResult(message=stderr, passed=False)
         error_lines: list[str] = []
 
-        # Substract lineno in merged code by lineno_delta, so that the lineno in
+        # Substract lineno in merged code by user_code_line_len, so that the lineno in
         # error message matches those in the test code editor. Fixed #20.
-        lineno_delta = len(user_code.splitlines())
+        user_code_lines_len = len(user_code.splitlines())
         for line in stdout.splitlines():
             m = re.match(cls.PYRIGHT_MESSAGE_REGEX, line)
             if m is None:
                 continue
-            line_number, message = int(m.group(1)), m.group(2)
+            line_number, message = int(m["line_number"]), m["message"]
             if line_number in error_line_seen_in_err_msg:
                 # Each reported error should be attached to a specific line,
                 # If it is commented with # expect-type-error, let it pass.
                 error_line_seen_in_err_msg[line_number] = True
                 continue
             # Error could be thrown from user code too, in which case delta shouldn't be applied.
-            error_lines.append(
-                f"{line_number if line_number <= lineno_delta else line_number - lineno_delta}:{message}"
-            )
+            error_line = f"%s:{message}"
+
+            if line_number <= user_code_lines_len:
+                error_lines.append(error_line % line_number)
+            elif line_number <= user_code_lines_len + len(test_code.splitlines()):
+                error_lines.append(error_line % (line_number - user_code_lines_len))
+            else:
+                error_lines.append(error_line % "[pyright-config]")
 
         # If there are any lines that are expected to fail but not reported by pyright,
         # they should be considered as errors.
         for line_number, seen in error_line_seen_in_err_msg.items():
             if not seen:
                 error_lines.append(
-                    f"{line_number - lineno_delta}: error: Expected type error but instead passed"
+                    f"{line_number - user_code_lines_len}: error: Expected type error but instead passed"
                 )
 
-        passed = len(error_lines) == 0
-        if passed:
-            error_lines.append("\nAll tests passed")
-        else:
-            error_lines.append(f"\nFound {len(error_lines)} errors")
+        # Error for pyright-config will not fail the challenge
+        passed = True
+        for error_line in error_lines:
+            if error_line.startswith("[pyright-config]"):
+                continue
+            passed = False
+
+        error_lines.append(f"\nFound {len(error_lines)} errors")
 
         return TypeCheckResult(message="\n".join(error_lines), passed=passed)
 

views/views.py

@@ -59,7 +59,7 @@ def get_challenge(level: str, name: str):
         "level": challenge.level,
         "challenges_groupby_level": challenge_manager.challenges_groupby_level,
         "code_under_test": challenge.user_code,
-        "test_code": challenge.test_code,
+        "test_code": challenge.test_code.partition("\n## End of test code ##\n")[0],
         "hints_for_display": render_hints(challenge.hints) if challenge.hints else None,
         "python_info": platform.python_version(),
     }