feat: add grade command

Author: krerkkiat

CHANGELOG.md

@@ -9,6 +9,17 @@ Note that the log for version before v0.1.1 may not be in the mentioned format.
 
 ## [Unreleased]
 
+## v0.1.2a1 - 2024-10-01
+
+### Added
+
+- Add `grade` command that runs all test in the homework.
+- Add `util.FindProblemList` that parse a README.md file for `:problem-list:` token and the list of problem names.
+
+### Changed
+
+- Fix some mypy error messages for function's return type and argument type.
+
 ## v0.1.1 - 2024-09-25
 
 Please see changelog from v0.1.1a1 to v0.1.1rc3.

MAINTENANCE.md

@@ -67,12 +67,14 @@ make html
 
 ## Deployment flow
 
+- Check that unit testing on Windows and MacOs do not fail.
+
 - Run tox.
 - Update `grading_lib/VERSION` file.
 - Update `CHANGELOG.md` file.
-- Make a commit
-- Tag the commit
-- Push
+- Make a commit.
+- Tag the commit.
+- Push to GitHub.
 - Approve the deployment flow on GitHub Actions.
 
 ### Manually publish to PyPI

grading_lib/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2a1

grading_lib/cli/__init__.py

@@ -1,8 +1,14 @@
+import copy
+import importlib
+import os
+import sys
+import unittest
 from pathlib import Path
 
 import click
 
-from ..util import get_problem_total_points, load_problems_metadata
+from ..common import MinimalistTestResult, MinimalistTestRunner
+from ..util import FindProblemList, get_problem_total_points, load_problems_metadata
 from .dev import dev
 from .internal import internal
 
@@ -71,5 +77,61 @@ def rebase_todo_injector_command(todo_items_file_path: str, path: str) -> None:
             out_f.write(todo_items_content)
 
 
+@cli.command(name="grade")
+@click.argument("path", default=".", type=click.Path(exists=True))
+def grade_command(path: str | Path) -> None:
+    """Grade problems at path"""
+    # Steps:
+    # 1. Using mistletoe, parse the README.md in the path for the problem order.
+    #    By extracting the list after the inline code token with `:problem-list:` on a heading token.
+    # 2. Prepare the tests to execute in that order. Remove or skip the test where the students did not change the content of the target file.
+    if isinstance(path, str):
+        path = Path(path)
+    problem_names = FindProblemList.from_file(path / "README.md")
+
+    if len(problem_names) == 0:
+        print("No problem found.")
+
+    current_directory = os.getcwd()
+    current_sys_path = copy.copy(sys.path)
+    test_programs = []
+    for idx, problem_name in enumerate(problem_names, start=1):
+        print(f"{idx} - Grading {problem_name} ")
+
+        try:
+            os.chdir(problem_name)
+
+            # Prepare sys.path for module import.
+            sys.path.append(os.getcwd())
+            importlib.invalidate_caches()
+            mod = importlib.import_module("scripts.grade")
+
+            runner = MinimalistTestRunner(resultclass=MinimalistTestResult)
+            test_program = unittest.main(
+                mod, testRunner=runner, argv=[sys.argv[0]], exit=False
+            )
+            test_programs.append((problem_name, test_program))
+        finally:
+            os.chdir(current_directory)
+            # Without copy, sys.path will be a ref to current_sys_path.
+            sys.path = copy.copy(current_sys_path)
+            if "scripts.grade" in sys.modules.keys():
+                # This will be re-used by Python since we have the same module name in
+                # every problem.
+                del sys.modules["scripts.grade"]
+
+        print("\n\n")
+
+    # Summary.
+    print("==== Summary ====")
+    total_points = sum([test_program.result.total_points for _, test_program in test_programs])
+    student_total_points = 0.0
+    for idx, item in enumerate(test_programs, start=1):
+        problem_name, test_program = item
+        print(f"{idx:>3} - {problem_name:<30}{test_program.result.points:>5} / {test_program.result.total_points:>5}")
+        student_total_points += test_program.result.points
+    print(f"Total: {student_total_points:>5} / {total_points:>5}")
+
+
 cli.add_command(dev)
 cli.add_command(internal)

grading_lib/util.py

@@ -1,7 +1,16 @@
 from pathlib import Path
 from typing import Any
 
+import mistletoe
 import tomli
+from mistletoe.block_token import (
+    BlockToken,
+    Heading,
+    List,
+    Paragraph,
+    SetextHeading,
+)
+from mistletoe.span_token import InlineCode, SpanToken
 
 
 def load_problems_metadata(path: Path = Path(".")) -> list[dict[str, Any]]:
@@ -23,3 +32,60 @@ def get_problem_total_points(problem: dict[str, Any]) -> float:
     for _, test in problem["problem"]["tests"].items():
         total_points += test["points"]
     return total_points
+
+
+class FindProblemList:
+    """
+    Parse the AST of the markdown file from mistletoe for the problem list.
+
+    The section that has the problem list must be marked with "`::problem-list`"
+    with
+    """
+
+    def __init__(self):
+        self.after_problem_list_marker = False
+        self.in_a_list = False
+        self.problem_names = []
+
+    @classmethod
+    def from_file(cls, file_path: Path) -> list[str]:
+        with open(file_path) as f:
+            data = f.read()
+            document = mistletoe.Document(data)
+            obj = cls()
+            obj.visit_block(document)
+            return obj.problem_names
+
+    def visit_text(self, token: SpanToken) -> None:
+        if isinstance(token, InlineCode):
+            if hasattr(token, "children") and len(token.children) != 0:
+                if self.after_problem_list_marker and self.in_a_list:
+                    # print(token.children[0].content)
+                    self.problem_names.append(token.children[0].content)
+                else:
+                    # print(token.children[0].content)
+                    if token.children[0].content == ":problem-list:":
+                        # print("found marker")
+                        self.after_problem_list_marker = True
+
+        elif hasattr(token, "children") and token.children is not None:
+            for child in token.children:
+                self.visit_text(child)
+
+    def visit_block(self, token: BlockToken) -> None:
+        if isinstance(token, Paragraph | SetextHeading | Heading):
+            for child in token.children:
+                self.visit_text(child)
+
+        if isinstance(token, List):
+            self.in_a_list = True
+            for child in token.children:
+                self.visit_block(child)
+            self.in_a_list = False
+
+            if self.after_problem_list_marker:
+                self.after_problem_list_marker = False
+
+        for child in token.children:
+            if isinstance(child, BlockToken):
+                self.visit_block(child)

pyproject.toml

@@ -21,6 +21,7 @@ dependencies = [
     "click",
     "tomli",
     "GitPython",
+    "mistletoe",
     "typing_extensions >=4.12, <5",
     'importlib-metadata; python_version>="3.10"',
 ]

tests/test_repository.py

@@ -5,12 +5,12 @@
 from grading_lib.repository import Repository, RepositoryBaseTestCase
 
 
-def test_Repository_init_with_an_empty_folder(tmp_path) -> None:
+def test_Repository_init_with_an_empty_folder(tmp_path: pytest.Fixture) -> None:
     repo = Repository(tmp_path)
     assert isinstance(repo.working_tree_dir, Path)
 
 
-def test_Repository_create_and_add_random_file(tmp_path) -> None:
+def test_Repository_create_and_add_random_file(tmp_path: pytest.Fixture) -> None:
     repo = Repository(tmp_path)
     assert isinstance(repo.working_tree_dir, Path)
 
@@ -22,7 +22,7 @@ def test_Repository_create_and_add_random_file(tmp_path) -> None:
     assert len(repo.repo.index.entries) == 2
 
 
-def test_RepositoryBaseTestCase_assertHasOnlyGitCommand(tmp_path) -> None:
+def test_RepositoryBaseTestCase_assertHasOnlyGitCommand(tmp_path: pytest.Fixture) -> None:
     class Child(RepositoryBaseTestCase):
         pass
 

tests/test_util.py

@@ -0,0 +1,42 @@
+
+from grading_lib.util import FindProblemList
+
+
+def test_FindProblemList(tmp_path) -> None:
+    with open(tmp_path / "README.md", "w") as f:
+        f.write("""
+# Homework 0
+
+## Problem List `:problem-list:`
+
+Some paragraph.
+
+- `problem-a`
+- `problem-b`
+
+## Fake Problem List
+
+- `problem-c`
+""")
+
+    problem_names = FindProblemList.from_file(tmp_path / "README.md")
+    assert len(problem_names) == 2
+
+    with open(tmp_path / "README2.md", "w") as f:
+        f.write("""
+# Homework 0
+
+## Problem List
+
+Some paragraph.
+
+- `problem-a`
+- `problem-b`
+
+## Fake Problem List
+
+- `problem-c`
+""")
+
+    problem_names = FindProblemList.from_file(tmp_path / "README2.md")
+    assert len(problem_names) == 0