feat: add utilities, simplify test case

Author: krerkkiat

CHANGELOG.md

@@ -1,7 +1,26 @@
 <!-- This file is included in the documentation so the header is removed. -->
 
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://packaging.python.org/en/latest/discussions/versioning/).
+
+Note that the log for version before v0.1.1 may not be in the mentioned format.
+
 ## Unreleased
 
+### Added
+
+- Add `common.points` for decorating the test method for points.
+- Add `common.file_has_correct_sha512_checksum` for checking file checksum.
+- Add `common.MinimalistTestRunner` that show collected points at the end.
+- Add `repository.RepositoryBaseTestCase.assertHasOnlyGitCommand` that check if file contain only Git command and nothing else.
+
+## Changed
+
+- `common.MinimalistTestResult` now track points if they are presence.
+- Some test cases now use the builtin `tmp_path` instead of our own version of it.
+
 ## v0.1.1rc2
 
 - Fix gunzip is not available on Windows.

MAINTENANCE.md

@@ -44,7 +44,7 @@ Run the test for various python's versions, run
 tox
 ```
 
-## Running test coverage
+With test coverage
 
 ```console
 pytest --cov=grading_lib tests/

grading_lib/common.py

@@ -1,15 +1,18 @@
 import datetime
+import hashlib
 import os
 import subprocess
 import sys
 import tempfile
 import time
+import typing as ty
 import unittest
 from collections import namedtuple
 from pathlib import Path
 from typing import Any, TypeVar
 
 T = TypeVar("T")
+P = ty.ParamSpec("P")
 
 COMMAND_FAILED_TEXT_TEMPLATE = "An error occurred while trying to run a command '{command}'. The command's output is\n\n{output}"
 FILE_NOT_EXIST_TEXT_TEMPLATE = "File '{path}' does not exist"
@@ -18,6 +21,29 @@
 NAME_POOL = ["herta", "cat", "dog", "dolphin", "falcon", "dandilion", "fox", "jett"]
 
 
+def points(value: float) -> ty.Callable[P, T]:
+    """Assign points to the test case."""
+
+    def decorator(test_item: T) -> T:
+        test_item.__gradinglib_points = value
+        return test_item
+
+    return decorator
+
+
+def file_has_correct_sha512_checksum(expected_checksum: str, file_path: Path) -> bool:
+    """Return True if the file has the same SHA512 checksum."""
+
+    with open(file_path, "rb") as f:
+        data = f.read()
+        m = hashlib.sha512()
+        m.update(data)
+        checksum = m.hexdigest()
+        return checksum == expected_checksum
+
+    return False
+
+
 def is_debug_mode(
     variable_name: str = "DEBUG",
     vals_for_true: tuple[str, ...] = ("true", "t", "on", "1"),
@@ -120,13 +146,29 @@ def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.dots = False
 
+        # Points tracking.
+        self.points = 0.0
+        self.total_points = 0.0
+
     def getDescription(self, test: unittest.TestCase) -> str:
         doc_first_line = test.shortDescription()
         if self.descriptions and doc_first_line:
             return doc_first_line
         else:
             return str(test)
 
+    def startTest(self, test):
+        super().startTest(test)
+        test_method = getattr(test, test._testMethodName)
+        if hasattr(test_method, "__gradinglib_points"):
+            self.total_points += getattr(test_method, "__gradinglib_points")
+
+    def addSuccess(self, test):
+        super().addSuccess(test)
+        test_method = getattr(test, test._testMethodName)
+        if hasattr(test_method, "__gradinglib_points"):
+            self.points += getattr(test_method, "__gradinglib_points")
+
     def addFailure(self, test: unittest.TestCase, err) -> None:
         self.failures.append((test, str(err[1]) + "\n"))
         self._mirrorOutput = True
@@ -138,6 +180,14 @@ def addFailure(self, test: unittest.TestCase, err) -> None:
             self.stream.flush()
 
 
+class MinimalistTestRunner(unittest.TextTestRunner):
+    def run(self, test):
+        result = super().run(test)
+        self.stream.writeln(f"POINTS: {result.points} / {result.total_points}")
+        self.stream.flush()
+        return result
+
+
 class BaseTestCaseMeta(type):
     def __new__(cls: T, name: str, bases: list[Any], attrs: dict[str, Any]) -> T:
         if "with_temporary_dir" not in attrs:

grading_lib/qa.py

@@ -10,7 +10,7 @@
 def import_as_non_testcase(
     module_name: str, name: str, package: str | None = None
 ) -> ty.Any:
-    """Import the name from a module and prevent pytest from running it as testcase."""
+    """Import the name from a module and prevent pytest from running it as a testcase."""
     mod = import_module(module_name, package=package)
     cls = mod.__dict__[name]
     cls.__test__ = False

grading_lib/repository.py

@@ -44,7 +44,12 @@ class Repository:
     :raise ValueError: When the repository does not have a working tree directory.
     """
 
-    def __init__(self, path: str | Path, *args, **kwargs) -> None:
+    def __init__(
+        self: Self,
+        path: str | Path,
+        *args,
+        **kwargs,
+    ) -> None:
         self.repo: Repo
         self.temp_dir: tempfile.TemporaryDirectory[str] | None = None
 
@@ -198,6 +203,21 @@ def visualize(self) -> str:
 
 
 class RepositoryBaseTestCase(BaseTestCase):
+    def assertHasOnlyGitCommand(
+        self,
+        path: Path,
+        msg_template="{path!s} appears to have non git command(s). Please comment out unrelated commands.",
+    ) -> None:
+        with open(path) as f:
+            lines = f.readlines()
+            for raw_line in lines:
+                line = raw_line.strip()
+                if len(line) == 0 or line[0] == "#":
+                    continue
+
+                if not line.startswith("git"):
+                    self.fail(msg_template.format(path=path))
+
     def assertHasTagWithNameAt(
         self, repo: Repository, name: str, commit_hash: str
     ) -> None:
@@ -208,7 +228,7 @@ def assertHasTagWithNameAt(
                 return
 
         tags_text = "\n".join(tag_ref.path for tag_ref in tag_refs)
-        raise self.failureException(
+        self.fail(
             f"Expect to see a tag '{name}' at commit '{commit_hash}', but found none. Tags at commit {commit_hash}:\n{tags_text}"
         )
 
@@ -234,6 +254,6 @@ def assertHasTagWithNameAndMessageAt(
                 text = f"{tag_ref.path}: {tag_ref.tag.message}"
             tags_texts.append(text)
         tags_text = "\n".join(tags_texts)
-        raise self.failureException(
+        self.fail(
             f"Expect to see a tag '{name}' with message '{message}' at commit '{commit_hash}', but found none. Tags at commit {commit_hash}:\n{tags_text}"
         )

pyproject.toml

@@ -55,6 +55,7 @@ select = [
     "RUF", # ruff-specific rules
     "UP",  # pyupgrade
     "W",   # pycodestyle warning
+    "T10", # flake8-debugger e.g. breakpoint() is presence
 ]
 ignore = [
     "E501"  # Line too long

tests/test_common.py

@@ -1,5 +1,4 @@
 import os
-import tempfile
 import time
 from pathlib import Path
 
@@ -8,6 +7,7 @@
 from grading_lib import is_debug_mode
 from grading_lib.common import (
     BaseTestCase,
+    file_has_correct_sha512_checksum,
     get_mtime_as_datetime,
     get_seed_from_env,
     has_file_changed,
@@ -50,52 +50,41 @@ def test_get_seed_from_env() -> None:
     assert res != 0
 
 
-@pytest.fixture
-def a_temp_file():
-    f = tempfile.NamedTemporaryFile()
-    f.seek(0)
-    yield f
-    f.close()
-
-
-def test_has_file_changed(a_temp_file) -> None:
-    path = Path(a_temp_file.name)
+def test_has_file_changed(tmp_path) -> None:
+    temp_file = tmp_path / "README.md"
+    temp_file.write_text("Hello World!")
 
     # Take snapshot of the mtime.
-    last_known_mtime = get_mtime_as_datetime(path)
-    assert not has_file_changed(last_known_mtime, path)
+    last_known_mtime = get_mtime_as_datetime(temp_file)
+    assert not has_file_changed(last_known_mtime, temp_file)
 
     # Modify the mtime and test.
     time.sleep(1.2)
-    path.touch()
-    assert has_file_changed(last_known_mtime, path)
+    temp_file.touch()
+    assert has_file_changed(last_known_mtime, temp_file)
 
     # Take another snapshot.
-    last_known_mtime = get_mtime_as_datetime(str(path))
-    assert not has_file_changed(last_known_mtime, path)
-
+    last_known_mtime = get_mtime_as_datetime(str(temp_file))
+    assert not has_file_changed(last_known_mtime, temp_file)
 
-def test_populate_folder_with_filenames() -> None:
-    with tempfile.TemporaryDirectory() as tmpdir:
-        tmpdir_path = Path(tmpdir)
 
-        with pytest.raises(ValueError):
-            (tmpdir_path / "a.txt").touch()
-            populate_folder_with_filenames(tmpdir_path / "a.txt", ["a", "b", "c"])
+def test_populate_folder_with_filenames(tmp_path) -> None:
+    expected_files = ["a", "b", "c"]
+    populate_folder_with_filenames(tmp_path, expected_files)
 
-        with pytest.raises(ValueError):
-            populate_folder_with_filenames(tmpdir_path / "src", ["a", "b", "c"])
+    # Order should not matter, we just want check that all files are created.
+    result_names = sorted([item.name for item in tmp_path.iterdir()])
 
-    with tempfile.TemporaryDirectory() as tmpdir:
-        tmpdir_path = Path(tmpdir)
+    assert result_names == expected_files
 
-        expected_files = ["a", "b", "c"]
-        populate_folder_with_filenames(tmpdir, expected_files)
 
-        # Order should not matter, we just want check that all files are created.
-        result_names = sorted([item.name for item in tmpdir_path.iterdir()])
+def test_populate_folder_with_filenames_error_handling(tmp_path) -> None:
+    with pytest.raises(ValueError):
+        (tmp_path / "a.txt").touch()
+        populate_folder_with_filenames(tmp_path / "a.txt", ["a", "b", "c"])
 
-        assert result_names == expected_files
+    with pytest.raises(ValueError):
+        populate_folder_with_filenames(tmp_path / "src", ["a", "b", "c"])
 
 
 def test_run_executable() -> None:
@@ -126,16 +115,30 @@ class ChildClsWithTempDir(BaseTestCase):
         instance.tearDown()
 
 
-def test_BaseTestCase_assertArchiveFileIsGzip() -> None:
+def test_BaseTestCase_assertArchiveFileIsGzip(tmp_path) -> None:
     class ChildClsWithTempDir(BaseTestCase):
         with_temporary_dir = True
 
     instance = ChildClsWithTempDir()
     instance.setUp()
 
     try:
-        with tempfile.NamedTemporaryFile() as tmp_file:
-            with pytest.raises(AssertionError):
-                instance.assertArchiveFileIsGzip(tmp_file.name)
+        tmp_file = tmp_path / "sdist.tar.gz"
+        tmp_file.write_bytes(b"\x1f\x65")
+        with pytest.raises(AssertionError):
+            instance.assertArchiveFileIsGzip(str(tmp_file))
     finally:
         instance.tearDown()
+
+
+def test_file_has_correct_sha512_checksum(tmp_path):
+    file_path = tmp_path / "repo.tar.gz"
+    with open(file_path, "wb") as f:
+        f.write(
+            b";fb\xc80^\xd1u*p\x00\xf50\xd1\xc1\x17\xb6\xec\x13\xb6\x1fd\xd5\xe2\xe1\xa0\xc4\xe5a\x0f"
+        )
+
+    assert file_has_correct_sha512_checksum(
+        "f58345b442700529c9f488df0eb76b805bd26fc347b83f9ff5aead0e06fee6b7fc480a556578be9a202813da0c322b48c5004a9c764f1f6b051a6467827338c8",
+        file_path,
+    )

tests/test_repository.py

@@ -1,29 +1,17 @@
-import sys
-import tempfile
 from pathlib import Path
 
 import pytest
 
-from grading_lib.repository import Repository
+from grading_lib.repository import Repository, RepositoryBaseTestCase
 
 
-@pytest.fixture
-def an_empty_folder():
-    if sys.version_info < (3, 12, 0):
-        temp_dir = tempfile.TemporaryDirectory()
-    else:
-        temp_dir = tempfile.TemporaryDirectory(delete=False)
-    yield temp_dir
-    temp_dir.cleanup()
-
-
-def test_Repository_init_with_an_empty_folder(an_empty_folder) -> None:
-    repo = Repository(an_empty_folder.name)
+def test_Repository_init_with_an_empty_folder(tmp_path) -> None:
+    repo = Repository(tmp_path)
     assert isinstance(repo.working_tree_dir, Path)
 
 
-def test_Repository_create_and_add_random_file(an_empty_folder) -> None:
-    repo = Repository(an_empty_folder.name)
+def test_Repository_create_and_add_random_file(tmp_path) -> None:
+    repo = Repository(tmp_path)
     assert isinstance(repo.working_tree_dir, Path)
 
     repo.create_and_add_random_file(name="a.txt")
@@ -32,3 +20,31 @@ def test_Repository_create_and_add_random_file(an_empty_folder) -> None:
 
     repo.create_and_add_random_file()
     assert len(repo.repo.index.entries) == 2
+
+
+def test_RepositoryBaseTestCase_assertHasOnlyGitCommand(tmp_path) -> None:
+    class Child(RepositoryBaseTestCase):
+        pass
+
+    instance = Child()
+    instance.setUp()
+    try:
+        file_path = tmp_path / "answer.sh"
+        with open(file_path, "w") as f:
+            f.write("# Comment\ntar -xzf repo.tar.gz\ngit status")
+
+        with pytest.raises(AssertionError):
+            instance.assertHasOnlyGitCommand(file_path)
+    finally:
+        instance.tearDown()
+
+    instance.setUp()
+    try:
+        file_path = tmp_path / "answer.sh"
+        with open(file_path, "w") as f:
+            f.write("# Comment\n# tar -xzf repo.tar.gz\ngit status")
+
+        instance.assertHasOnlyGitCommand(file_path)
+
+    finally:
+        instance.tearDown()