Merge pull request #11 from NicolasLacroix/develop

Merge : Memory testing for first release

Author: NicolasLacroix

.github/workflows/pylint.yml

@@ -17,6 +17,6 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pylint
-    - name: Test with pytest
+    - name: Lint with PyLint
       run: |
-        pylint `ls -R|grep .py$|xargs`
+        pylint -j 0 pyunitperf --exit-zero

.github/workflows/python-package.yml

@@ -5,7 +5,7 @@ name: Python package
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, develop ]
   pull_request:
     branches: [ master ]
 
@@ -26,14 +26,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8 pytest
+        pip install pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
       run: |
-        pytest
+        pytest --pyargs pyunitperf/tests

README.md

@@ -1,2 +1,78 @@
 # PyUnitPerf
-A simple API for unit testing a Python project's performances :wheelchair: :zap:.
+
+![Python package](https://github.com/NicolasLacroix/PyUnitPerf/workflows/Python%20package/badge.svg?branch=master)
+![Python versions](https://img.shields.io/badge/python-3.5,%203.6%2C%203.7%2C%203.8-blue?logo=python)
+![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)
+
+A simple and lightweight API for unit testing a Python 🐍 project's performances :zap::wheelchair:.
+
+From **Python 3.5 to 3.8**, easily improve your project's performances tests through the use of dedicated **decorators**.
+
+### Table of contents
+
+- [Memory testing](#Memory-testing)
+  - [Memory overload assertions](#Memory-overload-assertions)
+  - [Memory leaks assertions](#Memory-leaks-assertions)
+
+### Memory testing
+
+#### Memory overload assertions
+
+The memory overload assertions are possible thanks to the `@memory_not_exceed` decorator.
+
+A simple *TestCase* associated to this decorator does the job :
+
+```python
+class TestLimitMemoryUsage(unittest.TestCase):
+    """
+    This class illustrates the use of the memory_not_exceed decorator.
+    """
+    @memory_not_exceed(threshold=0)
+    def test_memory_usage_exceed(self):
+        """
+        This test won't pass due to a very low threshold.
+        """
+        return list(range(1000)) * 1
+
+    @memory_not_exceed(threshold=1000)
+    def test_memory_usage_not_exceed(self):
+        """
+        This test passes due to a very high threshold.
+        """
+        return list(range(1000)) * 1
+```
+
+#### Memory leaks assertions
+
+The memory leaks assertions are possible thanks to the `@memory_not_leak` decorator.
+
+Once again, a simple *TestCase* associated to this decorator does the job :
+
+```python
+class TetsMemoryLeaksDetection(unittest.TestCase):
+    """
+    This class illustrates the use of the memory_not_leak decorator.
+    """
+    leaking_list = []
+
+    def leak(self):
+        """
+        Generates a memory leak involving the leaking_list.
+        """
+        self.leaking_list.append("will leak")
+
+    @memory_not_leak()
+    def test_memory_leaks(self):
+        """
+        This test won't pass due to a memory leak.
+        """
+        self.leak()
+
+    @memory_not_leak()
+    def test_memory_not_leak(self):
+        """
+        This test passes due to the absence of memory leak.
+        """
+        valid_list = []
+        valid_list.append("won't leak")
+```

examples/limit_memory_usage.py

@@ -0,0 +1,30 @@
+"""
+This file contains a class that illustrates the use of the
+memory limit assertions using the pyunitperf.memory package.
+"""
+import unittest
+
+from pyunitperf.memory import memory_not_exceed
+
+
+class TestLimitMemoryUsage(unittest.TestCase):
+    """
+    This class illustrates the use of the memory_not_exceed decorator.
+    """
+    @memory_not_exceed(threshold=0)
+    def test_memory_usage_exceed(self):
+        """
+        This test won't pass due to a very low threshold.
+        """
+        return list(range(1000)) * 1
+
+    @memory_not_exceed(threshold=1000)
+    def test_memory_usage_not_exceed(self):
+        """
+        This test passes due to a very high threshold.
+        """
+        return list(range(1000)) * 1
+
+
+if __name__ == '__main__':
+    unittest.main()

examples/memory_leaks_detection.py

@@ -0,0 +1,39 @@
+"""
+This file contains a class that illustrates the use of the
+memory leaks assertions using the pyunitperf.memory package.
+"""
+import unittest
+
+from pyunitperf.memory import memory_not_leak
+
+
+class TetsMemoryLeaksDetection(unittest.TestCase):
+    """
+    This class illustrates the use of the memory_not_leak decorator.
+    """
+    leaking_list = []
+
+    def leak(self):
+        """
+        Generates a memory leak involving the leaking_list.
+        """
+        self.leaking_list.append("will leak")
+
+    @memory_not_leak()
+    def test_memory_leaks(self):
+        """
+        This test won't pass due to a memory leak.
+        """
+        self.leak()
+
+    @memory_not_leak()
+    def test_memory_not_leak(self):
+        """
+        This test passes due to the absence of memory leak.
+        """
+        valid_list = []
+        valid_list.append("won't leak")
+
+
+if __name__ == '__main__':
+    unittest.main()

pyunitperf/__init__.py

@@ -0,0 +1,162 @@
+"""
+This package is dedicated to all the memory aspects testing.
+"""
+import tracemalloc
+from tracemalloc import Snapshot, Statistic, StatisticDiff
+from copy import deepcopy
+from functools import wraps
+from typing import Callable, Iterable, Optional, List
+import collections.abc as abc
+
+ExcludeType = Optional[Iterable[str]]
+
+DEFAULT_FILTERS = {
+    tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
+    tracemalloc.Filter(False, "<unknown>"),
+    tracemalloc.Filter(False, tracemalloc.__file__),
+    tracemalloc.Filter(False, __file__)
+}
+
+def _filter_snapshot(snapshot: Snapshot, exclude: ExcludeType = None) -> tracemalloc.Snapshot:
+    """
+    Filters the given snapshot using the exclude value and the DEFAULT_FILTERS.
+    :param snapshot: snapshot to be filtered
+    :param exclude: element(s) to be excluded from the snapshot
+    :return: the filtered snapshot
+    """
+    filters = deepcopy(DEFAULT_FILTERS)
+    if isinstance(exclude, str):
+        filters.add(tracemalloc.Filter(False, exclude))
+    elif isinstance(exclude, abc.Iterable):
+        filters.update(
+            {tracemalloc.Filter(False, e) for e in exclude if isinstance(e, str)}
+        )
+    return snapshot.filter_traces(tuple(filters))
+
+
+def _get_overload(snapshot: Snapshot, threshold: float, key_type: str = "lineno") -> \
+        List[Statistic]:
+    """
+    Returns a list of statistics that exceed the given threshold in KiB.
+    :param snapshot: snapshot to analyze
+    :param threshold: threshold not to exceed (in KiB)
+    :param key_type: key used to order the snapshot's statistics
+
+    :return: a list of statistics that exceed the given threshold
+    """
+    stats = snapshot.statistics(key_type)
+    return [stat for stat in stats if stat.size / 1024 > threshold]
+
+
+def _get_leaks(snapshot1: Snapshot, snapshot2: Snapshot, threshold: float,
+               key_type: str = "lineno") -> List[StatisticDiff]:
+    """
+    Returns a list of statistics that contains detected memory leaks above
+    the given threshold in bytes.
+    :param snapshot1: first snapshot (before the tested function execution)
+    :param snapshot2: second snapshot (after the tested function execution)
+    :param threshold: threshold to be exceeded to retain the leaks (in B)
+    :param key_type: key used to order the snapshot's statistics
+    :return: a list of statistics that contains detected memory leaks
+    """
+    return [s for s in snapshot2.compare_to(snapshot1, key_type) if s.size_diff / 1024 >= threshold]
+
+
+def _get_failure_details(stats: List[Statistic]) -> str:
+    """
+    Returns a string containing details about the given stats which led to a failure.
+    :param stats: statistics to describe
+    :return: a string containing details about the given stats
+             which led to a failure
+    """
+    details = "\n"
+    for stat in stats:
+        formatted_traceback = "\n".join(stat.traceback.format())
+        details += "{} failed with size : [{} KiB]\n".format(formatted_traceback, stat.size / 1024)
+    return details
+
+
+def _get_leaks_details(stats: List[StatisticDiff]) -> str:
+    """
+    Returns a string containing details about the given stats which led to a failure.
+    :param stats: statistics to describe
+    :return: a string containing details about the given stats
+             which led to a failure
+    """
+    failed_sentence = "{} failed with diff : [{} B]\n"
+    details = "\n"
+    for stat in stats:
+        formatted_traceback = "\n".join(stat.traceback.format())
+        details += failed_sentence.format(formatted_traceback, stat.size_diff / 1024)
+    return details
+
+
+def memory_not_exceed(threshold: float, exclude: ExcludeType = None) -> Callable:
+    """
+    Tests that the memory taken up by the given function
+    doesn't exceed the given threshold in KiB.
+    :param threshold: threshold in KiB
+    :param exclude: element(s) to be excluded from the snapshot
+    :return: the memeory_not_exceed's decorator
+    """
+
+    def memory_not_exceed_decorator(func: Callable) -> Callable:
+        """
+        memory_not_exceed's decorator.
+        :param func: function to call
+        :return: the memory_not_exceed's wrapper
+        """
+
+        @wraps(func)
+        def memory_not_exceed_wrapper(*args) -> None:
+            """
+            memory_not_exceed's wrapper.
+            :param args: wrapper's arguments
+            """
+            tracemalloc.start()
+            func(*args)
+            snapshot = tracemalloc.take_snapshot()
+            tracemalloc.stop()
+            snapshot = _filter_snapshot(snapshot, exclude=exclude)
+            overload = _get_overload(snapshot, threshold=threshold)
+            assert not overload, _get_failure_details(overload)
+
+        return memory_not_exceed_wrapper
+
+    return memory_not_exceed_decorator
+
+
+def memory_not_leak(threshold: float = 0, exclude: ExcludeType = None) -> Callable:
+    """
+    Tests that the memory taken up by the given function
+    doesn't leak.
+    :param threshold: threshold in B
+    :param exclude: element(s) to be excluded from the snapshot
+    :return: the memory_not_leak's decorator
+    """
+
+    def memory_not_leak_decorator(func: Callable) -> Callable:
+        """
+        memory_not_leak's decorator.
+        :param func: function to call
+        :return: the memory_not_leak's wrapper
+        """
+        @wraps(func)
+        def memory_not_leak_wrapper(*args) -> None:
+            """
+            memory_not_leak's wrapper.
+            :param args: wrapper's arguments
+            """
+            tracemalloc.start()
+            snapshot1 = tracemalloc.take_snapshot()
+            func(*args)
+            snapshot2 = tracemalloc.take_snapshot()
+            tracemalloc.stop()
+            snapshot1 = _filter_snapshot(snapshot1, exclude=exclude)
+            snapshot2 = _filter_snapshot(snapshot2, exclude=exclude)
+            leaks = _get_leaks(snapshot1, snapshot2, threshold)
+            assert not leaks, _get_leaks_details(leaks)
+
+        return memory_not_leak_wrapper
+
+    return memory_not_leak_decorator