From 8238d052e524e5ccd3f0478415263f2602593c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Wed, 20 Apr 2022 12:50:24 +0200 Subject: [PATCH 01/12] First markdown parser tests --- .github/workflows/markdown-parser.yml | 102 +++++++++++++++++++++++ markdown-parser/README.md | 7 ++ markdown-parser/jupyter_lab_config.py | 21 +++++ markdown-parser/requirements.txt | 5 ++ markdown-parser/tests/__init__.py | 0 markdown-parser/tests/conftest.py | 56 +++++++++++++ markdown-parser/tests/test_jupyter.py | 25 ++++++ markdown-parser/tests/test_jupyterlab.py | 24 ++++++ markdown-parser/tests/test_nbconvert.py | 18 ++++ markdown-parser/tests/utils.py | 59 +++++++++++++ 10 files changed, 317 insertions(+) create mode 100644 .github/workflows/markdown-parser.yml create mode 100644 markdown-parser/README.md create mode 100644 markdown-parser/jupyter_lab_config.py create mode 100644 markdown-parser/requirements.txt create mode 100644 markdown-parser/tests/__init__.py create mode 100644 markdown-parser/tests/conftest.py create mode 100644 markdown-parser/tests/test_jupyter.py create mode 100644 markdown-parser/tests/test_jupyterlab.py create mode 100644 markdown-parser/tests/test_nbconvert.py create mode 100644 markdown-parser/tests/utils.py diff --git a/.github/workflows/markdown-parser.yml b/.github/workflows/markdown-parser.yml new file mode 100644 index 00000000..cc140378 --- /dev/null +++ b/.github/workflows/markdown-parser.yml @@ -0,0 +1,102 @@ +name: Markdown parser tests + +on: + pull_request: + push: + branches: ["master"] + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-20.04 + + steps: + - name: Checkout benchmarks project + uses: actions/checkout@v2 + with: + path: benchmarks + + # - name: Install node + # uses: actions/setup-node@v2 + # with: + # node-version: "14.x" + + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Cache pip on Linux + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-3.8-${{ hashFiles('**/requirements.txt', 'setup.cfg') }} + restore-keys: | + ${{ runner.os }}-pip-3.8 + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: Cache yarn + uses: actions/cache@v1 + id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install dependencies + working-directory: benchmarks/markdown-parser + run: | + set -ex + python -m pip install -r requirements.txt + python -m playwright install chromium + + # - name: Checkout JupyterLab + # uses: actions/checkout@v2 + # with: + # repository: jupyterlab/jupyterlab + # ref: master + # path: reference + + # - name: Install dependencies + # run: | + # set -ex + # echo "OLD_REF_SHA=$(git log -n1 --format='%H')" >> $GITHUB_ENV + # bash ./scripts/ci_install.sh + # # Build dev mode + # jlpm run build + # working-directory: reference + + - name: Launch JupyterLab + shell: bash + run: | + set -ex + python -m jupyterlab --config jupyter_lab_config.py 2>&1 > /tmp/jupyterlab_server.log & + working-directory: benchmarks/markdown-parser + + - name: Wait for JupyterLab + uses: ifaxity/wait-on-action@v1 + with: + resource: http-get://localhost:9999/lab + timeout: 360000 + + - name: Tests + working-directory: benchmarks/markdown-parser + run: | + set -ex + pytest -rap -vv + + - name: Kill the server + if: always() + shell: bash + run: | + kill -s SIGKILL $(pgrep jupyter-lab) + + - name: Print JupyterLab logs + if: always() + run: | + echo "::group::Server log" + cat /tmp/jupyterlab_server.log + echo "::endgroup::" diff --git a/markdown-parser/README.md b/markdown-parser/README.md new file mode 100644 index 00000000..7cc34e01 --- /dev/null +++ b/markdown-parser/README.md @@ -0,0 +1,7 @@ +# Markdown parser tests + +This folder contains tests to validate markdown parsers. + +The tests are written using [pytest](https://docs.pytest.org). To test web frontend parsers, [playwright](https://playwright.dev/python/docs/intro) is used to evaluate the markdown conversion to HTML. + +> The test database is downloaded from [GitHub flavored Commonmark](https://github.com/github/cmark-gfm). diff --git a/markdown-parser/jupyter_lab_config.py b/markdown-parser/jupyter_lab_config.py new file mode 100644 index 00000000..d7e46661 --- /dev/null +++ b/markdown-parser/jupyter_lab_config.py @@ -0,0 +1,21 @@ +import getpass +from tempfile import mkdtemp + +# Test if we are running in a docker +if getpass.getuser() == "jovyan": + c.ServerApp.ip = "0.0.0.0" + +c.ServerApp.port = 9999 +c.ServerApp.open_browser = False + +c.ServerApp.root_dir = mkdtemp(prefix='markdown-parser-lab-') + +c.ServerApp.token = "" +c.ServerApp.password = "" +c.ServerApp.disable_check_xsrf = True + +# c.LabApp.dev_mode = True +# c.LabApp.extensions_in_dev_mode = True +c.LabApp.expose_app_in_browser = True + +c.RetroApp.expose_app_in_browser = True diff --git a/markdown-parser/requirements.txt b/markdown-parser/requirements.txt new file mode 100644 index 00000000..b631a742 --- /dev/null +++ b/markdown-parser/requirements.txt @@ -0,0 +1,5 @@ +jupyterlab[test] +nbconvert +playwright +pytest +requests diff --git a/markdown-parser/tests/__init__.py b/markdown-parser/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/markdown-parser/tests/conftest.py b/markdown-parser/tests/conftest.py new file mode 100644 index 00000000..9a8e76b5 --- /dev/null +++ b/markdown-parser/tests/conftest.py @@ -0,0 +1,56 @@ +# import io +# import logging +# from tempfile import mkdtemp +# import pytest +# from jupyter_server.serverapp import ServerApp +# from jupyterlab.labapp import LabApp +# from traitlets.config import Config + + +# @pytest.fixture(scope="session") +# def jupyterlab_server(): +# ServerApp.clear_instance() +# c = Config() +# c.NotebookNotary.db_file = ":memory:" +# c.setdefault( +# "ServerApp", Config() +# ) +# c.setdefault("LabApp", Config()) +# c.ServerApp.port = 9999 +# c.ServerApp.open_browser = False + +# c.ServerApp.root_dir = mkdtemp(prefix='markdown-parser-lab-') + +# c.ServerApp.token = "" +# c.ServerApp.password = "" +# c.ServerApp.disable_check_xsrf = True + +# # c.LabApp.dev_mode = True +# # c.LabApp.extensions_in_dev_mode = True +# c.LabApp.expose_app_in_browser = True + +# server_app = ServerApp.instance(log_level="DEBUG", config=c) + +# server_app.init_signal = lambda: None +# server_app.log.propagate = True +# server_app.log.handlers = [] + +# server_app.initialize(argv=[], new_httpserver=False) +# # Reroute all logging StreamHandlers away from stdin/stdout since pytest hijacks +# # these streams and closes them at unfortunate times. +# stream_handlers = [ +# h for h in server_app.log.handlers if isinstance(h, logging.StreamHandler) +# ] +# for handler in stream_handlers: +# handler.setStream(io.StringIO()) +# server_app.log.propagate = True +# server_app.log.handlers = [] +# # Start app without ioloop +# server_app.start_app() + +# labapp = LabApp() +# labapp._link_jupyter_server_extension(server_app) +# labapp.initialize() +# yield labapp +# server_app.remove_server_info_file() +# server_app.remove_browser_open_files() diff --git a/markdown-parser/tests/test_jupyter.py b/markdown-parser/tests/test_jupyter.py new file mode 100644 index 00000000..5a117cdd --- /dev/null +++ b/markdown-parser/tests/test_jupyter.py @@ -0,0 +1,25 @@ +import pytest +from playwright.sync_api import sync_playwright +from nbconvert.filters.markdown import markdown2html + +from .utils import commonmark_gfm_tests, get_jupyterlab_rendered_markdown + +counter = 0 + + +def id_gfm_foo(test): + global counter + counter += 1 + return f"Test {counter} " + test.get("section", "") + + +@pytest.mark.parametrize("gfm", commonmark_gfm_tests(), ids=id_gfm_foo) +def test_nbconvert_jupyterlab(gfm): + with sync_playwright() as p: + browser = p.chromium.launch() + page = browser.new_page() + page.goto("http://127.0.0.1:9999/") + lab_html = page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) + print(repr(gfm["markdown"])) + assert lab_html == markdown2html(gfm["markdown"]) + browser.close() diff --git a/markdown-parser/tests/test_jupyterlab.py b/markdown-parser/tests/test_jupyterlab.py new file mode 100644 index 00000000..3d4add1c --- /dev/null +++ b/markdown-parser/tests/test_jupyterlab.py @@ -0,0 +1,24 @@ +import pytest +from playwright.sync_api import sync_playwright + +from .utils import commonmark_gfm_tests, get_jupyterlab_rendered_markdown + +counter = 0 + + +def id_gfm_foo(test): + global counter + counter += 1 + return f"Test {counter} " + test.get("section", "") + + +@pytest.mark.parametrize("gfm", commonmark_gfm_tests(), ids=id_gfm_foo) +def test_gfm_jupyterlab_renderer(gfm): + with sync_playwright() as p: + browser = p.chromium.launch() + page = browser.new_page() + page.goto("http://127.0.0.1:9999/") + lab_html = page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) + print(repr(gfm["markdown"])) + assert lab_html.replace('\n', '') == gfm["html"].replace('\n', '') + browser.close() diff --git a/markdown-parser/tests/test_nbconvert.py b/markdown-parser/tests/test_nbconvert.py new file mode 100644 index 00000000..9b7f3da3 --- /dev/null +++ b/markdown-parser/tests/test_nbconvert.py @@ -0,0 +1,18 @@ +import pytest +from nbconvert.filters.markdown import markdown2html + +from .utils import commonmark_gfm_tests + +counter = 0 + + +def id_gfm_foo(test): + global counter + counter += 1 + return f"Test {counter} " + test.get("section", "") + + +@pytest.mark.parametrize("gfm", commonmark_gfm_tests(), ids=id_gfm_foo) +def test_gfm_nbconvert_markdown2html(gfm): + print(repr(gfm["markdown"])) + assert markdown2html(gfm["markdown"]).replace('\n', '') == gfm["html"].replace('\n', '') diff --git a/markdown-parser/tests/utils.py b/markdown-parser/tests/utils.py new file mode 100644 index 00000000..c30ea199 --- /dev/null +++ b/markdown-parser/tests/utils.py @@ -0,0 +1,59 @@ +import functools +from pathlib import Path +import sys +import tempfile +import zipfile +import requests + +# Commonmark GitHub flavored project +COMMONMARK_GFM_URL = "https://github.com/github/cmark-gfm/archive/refs/heads/master.zip" +SPEC_EXTRACTOR = "cmark-gfm-master/test/spec_tests.py" +SPEC_FILES = ["spec.txt"] + + +@functools.lru_cache(1) +def commonmark_gfm_tests(): + tests = [] + r = requests.get(COMMONMARK_GFM_URL) + + with tempfile.TemporaryDirectory() as tmpdir: + with tempfile.NamedTemporaryFile(mode="wb", suffix=".zip") as tfile: + tfile.write(r.content) + + with zipfile.ZipFile(tfile.name, mode="r") as fzip: + fzip.extractall(tmpdir) + + tmp_path = Path(tmpdir) + test_folder = (tmp_path / SPEC_EXTRACTOR).parent + + sys.path.insert(0, str(test_folder)) + + from spec_tests import get_tests + + for testfile in SPEC_FILES: + tests.extend(get_tests(str(test_folder / testfile))) + + sys.path.remove(str(test_folder)) + + return tests + +# Javascript function to render a markdown string `md` +# to HTML using the JupyterLab parser. +get_jupyterlab_rendered_markdown = """async (md) => { + const app = window.jupyterlab ?? window.jupyterapp; + + const pluginId = '@jupyterlab/rendermime-extension:plugin'; + + const plugin = app._pluginMap[pluginId]; + + if (!plugin.activated) { + await app.activatePlugin(pluginId); + } + + const renderer = plugin.service.createRenderer('text/markdown'); + await renderer.renderModel( + plugin.service.createModel({ data: { 'text/markdown': md } }) + ); + + return renderer.node.innerHTML; +}""" From 03a8e78e4f6f473a82ba00ce504077de6d49c3ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Wed, 20 Apr 2022 13:02:11 +0200 Subject: [PATCH 02/12] [skip ci] Clean-up --- markdown-parser/tests/conftest.py | 56 ------------------------ markdown-parser/tests/test_jupyter.py | 10 +---- markdown-parser/tests/test_jupyterlab.py | 10 +---- markdown-parser/tests/test_nbconvert.py | 10 +---- 4 files changed, 3 insertions(+), 83 deletions(-) delete mode 100644 markdown-parser/tests/conftest.py diff --git a/markdown-parser/tests/conftest.py b/markdown-parser/tests/conftest.py deleted file mode 100644 index 9a8e76b5..00000000 --- a/markdown-parser/tests/conftest.py +++ /dev/null @@ -1,56 +0,0 @@ -# import io -# import logging -# from tempfile import mkdtemp -# import pytest -# from jupyter_server.serverapp import ServerApp -# from jupyterlab.labapp import LabApp -# from traitlets.config import Config - - -# @pytest.fixture(scope="session") -# def jupyterlab_server(): -# ServerApp.clear_instance() -# c = Config() -# c.NotebookNotary.db_file = ":memory:" -# c.setdefault( -# "ServerApp", Config() -# ) -# c.setdefault("LabApp", Config()) -# c.ServerApp.port = 9999 -# c.ServerApp.open_browser = False - -# c.ServerApp.root_dir = mkdtemp(prefix='markdown-parser-lab-') - -# c.ServerApp.token = "" -# c.ServerApp.password = "" -# c.ServerApp.disable_check_xsrf = True - -# # c.LabApp.dev_mode = True -# # c.LabApp.extensions_in_dev_mode = True -# c.LabApp.expose_app_in_browser = True - -# server_app = ServerApp.instance(log_level="DEBUG", config=c) - -# server_app.init_signal = lambda: None -# server_app.log.propagate = True -# server_app.log.handlers = [] - -# server_app.initialize(argv=[], new_httpserver=False) -# # Reroute all logging StreamHandlers away from stdin/stdout since pytest hijacks -# # these streams and closes them at unfortunate times. -# stream_handlers = [ -# h for h in server_app.log.handlers if isinstance(h, logging.StreamHandler) -# ] -# for handler in stream_handlers: -# handler.setStream(io.StringIO()) -# server_app.log.propagate = True -# server_app.log.handlers = [] -# # Start app without ioloop -# server_app.start_app() - -# labapp = LabApp() -# labapp._link_jupyter_server_extension(server_app) -# labapp.initialize() -# yield labapp -# server_app.remove_server_info_file() -# server_app.remove_browser_open_files() diff --git a/markdown-parser/tests/test_jupyter.py b/markdown-parser/tests/test_jupyter.py index 5a117cdd..52aa8103 100644 --- a/markdown-parser/tests/test_jupyter.py +++ b/markdown-parser/tests/test_jupyter.py @@ -4,16 +4,8 @@ from .utils import commonmark_gfm_tests, get_jupyterlab_rendered_markdown -counter = 0 - -def id_gfm_foo(test): - global counter - counter += 1 - return f"Test {counter} " + test.get("section", "") - - -@pytest.mark.parametrize("gfm", commonmark_gfm_tests(), ids=id_gfm_foo) +@pytest.mark.parametrize("gfm", commonmark_gfm_tests()) def test_nbconvert_jupyterlab(gfm): with sync_playwright() as p: browser = p.chromium.launch() diff --git a/markdown-parser/tests/test_jupyterlab.py b/markdown-parser/tests/test_jupyterlab.py index 3d4add1c..8b2271aa 100644 --- a/markdown-parser/tests/test_jupyterlab.py +++ b/markdown-parser/tests/test_jupyterlab.py @@ -3,16 +3,8 @@ from .utils import commonmark_gfm_tests, get_jupyterlab_rendered_markdown -counter = 0 - -def id_gfm_foo(test): - global counter - counter += 1 - return f"Test {counter} " + test.get("section", "") - - -@pytest.mark.parametrize("gfm", commonmark_gfm_tests(), ids=id_gfm_foo) +@pytest.mark.parametrize("gfm", commonmark_gfm_tests()) def test_gfm_jupyterlab_renderer(gfm): with sync_playwright() as p: browser = p.chromium.launch() diff --git a/markdown-parser/tests/test_nbconvert.py b/markdown-parser/tests/test_nbconvert.py index 9b7f3da3..9467a22b 100644 --- a/markdown-parser/tests/test_nbconvert.py +++ b/markdown-parser/tests/test_nbconvert.py @@ -3,16 +3,8 @@ from .utils import commonmark_gfm_tests -counter = 0 - -def id_gfm_foo(test): - global counter - counter += 1 - return f"Test {counter} " + test.get("section", "") - - -@pytest.mark.parametrize("gfm", commonmark_gfm_tests(), ids=id_gfm_foo) +@pytest.mark.parametrize("gfm", commonmark_gfm_tests()) def test_gfm_nbconvert_markdown2html(gfm): print(repr(gfm["markdown"])) assert markdown2html(gfm["markdown"]).replace('\n', '') == gfm["html"].replace('\n', '') From 3a39553a774b2adf0d7242055d3de29a6261a1e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Wed, 20 Apr 2022 13:35:52 +0200 Subject: [PATCH 03/12] Speed tests by using the same web page --- markdown-parser/requirements.txt | 1 + markdown-parser/tests/conftest.py | 82 ++++++++++++++++++++++++ markdown-parser/tests/test_jupyter.py | 13 ++-- markdown-parser/tests/test_jupyterlab.py | 13 ++-- 4 files changed, 91 insertions(+), 18 deletions(-) create mode 100644 markdown-parser/tests/conftest.py diff --git a/markdown-parser/requirements.txt b/markdown-parser/requirements.txt index b631a742..a5e1e7c9 100644 --- a/markdown-parser/requirements.txt +++ b/markdown-parser/requirements.txt @@ -2,4 +2,5 @@ jupyterlab[test] nbconvert playwright pytest +pytest-playwright requests diff --git a/markdown-parser/tests/conftest.py b/markdown-parser/tests/conftest.py new file mode 100644 index 00000000..e2d346a6 --- /dev/null +++ b/markdown-parser/tests/conftest.py @@ -0,0 +1,82 @@ +import os +import pytest +from playwright.sync_api import Error +from slugify import slugify + + +def _build_artifact_test_folder(pytestconfig, request, folder_or_file_name): + # Taken from pytest-playwright + output_dir = pytestconfig.getoption("--output") + return os.path.join(output_dir, slugify(request.node.nodeid), folder_or_file_name) + + +@pytest.fixture(scope="session") +def jupyterlab_page(browser, browser_context_args, pytestconfig, request): + # Merge pytest-playwright fixture context and page at scope module + pages = [] + context = browser.new_context(**browser_context_args) + context.on("page", lambda page: pages.append(page)) + + tracing_option = pytestconfig.getoption("--tracing") + capture_trace = tracing_option in ["on", "retain-on-failure"] + if capture_trace: + context.tracing.start( + name=slugify(request.node.nodeid), + screenshots=True, + snapshots=True, + sources=True, + ) + + page = context.new_page() + page.goto("/lab") + yield page + + # If request.node is missing rep_call, then some error happened during execution + # that prevented teardown, but should still be counted as a failure + failed = request.node.rep_call.failed if hasattr(request.node, "rep_call") else True + + if capture_trace: + retain_trace = tracing_option == "on" or ( + failed and tracing_option == "retain-on-failure" + ) + if retain_trace: + trace_path = _build_artifact_test_folder(pytestconfig, request, "trace.zip") + context.tracing.stop(path=trace_path) + else: + context.tracing.stop() + + screenshot_option = pytestconfig.getoption("--screenshot") + capture_screenshot = screenshot_option == "on" or ( + failed and screenshot_option == "only-on-failure" + ) + if capture_screenshot: + for index, page in enumerate(pages): + human_readable_status = "failed" if failed else "finished" + screenshot_path = _build_artifact_test_folder( + pytestconfig, request, f"test-{human_readable_status}-{index+1}.png" + ) + try: + page.screenshot(timeout=5000, path=screenshot_path) + except Error: + pass + + context.close() + + video_option = pytestconfig.getoption("--video") + preserve_video = video_option == "on" or ( + failed and video_option == "retain-on-failure" + ) + if preserve_video: + for page in pages: + video = page.video + if not video: + continue + try: + video_path = video.path() + file_name = os.path.basename(video_path) + video.save_as( + path=_build_artifact_test_folder(pytestconfig, request, file_name) + ) + except Error: + # Silent catch empty videos. + pass diff --git a/markdown-parser/tests/test_jupyter.py b/markdown-parser/tests/test_jupyter.py index 52aa8103..a4073237 100644 --- a/markdown-parser/tests/test_jupyter.py +++ b/markdown-parser/tests/test_jupyter.py @@ -6,12 +6,7 @@ @pytest.mark.parametrize("gfm", commonmark_gfm_tests()) -def test_nbconvert_jupyterlab(gfm): - with sync_playwright() as p: - browser = p.chromium.launch() - page = browser.new_page() - page.goto("http://127.0.0.1:9999/") - lab_html = page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) - print(repr(gfm["markdown"])) - assert lab_html == markdown2html(gfm["markdown"]) - browser.close() +def test_nbconvert_jupyterlab(jupyterlab_page, gfm): + lab_html = jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) + print(repr(gfm["markdown"])) + assert lab_html == markdown2html(gfm["markdown"]) diff --git a/markdown-parser/tests/test_jupyterlab.py b/markdown-parser/tests/test_jupyterlab.py index 8b2271aa..43b77391 100644 --- a/markdown-parser/tests/test_jupyterlab.py +++ b/markdown-parser/tests/test_jupyterlab.py @@ -5,12 +5,7 @@ @pytest.mark.parametrize("gfm", commonmark_gfm_tests()) -def test_gfm_jupyterlab_renderer(gfm): - with sync_playwright() as p: - browser = p.chromium.launch() - page = browser.new_page() - page.goto("http://127.0.0.1:9999/") - lab_html = page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) - print(repr(gfm["markdown"])) - assert lab_html.replace('\n', '') == gfm["html"].replace('\n', '') - browser.close() +def test_gfm_jupyterlab_renderer(jupyterlab_page, gfm): + lab_html = jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) + print(repr(gfm["markdown"])) + assert lab_html.replace("\n", "") == gfm["html"].replace("\n", "") From fceeb6a7e82e74dc049a40490fe27e4c93b28e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Wed, 20 Apr 2022 13:40:39 +0200 Subject: [PATCH 04/12] Set base-url in CI --- .github/workflows/markdown-parser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/markdown-parser.yml b/.github/workflows/markdown-parser.yml index cc140378..5b02aa36 100644 --- a/.github/workflows/markdown-parser.yml +++ b/.github/workflows/markdown-parser.yml @@ -86,7 +86,7 @@ jobs: working-directory: benchmarks/markdown-parser run: | set -ex - pytest -rap -vv + pytest -rap -vv --base-url http://localhost:9999 - name: Kill the server if: always() From 6bbd444236cb22f8a1daeac4c4f1b5610e363ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Wed, 20 Apr 2022 15:04:53 +0200 Subject: [PATCH 05/12] Fix some doc string --- markdown-parser/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/markdown-parser/tests/conftest.py b/markdown-parser/tests/conftest.py index e2d346a6..112541a2 100644 --- a/markdown-parser/tests/conftest.py +++ b/markdown-parser/tests/conftest.py @@ -12,7 +12,7 @@ def _build_artifact_test_folder(pytestconfig, request, folder_or_file_name): @pytest.fixture(scope="session") def jupyterlab_page(browser, browser_context_args, pytestconfig, request): - # Merge pytest-playwright fixture context and page at scope module + # Merge pytest-playwright fixtures context and page at scope session and load the JupyterLab page pages = [] context = browser.new_context(**browser_context_args) context.on("page", lambda page: pages.append(page)) From 861cec81d738236bff32704a75d31fe7038796a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Thu, 21 Apr 2022 10:14:03 +0200 Subject: [PATCH 06/12] Add reports generation --- .github/workflows/markdown-parser.yml | 12 +++++++- markdown-parser/tests/conftest.py | 39 ++++++++++++++++++++++++ markdown-parser/tests/test_jupyter.py | 28 +++++++++++++---- markdown-parser/tests/test_jupyterlab.py | 26 +++++++++++++--- markdown-parser/tests/test_nbconvert.py | 26 +++++++++++++--- 5 files changed, 115 insertions(+), 16 deletions(-) diff --git a/.github/workflows/markdown-parser.yml b/.github/workflows/markdown-parser.yml index 5b02aa36..d82b12c2 100644 --- a/.github/workflows/markdown-parser.yml +++ b/.github/workflows/markdown-parser.yml @@ -86,7 +86,17 @@ jobs: working-directory: benchmarks/markdown-parser run: | set -ex - pytest -rap -vv --base-url http://localhost:9999 + mkdir -p reports + # For now limit ot JupyterLab vs GFM + pytest -rap -vv --base-url http://localhost:9999 --report-dir "$PWD/reports" tests/test_jupyterlab + + - name: Upload pytest reports + if: always() + uses: actions/upload-artifact@v2 + with: + name: markdown-parser-report-${{ github.run_number }} + path: | + benchmarks/markdown-parser/reports - name: Kill the server if: always() diff --git a/markdown-parser/tests/conftest.py b/markdown-parser/tests/conftest.py index 112541a2..f2dc3ea0 100644 --- a/markdown-parser/tests/conftest.py +++ b/markdown-parser/tests/conftest.py @@ -1,4 +1,5 @@ import os +import pathlib import pytest from playwright.sync_api import Error from slugify import slugify @@ -80,3 +81,41 @@ def jupyterlab_page(browser, browser_context_args, pytestconfig, request): except Error: # Silent catch empty videos. pass + + +def pytest_addoption(parser): + parser.addoption( + "--report-dir", + help="Directory in which the reports must be saved.", + ) + + +@pytest.fixture(scope="module") +def md_report(request): + test_reports = [] + + yield test_reports + + print(test_reports) + + if len(test_reports[0]) > 0: + filename = pathlib.Path(request.config.getoption("report_dir")) / ( + request.module.__name__.replace(".", "_") + "_report.md" + ) + + with filename.open("w") as f: + headers = test_reports[0] + f.writelines( + [ + f"# {request.module.__name__}\n", + "\n", + "| " + " | ".join(headers) + " |\n", + "| " + " | ".join(["---"] * len(headers)) + " |\n", + ] + ) + f.writelines( + map( + lambda e: "| " + " | ".join(map(str, e.values())) + " |\n", + test_reports, + ) + ) diff --git a/markdown-parser/tests/test_jupyter.py b/markdown-parser/tests/test_jupyter.py index a4073237..6b0c9904 100644 --- a/markdown-parser/tests/test_jupyter.py +++ b/markdown-parser/tests/test_jupyter.py @@ -1,12 +1,28 @@ import pytest -from playwright.sync_api import sync_playwright -from nbconvert.filters.markdown import markdown2html +from nbconvert.filters.markdown import markdown2html_mistune from .utils import commonmark_gfm_tests, get_jupyterlab_rendered_markdown @pytest.mark.parametrize("gfm", commonmark_gfm_tests()) -def test_nbconvert_jupyterlab(jupyterlab_page, gfm): - lab_html = jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) - print(repr(gfm["markdown"])) - assert lab_html == markdown2html(gfm["markdown"]) +def test_nbconvert_jupyterlab(jupyterlab_page, md_report, gfm): + given = gfm["markdown"] + test = markdown2html_mistune(given) + ref = jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) + + success = True + try: + assert test == ref + except Exception as e: + success = False + raise e + finally: + md_report.append({ + "id": gfm["example"], + "section": gfm["section"], + "failed": "" if success else "X", + "markdown": repr(given).replace("'", "`"), + "JupyterLab": repr(ref).replace("'", "`"), + "nbconvert - mistune": repr(test).replace("'", "`"), + "comments": "" + }) diff --git a/markdown-parser/tests/test_jupyterlab.py b/markdown-parser/tests/test_jupyterlab.py index 43b77391..d2d8bc74 100644 --- a/markdown-parser/tests/test_jupyterlab.py +++ b/markdown-parser/tests/test_jupyterlab.py @@ -1,11 +1,27 @@ import pytest -from playwright.sync_api import sync_playwright from .utils import commonmark_gfm_tests, get_jupyterlab_rendered_markdown @pytest.mark.parametrize("gfm", commonmark_gfm_tests()) -def test_gfm_jupyterlab_renderer(jupyterlab_page, gfm): - lab_html = jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) - print(repr(gfm["markdown"])) - assert lab_html.replace("\n", "") == gfm["html"].replace("\n", "") +def test_gfm_jupyterlab_renderer(jupyterlab_page, md_report, gfm): + given = gfm["markdown"] + test = jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) + ref = gfm["html"] + + success = True + try: + assert test.replace('\n', '') == ref.replace('\n', '') + except Exception as e: + success = False + raise e + finally: + md_report.append({ + "id": gfm["example"], + "section": gfm["section"], + "failed": "" if success else "X", + "markdown": repr(given).replace("'", "`"), + "commonmark-gfm": repr(ref).replace("'", "`"), + "JupyterLab": repr(test).replace("'", "`"), + "comments": "" + }) diff --git a/markdown-parser/tests/test_nbconvert.py b/markdown-parser/tests/test_nbconvert.py index 9467a22b..4dc38e45 100644 --- a/markdown-parser/tests/test_nbconvert.py +++ b/markdown-parser/tests/test_nbconvert.py @@ -1,10 +1,28 @@ import pytest -from nbconvert.filters.markdown import markdown2html +from nbconvert.filters.markdown import markdown2html_mistune from .utils import commonmark_gfm_tests @pytest.mark.parametrize("gfm", commonmark_gfm_tests()) -def test_gfm_nbconvert_markdown2html(gfm): - print(repr(gfm["markdown"])) - assert markdown2html(gfm["markdown"]).replace('\n', '') == gfm["html"].replace('\n', '') +def test_gfm_nbconvert_markdown2html(md_report, gfm): + given = gfm["markdown"] + test = markdown2html_mistune(given) + ref = gfm["html"] + + success = True + try: + assert test.replace('\n', '') == ref.replace('\n', '') + except Exception as e: + success = False + raise e + finally: + md_report.append({ + "id": gfm["example"], + "section": gfm["section"], + "failed": "" if success else "X", + "markdown": repr(given).replace("'", "`"), + "commonmark-gfm": repr(ref).replace("'", "`"), + "nbconvert - mistune": repr(test).replace("'", "`"), + "comments": "" + }) From 8e3d63e052087f24a533fffc9779f0781f755aad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Thu, 21 Apr 2022 10:34:33 +0200 Subject: [PATCH 07/12] Use commonmark normalization --- markdown-parser/tests/conftest.py | 8 +++++-- markdown-parser/tests/test_jupyter.py | 8 +++++-- markdown-parser/tests/test_jupyterlab.py | 30 ++++++++++++++---------- markdown-parser/tests/test_nbconvert.py | 30 ++++++++++++++---------- markdown-parser/tests/utils.py | 1 + 5 files changed, 49 insertions(+), 28 deletions(-) diff --git a/markdown-parser/tests/conftest.py b/markdown-parser/tests/conftest.py index f2dc3ea0..ec42e65a 100644 --- a/markdown-parser/tests/conftest.py +++ b/markdown-parser/tests/conftest.py @@ -84,6 +84,7 @@ def jupyterlab_page(browser, browser_context_args, pytestconfig, request): def pytest_addoption(parser): + """Add option to set the comparison reports""" parser.addoption( "--report-dir", help="Directory in which the reports must be saved.", @@ -92,12 +93,15 @@ def pytest_addoption(parser): @pytest.fixture(scope="module") def md_report(request): + """Generate a comparison report for each test module. + + Each test must return a dictionary with the same keys. Each keys will be + a table header and each test will be a table row. + """ test_reports = [] yield test_reports - print(test_reports) - if len(test_reports[0]) > 0: filename = pathlib.Path(request.config.getoption("report_dir")) / ( request.module.__name__.replace(".", "_") + "_report.md" diff --git a/markdown-parser/tests/test_jupyter.py b/markdown-parser/tests/test_jupyter.py index 6b0c9904..d72b5924 100644 --- a/markdown-parser/tests/test_jupyter.py +++ b/markdown-parser/tests/test_jupyter.py @@ -6,9 +6,13 @@ @pytest.mark.parametrize("gfm", commonmark_gfm_tests()) def test_nbconvert_jupyterlab(jupyterlab_page, md_report, gfm): + + # Import normalize helper from github/cmark-gfm through its addition to sys.path in commonmark_gfm_tests + from normalize import normalize_html + given = gfm["markdown"] - test = markdown2html_mistune(given) - ref = jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) + test = normalize_html(markdown2html_mistune(given)) + ref = normalize_html(jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"])) success = True try: diff --git a/markdown-parser/tests/test_jupyterlab.py b/markdown-parser/tests/test_jupyterlab.py index d2d8bc74..7f94cbb4 100644 --- a/markdown-parser/tests/test_jupyterlab.py +++ b/markdown-parser/tests/test_jupyterlab.py @@ -5,23 +5,29 @@ @pytest.mark.parametrize("gfm", commonmark_gfm_tests()) def test_gfm_jupyterlab_renderer(jupyterlab_page, md_report, gfm): + + # Import normalize helper from github/cmark-gfm through its addition to sys.path in commonmark_gfm_tests + from normalize import normalize_html + given = gfm["markdown"] - test = jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"]) - ref = gfm["html"] + test = normalize_html(jupyterlab_page.evaluate(get_jupyterlab_rendered_markdown, gfm["markdown"])) + ref = normalize_html(gfm["html"]) success = True try: - assert test.replace('\n', '') == ref.replace('\n', '') + assert test == ref except Exception as e: success = False raise e finally: - md_report.append({ - "id": gfm["example"], - "section": gfm["section"], - "failed": "" if success else "X", - "markdown": repr(given).replace("'", "`"), - "commonmark-gfm": repr(ref).replace("'", "`"), - "JupyterLab": repr(test).replace("'", "`"), - "comments": "" - }) + md_report.append( + { + "id": gfm["example"], + "section": gfm["section"], + "failed": "" if success else "X", + "markdown": repr(given).replace("'", "`"), + "commonmark-gfm": repr(ref).replace("'", "`"), + "JupyterLab": repr(test).replace("'", "`"), + "comments": "", + } + ) diff --git a/markdown-parser/tests/test_nbconvert.py b/markdown-parser/tests/test_nbconvert.py index 4dc38e45..205588a2 100644 --- a/markdown-parser/tests/test_nbconvert.py +++ b/markdown-parser/tests/test_nbconvert.py @@ -6,23 +6,29 @@ @pytest.mark.parametrize("gfm", commonmark_gfm_tests()) def test_gfm_nbconvert_markdown2html(md_report, gfm): + + # Import normalize helper from github/cmark-gfm through its addition to sys.path in commonmark_gfm_tests + from normalize import normalize_html + given = gfm["markdown"] - test = markdown2html_mistune(given) - ref = gfm["html"] + test = normalize_html(markdown2html_mistune(given)) + ref = normalize_html(gfm["html"]) success = True try: - assert test.replace('\n', '') == ref.replace('\n', '') + assert test == ref except Exception as e: success = False raise e finally: - md_report.append({ - "id": gfm["example"], - "section": gfm["section"], - "failed": "" if success else "X", - "markdown": repr(given).replace("'", "`"), - "commonmark-gfm": repr(ref).replace("'", "`"), - "nbconvert - mistune": repr(test).replace("'", "`"), - "comments": "" - }) + md_report.append( + { + "id": gfm["example"], + "section": gfm["section"], + "failed": "" if success else "X", + "markdown": repr(given).replace("'", "`"), + "commonmark-gfm": repr(ref).replace("'", "`"), + "nbconvert - mistune": repr(test).replace("'", "`"), + "comments": "", + } + ) diff --git a/markdown-parser/tests/utils.py b/markdown-parser/tests/utils.py index c30ea199..96c9519a 100644 --- a/markdown-parser/tests/utils.py +++ b/markdown-parser/tests/utils.py @@ -29,6 +29,7 @@ def commonmark_gfm_tests(): sys.path.insert(0, str(test_folder)) from spec_tests import get_tests + from normalize import normalize_html for testfile in SPEC_FILES: tests.extend(get_tests(str(test_folder / testfile))) From f90969f9497c72fe8dd1b9436a98763715f194c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Thu, 21 Apr 2022 10:41:01 +0200 Subject: [PATCH 08/12] Debug CI --- .github/workflows/markdown-parser.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/markdown-parser.yml b/.github/workflows/markdown-parser.yml index d82b12c2..6d0c194b 100644 --- a/.github/workflows/markdown-parser.yml +++ b/.github/workflows/markdown-parser.yml @@ -86,9 +86,10 @@ jobs: working-directory: benchmarks/markdown-parser run: | set -ex + echo $PWD mkdir -p reports # For now limit ot JupyterLab vs GFM - pytest -rap -vv --base-url http://localhost:9999 --report-dir "$PWD/reports" tests/test_jupyterlab + pytest -rap -vv --base-url "http://localhost:9999" --report-dir "$PWD/reports" tests/test_jupyterlab - name: Upload pytest reports if: always() From 9269f8942a4370470ed55071a16319f8f68f0a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Thu, 21 Apr 2022 10:45:56 +0200 Subject: [PATCH 09/12] Force rootdir --- .github/workflows/markdown-parser.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/markdown-parser.yml b/.github/workflows/markdown-parser.yml index 6d0c194b..6dd0f310 100644 --- a/.github/workflows/markdown-parser.yml +++ b/.github/workflows/markdown-parser.yml @@ -34,17 +34,17 @@ jobs: restore-keys: | ${{ runner.os }}-pip-3.8 - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Cache yarn - uses: actions/cache@v1 - id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- + # - name: Get yarn cache directory path + # id: yarn-cache-dir-path + # run: echo "::set-output name=dir::$(yarn cache dir)" + # - name: Cache yarn + # uses: actions/cache@v1 + # id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + # with: + # path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + # key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + # restore-keys: | + # ${{ runner.os }}-yarn- - name: Install dependencies working-directory: benchmarks/markdown-parser @@ -86,10 +86,9 @@ jobs: working-directory: benchmarks/markdown-parser run: | set -ex - echo $PWD mkdir -p reports # For now limit ot JupyterLab vs GFM - pytest -rap -vv --base-url "http://localhost:9999" --report-dir "$PWD/reports" tests/test_jupyterlab + pytest -rap -vv --rootdir "$PWD" --base-url "http://localhost:9999" --report-dir "$PWD/reports" tests/test_jupyterlab - name: Upload pytest reports if: always() From 6bd4c3b229190760fbfd97c3781814d204c82356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Thu, 21 Apr 2022 10:51:41 +0200 Subject: [PATCH 10/12] Move addoption up --- markdown-parser/conftest.py | 6 ++++++ markdown-parser/tests/conftest.py | 8 -------- 2 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 markdown-parser/conftest.py diff --git a/markdown-parser/conftest.py b/markdown-parser/conftest.py new file mode 100644 index 00000000..d699ea1c --- /dev/null +++ b/markdown-parser/conftest.py @@ -0,0 +1,6 @@ +def pytest_addoption(parser): + """Add option to set the comparison reports""" + parser.addoption( + "--report-dir", + help="Directory in which the reports must be saved.", + ) diff --git a/markdown-parser/tests/conftest.py b/markdown-parser/tests/conftest.py index ec42e65a..76ead114 100644 --- a/markdown-parser/tests/conftest.py +++ b/markdown-parser/tests/conftest.py @@ -83,14 +83,6 @@ def jupyterlab_page(browser, browser_context_args, pytestconfig, request): pass -def pytest_addoption(parser): - """Add option to set the comparison reports""" - parser.addoption( - "--report-dir", - help="Directory in which the reports must be saved.", - ) - - @pytest.fixture(scope="module") def md_report(request): """Generate a comparison report for each test module. From ed6b83f4583d226ff763faeb2b2dfe8caa7fd7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Thu, 21 Apr 2022 10:58:01 +0200 Subject: [PATCH 11/12] Fix test filename --- .github/workflows/markdown-parser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/markdown-parser.yml b/.github/workflows/markdown-parser.yml index 6dd0f310..e5a3f3ad 100644 --- a/.github/workflows/markdown-parser.yml +++ b/.github/workflows/markdown-parser.yml @@ -88,7 +88,7 @@ jobs: set -ex mkdir -p reports # For now limit ot JupyterLab vs GFM - pytest -rap -vv --rootdir "$PWD" --base-url "http://localhost:9999" --report-dir "$PWD/reports" tests/test_jupyterlab + pytest -rap -vv --rootdir "$PWD" --base-url "http://localhost:9999" --report-dir "$PWD/reports" tests/test_jupyterlab.py - name: Upload pytest reports if: always() From a11a37fbdf63fdf860d1594b4cf143475799b266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Wed, 1 Mar 2023 12:01:41 +0100 Subject: [PATCH 12/12] Force trigger --- markdown-parser/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/markdown-parser/README.md b/markdown-parser/README.md index 7cc34e01..d671d9c9 100644 --- a/markdown-parser/README.md +++ b/markdown-parser/README.md @@ -5,3 +5,4 @@ This folder contains tests to validate markdown parsers. The tests are written using [pytest](https://docs.pytest.org). To test web frontend parsers, [playwright](https://playwright.dev/python/docs/intro) is used to evaluate the markdown conversion to HTML. > The test database is downloaded from [GitHub flavored Commonmark](https://github.com/github/cmark-gfm). +