refactor: improve performance; Python 3.11 min (#101)

* refactor: improve performance

Improves performance by making better use of concurrency for extracting
data from Markdown files and processing images

* docs: update dependencies

* breaking: make minimum Python 3.11

Updates the minimum Python version to 3.11 for its built-in tomllib and
removes the toml dependency.

toml wasn't pickling properly, causing issues with multiprocessing

* breaking: change non-schema variable prefix

Changes the non-schema front matter variable prefix from ~ to _ so they
don't have to be quoted. tomllib throws an error for an unquoted key
starting with "~"

* docs: add note about non-schema front matter

Adds a note to the Markdown docs about non-schema front matter.

Also adds a link to the settings file in GitHub to show the default
values

Author: johnfraney

.github/workflows/github-actions-nox.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.11", "3.12", "3.13"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

blurry/__init__.py

@@ -63,7 +63,6 @@ def process_non_markdown_file(
 ):
     # Process Jinja files
     if ".jinja" in filepath.suffixes:
-        # Process file
         process_jinja_file(filepath, jinja_env, file_data_by_directory)
         return
 
@@ -196,22 +195,31 @@ def gather_file_data_by_directory() -> DirectoryFileData:
     file_data_by_directory: DirectoryFileData = {}
     content_directory = get_content_directory()
 
-    for filepath in content_directory.glob("**/*.md"):
-        # Extract filepath for storing context data and writing out
-        relative_filepath = filepath.relative_to(content_directory)
-        directory = relative_filepath.parent
-
-        # Convert Markdown file to HTML
-        body, front_matter = convert_markdown_file_to_html(filepath)
-        file_data = MarkdownFileData(
-            body=body,
-            front_matter=front_matter,
-            path=relative_filepath,
-        )
-        try:
-            file_data_by_directory[directory].append(file_data)
-        except KeyError:
-            file_data_by_directory[directory] = [file_data]
+    markdown_future_to_path: dict[concurrent.futures.Future, Path] = {}
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        for filepath in content_directory.rglob("*.md"):
+            # Extract filepath for storing context data and writing out
+            relative_filepath = filepath.relative_to(content_directory)
+
+            # Convert Markdown file to HTML
+            future = executor.submit(convert_markdown_file_to_html, filepath)
+            markdown_future_to_path[future] = relative_filepath
+
+        for future in concurrent.futures.as_completed(markdown_future_to_path):
+            body, front_matter = future.result()
+            relative_filepath = markdown_future_to_path[future]
+            file_data = MarkdownFileData(
+                body=body,
+                front_matter=front_matter,
+                path=relative_filepath,
+            )
+            parent_directory = relative_filepath.parent
+            try:
+                file_data_by_directory[parent_directory].append(file_data)
+            except KeyError:
+                file_data_by_directory[parent_directory] = [file_data]
+
+    concurrent.futures.wait(markdown_future_to_path)
 
     return sort_directory_file_data_by_date(file_data_by_directory)
 

blurry/images.py

@@ -1,4 +1,5 @@
 import concurrent.futures
+from concurrent.futures import Future
 from pathlib import Path
 
 from wand.image import Image
@@ -43,38 +44,53 @@ def convert_image_to_avif(image_path: Path, target_path: Path | None = None):
 
 
 def clone_and_resize_image(
-    image: Image, target_width: int, resized_image_destination: Path
+    image_path: Path, target_width: int, resized_image_destination: Path
 ):
     if resized_image_destination.exists():
         return
-    image.transform(resize=str(target_width))
-    image.save(filename=resized_image_destination)
+    with Image(filename=str(image_path)) as image:
+        image.transform(resize=str(target_width))
+        image.save(filename=resized_image_destination)
 
 
 async def generate_images_for_srcset(image_path: Path):
     BUILD_DIR = get_build_directory()
     CONTENT_DIR = get_content_directory()
-    filepaths_to_convert_to_avif = []
+    image_futures: list[Future] = []
 
     build_path = BUILD_DIR / image_path.resolve().relative_to(CONTENT_DIR)
     with concurrent.futures.ThreadPoolExecutor() as executor:
-        with Image(filename=str(image_path)) as img:
-            width = img.width
-
-            for target_width in get_widths_for_image_width(width):
-                new_filepath = add_image_width_to_path(image_path, target_width)
-                relative_filepath = new_filepath.resolve().relative_to(CONTENT_DIR)
-                build_filepath = BUILD_DIR / relative_filepath
-                # We convert the resized images to AVIF, so do this synchronously
-                clone_and_resize_image(img, target_width, build_filepath)
-                filepaths_to_convert_to_avif.append(build_filepath)
-
         # Convert original image
-        executor.submit(
+        full_sized_avif_future = executor.submit(
             convert_image_to_avif, image_path=image_path, target_path=build_path
         )
-        # Generate AVIF files for resized images
-        executor.map(convert_image_to_avif, filepaths_to_convert_to_avif)
+        image_futures.append(full_sized_avif_future)
+
+        # Store resized image futures so AVIF versions can be created once they're done
+        resize_future_to_filepath: dict[Future, Path] = {}
+
+        width = Image(filename=str(image_path)).width
+
+        for target_width in get_widths_for_image_width(width):
+            new_filepath = add_image_width_to_path(image_path, target_width)
+            new_filepath_in_build = BUILD_DIR / new_filepath.relative_to(CONTENT_DIR)
+
+            resized_original_image_type_future = executor.submit(
+                clone_and_resize_image, image_path, target_width, new_filepath_in_build
+            )
+            resize_future_to_filepath[
+                resized_original_image_type_future
+            ] = new_filepath_in_build
+
+        # Create AVIF versions of resized images as they're ready
+        for future in concurrent.futures.as_completed(resize_future_to_filepath):
+            resized_image_filepath = resize_future_to_filepath[future]
+            resized_avif_future = executor.submit(
+                convert_image_to_avif, resized_image_filepath
+            )
+            image_futures.append(resized_avif_future)
+
+        concurrent.futures.wait(image_futures + list(resize_future_to_filepath.keys()))
 
 
 def get_widths_for_image_width(image_width: int) -> list[int]:

blurry/markdown/front_matter.py

@@ -1,10 +1,10 @@
 import re
+import tomllib
 from collections.abc import MutableMapping
 from pathlib import Path
 from typing import Any
 from typing import TypeGuard
 
-import toml
 from mistune import BlockState
 from mistune import Markdown
 
@@ -30,7 +30,7 @@ def get_data(doc: str) -> tuple[str, MutableMapping]:
     if toml_block_match:
         toml_part = toml_block_match.group(1)
         try:
-            data = toml.loads(toml_part)
+            data = tomllib.loads(toml_part)
         except Exception as e:
             print("Parsing TOML failed: ", e)
         try:

blurry/settings.py

@@ -1,8 +1,7 @@
+import tomllib
 from os import environ
 from typing import TypedDict
 
-import toml
-
 from blurry.constants import CURR_DIR
 from blurry.constants import ENV_VAR_PREFIX
 from blurry.constants import SETTINGS_FILENAME
@@ -44,14 +43,14 @@ class Settings(TypedDict):
     "VIDEO_EXTENSIONS": ["mp4", "webm", "mkv"],
     "USE_HTTP": False,
     "RUNSERVER": False,
-    "FRONTMATTER_NON_SCHEMA_VARIABLE_PREFIX": "~",
+    "FRONTMATTER_NON_SCHEMA_VARIABLE_PREFIX": "_",
     "TEMPLATE_SCHEMA_TYPES": {},
 }
 
 
 def update_settings():
     try:
-        blurry_config = toml.load(open(SETTINGS_FILENAME))
+        blurry_config = tomllib.load(open(SETTINGS_FILENAME, "rb"))
         user_settings = blurry_config["blurry"]
         for setting, value in user_settings.items():
             SETTINGS[setting.upper()] = value

docs/content/configuration/settings.md

@@ -20,3 +20,5 @@ The setting hierarchy is:
 See the `Settings` type for Blurry's available settings:
 
 @python<blurry.settings.Settings>
+
+The default values are visible at <https://github.com/blurry-dev/blurry/blob/main/blurry/settings.py>

docs/content/content/markdown.md

@@ -20,13 +20,18 @@ name = "About Blurry"
 abstract = "Learn about Blurry, a static site generator build for page speed and SEO"
 datePublished = 2023-01-07
 image = "../images/blurry-logo.png"
+_is_this_available_in_a_template = True
 +++
 
 # About Blurry
 
 Regular Markdown content can go here.
 ```
 
+:::{info}
+Variables that are not in a Schema.org type but are useful in templates should start with the `FRONTMATTER_NON_SCHEMA_VARIABLE_PREFIX` [setting](../configuration/settings.md), which defaults to an underscore (`_`).
+:::
+
 ## Customizations
 
 On top of [Mistune's built-in plugins](https://mistune.lepture.com/en/latest/plugins.html), Blurry ships with a number of Markdown customizations.