Merge pull request #253 from machow/feat-interlinks-fast

Feat interlinks fast

Author: machow

_extensions/interlinks/_extension.yml

@@ -1,6 +1,6 @@
 title: Interlinks
 author: Michael Chow
-version: 1.0.0
+version: 1.1.0
 quarto-required: ">=1.2.0"
 contributes:
   filters:

_extensions/interlinks/interlinks.lua

@@ -1,11 +1,77 @@
+local function read_inv_text(filename)
+    -- read file
+    local file = io.open(filename, "r")
+    if file == nil then
+        return nil
+    end
+    local str = file:read("a")
+    file:close()
+
+
+    local project = str:match("# Project: (%S+)")
+    local version = str:match("# Version: (%S+)")
+
+    local data = {project = project, version = version, items = {}}
+
+    local ptn_data =
+        "^" ..
+        "(.-)%s+" ..        -- name
+        "([%S:]-):" ..      -- domain
+        "([%S]+)%s+" ..     -- role
+        "(%-?%d+)%s+" ..     -- priority
+        "(%S*)%s+" ..       -- uri
+        "(.-)\r?$"         -- dispname
+
+
+    -- Iterate through each line in the file content
+    for line in str:gmatch("[^\r\n]+") do
+        if not line:match("^#") then
+            -- Match each line against the pattern
+            local name, domain, role, priority, uri, dispName = line:match(ptn_data)
+
+            -- if name is nil, raise an error
+            if name == nil then
+                error("Error parsing line: " .. line)
+            end
+
+            data.items[#data.items + 1] = {
+                name = name,
+                domain = domain,
+                role = role,
+                priority = priority,
+                uri = uri,
+                dispName = dispName
+            }
+        end
+    end
+    return data
+end
+
 local function read_json(filename)
+
     local file = io.open(filename, "r")
     if file == nil then
         return nil
     end
     local str = file:read("a")
     file:close()
-    return quarto.json.decode(str)
+
+    local decoded = quarto.json.decode(str)
+    return decoded
+end
+
+local function read_inv_text_or_json(base_name)
+    local file = io.open(base_name .. ".txt", "r")
+    if file then
+        -- TODO: refactors so we don't just close the file immediately
+        io.close(file)
+        json = read_inv_text(base_name .. ".txt")
+
+    else
+        json = read_json(base_name .. ".json")
+    end
+
+    return json
 end
 
 local inventory = {}
@@ -167,11 +233,12 @@ return {
             local json
             local prefix
             for k, v in pairs(meta.interlinks.sources) do
-                json = read_json(quarto.project.offset .. "/_inv/" .. k .. "_objects.json")
+                local base_name = quarto.project.offset .. "/_inv/" .. k .. "_objects"
+                json = read_inv_text_or_json(base_name)
                 prefix = pandoc.utils.stringify(v.url)
                 fixup_json(json, prefix)
             end
-            json = read_json(quarto.project.offset .. "/objects.json")
+            json = read_inv_text_or_json(quarto.project.offset .. "/objects")
             if json ~= nil then
                 fixup_json(json, "/")
             end

docs/get-started/interlinks.qmd

@@ -54,6 +54,22 @@ Notice 2 important pieces in this config:
 By default, downloaded inventory files will be saved in the `_inv` folder of your
 documentation directory.
 
+### Experimental fast option
+
+Use the experimental `fast: true` option to speed up the interlinks filter.
+
+```yaml
+interlinks:
+  fast: true
+  sources:
+```
+
+By default inventory files are saved as JSON, but this option keeps them as text files,
+and attempts to parse them much faster.
+
+:::{.callout-warning}
+Be sure to install the latest version of the interlinks filter, using `quarto add machow/quartodoc`.
+:::
 
 ## Running the interlinks filter
 

quartodoc/__main__.py

@@ -212,30 +212,44 @@ def build(config, filter, dry_run, watch, verbose):
 @click.command()
 @click.argument("config", default="_quarto.yml")
 @click.option("--dry-run", is_flag=True, default=False)
-def interlinks(config, dry_run):
+@click.option("--fast", is_flag=True, default=False)
+def interlinks(config, dry_run, fast):
+    # config loading ----
     cfg = yaml.safe_load(open(config))
-    interlinks = cfg.get("interlinks", None)
-
-    cache = cfg.get("cache", "_inv")
+    interlinks = cfg.get("interlinks", {})
 
     p_root = Path(config).parent
 
-    if interlinks is None:
+    if not interlinks:
         print("No interlinks field found in your quarto config. Quitting.")
         return
 
+    # interlinks config settings ----
+    cache = p_root / "_inv"
+    cfg_fast = interlinks.get("fast", False)
+
+    fast = cfg_fast or fast
+
     for k, v in interlinks["sources"].items():
-        # TODO: user shouldn't need to include their own docs in interlinks
+        # don't include user's own docs (users don't need to specify their own docs in
+        # the interlinks config anymore, so this is for backwards compat).
         if v["url"] == "/":
             continue
 
         url = v["url"] + v.get("inv", "objects.inv")
         inv = soi.Inventory(url=url)
 
-        p_dst = p_root / cache / f"{k}_objects.json"
+        p_dst = p_root / cache / f"{k}_objects"
         p_dst.parent.mkdir(exist_ok=True, parents=True)
 
-        convert_inventory(inv, p_dst)
+        if fast:
+            # use sphobjinv to dump inv in txt format
+            df = inv.data_file()
+            soi.writebytes(p_dst.with_suffix(".txt"), df)
+
+        else:
+            # old behavior of converting to custom json format
+            convert_inventory(inv, p_dst.with_suffix(".json"))
 
 
 cli.add_command(build)

quartodoc/autosummary.py

@@ -446,6 +446,7 @@ def __init__(
         source_dir: "str | None" = None,
         dynamic: bool | None = None,
         parser="numpy",
+        _fast_inventory=False,
     ):
         self.layout = self.load_layout(
             sections=sections, package=package, options=options
@@ -467,6 +468,8 @@ def __init__(
         self.source_dir = str(Path(source_dir).absolute()) if source_dir else None
         self.dynamic = dynamic
 
+        self._fast_inventory = _fast_inventory
+
     def load_layout(self, sections: dict, package: str, options=None):
         # TODO: currently returning the list of sections, to make work with
         # previous code. We should make Layout a first-class citizen of the
@@ -522,7 +525,16 @@ def build(self, filter: str = "*"):
 
         _log.info("Creating inventory file")
         inv = self.create_inventory(items)
-        convert_inventory(inv, self.out_inventory)
+        if self._fast_inventory:
+            # dump the inventory file directly as text
+            # TODO: copied from __main__.py, should add to inventory.py
+            import sphobjinv as soi
+
+            df = inv.data_file()
+            soi.writebytes(Path(self.out_inventory).with_suffix(".txt"), df)
+
+        else:
+            convert_inventory(inv, self.out_inventory)
 
         # sidebar ----
 
@@ -647,12 +659,18 @@ def from_quarto_config(cls, quarto_cfg: "str | dict"):
 
             quarto_cfg = yaml.safe_load(open(quarto_cfg))
 
-        cfg = quarto_cfg["quartodoc"]
+        cfg = quarto_cfg.get("quartodoc")
+        if cfg is None:
+            raise KeyError("No `quartodoc:` section found in your _quarto.yml.")
         style = cfg.get("style", "pkgdown")
-
         cls_builder = cls._registry[style]
 
-        return cls_builder(**{k: v for k, v in cfg.items() if k != "style"})
+        _fast_inventory = quarto_cfg.get("interlinks", {}).get("fast", False)
+
+        return cls_builder(
+            **{k: v for k, v in cfg.items() if k != "style"},
+            _fast_inventory=_fast_inventory,
+        )
 
 
 class BuilderPkgdown(Builder):