Implement preview command (#139)

* Implement dev command

* Make quality

* Update src/doc_builder/autodoc.py

Co-authored-by: Sylvain Gugger <[email protected]>

* Update src/doc_builder/build_doc.py

Co-authored-by: Sylvain Gugger <[email protected]>

* Update src/doc_builder/build_doc.py

Co-authored-by: Sylvain Gugger <[email protected]>

* `src_py_path_mapping` -> `source_files_mapping`

* `src_py` -> `source_files`

* Add soft-dependency `watchdog`

* Update is_watchdog_available

* Command `preview`

* Rm mac-specific `open` command

* Reserved sveltekit keywords

* Simpler routing base for preview mode

* make style

* rm -rf

* superlcass FileSystemEventHandler init

* try catch subsequent builds

* Make style

* Open localhost:3000 automicalyy

* just log

* debug file watcher

* formatted string

* fix hardcoded val

* Update kit readme

* Chore

* Fix relative_path bug

* Stop vite auto server open

* Better automatic browser open

* Cleaner delete file

* Add preview readme

* doc typo

* Support markdown

* Fix

Co-authored-by: Sylvain Gugger <[email protected]>

Author: mishig25

README.md

@@ -17,6 +17,19 @@ cd doc-builder
 pip install -e .
 ```
 
+## Previewing
+
+To preview the docs, use the following command:
+
+```bash
+doc-builder preview {package_name} {path_to_docs}
+```
+
+For example:
+```bash
+doc-builder preview datasets ~/Desktop/datasets/docs/source/
+```
+
 ## Doc building
 
 To build the documentation of a given package, use the following command:

kit/README.md

@@ -13,4 +13,4 @@ The generated html files and assets are in the `build` folder.
 
 ## Previewing the docs
 
-Instead of `npm run build`, do `npm run dev`. Then go to http://localhost:3000/docs/transformers/master/en (or replace `master` with the correct prefix).
+Instead of `npm run build`, do `npm run dev`. Then go to http://localhost:3000 (or replace `master` with the correct prefix).

kit/package.json

@@ -2,7 +2,7 @@
 	"name": "kit",
 	"version": "0.0.1",
 	"scripts": {
-		"dev": "svelte-kit dev",
+		"dev": "svelte-kit dev -o",
 		"build": "rm -Rf src/routes/__pycache__ && svelte-kit build && ./postbuild.sh",
 		"package": "svelte-kit package",
 		"preview": "svelte-kit preview",

kit/svelte.config.js

@@ -30,17 +30,18 @@ const config = {
 		vite: {
 			build: {
 				sourcemap: Boolean(process.env.DOCS_SOURCEMAP)
-			}
+			},
 		},
 
 		paths: {
-			base:
-				"/docs/" +
-				(process.env.DOCS_LIBRARY || "transformers") +
-				"/" +
-				(process.env.DOCS_VERSION || "master") +
-				"/" +
-				(process.env.DOCS_LANGUAGE || "en")
+			base: process.argv.includes("dev")
+				? ""
+				: "/docs/" +
+				  (process.env.DOCS_LIBRARY || "transformers") +
+				  "/" +
+				  (process.env.DOCS_VERSION || "master") +
+				  "/" +
+				  (process.env.DOCS_LANGUAGE || "en")
 		}
 	},
 

src/doc_builder/autodoc.py

@@ -287,6 +287,19 @@ def get_source_link(obj, page_info):
     return f"{base_link}{module}.py#L{line_number}"
 
 
+def get_source_path(object_name, package):
+    """
+    Find a path to file in which given object was defined.
+
+    Args:
+    - object_name (`str`): The name of the object to retrieve.
+    - package (`types.ModuleType`): The package to look into.
+    """
+    obj = obj = find_object_in_package(object_name=object_name, package=package)
+    obj_path = inspect.getfile(obj)
+    return obj_path
+
+
 def document_object(object_name, package, page_info, full_name=True):
     """
     Writes the document of a function, class or method.

src/doc_builder/build_doc.py

@@ -24,7 +24,7 @@
 import yaml
 from tqdm import tqdm
 
-from .autodoc import autodoc, find_object_in_package, remove_example_tags, resolve_links_in_text
+from .autodoc import autodoc, find_object_in_package, get_source_path, remove_example_tags, resolve_links_in_text
 from .convert_md_to_mdx import convert_md_to_mdx
 from .convert_rst_to_mdx import convert_rst_to_mdx, find_indent, is_empty_line
 from .convert_to_notebook import generate_notebooks_from_file
@@ -87,6 +87,7 @@ def resolve_autodoc(content, package, return_anchors=False, page_info=None):
     is_inside_codeblock = False
     lines = content.split("\n")
     new_lines = []
+    source_files = None
     if return_anchors:
         anchors = []
         errors = []
@@ -125,6 +126,8 @@ def resolve_autodoc(content, package, return_anchors=False, page_info=None):
                 errors.extend(doc[2])
                 doc = doc[0]
             new_lines.append(doc)
+
+            source_files = get_source_path(object_name, package)
         else:
             new_lines.append(lines[idx])
             if lines[idx].startswith("```"):
@@ -136,7 +139,7 @@ def resolve_autodoc(content, package, return_anchors=False, page_info=None):
     new_content = "\n".join(new_lines)
     new_content = remove_example_tags(new_content)
 
-    return (new_content, anchors, errors) if return_anchors else new_content
+    return (new_content, anchors, source_files, errors) if return_anchors else new_content
 
 
 def build_mdx_files(package, doc_folder, output_dir, page_info):
@@ -153,6 +156,7 @@ def build_mdx_files(package, doc_folder, output_dir, page_info):
     output_dir = Path(output_dir)
     os.makedirs(output_dir, exist_ok=True)
     anchor_mapping = {}
+    source_files_mapping = {}
 
     if "package_name" not in page_info:
         page_info["package_name"] = package.__name__
@@ -171,9 +175,11 @@ def build_mdx_files(package, doc_folder, output_dir, page_info):
                     content = reader.read()
                 content = convert_md_to_mdx(content, page_info)
                 content = resolve_open_in_colab(content, page_info)
-                content, new_anchors, errors = resolve_autodoc(
+                content, new_anchors, source_files, errors = resolve_autodoc(
                     content, package, return_anchors=True, page_info=page_info
                 )
+                if source_files is not None:
+                    source_files_mapping[source_files] = str(file)
                 with open(dest_file, "w", encoding="utf-8") as writer:
                     writer.write(content)
                 # Make sure we clean up for next page.
@@ -186,9 +192,11 @@ def build_mdx_files(package, doc_folder, output_dir, page_info):
                     content = reader.read()
                 content = convert_rst_to_mdx(content, page_info)
                 content = resolve_open_in_colab(content, page_info)
-                content, new_anchors, errors = resolve_autodoc(
+                content, new_anchors, source_files, errors = resolve_autodoc(
                     content, package, return_anchors=True, page_info=page_info
                 )
+                if source_files is not None:
+                    source_files_mapping[source_files] = str(file)
                 with open(dest_file, "w", encoding="utf-8") as writer:
                     writer.write(content)
                 # Make sure we clean up for next page.
@@ -213,7 +221,7 @@ def build_mdx_files(package, doc_folder, output_dir, page_info):
             "The deployment of the documentation will fail because of the following errors:\n" + "\n".join(all_errors)
         )
 
-    return anchor_mapping
+    return anchor_mapping, source_files_mapping
 
 
 def resolve_links(doc_folder, package, mapping, page_info):
@@ -351,6 +359,7 @@ def build_doc(
     language="en",
     notebook_dir=None,
     is_python_module=False,
+    watch_mode=False,
 ):
     """
     Build the documentation of a package.
@@ -368,6 +377,9 @@ def build_doc(
             If provided, where to save the notebooks generated from the doc file with an [[open-in-colab]] marker.
         is_python_module (`bool`, *optional*, defaults to `False`):
             Whether the docs being built are for python module. (For example, HF Course is not a python module).
+        watch_mode (`bool`, *optional*, default to `False`):
+            If `True`, disables the toc tree check and sphinx objects.inv builds since they are not needed
+            when this mode is active.
     """
     page_info = {"version": version, "language": language, "package_name": package_name}
     if clean and Path(output_dir).exists():
@@ -376,11 +388,13 @@ def build_doc(
     read_doc_config(doc_folder)
 
     package = importlib.import_module(package_name) if is_python_module else None
-    anchors_mapping = build_mdx_files(package, doc_folder, output_dir, page_info)
-    sphinx_refs = check_toc_integrity(doc_folder, output_dir)
-    sphinx_refs.extend(convert_anchors_mapping_to_sphinx_format(anchors_mapping, package))
+    anchors_mapping, source_files_mapping = build_mdx_files(package, doc_folder, output_dir, page_info)
+    if not watch_mode:
+        sphinx_refs = check_toc_integrity(doc_folder, output_dir)
+        sphinx_refs.extend(convert_anchors_mapping_to_sphinx_format(anchors_mapping, package))
     if is_python_module:
-        build_sphinx_objects_ref(sphinx_refs, output_dir, page_info)
+        if not watch_mode:
+            build_sphinx_objects_ref(sphinx_refs, output_dir, page_info)
         resolve_links(output_dir, package, anchors_mapping, page_info)
     generate_frontmatter(output_dir)
 
@@ -390,6 +404,8 @@ def build_doc(
                 os.remove(nb_file)
         build_notebooks(doc_folder, notebook_dir, package=package, mapping=anchors_mapping, page_info=page_info)
 
+    return source_files_mapping
+
 
 def check_toc_integrity(doc_folder, output_dir):
     """

src/doc_builder/commands/doc_builder_cli.py

@@ -18,6 +18,7 @@
 
 from doc_builder.commands.build import build_command_parser
 from doc_builder.commands.convert_doc_file import convert_command_parser
+from doc_builder.commands.preview import preview_command_parser
 from doc_builder.commands.style import style_command_parser
 
 
@@ -29,6 +30,7 @@ def main():
     convert_command_parser(subparsers=subparsers)
     build_command_parser(subparsers=subparsers)
     style_command_parser(subparsers=subparsers)
+    preview_command_parser(subparsers=subparsers)
 
     # Let's go
     args = parser.parse_args()