update doc sources and flow and add justfile

Author: JosiahParry

README.md

@@ -4,10 +4,16 @@ This repository contains the public facing source of https://developers.arcgis.c
 
 We use this repository to build out the internal docs infrastructure. This way, you, the community can contribute. 
 
-## Building the docs
+## Building docs
 
-To build **API Reference** run `dev/generate-api-ref.R`. This will clone all of the repositories and build the documentation using pkgdown and some other magic. 
+```bash
+# update api reference
+just doc-ref
+
+# update articles
+just doc-pages
+```
 
-## Building the articles
+To build **API Reference** run `dev/generate-api-ref.R`. This will clone all of the repositories and build the documentation using pkgdown and some other magic. 
 
-To build all other documentation run `dev/generate-docs.R`. This will render `.qmd` files using R Markdown and perform syntax hilighting with downlit. 
+To build all other documentation run `dev/generate-docs.R`. This will render `.qmd` files using R Markdown and perform syntax hilighting with downlit. 

_docs.zip

@@ -16,21 +16,28 @@ pkgs <- c(
 )
 
 generate_api_ref <- function(pkg) {
+  cli::cli_h1("Processing {.pkg {pkg}}")
+
   tmp <- tempdir()
   clone_dir <- file.path(tmp, pkg)
 
   if (file.exists(clone_dir)) {
     unlink(clone_dir, recursive = TRUE)
   }
+
   # clone into a temporary directory
+  cli::cli_progress_step("Cloning {.pkg {pkg}} from GitHub")
   gert::git_clone(
     sprintf("https://github.com/r-arcgis/%s", pkg),
     path = clone_dir
   )
 
   # init pkgdown in the temporary folder
+  cli::cli_progress_step("Initialising pkgdown site")
   pkgdown::init_site(clone_dir)
+
   # generate reference docs
+  cli::cli_progress_step("Building reference docs")
   pkgdown::build_reference(clone_dir, preview = FALSE)
 
   all_pkg_files <- list.files(
@@ -40,6 +47,7 @@ generate_api_ref <- function(pkg) {
   )
 
   # create the api reference directory for the package
+  cli::cli_progress_step("Creating output directories")
   dir.create(
     file.path("_api_ref", pkg),
     recursive = TRUE
@@ -73,6 +81,7 @@ generate_api_ref <- function(pkg) {
     vapply(strsplit(to_copy, "reference/"), `[[`, character(1), 2)
   )
 
+  cli::cli_progress_step("Copying {length(to_copy)} non-HTML asset{?s}")
   file.copy(to_copy, copy_dest, overwrite = TRUE)
 
   # this function extract the html content
@@ -83,20 +92,20 @@ generate_api_ref <- function(pkg) {
   }
 
   # apply to all html files
+  cli::cli_progress_step("Extracting content from {length(fn_ref_html)} HTML file{?s}")
   extracted <- lapply(fn_ref_html, extract_fn_ref)
   out_ref_paths <- file.path("_api_ref", pkg, basename(fn_ref_html))
 
   # write the html files
+  cli::cli_progress_step("Writing {length(out_ref_paths)} reference page{?s}")
   Map(
-    function(.html, .path) {
-      cli::cli_alert_info("Writing {.file {(.path)}}")
-      writeLines(.html, .path)
-    },
+    function(.html, .path) writeLines(.html, .path),
     extracted,
     out_ref_paths
   )
 
   # Process the index file
+  cli::cli_progress_step("Processing package index")
   index_html <-
     rvest::read_html(file.path(clone_dir, "docs", "reference", "index.html")) |>
     rvest::html_nodes("#main") |>
@@ -110,11 +119,14 @@ generate_api_ref <- function(pkg) {
   )
 
   unlink(tmp)
+  cli::cli_alert_success("Done with {.pkg {pkg}}")
 }
 
+cli::cli_h1("Generating API references")
 for (pkg in pkgs) {
   generate_api_ref(pkg)
 }
+cli::cli_alert_success("All packages processed")
 
 # NOTE url paths are adjusted manually using find and replace all w/ regex in vs code
 # search: https://r\.esri\.com/([^/]+)/reference/([^/]+)\.html
@@ -190,10 +202,15 @@ as_topic_nav <- function(pkg_indices) {
     yaml::as.yaml(indent.mapping.sequence = TRUE)
 }
 
+cli::cli_h1("Building topic navigation")
+cli::cli_progress_step("Generating nav YAML from {length(pkg_indices)} package index{?es}")
 as_topic_nav(pkg_indices) |>
   brio::write_file("_api_ref/nav.yml")
+cli::cli_alert_success("Written {.file _api_ref/nav.yml}")
 
+cli::cli_progress_step("Zipping {.file api-ref.zip}")
 zip::zip(
   "api-ref.zip",
   list.files("_api_ref", recursive = TRUE, full.names = TRUE)
 )
+cli::cli_alert_success("Done — {.file api-ref.zip} created")

api-ref.zip

@@ -19,6 +19,7 @@ render_qmd_to_md <- function(in_path, out_path, work_dir = dirname(in_path)) {
   # #> [1] "Hello world"
 
   # extract header from the .qmd file
+  cli::cli_progress_step("Extracting YAML front matter")
   header <- rmarkdown::yaml_front_matter(in_path)
   if (length(header) > 0) {
     md_header <- c("---", paste(names(header), header, sep = ": "), "---", "")
@@ -32,6 +33,7 @@ render_qmd_to_md <- function(in_path, out_path, work_dir = dirname(in_path)) {
   knitr::opts_chunk$set(comment = "#>")
 
   # render the section alone
+  cli::cli_progress_step("Rendering {.file {in_path}} via rmarkdown")
   out <- rmarkdown::render(
     in_path,
     rmarkdown::github_document(),
@@ -46,17 +48,19 @@ render_qmd_to_md <- function(in_path, out_path, work_dir = dirname(in_path)) {
 
   md_body <- NULL
   # add autolinking and syntax highlighting (we will have to choose colors manually later)
+  cli::cli_progress_step("Applying downlit autolinking")
   tryCatch(
     {
       downlit::downlit_md_path(tmp, out_path = out_path)
       md_body <- readLines(out_path)
     },
     error = function(e) {
-      cli::cli_alert_danger("Failed apply downlit to {.file {in_path}}")
+      cli::cli_alert_danger("Failed to apply downlit to {.file {in_path}}")
       md_body <<- readLines(tmp)
       # file.copy(tmp, out_path, TRUE)
     }
   )
+  cli::cli_progress_step("Writing {.file {out_path}}")
   final_content <- c(md_header, "", md_body)
   writeLines(final_content, out_path)
 }
@@ -92,6 +96,8 @@ render_with_retries <- function(
   return(success)
 }
 
+cli::cli_h1("Collecting files")
+
 all_files <- list.files(
   c("images", "docs"),
   full.names = TRUE,
@@ -101,6 +107,7 @@ all_files <- list.files(
 
 # all quarto docs
 in_fps <- list.files(pattern = "*.qmd", recursive = TRUE)
+cli::cli_alert_info("Found {length(in_fps)} .qmd file{?s} and {length(all_files)} total asset{?s}")
 
 # remove quarto docs from all_files
 to_copy <- setdiff(all_files, in_fps)
@@ -116,33 +123,38 @@ out_fps <- paste0(
 )
 
 # create directories
+cli::cli_progress_step("Creating output directories")
 for (dirp in unique(dirname(c(out_fps, copy_dest)))) {
   if (!dir.exists(dirp)) {
     dir.create(dirp, recursive = TRUE)
   }
 }
 
 # copy all of the non-quarto files
+cli::cli_progress_step("Copying {length(to_copy)} non-Quarto asset{?s}")
 file.copy(
   to_copy,
   copy_dest,
   overwrite = TRUE
 )
 
+cli::cli_progress_step("Processing images")
 source("dev/imgs.R")
 
+cli::cli_h1("Rendering documents")
 
 # render all of the files
 failed_files <- c()
-for (i in 1:length(in_fps)) {
+n <- length(in_fps)
+for (i in seq_along(in_fps)) {
   ip <- in_fps[[i]]
   op <- out_fps[[i]]
   # check if rendered md already exists
   if (file.exists(op)) {
-    cli::cli_alert_info("Skipping {.file {ip}}. Output exists at {.file {op}}")
+    cli::cli_alert_info("[{i}/{n}] Skipping {.file {ip}} — output already exists")
     next
   }
-  cli::cli_alert_info("Rendering # file {i}: {.file {ip}} to {.file {op}}")
+  cli::cli_h2("[{i}/{n}] {ip}")
   # always unset the arcgis token first
   arcgisutils::unset_arc_token()
   # render and retry up to 3 times (default) if it fails
@@ -153,13 +165,15 @@ for (i in 1:length(in_fps)) {
   }
 }
 
+cli::cli_h1("Summary")
 if (length(failed_files) > 0) {
-  cli::cli_alert_danger("The following files failed after retries:")
-  print(failed_files)
+  cli::cli_alert_danger("{length(failed_files)} file{?s} failed after retries:")
+  cli::cli_ul(failed_files)
 } else {
-  cli::cli_alert_success("All files rendered or copied successfully.")
+  cli::cli_alert_success("All {n} file{?s} rendered or skipped successfully")
 }
 
+cli::cli_progress_step("Zipping {.file _docs.zip}")
 zip::zip(
   "_docs.zip",
   list.files(
@@ -168,3 +182,4 @@ zip::zip(
     recursive = TRUE
   )
 )
+cli::cli_alert_success("Done — {.file _docs.zip} created")

dev/attachments.qmd

@@ -0,0 +1,8 @@
+default:
+  just --list
+
+doc-ref:
+  R -f dev/generate-api-ref.R
+
+doc-pages:
+  R -f dev/generate-docs.R