Merge pull request #16 from 360-info/dev

v0.0.2

Author: jimjam-slam

.quartoignore

@@ -1,2 +1,6 @@
 README.md
+NEWS.md
 LICENSE
+docs/
+.quarto/
+!.gitignore

NEWS.md

@@ -0,0 +1,12 @@
+## Sverto 0.0.2
+
+- Bump minimum Quarto version to 1.3.0.
+- Fixes for compatibility with newer Quarto 1.3 pre-releases
+  - Quarto's switch from Pandoc 2 to Pandoc 3 caused some issues with the way Sverto identifies Svelte import statements. This should no longer be a problem.
+- We now take advantage of the improved `.quartoignore` functionality in Quarto 1.3 to:
+  1. avoid copying the `docs` folder in with the project template; and
+  2. include the `.gitignore` with the template
+
+## 0.0.1
+
+- Initial release

_extensions/sverto/_extension.yml

@@ -1,7 +1,7 @@
 title: Sverto
 author: 360info
-version: 0.0.1
-quarto-version: ">=1.2.0"
+version: 0.0.2
+quarto-version: ">=1.3.0"
 contributes:
   project:
     project:

_extensions/sverto/create-imports.lua

@@ -1,3 +1,11 @@
+-- create-imports: a project pre-render script that:
+-- * replaces svelte_import() ojs statements in qmd files, saving to /.sverto
+-- * writes svelte import paths to /.sverto s othey can be compiled
+
+-- some content from quarto's qmd-reader.lua
+-- (c) 2023 rstudio, pbc. licensed under gnu gpl v2:
+-- https://github.com/quarto-dev/quarto-cli/blob/main/COPYRIGHT
+
 -- return contents of named file
 function read_file(name)
   local file = io.open(name, "r")
@@ -53,10 +61,114 @@ function path_dir(path)
     return path:match("(.*".. get_path_sep() ..")") or ""
 end
 
--- TODO - 
+-- content following from quarto's qmd-reader.lua
+-- (c) 2023 rstudio, pbc. licensed under gnu gpl v2:
+-- https://github.com/quarto-dev/quarto-cli/blob/main/COPYRIGHT
+
+function random_string(size)
+  -- we replace invalid tags with random strings of the same size
+  -- to safely allow code blocks inside pipe tables
+  -- note that we can't use uppercase letters here
+  -- because pandoc canonicalizes classes to lowercase.
+  local chars = "abcdefghijklmnopqrstuvwxyz"
+  local lst = {}
+  for _ = 1,size do
+    local ix = math.random(1, #chars)
+    table.insert(lst, string.sub(chars, ix, ix))
+  end
+  return table.concat(lst, "")
+end
+
+function find_invalid_tags(str)
+  -- [^.=\n]
+  --   we disallow "." to avoid catching {.python}
+  --   we disallow "=" to avoid catching {foo="bar"}
+  --   we disallow "\n" to avoid multiple lines
+
+  -- no | in lua patterns...
+
+  -- (c standard, 7.4.1.10, isspace function)
+  -- %s catches \n and \r, so we must use [ \t\f\v] instead
+
+  local patterns = {
+    "^[ \t\f\v]*(```+[ \t\f\v]*)(%{+[^.=\n\r]*%}+)", 
+    "\n[ \t\f\v]*(```+[ \t\f\v]*)(%{+[^.=\n\r]+%}+)"
+  }
+  local function find_it(init)
+    for _, pattern in ipairs(patterns) do
+      local range_start, range_end, ticks, tag = str:find(pattern, init)
+      if range_start ~= nil then
+        return range_start, range_end, ticks, tag
+      end
+    end
+    return nil
+  end
+
+  local init = 1
+  local range_start, range_end, ticks, tag = find_it(init)
+  local tag_set = {}
+  local tags = {}
+  while tag ~= nil do
+    init = range_end + 1
+    if not tag_set[tag] then
+      tag_set[tag] = true
+      table.insert(tags, tag)
+    end
+    range_start, range_end, ticks, tag = find_it(init)
+  end
+  return tags
+end
+
+function escape_invalid_tags(str)
+  local tags = find_invalid_tags(str)
+  -- we must now replace the tags in a careful order. Specifically,
+  -- we can't replace a key that's a substring of a larger key without
+  -- first replacing the larger key.
+  --
+  -- ie. if we replace {python} before {{python}}, Bad Things Happen.
+  -- so we sort the tags by descending size, which suffices
+  table.sort(tags, function(a, b) return #b < #a end)
+
+  local replacements = {}
+  for _, k in ipairs(tags) do
+    local replacement
+    local attempts = 1
+    repeat
+      replacement = random_string(#k)
+      attempts = attempts + 1
+    until str:find(replacement, 1, true) == nil or attempts == 100
+    if attempts == 100 then
+      print("Internal error, could not find safe replacement for "..k.." after 100 tries")
+      print("Please file a bug at https://github.com/quarto-dev/quarto-cli")
+      os.exit(1)
+    end
+    -- replace all lua special pattern characters with their
+    -- escaped versions
+    local safe_pattern = k:gsub("([%^%$%(%)%%%.%[%]%*%+%-%?])", "%%%1")
+    replacements[replacement] = k
+    local patterns = {
+      "^([ \t\f\v]*```+[ \t\f\v]*)" .. safe_pattern,
+      "(\n[ \t\f\v]*```+[ \t\f\v]*)" .. safe_pattern
+    }
+
+    str = str:gsub(patterns[1], "%1" .. replacement):gsub(patterns[2], "%1" .. replacement)
+  end
+  return str, replacements
+end
+
+function unescape_invalid_tags(str, tags)
+  for replacement, k in pairs(tags) do
+    -- replace all lua special replacement characters with their
+    -- escaped versions, so that when we restore the behavior,
+    -- we don't accidentally create a pattern
+    local result = k:gsub("([$%%])", "%%%1")
+    str = str:gsub(replacement, result)
+  end
+  return str
+end
 
 local preprocess_qmd_filter = {
-  
+
   -- search for `import_svelte("X.svelte")` refs in codeblocks and switch them
   -- to `import("X.js")`
   CodeBlock = function(block)
@@ -114,15 +226,62 @@ end
 -- transform each input qmd, saving the transformation in .sverto/[path]
 -- (write the identified .svelte files out to a file too!)
 for key, qmd_path in ipairs(in_files) do
-  
-  local doc = pandoc.read(read_file(qmd_path))
+
+  -- before we read the file in with pandoc.read, let's read it in as a raw
+  -- string and do the quarto team's qmd-reader processing on it. THEN we can
+  -- read it with pandoc.read
+
+  print(">>> PROCESSING " .. qmd_path)
+
+  local raw_doc = read_file(qmd_path)
 
   -- store the current qmd_path on disk so the filter can access it
   write_file(".sverto/.sverto-current-qmd-folder", path_dir(qmd_path))
+
+  -- escape invalid tags
+  local txt, tags = escape_invalid_tags(tostring(raw_doc))
+
+  -- some extension + format stuff that we can maybe ignore?
+  
+  -- for k, v in pairs(opts.extensions) do
+  --   extensions[v] = true
+  -- end
+
+  -- if param("user-defined-from") then
+  --   local user_format = _quarto.format.parse_format(param("user-defined-from"))
+  --   for k, v in pairs(user_format.extensions) do
+  --     extensions[k] = v
+  --   end
+  -- end
+
+  -- -- Format flavor, i.e., which extensions should be enabled/disabled
+  -- local flavor = {
+  --   format = "markdown",
+  --   extensions = extensions,
+  -- }
+
+  local function restore_invalid_tags(tag)
+    return tags[tag] or tag
+  end
+
+  -- NOW we read in with pandoc (hopefully ending up with real code blocks)
+  -- and restore them
+  -- local doc = pandoc.read(txt, flavor, opts):walk {
+  local doc = pandoc.read(txt, "markdown")
+  
+  local restored_doc = doc:walk {
+    CodeBlock = function (cb)
+      cb.classes = cb.classes:map(restore_invalid_tags)
+      cb.text = unescape_invalid_tags(cb.text, tags)
+      return cb
+    end
+  }
+
+  -- local doc = pandoc.read(read_file(qmd_path))
 
   -- pre-process the qmd, populating `svelte_files` in the process
   -- local svelte_files = {}
-  local transformed_doc = doc:walk(preprocess_qmd_filter)
+  local transformed_doc = restored_doc:walk(preprocess_qmd_filter)
   create_dir_recursively(".sverto/" .. path_dir(qmd_path))
   write_file(".sverto/" .. qmd_path, pandoc.write(transformed_doc, "markdown"))
 
@@ -132,3 +291,4 @@ end
 write_file(".sverto/.sverto-outdir", os.getenv("QUARTO_PROJECT_OUTPUT_DIR"))
 
 -- TODO - if there's no {{< import .sverto/file.qmd >}} block, add it?
+