Use a safer, two-stage transformation in readqmd.lua (#13792)

Use a safer two-stage transformation to send shortcodes through the Markdown parsing layer.

This avoids us clobbering the existing Markdown structure when shortcodes are invoked inside link targets. Pandoc 3.7 and later are more strict in where spaces can appear inside links and image targets, and this transformation should now work in that setting.

Author: cscheid

src/resources/pandoc/datadir/lpegshortcode.lua

@@ -313,12 +313,64 @@ local function wrap_lpeg_match(pattern, txt)
   return txt
 end
 
+-- Convert a string to its hexadecimal representation
+local function string_to_hex(str)
+    return (str:gsub('.', function(c)
+        return string.format('%02X', string.byte(c))
+    end))
+end
+
+local md_shortcode_2_uuid         = "b58fc729-690b-4000-b19f-365a4093b2ff"
+local md_shortcode_2_uuid_pattern = "b58fc729%-690b%-4000%-b19f%-365a4093b2ff%-"
+local function md_escaped_shortcode_2_fun(s)
+  return table.concat({
+    md_shortcode_2_uuid,
+    "-",
+    string_to_hex("{{{<" .. s .. ">}}}"),
+    "-"
+  })
+end
+
+local function md_shortcode_2_fun(open, space, lst, close)
+  local raw = open .. space
+  for i = 1, #lst do
+    local un = unshortcode:match(lst[i]) 
+    raw = raw .. (un or lst[i])
+  end
+  raw = raw .. close
+  return table.concat({
+    md_shortcode_2_uuid,
+    "-",
+    string_to_hex(raw),
+    "-"
+  });
+end
+
+-- This new transformation into a plain UUID-guarded string,
+-- is designed to survive the pandoc markdown reader barrier under Pandoc 3.7 and later.
+-- we still need the first shortcode transformation to actually convert
+-- to a span when it's safe to do so, but this transformation
+-- is safe to use in all contexts (including link and image targets).
+local md_shortcode_2 = make_shortcode_parser({
+  escaped = md_escaped_shortcode_2_fun,
+  string = md_string_param,
+  keyvalue = md_keyvalue_param,
+  shortcode = md_shortcode_2_fun,
+  ignore_pattern = lpeg.P("{.hidden .quarto-markdown-envelope-contents render-id=\"") * (lpeg.P(1) - lpeg.P("\"}"))^1 * lpeg.P("\"}")
+})
+
 return {
   lpegs = {
     md_shortcode = md_shortcode,
+    md_shortcode_2 = md_shortcode_2,
+    md_shortcode_2_uuid = md_shortcode_2_uuid_pattern,
     unshortcode = unshortcode -- for undoing shortcodes in non-markdown contexts
   },
 
+  parse_md_shortcode_2 = function(txt)
+    return wrap_lpeg_match(md_shortcode_2, txt)
+  end,
+
   parse_md_shortcode = function(txt)
     return wrap_lpeg_match(md_shortcode, txt)
   end,

src/resources/pandoc/datadir/readqmd.lua

@@ -113,23 +113,20 @@ local function unescape_invalid_tags(str, tags)
   return str
 end
 
-local function urldecode(url)
-  if url == nil then
-    return
-  end
-    url = url:gsub("+", " ")
-    url = url:gsub("%%(%x%x)", function(x)
-      return string.char(tonumber(x, 16))
-    end)
-    url = url:gsub('%&quot%;', '"')
-  return url
+-- Convert a hexadecimal string back to the original string
+local function hex_to_string(hex)
+    return (hex:gsub('..', function(cc)
+        return string.char(tonumber(cc, 16))
+    end))
 end
 
 local function readqmd(txt, opts)
+  local uuid_pattern = "b58fc729%-690b%-4000%-b19f%-365a4093b2ff%-([A-Fa-f0-9]+)%-"
   local tags
   txt = md_fenced_div.attempt_to_fix_fenced_div(txt)
   txt, tags = escape_invalid_tags(txt)
-  txt = md_shortcode.parse_md_shortcode(txt)
+  txt = md_shortcode.parse_md_shortcode_2(txt)
+  print(txt)
   local flavor = {
     format = "markdown",
     extensions = {},
@@ -151,17 +148,27 @@ local function readqmd(txt, opts)
   -- so we need to undo that damage here
 
   local unshortcode_text = function (c)
-    if c.text:match("data%-is%-shortcode%=%\"1%\"") then
-      c.text = md_shortcode.unparse_md_shortcode(c.text)
-    end
+    c.text = c.text:gsub(uuid_pattern, hex_to_string)
     return c
   end
 
   local function filter_attrs(el)
     for k,v in pairs(el.attributes) do
-      if type(v) == "string" and v:match("data%-is%-shortcode%=%\"1%\"") then
-        local new_v = md_shortcode.unparse_md_shortcode(v)
-        el.attributes[k] = new_v
+      if type(v) == "string" then
+        local new_str = v:gsub(uuid_pattern, hex_to_string)
+        -- we avoid always assigning to slightly workaround 
+        -- what appears to be a foundational problem with Pandoc's Lua API
+        -- while accessing attributes with repeated keys.
+        -- Quarto is still going to be broken for the case
+        -- where there are shortcodes inside values of attributes with
+        -- repeated keys:
+        --
+        -- []{k='{{< meta k1 >}}' k='{{< meta k2 >}}'}
+        --
+        -- But I don't know how to work around this.
+        if new_str ~= v then
+          el.attributes[k] = new_str
+        end
       end
     end
     return el
@@ -170,9 +177,7 @@ local function readqmd(txt, opts)
   local doc = pandoc.read(txt or "", flavor, opts):walk {
     CodeBlock = function (cb)
       cb.classes = cb.classes:map(restore_invalid_tags)
-      if cb.text:match("data%-is%-shortcode%=%\"1%\"") then
-        cb.text = md_shortcode.unparse_md_shortcode(cb.text)
-      end
+      cb.text = cb.text:gsub(uuid_pattern, hex_to_string)
       cb.text = unescape_invalid_tags(cb.text, tags)
       return cb
     end,
@@ -184,20 +189,61 @@ local function readqmd(txt, opts)
     Div = filter_attrs,
     Link = function (l)
       l = filter_attrs(l)
-      if l.target:match("data%-is%-shortcode%=%%221%%22") then
-        l.target = md_shortcode.unparse_md_shortcode(urldecode(l.target))
-        return l
-      end
+      l.target = l.target:gsub(uuid_pattern, hex_to_string)
       return l
     end,
     Image = function (i)
       i = filter_attrs(i)
-      if i.src:match("data%-is%-shortcode%=%%221%%22") then
-        i.src = md_shortcode.unparse_md_shortcode(urldecode(i.src))
-        return i
-      end
+      -- Replace UUID-encoded shortcodes in i.src
+      i.src = i.src:gsub(uuid_pattern, hex_to_string)
       return i
     end,
+    Str = function(str_node)
+      local str = str_node.text
+      -- Quick check: if UUID not present at all, return as-is
+      if not str:find("b58fc729-690b-4000-b19f-365a4093b2ff", 1, true) then
+        return nil
+      end
+
+      local result = pandoc.Inlines{}
+      local pos = 1
+
+      while true do
+        local match_start, match_end, hex_content = str:find(uuid_pattern, pos)
+
+        if not match_start then
+          -- No more matches; append remaining string if any
+          if pos <= #str then
+            table.insert(result, pandoc.Str(str:sub(pos)))
+          end
+          break
+        end
+
+        -- Append prefix before the match as a Str node (if non-empty)
+        if match_start > pos then
+          table.insert(result, pandoc.Str(str:sub(pos, match_start - 1)))
+        end
+
+        -- Convert hex to original shortcode string
+        local shortcode_text = hex_to_string(hex_content)
+
+        -- Parse the shortcode to markdown span syntax
+        local parsed_md = md_shortcode.parse_md_shortcode(shortcode_text) or ""
+
+        -- Convert to Pandoc inlines via pandoc.read
+        local doc = pandoc.read(parsed_md, "markdown")
+        local inlines = doc.blocks[1] and doc.blocks[1].content or pandoc.Inlines{}
+        -- Append the inlines to result
+        for _, inline in ipairs(inlines) do
+          table.insert(result, inline)
+        end
+
+        -- Move position past the match
+        pos = match_end + 1
+      end
+
+      return result
+    end
   }
   return doc
 end