change parsing to handle values after = and arrays

Author: Jesse Haigh

Sources/SwiftDocC/Checker/Checkers/InvalidCodeBlockOption.swift

@@ -31,33 +31,29 @@ public struct InvalidCodeBlockOption: Checker {
     }
 
     public mutating func visitCodeBlock(_ codeBlock: CodeBlock) {
-        let info = codeBlock.language?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
-        guard !info.isEmpty else { return }
+        let (lang, tokens) = tokenizeLanguageString(codeBlock.language)
 
-        // TODO this will also fail on parsing highlight values with commas inside the array
-        let tokens = info
-            .split(separator: ",")
-            .map { $0.trimmingCharacters(in: .whitespaces) }
-            .filter { !$0.isEmpty }
+        func matches(token: RenderBlockContent.CodeListing.OptionName, value: String?) {
+            guard token == .unknown, let value = value else { return }
 
-        guard !tokens.isEmpty else { return }
-
-        for token in tokens {
-            // if the token is an exact match, we don't need to do anything
-            guard !knownOptions.contains(token) else { continue }
-
-            let matches = NearMiss.bestMatches(for: knownOptions, against: token)
+            let matches = NearMiss.bestMatches(for: knownOptions, against: value)
 
             if !matches.isEmpty {
-                let diagnostic = Diagnostic(source: sourceFile, severity: .warning, range: codeBlock.range, identifier: "org.swift.docc.InvalidCodeBlockOption", summary: "Unknown option \(token.singleQuoted) in code block.")
+                let diagnostic = Diagnostic(source: sourceFile, severity: .warning, range: codeBlock.range, identifier: "org.swift.docc.InvalidCodeBlockOption", summary: "Unknown option \(value.singleQuoted) in code block.")
                 let possibleSolutions = matches.map { candidate in
                     Solution(
-                        summary: "Replace \(token.singleQuoted) with \(candidate.singleQuoted).",
+                        summary: "Replace \(value.singleQuoted) with \(candidate.singleQuoted).",
                         replacements: []
                     )
                 }
                 problems.append(Problem(diagnostic: diagnostic, possibleSolutions: possibleSolutions))
             }
         }
+
+        for (token, value) in tokens {
+            matches(token: token, value: value)
+        }
+        // check if first token (lang) might be a typo
+        matches(token: .unknown, value: lang)
     }
 }

Sources/SwiftDocC/Model/Rendering/Content/RenderBlockContent.swift

@@ -132,6 +132,7 @@ public enum RenderBlockContent: Equatable {
             case nocopy
             case wrap
             case highlight
+            case unknown
 
             init?<S: StringProtocol>(caseInsensitive raw: S) {
                 self.init(rawValue: raw.lowercased())

Sources/SwiftDocC/Model/Rendering/RenderContentCompiler.swift

@@ -49,51 +49,10 @@ struct RenderContentCompiler: MarkupVisitor {
         // Default to the bundle's code listing syntax if one is not explicitly declared in the code block.
 
         if FeatureFlags.current.isExperimentalCodeBlockEnabled {
-
-            func parseLanguageString(_ input: String?) -> (lang: String? , tokens: [(RenderBlockContent.CodeListing.OptionName, Substring?)]) {
-                guard let input else { return (lang: nil, tokens: []) }
-                // TODO this fails on parsing highlight values with commas inside the array
-                let parts = input
-                    .split(separator: ",")
-                    .map { $0.trimmingCharacters(in: .whitespaces) }
-                var lang: String? = nil
-                var options: [(RenderBlockContent.CodeListing.OptionName, Substring?)] = []
-
-                for part in parts {
-                    if let eq = part.firstIndex(of: "=") {
-                        let name = part[..<eq].trimmingCharacters(in: .whitespaces)
-                        let value = part[part.index(after: eq)...]
-                        if let option = RenderBlockContent.CodeListing.OptionName(caseInsensitive: name) {
-                            options.append((option, value))
-                        } else if lang == nil {
-                            lang = String(part)
-                        }
-                    } else {
-                        if let option = RenderBlockContent.CodeListing.OptionName(caseInsensitive: part) {
-                            options.append((option, nil))
-                        } else if lang == nil {
-                            lang = String(part)
-                        }
-                    }
-                }
-                return (lang, options)
-            }
-
-            func parseHighlight(_ value: Substring?) -> [Int]? {
-                guard var s = value.map(String.init) else { return nil }
-                s = s.trimmingCharacters(in: .whitespaces)
-                if s.hasPrefix("[") && s.hasSuffix("]") {
-                    s.removeFirst()
-                    s.removeLast()
-                }
-                let ints = s.split(separator: ",").compactMap{ Int($0.trimmingCharacters(in: .whitespaces)) }
-                return ints.isEmpty ? nil : ints
-            }
-
-            let options = parseLanguageString(codeBlock.language)
+            let (lang, tokens) = tokenizeLanguageString(codeBlock.language)
 
             var listing = RenderBlockContent.CodeListing(
-                syntax: options.lang ?? bundle.info.defaultCodeListingLanguage,
+                syntax: lang ?? bundle.info.defaultCodeListingLanguage,
                 code: codeBlock.code.splitByNewlines,
                 metadata: nil,
                 copyToClipboard: true, // default value
@@ -102,7 +61,7 @@ struct RenderContentCompiler: MarkupVisitor {
             )
 
             // apply code block options
-            for (option, value) in options.tokens {
+            for (option, value) in tokens {
                 switch option {
                 case .nocopy:
                     listing.copyToClipboard = false
@@ -114,6 +73,8 @@ struct RenderContentCompiler: MarkupVisitor {
                     }
                 case .highlight:
                     listing.highlight = parseHighlight(value) ?? []
+                case .unknown:
+                    break
                 }
             }
 

Sources/SwiftDocC/Utility/ParseLanguageString.swift

@@ -0,0 +1,90 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2025 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See https://swift.org/LICENSE.txt for license information
+ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+public func parseHighlight(_ value: String?) -> [Int]? {
+    guard var s = value?.trimmingCharacters(in: .whitespaces), !s.isEmpty else { return [] }
+
+    if s.hasPrefix("[") && s.hasSuffix("]") {
+        s.removeFirst()
+        s.removeLast()
+    }
+
+    return s.split(separator: ",").compactMap { Int($0.trimmingCharacters(in: .whitespaces)) }
+}
+
+/// A function that parses the language line options on code blocks, returning the language and tokens, an array of OptionName and option values
+public func tokenizeLanguageString(_ input: String?) -> (lang: String?, tokens: [(RenderBlockContent.CodeListing.OptionName, String?)]) {
+    guard let input else { return (lang: nil, tokens: []) }
+
+    let parts = parseLanguageString(input)
+    var tokens: [(RenderBlockContent.CodeListing.OptionName, String?)] = []
+    var lang: String? = nil
+
+    for (index, part) in parts.enumerated() {
+        if let eq = part.firstIndex(of: "=") {
+            let key = part[..<eq].trimmingCharacters(in: .whitespaces).lowercased()
+            let value = part[part.index(after: eq)...].trimmingCharacters(in: .whitespaces)
+            if key == "wrap" {
+                tokens.append((.wrap, value))
+            } else if key == "highlight" {
+                tokens.append((.highlight, value))
+            } else {
+                tokens.append((.unknown, key))
+            }
+        } else {
+            let key = part.trimmingCharacters(in: .whitespaces).lowercased()
+            if key == "nocopy" {
+                tokens.append((.nocopy, nil as String?))
+            } else if key == "wrap" {
+                tokens.append((.wrap, nil as String?))
+            } else if key == "highlight" {
+                tokens.append((.highlight, nil as String?))
+            } else if index == 0 && !key.contains("[") && !key.contains("]") {
+                lang = key
+            } else {
+                tokens.append((.unknown, key))
+            }
+        }
+    }
+    return (lang, tokens)
+}
+
+func parseLanguageString(_ input: String?) -> [Substring] {
+
+    guard let input else { return [] }
+    var parts: [Substring] = []
+    var start = input.startIndex
+    var i = input.startIndex
+
+    var bracketDepth = 0
+
+    while i < input.endIndex {
+        let c = input[i]
+
+        if c == "[" { bracketDepth += 1 }
+        else if c == "]" { bracketDepth = max(0, bracketDepth - 1) }
+        else if c == "," && bracketDepth == 0 {
+            let seq = input[start..<i]
+            if !seq.isEmpty {
+                parts.append(seq)
+            }
+            input.formIndex(after: &i)
+            start = i
+            continue
+        }
+        input.formIndex(after: &i)
+    }
+    let tail = input[start..<input.endIndex]
+    if !tail.isEmpty {
+        parts.append(tail)
+    }
+
+    return parts
+}

Tests/SwiftDocCTests/Checker/Checkers/InvalidCodeBlockOptionTests.swift

@@ -24,7 +24,7 @@ let a = 1
         var checker = InvalidCodeBlockOption(sourceFile: nil)
         checker.visit(document)
         XCTAssertTrue(checker.problems.isEmpty)
-        XCTAssertEqual(RenderBlockContent.CodeListing.knownOptions, ["highlight", "nocopy", "wrap"])
+        XCTAssertEqual(RenderBlockContent.CodeListing.knownOptions, ["highlight", "nocopy", "unknown", "wrap"])
     }
 
     func testOption() {
@@ -67,7 +67,7 @@ let c = 3
 let d = 4
 ```         
 
-```unknown, nocpoy
+```haskell, nocpoy
 let e = 5
 ```
 

Tests/SwiftDocCTests/Rendering/RenderContentCompilerTests.swift

@@ -272,6 +272,31 @@ class RenderContentCompilerTests: XCTestCase {
         XCTAssertEqual(codeListing.copyToClipboard, false)
     }
 
+    func testCopyToClipboardNoLang() async throws {
+        enableFeatureFlag(\.isExperimentalCodeBlockEnabled)
+
+        let (bundle, context) = try await testBundleAndContext()
+        var compiler = RenderContentCompiler(context: context, bundle: bundle, identifier: ResolvedTopicReference(bundleID: bundle.id, path: "/path", fragment: nil, sourceLanguage: .swift))
+
+        let source = #"""
+        ```nocopy
+        let x = 1
+        ```
+        """#
+        let document = Document(parsing: source)
+
+        let result = document.children.flatMap { compiler.visit($0) }
+
+        let renderCodeBlock = try XCTUnwrap(result[0] as? RenderBlockContent)
+        guard case let .codeListing(codeListing) = renderCodeBlock else {
+            XCTFail("Expected RenderBlockContent.codeListing")
+            return
+        }
+
+        XCTAssertEqual(codeListing.syntax, nil)
+        XCTAssertEqual(codeListing.copyToClipboard, false)
+    }
+
     func testCopyToClipboardNoFeatureFlag() async throws {
         let (bundle, context) = try await testBundleAndContext()
         var compiler = RenderContentCompiler(context: context, bundle: bundle, identifier: ResolvedTopicReference(bundleID: bundle.id, path: "/path", fragment: nil, sourceLanguage: .swift))
@@ -432,6 +457,6 @@ class RenderContentCompilerTests: XCTestCase {
         }
 
         XCTAssertEqual(codeListing.syntax, "swift")
-        //XCTAssertEqual(codeListing.highlight, [1, 2, 3])
+        XCTAssertEqual(codeListing.highlight, [1, 2, 3])
     }
 }