Merge pull request #361 from EvotecIT/ParsingImprovements

Author: PrzemyslawKlys

PSParseHTML.psd1

@@ -4,7 +4,7 @@
     CmdletsToExport        = @('Close-HtmlBrowserSession', 'Compare-HTML', 'ConvertFrom-HTML', 'ConvertFrom-HtmlAttributes', 'ConvertFrom-HTMLCookie', 'ConvertFrom-HtmlForm', 'ConvertFrom-HtmlList', 'ConvertFrom-HtmlMeta', 'ConvertFrom-HtmlMicrodata', 'ConvertFrom-HtmlOpenGraph', 'ConvertFrom-HtmlTable', 'Convert-HTMLToText', 'Export-BrowserState', 'Export-HtmlBrowserSession', 'Export-HTMLOutline', 'Format-CSS', 'Format-HTML', 'Format-JavaScript', 'Get-HtmlBrowserConsoleLog', 'Get-HtmlBrowserContent', 'Get-HtmlBrowserCookie', 'Get-HtmlBrowserFormField', 'Get-HtmlBrowserInteractable', 'Get-HtmlBrowserLoginForm', 'Get-HtmlBrowserNetworkLog', 'Get-HTMLResource', 'Import-BrowserState', 'Import-HtmlBrowserSession', 'Invoke-HtmlBrowserClick', 'Invoke-HtmlBrowserDomScript', 'Invoke-HtmlBrowserLogin', 'Invoke-HtmlBrowserNavigation', 'Invoke-HtmlBrowserScript', 'Invoke-HTMLRendering', 'Measure-HtmlBrowserPerformance', 'Measure-HtmlDocumentStructure', 'New-HtmlBrowserCookie', 'Optimize-CSS', 'Optimize-Email', 'Optimize-HTML', 'Optimize-JavaScript', 'Register-HTMLRoute', 'Save-HtmlBrowserAttachment', 'Save-HtmlBrowserHar', 'Save-HtmlBrowserPdf', 'Save-HtmlBrowserScreenshot', 'Set-HtmlBrowserChecked', 'Set-HtmlBrowserClientOption', 'Set-HtmlBrowserCookie', 'Set-HtmlBrowserInput', 'Set-HtmlBrowserSelectOption', 'Show-HtmlBrowserHar', 'Start-HtmlBrowserTracing', 'Start-HtmlBrowserVideoCapture', 'Stop-HtmlBrowserTracing', 'Stop-HtmlBrowserVideoCapture', 'Submit-HtmlBrowserForm', 'Test-HtmlMicrodata', 'Unregister-HTMLRoute', 'Test-HtmlBrowser', 'Clear-HtmlBrowserCache')
     CompanyName            = 'Evotec'
     CompatiblePSEditions   = @('Desktop', 'Core')
-    Copyright              = '(c) 2011 - 2025 Przemyslaw Klys @ Evotec. All rights reserved.'
+    Copyright              = '(c) 2011 - 2026 Przemyslaw Klys @ Evotec. All rights reserved.'
     Description            = 'Module that allows to manipulate, parse, format and optimize HTML, JavaScript and CSS'
     DotNetFrameworkVersion = '4.7.2'
     FunctionsToExport      = @()

README.MD

@@ -1070,7 +1070,7 @@ Console.WriteLine($"\nSlowest page: {slowest.Url} ({slowest.PageLoadTime.TotalSe
 
 ### Playwright Auto-Setup
 
-Playwright browsers are automatically downloaded on first use. No manual setup required! The download happens once per system and is shared across all applications.
+Playwright browsers are automatically downloaded on first use. No manual setup required. The download is cached per-user (default locations below).
 
 #### How Auto-Download Works
 When you first use browser testing, Playwright automatically downloads required components:
@@ -1093,6 +1093,13 @@ When you first use browser testing, Playwright automatically downloads required
    - Subsequent runs use cached components - no re-download needed
    - You can manually ensure browsers are installed using `HtmlBrowser.EnsureInstalledAsync()`
 
+#### Linux: Avoiding sudo prompts
+On Linux, Playwright can also install OS-level dependencies when invoked with `--with-deps` (this typically requires root/sudo).
+
+By default, HtmlTinkerX only uses `--with-deps` when running as root to avoid unexpected sudo prompts during normal test execution. You can override this behavior by setting:
+- `HTMLTINKERX_PLAYWRIGHT_WITH_DEPS=1` to force `--with-deps`
+- `HTMLTINKERX_PLAYWRIGHT_WITH_DEPS=0` to never use `--with-deps`
+
 #### Cleaning Playwright Cache
 ```powershell
 # View cache size and clean if needed

Sources/HtmlTinkerX.Tests/HtmlBrowserInstallerTests.cs

@@ -16,7 +16,7 @@ namespace HtmlTinkerX.Tests;
 public class HtmlBrowserInstallerTests
 {
     [Fact]
-    public async Task EnsureInstalledAsync_InstallsDepsOnLinux()
+    public async Task EnsureInstalledAsync_InstallsDepsOnLinux_WhenEnabled()
     {
         if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
         {
@@ -27,6 +27,7 @@ public async Task EnsureInstalledAsync_InstallsDepsOnLinux()
         string tempDriver = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
         Environment.SetEnvironmentVariable("PLAYWRIGHT_BROWSERS_PATH", tempBrowsers);
         Environment.SetEnvironmentVariable("PLAYWRIGHT_DRIVER_SEARCH_PATH", tempDriver);
+        Environment.SetEnvironmentVariable("HTMLTINKERX_PLAYWRIGHT_WITH_DEPS", "1");
 
         var originalInstaller = HtmlBrowser.PlaywrightInstaller;
 
@@ -54,6 +55,53 @@ public async Task EnsureInstalledAsync_InstallsDepsOnLinux()
             HtmlBrowser.PlaywrightInstaller = originalInstaller;
             Environment.SetEnvironmentVariable("PLAYWRIGHT_BROWSERS_PATH", null);
             Environment.SetEnvironmentVariable("PLAYWRIGHT_DRIVER_SEARCH_PATH", null);
+            Environment.SetEnvironmentVariable("HTMLTINKERX_PLAYWRIGHT_WITH_DEPS", null);
+            if (Directory.Exists(tempBrowsers)) Directory.Delete(tempBrowsers, true);
+            if (Directory.Exists(tempDriver)) Directory.Delete(tempDriver, true);
+        }
+    }
+
+    [Fact]
+    public async Task EnsureInstalledAsync_DoesNotInstallDepsOnLinux_WhenDisabled()
+    {
+        if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            return;
+        }
+
+        string tempBrowsers = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+        string tempDriver = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+        Environment.SetEnvironmentVariable("PLAYWRIGHT_BROWSERS_PATH", tempBrowsers);
+        Environment.SetEnvironmentVariable("PLAYWRIGHT_DRIVER_SEARCH_PATH", tempDriver);
+        Environment.SetEnvironmentVariable("HTMLTINKERX_PLAYWRIGHT_WITH_DEPS", "0");
+
+        var originalInstaller = HtmlBrowser.PlaywrightInstaller;
+
+        try
+        {
+            // prepare fake driver so IsDriverPresent returns true
+            string baseDir = Path.Combine(tempDriver, ".playwright");
+            string platformId = PlatformExtensions.GetCurrentPlatform().ToPlatformId();
+            string nodeDir = Path.Combine(baseDir, "node", platformId);
+            Directory.CreateDirectory(nodeDir);
+            Directory.CreateDirectory(Path.Combine(baseDir, "package"));
+            File.WriteAllText(Path.Combine(nodeDir, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "node.exe" : "node"), "");
+            File.WriteAllText(Path.Combine(baseDir, ".version"), typeof(Microsoft.Playwright.Playwright).Assembly.GetName().Version?.ToString(3) ?? "1.52.0");
+
+            string[]? captured = null;
+            HtmlBrowser.PlaywrightInstaller = args => captured = args;
+
+            await HtmlBrowser.EnsureInstalledAsync(HtmlBrowserEngine.Chromium);
+
+            Assert.NotNull(captured);
+            Assert.DoesNotContain("--with-deps", captured);
+        }
+        finally
+        {
+            HtmlBrowser.PlaywrightInstaller = originalInstaller;
+            Environment.SetEnvironmentVariable("PLAYWRIGHT_BROWSERS_PATH", null);
+            Environment.SetEnvironmentVariable("PLAYWRIGHT_DRIVER_SEARCH_PATH", null);
+            Environment.SetEnvironmentVariable("HTMLTINKERX_PLAYWRIGHT_WITH_DEPS", null);
             if (Directory.Exists(tempBrowsers)) Directory.Delete(tempBrowsers, true);
             if (Directory.Exists(tempDriver)) Directory.Delete(tempDriver, true);
         }
@@ -272,4 +320,3 @@ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage reques
         }
     }
 }
-

Sources/HtmlTinkerX.Tests/HtmlHttpClientFactoryTests.cs

@@ -132,12 +132,12 @@ public async Task Create_WithCookieContainer_StoresCookies() {
             using HttpClient client = HtmlHttpClientFactory.Create(out CookieContainer container);
             HttpResponseMessage response = await client.GetAsync(url);
             response.EnsureSuccessStatusCode();
-            Cookie cookie = container.GetCookies(new System.Uri(url))["session"];
+            Cookie? cookie = container.GetCookies(new System.Uri(url))["session"];
             Assert.NotNull(cookie);
             Assert.Equal("abc", cookie.Value);
         } finally {
             server.Stop();
             server.Close();
         }
     }
-}
+}

Sources/HtmlTinkerX.Tests/HtmlOpenGraphTests.cs

@@ -21,6 +21,6 @@ public void ParseOpenGraph_ReturnsExpected() {
 
     [Fact]
     public void ParseOpenGraph_NullHtml_Throws() {
-        Assert.Throws<ArgumentNullException>(() => HtmlParser.ParseOpenGraph(null));
+        Assert.Throws<ArgumentNullException>(() => HtmlParser.ParseOpenGraph(null!));
     }
-}
+}

Sources/HtmlTinkerX.Tests/HtmlParserFormTests.cs

@@ -33,6 +33,6 @@ public void ParseFormsWithAngleSharp_ReturnsForms() {
     /// Throws when html content is null.
     /// </summary>
     public void ParseFormsWithAngleSharp_NullHtml_Throws() {
-        Assert.Throws<ArgumentNullException>(() => HtmlParser.ParseFormsWithAngleSharp(null));
+        Assert.Throws<ArgumentNullException>(() => HtmlParser.ParseFormsWithAngleSharp(null!));
     }
-}
+}

Sources/HtmlTinkerX.Tests/HtmlParserFromTableHelpersTests.cs

@@ -23,7 +23,7 @@ public void ParseTableRows_ReturnsRows() {
         var doc = HtmlParser.ParseWithAngleSharp(html);
         var table = doc.QuerySelector("table")!;
         var (meta, rows, start) = HtmlParserFromTable.ReadTableMetadata(table, 0, null, false, false);
-        var data = HtmlParserFromTable.ParseTableRows(rows, start, meta.Headers, null, false, null);
+        var data = HtmlParserFromTable.ParseTableRows(rows, start, meta.Headers, null, false, null, HtmlCellTextFormat.Compact);
         Assert.Single(data);
         var row = data[0];
         Assert.Equal("1", row["A"]);

Sources/HtmlTinkerX.Tests/HtmlParserListTests.cs

@@ -64,6 +64,6 @@ public void ParseListsWithHtmlAgilityPackDetailed_ReturnsMetadata() {
 
     [Fact]
     public void ParseListsWithAngleSharpDetailed_NullHtml_Throws() {
-        Assert.Throws<ArgumentNullException>(() => HtmlParser.ParseListsWithAngleSharpDetailed(null, " "));
+        Assert.Throws<ArgumentNullException>(() => HtmlParser.ParseListsWithAngleSharpDetailed(null!, " "));
     }
-}
+}

Sources/HtmlTinkerX/Enums/HtmlCellTextFormat.cs

@@ -0,0 +1,19 @@
+namespace HtmlTinkerX;
+
+/// <summary>
+/// Controls how text is extracted from table cells.
+/// </summary>
+public enum HtmlCellTextFormat {
+    /// <summary>
+    /// Single-line, collapses whitespace (legacy/default).
+    /// </summary>
+    Compact = 0,
+    /// <summary>
+    /// Preserves block boundaries and list items with new-lines and bullets.
+    /// </summary>
+    Lines = 1,
+    /// <summary>
+    /// Similar to Lines but uses Markdown-friendly bullets.
+    /// </summary>
+    Markdown = 2
+}

Sources/HtmlTinkerX/HtmlParser.cs

@@ -93,18 +93,20 @@ public static async Task<HtmlDocument> ParseUrlWithHtmlAgilityPackAsync(string u
     /// <param name="allProperties">Whether to pad rows with missing cells.</param>
     /// <param name="skipFooter">Whether to skip HTML table footer elements.</param>
     /// <param name="cleanHeaders">Whether to automatically clean special characters from header names.</param>
-    /// <param name="emptyValuePlaceholder">Value to use for empty cells.</param>
-    /// <returns>List of table parse results with metadata.</returns>
-    public static List<HtmlTableResult> ParseTablesWithAngleSharpDetailed(
-        string html,
-        IDictionary<string, string>? replaceContent = null,
-        IDictionary<string, string>? replaceHeaders = null,
-        bool allProperties = false,
-        bool skipFooter = false,
-        bool cleanHeaders = false,
-        string? emptyValuePlaceholder = null) {
-        return HtmlParserFromTable.ParseTablesWithAngleSharpDetailed(html, replaceContent, replaceHeaders, allProperties, skipFooter, cleanHeaders, emptyValuePlaceholder);
-    }
+    /// <param name="emptyValuePlaceholder">Value to use for empty cells.</param>
+    /// <param name="cellTextFormat">Controls how cell text is flattened (compact, lines, markdown).</param>
+    /// <returns>List of table parse results with metadata.</returns>
+    public static List<HtmlTableResult> ParseTablesWithAngleSharpDetailed(
+        string html,
+        IDictionary<string, string>? replaceContent = null,
+        IDictionary<string, string>? replaceHeaders = null,
+        bool allProperties = false,
+        bool skipFooter = false,
+        bool cleanHeaders = false,
+        string? emptyValuePlaceholder = null,
+        HtmlCellTextFormat cellTextFormat = HtmlCellTextFormat.Compact) {
+        return HtmlParserFromTable.ParseTablesWithAngleSharpDetailed(html, replaceContent, replaceHeaders, allProperties, skipFooter, cleanHeaders, emptyValuePlaceholder, cellTextFormat);
+    }
 
     /// <summary>
     /// Extracts table data from HTML markup using AngleSharp.
@@ -152,19 +154,21 @@ public static List<HtmlTableResult> ParseTablesWithAngleSharpDetailed(
     /// <param name="allProperties">Whether to pad rows with missing cells.</param>
     /// <param name="skipFooter">Whether to skip HTML table footer elements.</param>
     /// <param name="cleanHeaders">Whether to automatically clean special characters from header names.</param>
-    /// <param name="emptyValuePlaceholder">Value to use for empty cells.</param>
+    /// <param name="emptyValuePlaceholder">Value to use for empty cells.</param>
+    /// <param name="cellTextFormat">Controls how cell text is flattened (compact, lines, markdown).</param>
     /// <returns>List of table parse results with metadata.</returns>
-    public static List<HtmlTableResult> ParseTablesWithHtmlAgilityPackDetailed(
-        string html,
-        bool reverseTable = false,
-        IDictionary<string, string>? replaceContent = null,
-        IDictionary<string, string>? replaceHeaders = null,
-        bool allProperties = false,
-        bool skipFooter = false,
-        bool cleanHeaders = false,
-        string? emptyValuePlaceholder = null) {
-        return HtmlParserFromTable.ParseTablesWithHtmlAgilityPackDetailed(html, reverseTable, replaceContent, replaceHeaders, allProperties, skipFooter, cleanHeaders, emptyValuePlaceholder);
-    }
+    public static List<HtmlTableResult> ParseTablesWithHtmlAgilityPackDetailed(
+        string html,
+        bool reverseTable = false,
+        IDictionary<string, string>? replaceContent = null,
+        IDictionary<string, string>? replaceHeaders = null,
+        bool allProperties = false,
+        bool skipFooter = false,
+        bool cleanHeaders = false,
+        string? emptyValuePlaceholder = null,
+        HtmlCellTextFormat cellTextFormat = HtmlCellTextFormat.Compact) {
+        return HtmlParserFromTable.ParseTablesWithHtmlAgilityPackDetailed(html, reverseTable, replaceContent, replaceHeaders, allProperties, skipFooter, cleanHeaders, emptyValuePlaceholder, cellTextFormat);
+    }
 
     /// <summary>
     /// Extracts table data from HTML markup using HtmlAgilityPack.
@@ -465,4 +469,4 @@ public static void EnsureUniqueNames(IList<string> names) {
             }
         }
     }
-}
+}