Debug CI failure on plots tests (#312)

* Debug CI failure on plots tests

* Only run integration tests with external packages against Julia LTS+

Author: MichaelHatherly

.github/workflows/CI.yml

@@ -105,7 +105,7 @@ jobs:
           depwarn: "yes"
       - uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38 # v1.2.2
         if: matrix.os == 'ubuntu-latest' && matrix.version == '1.11'
-      - uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
+      - uses: codecov/codecov-action@fdcc8476540edceab3de004e990f80d881c6cc00 # v5.5.0
         if: matrix.os == 'ubuntu-latest' && matrix.version == '1.11'
         with:
           verbose: true

test/testsets/display.jl

@@ -1,16 +1,18 @@
-include("../utilities/prelude.jl")
+if VERSION >= v"1.10"
+    include("../utilities/prelude.jl")
 
-test_example(joinpath(@__DIR__, "../examples/display.qmd")) do json
-    cells = json["cells"]
-    @test length(cells) == 5
+    test_example(joinpath(@__DIR__, "../examples/display.qmd")) do json
+        cells = json["cells"]
+        @test length(cells) == 5
 
-    cell = cells[4]
-    outputs = cell["outputs"]
-    @test length(outputs) == 2
+        cell = cells[4]
+        outputs = cell["outputs"]
+        @test length(outputs) == 2
 
-    @test outputs[1]["output_type"] == "display_data"
-    @test haskey(outputs[1]["data"], "image/png")
+        @test outputs[1]["output_type"] == "display_data"
+        @test haskey(outputs[1]["data"], "image/png")
 
-    @test outputs[2]["output_type"] == "display_data"
-    @test haskey(outputs[2]["data"], "image/png")
+        @test outputs[2]["output_type"] == "display_data"
+        @test haskey(outputs[2]["data"], "image/png")
+    end
 end

test/testsets/integrations/CairoMakie.jl

@@ -1,16 +1,18 @@
-include("../../utilities/prelude.jl")
+if VERSION >= v"1.10"
+    include("../../utilities/prelude.jl")
 
-test_example(joinpath(@__DIR__, "../../examples/integrations/CairoMakie.qmd")) do json
-    cells = json["cells"]
-    cell = cells[6]
+    test_example(joinpath(@__DIR__, "../../examples/integrations/CairoMakie.qmd")) do json
+        cells = json["cells"]
+        cell = cells[6]
 
-    px_per_inch = 96
+        px_per_inch = 96
 
-    # the size metadata is just inches converted to CSS pixels, independent of pixel resolution
-    @test cell["outputs"][1]["metadata"]["image/png"] ==
-          Dict("width" => 4 * px_per_inch, "height" => 3 * px_per_inch)
+        # the size metadata is just inches converted to CSS pixels, independent of pixel resolution
+        @test cell["outputs"][1]["metadata"]["image/png"] ==
+              Dict("width" => 4 * px_per_inch, "height" => 3 * px_per_inch)
 
-    pngbytes = Base64.base64decode(cell["outputs"][1]["data"]["image/png"])
-    @test QuartoNotebookRunner.png_image_metadata(pngbytes; phys_correction = false) ==
-          (; width = 4 * 150, height = 3 * 150)
+        pngbytes = Base64.base64decode(cell["outputs"][1]["data"]["image/png"])
+        @test QuartoNotebookRunner.png_image_metadata(pngbytes; phys_correction = false) ==
+              (; width = 4 * 150, height = 3 * 150)
+    end
 end

test/testsets/integrations/CairoMakieFigFormat.jl

@@ -1,43 +1,48 @@
-include("../../utilities/prelude.jl")
-
-
-@testset "CairoMakie fig-formats" begin
-    s = read(
-        joinpath(@__DIR__, "../../examples/integrations/CairoMakieFigFormat.qmd"),
-        String,
-    )
-
-    env = abspath(joinpath(@__DIR__, "../../examples/integrations/CairoMakie/Project.toml"))
-
-    showable_mimes(::Union{Val{:png},Val{:jpeg},Val{:retina}}) = ["image/png", "text/html"]
-    showable_mimes(::Union{Val{:pdf},Val{:svg}}) = ["image/svg+xml", "application/pdf"]
-    not_showable_mimes(::Union{Val{:png},Val{:jpeg},Val{:retina}}) =
-        showable_mimes(Val(:pdf))
-    not_showable_mimes(::Union{Val{:pdf},Val{:svg}}) = ["text/html"] # png stays as a fallback that should not be chosen if svg or pdf are accepted
-
-    mktempdir() do dir
-        server = QuartoNotebookRunner.Server()
-        file = joinpath(dir, "temp.qmd")
-
-        @testset "$format" for format in [:png, :jpeg, :retina, :svg, :pdf]
-            open(file, "w") do io
-                # 1.6 doesn't support multiple patterns
-                for pattern in [
-                    "retina" => format,
-                    "CAIROMAKIE_ENV" => repr(env),
-                    "NOT_SHOWABLE_MIMES" => repr(not_showable_mimes(Val(format))),
-                    "SHOWABLE_MIMES" => repr(showable_mimes(Val(format))),
-                ]
-                    s = replace(s, pattern)
+if VERSION >= v"1.10"
+    include("../../utilities/prelude.jl")
+
+
+    @testset "CairoMakie fig-formats" begin
+        s = read(
+            joinpath(@__DIR__, "../../examples/integrations/CairoMakieFigFormat.qmd"),
+            String,
+        )
+
+        env = abspath(
+            joinpath(@__DIR__, "../../examples/integrations/CairoMakie/Project.toml"),
+        )
+
+        showable_mimes(::Union{Val{:png},Val{:jpeg},Val{:retina}}) =
+            ["image/png", "text/html"]
+        showable_mimes(::Union{Val{:pdf},Val{:svg}}) = ["image/svg+xml", "application/pdf"]
+        not_showable_mimes(::Union{Val{:png},Val{:jpeg},Val{:retina}}) =
+            showable_mimes(Val(:pdf))
+        not_showable_mimes(::Union{Val{:pdf},Val{:svg}}) = ["text/html"] # png stays as a fallback that should not be chosen if svg or pdf are accepted
+
+        mktempdir() do dir
+            server = QuartoNotebookRunner.Server()
+            file = joinpath(dir, "temp.qmd")
+
+            @testset "$format" for format in [:png, :jpeg, :retina, :svg, :pdf]
+                open(file, "w") do io
+                    # 1.6 doesn't support multiple patterns
+                    for pattern in [
+                        "retina" => format,
+                        "CAIROMAKIE_ENV" => repr(env),
+                        "NOT_SHOWABLE_MIMES" => repr(not_showable_mimes(Val(format))),
+                        "SHOWABLE_MIMES" => repr(showable_mimes(Val(format))),
+                    ]
+                        s = replace(s, pattern)
+                    end
+                    print(io, s)
                 end
-                print(io, s)
+
+                result = QuartoNotebookRunner.run!(server, file)
+                @test result.cells[6].outputs[1].data["text/plain"] == "true"
+                @test result.cells[8].outputs[1].data["text/plain"] == "true"
             end
 
-            result = QuartoNotebookRunner.run!(server, file)
-            @test result.cells[6].outputs[1].data["text/plain"] == "true"
-            @test result.cells[8].outputs[1].data["text/plain"] == "true"
+            QuartoNotebookRunner.close!(server)
         end
-
-        QuartoNotebookRunner.close!(server)
     end
 end

test/testsets/integrations/PlotlyJS.jl

@@ -1,27 +1,29 @@
-include("../../utilities/prelude.jl")
+if VERSION >= v"1.10"
+    include("../../utilities/prelude.jl")
 
-if Sys.iswindows()
-    # There are issues with the Kaleido_jll v0.2 build on Windows, skip for now.
-    @info "Skipping PlotlyJS.jl tests on Windows"
-else
-    test_example(joinpath(@__DIR__, "../../examples/integrations/PlotlyJS.qmd")) do json
-        cells = json["cells"]
-        preamble_cell = cells[5]
-        outputs = preamble_cell["outputs"]
-        @test length(outputs) == 1
-        data = outputs[1]["data"]
-        @test keys(data) == Set(["text/html", "text/plain"])
-        @test startswith(data["text/html"], "<script type=\"text/javascript\">")
-        @test occursin("require.undef(\"plotly\")", data["text/html"])
-        for nth in (6, 9)
-            cell = cells[nth]
-            outputs = cell["outputs"]
+    if Sys.iswindows()
+        # There are issues with the Kaleido_jll v0.2 build on Windows, skip for now.
+        @info "Skipping PlotlyJS.jl tests on Windows"
+    else
+        test_example(joinpath(@__DIR__, "../../examples/integrations/PlotlyJS.qmd")) do json
+            cells = json["cells"]
+            preamble_cell = cells[5]
+            outputs = preamble_cell["outputs"]
             @test length(outputs) == 1
             data = outputs[1]["data"]
-            @test haskey(data, "image/png")
-            @test haskey(data, "image/svg+xml")
-            @test haskey(data, "text/html")
-            @test startswith(data["text/html"], "<div>")
+            @test keys(data) == Set(["text/html", "text/plain"])
+            @test startswith(data["text/html"], "<script type=\"text/javascript\">")
+            @test occursin("require.undef(\"plotly\")", data["text/html"])
+            for nth in (6, 9)
+                cell = cells[nth]
+                outputs = cell["outputs"]
+                @test length(outputs) == 1
+                data = outputs[1]["data"]
+                @test haskey(data, "image/png")
+                @test haskey(data, "image/svg+xml")
+                @test haskey(data, "text/html")
+                @test startswith(data["text/html"], "<div>")
+            end
         end
     end
 end

test/testsets/integrations/Plots.jl

@@ -1,23 +1,25 @@
-include("../../utilities/prelude.jl")
+if VERSION >= v"1.10"
+    include("../../utilities/prelude.jl")
 
-test_example(joinpath(@__DIR__, "../../examples/integrations/Plots.qmd")) do json
-    cells = json["cells"]
-    cell = cells[6]
-    output = cell["outputs"][1]
+    test_example(joinpath(@__DIR__, "../../examples/integrations/Plots.qmd")) do json
+        cells = json["cells"]
+        cell = cells[6]
+        output = cell["outputs"][1]
 
-    @test !isempty(output["data"]["image/png"])
-    @test !isempty(output["data"]["image/svg+xml"])
-    @test !isempty(output["data"]["text/html"])
+        @test !isempty(output["data"]["image/png"])
+        @test !isempty(output["data"]["image/svg+xml"])
+        @test !isempty(output["data"]["text/html"])
 
-    @test cell["outputs"][1]["metadata"]["image/png"] ==
-          Dict("width" => 575, "height" => 432) # Plots does not seem to follow standard dpi rules so the values don't match Makie
-end
+        @test cell["outputs"][1]["metadata"]["image/png"] ==
+              Dict("width" => 575, "height" => 432) # Plots does not seem to follow standard dpi rules so the values don't match Makie
+    end
 
-test_example(joinpath(@__DIR__, "../../examples/integrations/PlotsPDF.qmd")) do json
-    cells = json["cells"]
+    test_example(joinpath(@__DIR__, "../../examples/integrations/PlotsPDF.qmd")) do json
+        cells = json["cells"]
 
-    cell = cells[4]
-    output = cell["outputs"][1]
+        cell = cells[4]
+        output = cell["outputs"][1]
 
-    @test !isempty(output["data"]["application/pdf"])
+        @test !isempty(output["data"]["application/pdf"])
+    end
 end

test/testsets/integrations/PythonCall.jl

@@ -1,43 +1,45 @@
-include("../../utilities/prelude.jl")
+if VERSION >= v"1.10"
+    include("../../utilities/prelude.jl")
 
-test_example(joinpath(@__DIR__, "../../examples/integrations/PythonCall.qmd")) do json
-    cells = json["cells"]
+    test_example(joinpath(@__DIR__, "../../examples/integrations/PythonCall.qmd")) do json
+        cells = json["cells"]
 
-    cell = cells[8]
-    @test occursin("PythonCall", cell["outputs"][1]["data"]["text/plain"])
+        cell = cells[8]
+        @test occursin("PythonCall", cell["outputs"][1]["data"]["text/plain"])
 
-    cell = cells[11]
-    @test occursin("'PythonCall'", cell["outputs"][1]["text"])
+        cell = cells[11]
+        @test occursin("'PythonCall'", cell["outputs"][1]["text"])
 
-    cell = cells[16]
-    @test occursin("5", cell["outputs"][1]["data"]["text/plain"])
+        cell = cells[16]
+        @test occursin("5", cell["outputs"][1]["data"]["text/plain"])
 
-    cell = cells[17]
-    @test occursin("Inline python code: something.", join(cell["source"]))
+        cell = cells[17]
+        @test occursin("Inline python code: something.", join(cell["source"]))
 
-    cell = cells[22]
-    @test occursin("1000", cell["outputs"][1]["data"]["text/plain"])
+        cell = cells[22]
+        @test occursin("1000", cell["outputs"][1]["data"]["text/plain"])
 
-    cell = cells[27]
-    @test occursin("150", cell["outputs"][1]["data"]["text/plain"])
+        cell = cells[27]
+        @test occursin("150", cell["outputs"][1]["data"]["text/plain"])
 
-    cell = cells[30]
-    @test isempty(cell["outputs"])
+        cell = cells[30]
+        @test isempty(cell["outputs"])
 
-    cell = cells[33]
-    traceback = cell["outputs"][1]["traceback"]
-    @test occursin("division by zero", traceback[1])
-    @test occursin("PythonCall.qmd:54", traceback[4])
-    @test occursin("PythonCall.qmd:58", traceback[end])
+        cell = cells[33]
+        traceback = cell["outputs"][1]["traceback"]
+        @test occursin("division by zero", traceback[1])
+        @test occursin("PythonCall.qmd:54", traceback[4])
+        @test occursin("PythonCall.qmd:58", traceback[end])
 
-    cell = cells[36]
-    traceback = cell["outputs"][1]["traceback"]
-    @test occursin("unmatched ']'", traceback[1])
-    @test occursin("PythonCall.qmd, line 62", traceback[1])
-    @test occursin("ast.py", traceback[end])
+        cell = cells[36]
+        traceback = cell["outputs"][1]["traceback"]
+        @test occursin("unmatched ']'", traceback[1])
+        @test occursin("PythonCall.qmd, line 62", traceback[1])
+        @test occursin("ast.py", traceback[end])
 
-    cell = cells[39]
-    output = cell["outputs"][1]
-    @test output["name"] == "stdout"
-    @test occursin("Define the builtin 'help'.", output["text"])
+        cell = cells[39]
+        output = cell["outputs"][1]
+        @test output["name"] == "stdout"
+        @test occursin("Define the builtin 'help'.", output["text"])
+    end
 end

test/testsets/integrations/RCall.jl

@@ -1,24 +1,26 @@
-include("../../utilities/prelude.jl")
+if VERSION >= v"1.10"
+    include("../../utilities/prelude.jl")
 
-test_example(joinpath(@__DIR__, "../../examples/integrations/RCall.qmd")) do json
-    cells = json["cells"]
-    cell = cells[4]
-    output = cell["outputs"][1]
-
-    @test !isempty(output["data"]["image/png"])
-
-    @test cell["outputs"][1]["metadata"]["image/png"] ==
-          Dict("width" => 600, "height" => 450)
-end
-
-# Don't run this on macOS CI, since that appears to be missing the required libs.
-if !(get(ENV, "CI", "false") == "true" && Sys.isapple())
-    test_example(joinpath(@__DIR__, "../../examples/integrations/RCallSVG.qmd")) do json
+    test_example(joinpath(@__DIR__, "../../examples/integrations/RCall.qmd")) do json
         cells = json["cells"]
         cell = cells[4]
         output = cell["outputs"][1]
 
-        @test !isempty(output["data"]["image/svg+xml"])
-        @test isempty(cell["outputs"][1]["metadata"])
+        @test !isempty(output["data"]["image/png"])
+
+        @test cell["outputs"][1]["metadata"]["image/png"] ==
+              Dict("width" => 600, "height" => 450)
+    end
+
+    # Don't run this on macOS CI, since that appears to be missing the required libs.
+    if !(get(ENV, "CI", "false") == "true" && Sys.isapple())
+        test_example(joinpath(@__DIR__, "../../examples/integrations/RCallSVG.qmd")) do json
+            cells = json["cells"]
+            cell = cells[4]
+            output = cell["outputs"][1]
+
+            @test !isempty(output["data"]["image/svg+xml"])
+            @test isempty(cell["outputs"][1]["metadata"])
+        end
     end
 end