Merge pull request #13893 from wietzesuijker/perf/zarr-v3-first-probe

Zarr: probe zarr.json before v2 files in OpenRootGroup()

Author: rouault

autotest/gdrivers/zarr_driver.py

@@ -6223,62 +6223,59 @@ def test_zarr_read_simple_sharding_network():
     chunk_block = b"\x01\x02\x03\x04" + (16384 - 4) * b"\x00"
 
     try:
-        # Loop twice: second iteration proves ClearMemoryCaches() lets the
-        # driver re-fetch everything cleanly.
-        for _ in range(2):
-            handler = webserver.SequentialHandler()
-            handler.add("GET", "/test.zarr/", 404)
-            handler.add("HEAD", "/test.zarr/.zmetadata", 404)
-            handler.add("HEAD", "/test.zarr/.zarray", 404)
-            handler.add("HEAD", "/test.zarr/.zgroup", 404)
-            handler.add(
-                "HEAD",
-                "/test.zarr/zarr.json",
-                200,
-                {"Content-Length": "%d" % len(zarr_json)},
-            )
-            handler.add(
-                "GET",
-                "/test.zarr/zarr.json",
-                200,
-                {"Content-Length": "%d" % len(zarr_json)},
-                zarr_json,
-            )
-            handler.add("HEAD", "/test.zarr/zarr.json.aux.xml", 404)
-            handler.add("HEAD", "/test.zarr/zarr.aux", 404)
-            handler.add("HEAD", "/test.zarr/zarr.AUX", 404)
-            handler.add("HEAD", "/test.zarr/zarr.json.aux", 404)
-            handler.add("HEAD", "/test.zarr/zarr.json.AUX", 404)
-            handler.add("HEAD", "/test.zarr/c/0/0", 200, {"Content-Length": "65536"})
-            handler.add(
-                "GET",
-                "/test.zarr/c/0/0",
-                206,
-                {
-                    "Content-Length": "16384",
-                    "Content-Range": "bytes 49152-65535/65536",
-                },
-                shard_tail,
-                expected_headers={"Range": "bytes=49152-65535"},
-            )
-            handler.add(
-                "GET",
-                "/test.zarr/c/0/0",
-                206,
-                {
-                    "Content-Length": "16384",
-                    "Content-Range": "bytes 0-16383/65536",
-                },
-                chunk_block,
-                expected_headers={"Range": "bytes=0-16383"},
-            )
-            with webserver.install_http_handler(handler):
-                ds = gdal.Open(
-                    'ZARR:"/vsicurl/http://localhost:%d/test.zarr"' % webserver_port
+        with gdaltest.config_options({"GDAL_PAM_ENABLED": "NO"}):
+            # Loop twice: second iteration proves ClearMemoryCaches() lets the
+            # driver re-fetch everything cleanly.
+            for _ in range(2):
+                handler = webserver.SequentialHandler()
+                handler.add("GET", "/test.zarr/", 404)
+                handler.add("HEAD", "/test.zarr/.zmetadata", 404)
+                # Probe zarr.json first; v2 probes (.zarray, .zgroup) skipped.
+                handler.add(
+                    "HEAD",
+                    "/test.zarr/zarr.json",
+                    200,
+                    {"Content-Length": "%d" % len(zarr_json)},
+                )
+                handler.add(
+                    "GET",
+                    "/test.zarr/zarr.json",
+                    200,
+                    {"Content-Length": "%d" % len(zarr_json)},
+                    zarr_json,
+                )
+                handler.add(
+                    "HEAD", "/test.zarr/c/0/0", 200, {"Content-Length": "65536"}
+                )
+                handler.add(
+                    "GET",
+                    "/test.zarr/c/0/0",
+                    206,
+                    {
+                        "Content-Length": "16384",
+                        "Content-Range": "bytes 49152-65535/65536",
+                    },
+                    shard_tail,
+                    expected_headers={"Range": "bytes=49152-65535"},
+                )
+                handler.add(
+                    "GET",
+                    "/test.zarr/c/0/0",
+                    206,
+                    {
+                        "Content-Length": "16384",
+                        "Content-Range": "bytes 0-16383/65536",
+                    },
+                    chunk_block,
+                    expected_headers={"Range": "bytes=0-16383"},
                 )
-                assert ds.GetRasterBand(1).ReadBlock(0, 0) == b"\x01\x02\x03\x04"
-            ds = None
-            gdal.ClearMemoryCaches()
+                with webserver.install_http_handler(handler):
+                    ds = gdal.Open(
+                        'ZARR:"/vsicurl/http://localhost:%d/test.zarr"' % webserver_port
+                    )
+                    assert ds.GetRasterBand(1).ReadBlock(0, 0) == b"\x01\x02\x03\x04"
+                ds = None
+                gdal.ClearMemoryCaches()
 
     finally:
         webserver.server_stop(webserver_process, webserver_port)

frmts/zarr/zarr_sharedresource.cpp

@@ -89,6 +89,14 @@ ZarrSharedResource::~ZarrSharedResource()
 
 std::shared_ptr<ZarrGroupBase> ZarrSharedResource::OpenRootGroup()
 {
+    // Probe zarr.json first so v3 datasets skip the v2 stat cascade.
+    const std::string osZarrJsonFilename(CPLFormFilenameSafe(
+        m_osRootDirectoryName.c_str(), "zarr.json", nullptr));
+    VSIStatBufL sStat;
+    const bool bHasZarrJson =
+        (VSIStatL(osZarrJsonFilename.c_str(), &sStat) == 0);
+
+    if (!bHasZarrJson)
     {
         auto poRG = ZarrV2Group::Create(shared_from_this(), std::string(), "/");
         // Prevents potential recursion
@@ -98,7 +106,6 @@ std::shared_ptr<ZarrGroupBase> ZarrSharedResource::OpenRootGroup()
 
         const std::string osZarrayFilename(CPLFormFilenameSafe(
             m_osRootDirectoryName.c_str(), ".zarray", nullptr));
-        VSIStatBufL sStat;
         const auto nErrorCount = CPLGetErrorCounter();
         if (VSIStatL(osZarrayFilename.c_str(), &sStat) == 0)
         {
@@ -184,10 +191,7 @@ std::shared_ptr<ZarrGroupBase> ZarrSharedResource::OpenRootGroup()
     m_poWeakRootGroup = poRG_V3;
     poRG_V3->SetUpdatable(m_bUpdatable);
 
-    const std::string osZarrJsonFilename(CPLFormFilenameSafe(
-        m_osRootDirectoryName.c_str(), "zarr.json", nullptr));
-    VSIStatBufL sStat;
-    if (VSIStatL(osZarrJsonFilename.c_str(), &sStat) == 0)
+    if (bHasZarrJson)
     {
         CPLJSONDocument oDoc;
         if (!oDoc.Load(osZarrJsonFilename))