Revised AzureBlobStore deletions.

Author: axl8713

geowebcache/azureblob/pom.xml

@@ -43,6 +43,11 @@
       <artifactId>azure-storage-blob</artifactId>
       <version>${azure.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.azure</groupId>
+      <artifactId>azure-storage-blob-batch</artifactId>
+      <version>${azure.version}</version>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>

geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStore.java

@@ -22,19 +22,21 @@
 import com.azure.storage.blob.models.BlobProperties;
 import com.azure.storage.blob.models.BlobStorageException;
 import com.azure.storage.blob.specialized.BlockBlobClient;
-import com.google.common.collect.AbstractIterator;
 import com.google.common.collect.Iterators;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.time.OffsetDateTime;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.concurrent.Callable;
 import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
@@ -53,13 +55,14 @@
 import org.geowebcache.storage.StorageException;
 import org.geowebcache.storage.TileObject;
 import org.geowebcache.storage.TileRange;
-import org.geowebcache.storage.TileRangeIterator;
 import org.geowebcache.util.TMSKeyBuilder;
 import org.springframework.http.HttpStatus;
 
 public class AzureBlobStore implements BlobStore {
 
-    static Logger log = Logging.getLogger(AzureBlobStore.class.getName());
+    private static final Logger LOG = Logging.getLogger(AzureBlobStore.class.getName());
+
+    private static final Pattern TILE_BLOB_NAME_REGEXP = Pattern.compile("(?<z>\\d+)/(?<x>\\d+)/(?<y>\\d+)\\.\\w+$");
 
     private final TMSKeyBuilder keyBuilder;
     private final BlobStoreListenerList listeners = new BlobStoreListenerList();
@@ -190,58 +193,105 @@ public boolean delete(TileRange tileRange) throws StorageException {
             return false;
         }
 
-        // open an iterator oer tile locations, to avoid memory accumulation
-        final Iterator<long[]> tileLocations = new AbstractIterator<>() {
-
-            // TileRange iterator with 1x1 meta tiling factor
-            private TileRangeIterator trIter = new TileRangeIterator(tileRange, new int[] {1, 1});
-
-            @Override
-            protected long[] computeNext() {
-                long[] gridLoc = trIter.nextMetaGridLocation(new long[3]);
-                return gridLoc == null ? endOfData() : gridLoc;
-            }
-        };
+        List<BlobItem> blobsToDelete = findTileBlobsToDelete(tileRange, coordsPrefix);
 
-        // if no listeners, we don't need to gather extra tile info, use a dedicated fast path
         if (listeners.isEmpty()) {
             // if there are no listeners, don't bother requesting every tile
             // metadata to notify the listeners
-            Iterator<String> keysIterator = Iterators.transform(
-                    tileLocations, tl -> keyBuilder.forLocation(coordsPrefix, tl, tileRange.getMimeType()));
             // split the iteration in parts to avoid memory accumulation
-            Iterator<List<String>> partition = Iterators.partition(keysIterator, DeleteManager.PAGE_SIZE);
+            List<String> keysToDelete =
+                    blobsToDelete.stream().map(BlobItem::getName).collect(Collectors.toList());
+
+            Iterator<List<String>> partition = Iterators.partition(keysToDelete.iterator(), DeleteManager.PAGE_SIZE);
 
             while (partition.hasNext() && !shutDown) {
-                List<String> locations = partition.next();
-                deleteManager.deleteParallel(locations);
+                deleteManager.deleteParallel(partition.next());
             }
 
         } else {
             // if we need to gather info, we'll end up just calling "delete" on each tile
             // this is run here instead of inside the delete manager as we need high level info
             // about tiles, e.g., TileObject, to inform the listeners
-            String layerName = tileRange.getLayerName();
-            String gridSetId = tileRange.getGridSetId();
-            String format = tileRange.getMimeType().getFormat();
-            Map<String, String> parameters = tileRange.getParameters();
-
-            Iterator<Callable<?>> tilesIterator = Iterators.transform(tileLocations, xyz -> {
-                TileObject tile = TileObject.createQueryTileObject(layerName, xyz, gridSetId, format, parameters);
-                tile.setParametersId(tileRange.getParametersId());
-                return (Callable<Object>) () -> delete(tile);
-            });
-            Iterator<List<Callable<?>>> partition = Iterators.partition(tilesIterator, DeleteManager.PAGE_SIZE);
-
-            // once a page of callables is ready, run them in parallel on the delete manager
-            while (partition.hasNext() && !shutDown) {
-                deleteManager.executeParallel(partition.next());
-            }
+            List<Callable<?>> tilesDeletions = blobsToDelete.stream()
+                    .map(blobItem -> {
+                        TileObject tile = createTileObject(blobItem, tileRange);
+                        tile.setParametersId(tileRange.getParametersId());
+                        return (Callable<Object>) () -> delete(tile);
+                    })
+                    .collect(Collectors.toList());
+
+            executeParallelDeletions(tilesDeletions);
         }
 
         return true;
     }
 
+    private List<BlobItem> findTileBlobsToDelete(TileRange tileRange, String coordsPrefix) {
+
+        List<BlobItem> blobsToDelete = new ArrayList<>();
+
+        for (int zoom = tileRange.getZoomStart(); zoom <= tileRange.getZoomStop(); zoom++) {
+
+            String zoomPrefix = coordsPrefix + "/" + zoom;
+
+            if (!client.prefixExists(zoomPrefix)) {
+                // empty level, skipping
+                continue;
+            }
+
+            long[] rangeBoundsAtZoom = tileRange.rangeBounds(zoom);
+
+            client.listBlobs(zoomPrefix)
+                    .filter(tb -> isTileBlobInBounds(tb, rangeBoundsAtZoom))
+                    .forEach(blobsToDelete::add);
+        }
+
+        return blobsToDelete;
+    }
+
+    private boolean isTileBlobInBounds(BlobItem tileBlob, long[] bounds) {
+        long minX = bounds[0];
+        long minY = bounds[1];
+        long maxX = bounds[2];
+        long maxY = bounds[3];
+
+        long[] index = extractTileIndex(tileBlob);
+        long tileX = index[0];
+        long tileY = index[1];
+
+        return tileX >= minX && tileX <= maxX && tileY >= minY && tileY <= maxY;
+    }
+
+    private TileObject createTileObject(BlobItem blobItem, TileRange tileRange) {
+        String layerName = tileRange.getLayerName();
+        String gridSetId = tileRange.getGridSetId();
+        String format = tileRange.getMimeType().getFormat();
+        Map<String, String> parameters = tileRange.getParameters();
+        return TileObject.createQueryTileObject(layerName, extractTileIndex(blobItem), gridSetId, format, parameters);
+    }
+
+    private long[] extractTileIndex(BlobItem blobItem) {
+        Matcher matcher = TILE_BLOB_NAME_REGEXP.matcher(blobItem.getName());
+
+        if (!matcher.find()) {
+            throw new IllegalArgumentException("Invalid tile blob name");
+        }
+
+        return new long[] {
+            Long.parseLong(matcher.group("x")), Long.parseLong(matcher.group("y")), Long.parseLong(matcher.group("z"))
+        };
+    }
+
+    private void executeParallelDeletions(List<Callable<?>> tilesDeletions) throws StorageException {
+        Iterator<List<Callable<?>>> tilesDeletionsPartitions =
+                Iterators.partition(tilesDeletions.iterator(), DeleteManager.PAGE_SIZE);
+
+        // once a page of callables is ready, run them in parallel on the delete manager
+        while (tilesDeletionsPartitions.hasNext() && !shutDown) {
+            deleteManager.executeParallel(tilesDeletionsPartitions.next());
+        }
+    }
+
     @Override
     public boolean get(TileObject obj) throws StorageException {
         final String key = keyBuilder.forTile(obj);
@@ -373,7 +423,7 @@ public boolean rename(String oldLayerName, String newLayerName) throws StorageEx
         // revisit: this seems to hold true only for GeoServerTileLayer, "standalone" TileLayers
         // return getName() from getId(), as in AbstractTileLayer. Unfortunately the only option
         // for non-GeoServerTileLayers would be copy and delete. Expensive.
-        log.fine("No need to rename layers, AzureBlobStore uses layer id as key root");
+        LOG.fine("No need to rename layers, AzureBlobStore uses layer id as key root");
         if (client.prefixExists(oldLayerName)) {
             listeners.sendLayerRenamed(oldLayerName, newLayerName);
         }

geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java

@@ -26,6 +26,8 @@
 import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.batch.BlobBatchClient;
+import com.azure.storage.blob.batch.BlobBatchClientBuilder;
 import com.azure.storage.blob.models.BlobDownloadContentResponse;
 import com.azure.storage.blob.models.BlobHttpHeaders;
 import com.azure.storage.blob.models.BlobItem;
@@ -55,6 +57,7 @@ public class AzureClient {
 
     private AzureBlobStoreData configuration;
     private final BlobContainerClient container;
+    private final BlobBatchClient batch;
 
     public AzureClient(AzureBlobStoreData configuration) throws StorageException {
         this.configuration = configuration;
@@ -64,6 +67,7 @@ public AzureClient(AzureBlobStoreData configuration) throws StorageException {
 
             String containerName = configuration.getContainer();
             this.container = getOrCreateContainer(serviceClient, containerName);
+            this.batch = new BlobBatchClientBuilder(serviceClient).buildClient();
         } catch (StorageException e) {
             throw e;
         } catch (RuntimeException e) {
@@ -284,6 +288,10 @@ public BlobContainerClient getContainer() {
         return container;
     }
 
+    public BlobBatchClient getBatch() {
+        return batch;
+    }
+
     public boolean deleteBlob(String key) {
         BlockBlobClient metadata = getBlockBlobClient(key);
         return metadata.deleteIfExists();