Nio skip container check (Azure#26850)

* Updated some docs and added convert from Uri method

* Added option to skip container check

* Wrote tests

* Changelog

* Added recordings

* Added some javadocs and fixed a test

Author: rickle-msft

sdk/storage/azure-storage-blob-nio/CHANGELOG.md

@@ -3,6 +3,8 @@
 ## 12.0.0-beta.15 (Unreleased)
 
 ### Features Added
+- Added `AzurePath.fromBlobUrl` to help convert from a blob url to an AzurePath
+- Added a configuration option `AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK` to skip the initial container check in cases where the authentication method used will not have necessary permissions.
 
 ### Breaking Changes
 

sdk/storage/azure-storage-blob-nio/README.md

@@ -129,6 +129,7 @@ committed or available to be read until the write stream is closed.
 
 The following sections provide several code snippets covering some of the most common Azure Storage Blob NIO tasks, including:
 
+- [URI format](#uri-format)
 - [Create a `FileSystem`](#create-a-filesystem)
 - [Create a directory](#create-a-directory)
 - [Iterate over directory contents](#iterate-over-directory-contents)
@@ -139,6 +140,19 @@ The following sections provide several code snippets covering some of the most c
 - [Read attributes on a file](#read-attributes-on-a-file)
 - [Write attributes to a file](#write-attributes-to-a-file)
 
+### URI format
+URIs are the fundamental way of identifying a resource. This package defines its URI format as follows:
+
+The scheme for this provider is `"azb"`, and the format of the URI to identify an `AzureFileSystem` is 
+`"azb://?endpoint=<endpoint>"`. The endpoint of the Storage account is used to uniquely identify the filesystem.
+
+The root component, if it is present, is the first element of the path and is denoted by a `':'` as the last character.
+Hence, only one instance of `':'` may appear in a path string, and it may only be the last character of the first 
+element in the path. The root component is used to identify which container a path belongs to.
+
+All other path elements, including separators, are considered as the blob name. `AzurePath#fromBlobUrl`
+may be used to convert a typical http url pointing to a blob into an `AzurePath` object pointing to the same resource.
+
 ### Create a `FileSystem`
 
 Create a `FileSystem` using the [`shared key`](#get-credentials) retrieved above.

sdk/storage/azure-storage-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileStore.java

@@ -24,7 +24,8 @@ public final class AzureFileStore extends FileStore {
     private final BlobContainerClient containerClient;
 
 
-    AzureFileStore(AzureFileSystem parentFileSystem, String containerName) throws IOException {
+    AzureFileStore(AzureFileSystem parentFileSystem, String containerName, Boolean skipConnectionCheck)
+        throws IOException {
         // A FileStore should only ever be created by a FileSystem.
         if (Objects.isNull(parentFileSystem)) {
             throw LoggingUtility.logError(logger, new IllegalStateException("AzureFileStore cannot be instantiated "
@@ -33,14 +34,16 @@ public final class AzureFileStore extends FileStore {
         this.parentFileSystem = parentFileSystem;
         this.containerClient = this.parentFileSystem.getBlobServiceClient().getBlobContainerClient(containerName);
 
-        try {
-            // This also serves as our connection check.
-            if (!this.containerClient.exists()) {
-                this.containerClient.create();
+        if (skipConnectionCheck == null || !skipConnectionCheck) {
+            try {
+                // This also serves as our connection check.
+                if (!this.containerClient.exists()) {
+                    this.containerClient.create();
+                }
+            } catch (Exception e) {
+                throw LoggingUtility.logError(logger, new IOException("There was an error in establishing the existence of "
+                    + "container: " + containerName, e));
             }
-        } catch (Exception e) {
-            throw LoggingUtility.logError(logger, new IOException("There was an error in establishing the existence of "
-                + "container: " + containerName, e));
         }
     }
 

sdk/storage/azure-storage-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystem.java

@@ -130,6 +130,11 @@ public final class AzureFileSystem extends FileSystem {
      */
     public static final String AZURE_STORAGE_FILE_STORES = "AzureStorageFileStores";
 
+    /**
+     * Expected type: Boolean
+     */
+    public static final String AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK = "AzureStorageSkipInitialContainerCheck";
+
     static final String PATH_SEPARATOR = "/";
 
     private static final Map<String, String> PROPERTIES =
@@ -432,9 +437,10 @@ private Map<String, FileStore> initializeFileStores(Map<String, ?> config) throw
                 + "null."));
         }
 
+        Boolean skipConnectionCheck = (Boolean) config.get(AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK);
         Map<String, FileStore> fileStores = new HashMap<>();
         for (String fileStoreName : fileStoreNames.split(",")) {
-            FileStore fs = new AzureFileStore(this, fileStoreName);
+            FileStore fs = new AzureFileStore(this, fileStoreName, skipConnectionCheck);
             if (this.defaultFileStore == null) {
                 this.defaultFileStore = fs;
             }

sdk/storage/azure-storage-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystemProvider.java

@@ -81,7 +81,7 @@
  * {@link FileSystemProvider}.
  * <p>
  * The scheme for this provider is {@code "azb"}, and the format of the URI to identify an {@code AzureFileSystem} is
- * {@code "azb://?endpoint=<endpoing>"}. The endpoint of the Storage account is used to uniquely identify the
+ * {@code "azb://?endpoint=<endpoint>"}. The endpoint of the Storage account is used to uniquely identify the
  * filesystem.
  * <p>
  * An {@link AzureFileSystem} is backed by an account. An {@link AzureFileStore} is backed by a container. Any number of
@@ -94,7 +94,8 @@
  * <p>
  * {@link #newFileSystem(URI, Map)} will check for the following keys in the configuration map and expect the named
  * types. Any entries not listed here will be ignored. Note that {@link AzureFileSystem} has public constants defined
- * for each of the keys for convenience.
+ * for each of the keys for convenience. Most values are documented in the blob package. Any values which are unique to
+ * nio will be documented here.
  * <ul>
  *     <li>{@code AzureStorageSharedKeyCredential:}{@link com.azure.storage.common.StorageSharedKeyCredential}</li>
  *     <li>{@code AzureStorageSasTokenCredential:}{@link com.azure.core.credential.AzureSasCredential}</li>
@@ -111,6 +112,9 @@
  *     <li>{@code AzureStorageMaxConcurrencyPerRequest:}{@link Integer}</li>
  *     <li>{@code AzureStorageDownloadResumeRetries:}{@link Integer}</li>
  *     <li>{@code AzureStorageFileStores:}{@link String}</li>
+ *     <li>{@code AzureStorageSkipInitialContainerCheck:}{@link Boolean}. Indicates that the initial check which
+ *     confirms the existence of the containers meant to act as file stores should be skipped. This can be usesful in
+ *     cases where a sas token that is scoped to only one file is used to authenticate.</li>
  * </ul>
  * <p>
  * Either an account key or a sas token must be specified. If both are provided, the account key will be preferred. If
@@ -226,7 +230,7 @@ public FileSystem newFileSystem(URI uri, Map<String, ?> config) throws IOExcepti
     /**
      * Returns an existing FileSystem created by this provider.
      * <p>
-     * The format of a {@code URI} identifying an file system is {@code "azb://?endpoint=&lt;endpoint&gt;"}.
+     * The format of a {@code URI} identifying a file system is {@code "azb://?endpoint=&lt;endpoint&gt;"}.
      * <p>
      * Trying to retrieve a closed file system will throw a {@link FileSystemNotFoundException}. Once closed, a
      * file system with the same identifier may be reopened.

sdk/storage/azure-storage-blob-nio/src/main/java/com/azure/storage/blob/nio/AzurePath.java

@@ -6,6 +6,7 @@
 import com.azure.core.util.logging.ClientLogger;
 import com.azure.storage.blob.BlobClient;
 import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobUrlParts;
 
 import java.io.File;
 import java.io.IOException;
@@ -33,8 +34,11 @@
  * An object that may be used to locate a file in a file system.
  * <p>
  * The root component, if it is present, is the first element of the path and is denoted by a {@code ':'} as the last
- * character. Hence, only one instance of {@code ':'} may appear in a path string and it may only be the last character
- * of the first element in the path. The root component is used to identify which container a path belongs to.
+ * character. Hence, only one instance of {@code ':'} may appear in a path string, and it may only be the last character
+ * of the first element in the path. The root component is used to identify which container a path belongs to. All other
+ * path elements, including separators, are considered as the blob name. {@link AzurePath#fromBlobUrl} may
+ * be used to convert a typical http url pointing to a blob into an {@code AzurePath} object pointing to the same
+ * resource.
  * <p>
  * Constructing a syntactically valid path does not ensure a resource exists at the given path. An error will
  * not be thrown until trying to access an invalid resource, e.g. trying to access a resource that does not exist.
@@ -742,6 +746,36 @@ public BlobClient toBlobClient() throws IOException {
         return containerClient.getBlobClient(blobName);
     }
 
+    /**
+     * A utility method to conveniently convert from a url to a storage resource to an {@code AzurePath} pointing to the
+     * same resource.
+     *
+     * The url must be well formatted. There must be an open filesystem corresponding to the account which contains the
+     * blob. Otherwise, a {@link java.nio.file.FileSystemNotFoundException} will be thrown.
+     *
+     * The url may point to either an account, container, or blob. If it points to an account, the path will be empty,
+     * but it will have an internal reference to the file system containing it, meaning instance methods may be
+     * performed on the path to construct a reference to another object. If it points to a container, there will be one
+     * element, which is the root element. Everything after the container, that is the blob name, will then be appended
+     * after the root element.
+     *
+     * IP style urls are not currently supported.
+     *
+     * The {@link AzureFileSystemProvider} can typically be obtained via {@link AzureFileSystem#provider()}.
+     *
+     * @param provider The installed {@link AzureFileSystemProvider} that manages open file systems for this jvm.
+     * @param url The url to the desired resource.
+     * @return An {@link AzurePath} which points to the resrouce identified by the url.
+     * @throws URISyntaxException If the url contains elements which are not well formatted.
+     */
+    public static AzurePath fromBlobUrl(AzureFileSystemProvider provider, String url) throws URISyntaxException {
+        BlobUrlParts parts = BlobUrlParts.parse(url);
+        URI fileSystemUri = hostToFileSystemUri(provider, parts.getScheme(), parts.getHost());
+        FileSystem parentFileSystem = provider.getFileSystem(fileSystemUri);
+        return new AzurePath((AzureFileSystem) parentFileSystem, fileStoreToRoot(parts.getBlobContainerName()),
+            parts.getBlobName() == null ? "" : parts.getBlobName());
+    }
+
     /**
      * @return Whether this path consists of only a root component.
      */
@@ -782,6 +816,18 @@ private String rootToFileStore(String root) {
         return root.substring(0, root.length() - 1); // Remove the ROOT_DIR_SUFFIX
     }
 
+    private static String fileStoreToRoot(String fileStore) {
+        if (fileStore == null || "".equals(fileStore)) {
+            return "";
+        }
+        return fileStore + ROOT_DIR_SUFFIX;
+    }
+
+    private static URI hostToFileSystemUri(AzureFileSystemProvider provider, String scheme, String host)
+        throws URISyntaxException {
+        return new URI(provider.getScheme() + "://?endpoint=" + scheme + "://" + host);
+    }
+
     static void ensureFileSystemOpen(Path p) {
         if (!p.getFileSystem().isOpen()) {
             throw LoggingUtility.logError(((AzurePath) p).logger,

sdk/storage/azure-storage-blob-nio/src/test/java/com/azure/storage/blob/nio/AzureFileStoreTest.groovy

@@ -26,7 +26,7 @@ class AzureFileStoreTest extends APISpec {
     def "Name"() {
         setup:
         def name = generateContainerName()
-        def store = new AzureFileStore(fs, name)
+        def store = new AzureFileStore(fs, name, false)
 
         expect:
         store.name() == name

sdk/storage/azure-storage-blob-nio/src/test/java/com/azure/storage/blob/nio/AzureFileSystemTest.groovy

@@ -100,6 +100,24 @@ class AzureFileSystemTest extends APISpec {
         thrown(IOException)
     }
 
+    def "Create skip container check"() {
+        setup:
+        config[AzureFileSystem.AZURE_STORAGE_SAS_TOKEN_CREDENTIAL] = new AzureSasCredential(
+            primaryBlobServiceClient.generateAccountSas(
+                new AccountSasSignatureValues(OffsetDateTime.now().plusDays(2),
+                    AccountSasPermission.parse("d"), new AccountSasService().setBlobAccess(true),
+                    new AccountSasResourceType().setContainer(true))))
+        config[AzureFileSystem.AZURE_STORAGE_FILE_STORES] = generateContainerName()
+        config[AzureFileSystem.AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK] = true
+
+        when:
+        // This would fail, but we skipped the check
+        new AzureFileSystem(new AzureFileSystemProvider(), environment.primaryAccount.blobEndpoint, config)
+
+        then:
+        notThrown(IOException)
+    }
+
     def "Close"() {
         setup:
         def provider = new AzureFileSystemProvider()

sdk/storage/azure-storage-blob-nio/src/test/java/com/azure/storage/blob/nio/AzurePathTest.groovy

@@ -6,6 +6,7 @@ package com.azure.storage.blob.nio
 import spock.lang.ResourceLock
 import spock.lang.Unroll
 
+import java.nio.file.FileSystemNotFoundException
 import java.nio.file.FileSystems
 
 @ResourceLock("AzurePathTest")
@@ -17,7 +18,9 @@ class AzurePathTest extends APISpec {
         def config = initializeConfigMap()
         config[AzureFileSystem.AZURE_STORAGE_SHARED_KEY_CREDENTIAL] = environment.primaryAccount.credential
         config[AzureFileSystem.AZURE_STORAGE_FILE_STORES] = "jtcazurepath1,jtcazurepath2"
-        fs = new AzureFileSystem(new AzureFileSystemProvider(), environment.primaryAccount.blobEndpoint, config)
+        def provider = new AzureFileSystemProvider()
+        fs = (AzureFileSystem) provider.newFileSystem(
+            new URI("azb://?endpoint=" + environment.primaryAccount.blobEndpoint), config)
     }
 
     def "GetFileSystem"() {
@@ -423,4 +426,39 @@ class AzurePathTest extends APISpec {
         then:
         thrown(IOException)
     }
+
+    @Unroll
+    def "FromBlobUrl"() {
+        setup:
+        // Adjust the parameterized urls to point at real resources
+        def scheme = environment.primaryAccount.blobEndpoint.startsWith("https") ? "https" : "http"
+        url = scheme + url
+        url = url.replace("myaccount", environment.primaryAccount.name)
+        url = url.replace("containername", "jtcazurepath1")
+
+        path = path.replace("myaccount", environment.primaryAccount.name)
+        path = path.replace("containername", "jtcazurepath1")
+
+        when:
+        def resultPath = AzurePath.fromBlobUrl((AzureFileSystemProvider) fs.provider(), url)
+
+        then:
+        resultPath.getFileSystem() == fs
+        resultPath.toString() == path
+
+        where:
+        url                                                                     | path
+        "://myaccount.blob.core.windows.net/containername/blobname"         | "containername:/blobname"
+        "://myaccount.blob.core.windows.net/containername/dirname/blobname" | "containername:/dirname/blobname"
+        "://myaccount.blob.core.windows.net/containername"                  | "containername:"
+        "://myaccount.blob.core.windows.net/"                               | ""
+    }
+
+    def "FromBlobUrl no open file system"() {
+        when:
+        AzurePath.fromBlobUrl(new AzureFileSystemProvider(), "http://myaccount.blob.core.windows.net/container/blob")
+
+        then:
+        thrown(FileSystemNotFoundException)
+    }
 }

sdk/storage/azure-storage-blob-nio/src/test/resources/session-records/AzureFileSystemTestCreateSkipContainerCheck.json

@@ -0,0 +1,4 @@
+{
+  "networkCallRecords" : [ ],
+  "variables" : [ "0b3431c000b3431c056410375795eb7d50060472881a", "0b3431c010b3431c05647858174b0c8e62d9f46b7b0c" ]
+}