diff --git a/eng/centralpackagemanagement/overrides/Azure.Storage.Blobs.Packages.props b/eng/centralpackagemanagement/overrides/Azure.Storage.Blobs.Packages.props new file mode 100644 index 000000000000..7c33b85bf879 --- /dev/null +++ b/eng/centralpackagemanagement/overrides/Azure.Storage.Blobs.Packages.props @@ -0,0 +1,5 @@ + + + + + diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net10.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net10.0.cs index b4377602322d..36acdd95455a 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net10.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net10.0.cs @@ -1299,18 +1299,22 @@ public partial class GetBlobsByHierarchyOptions { public GetBlobsByHierarchyOptions() { } public string Delimiter { get { throw null; } set { } } + public string EndBefore { get { throw null; } set { } } public string Prefix { get { throw null; } set { } } public string StartFrom { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } } + public bool UseApacheArrow { get { throw null; } set { } } } public partial class GetBlobsOptions { public GetBlobsOptions() { } + public string EndBefore { get { throw null; } set { } } public string Prefix { get { throw null; } set { } } public string StartFrom { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } } + public bool UseApacheArrow { get { throw null; } set { } } } public partial class GetBlobTagResult { diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs index b4377602322d..36acdd95455a 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs @@ -1299,18 +1299,22 @@ public partial class GetBlobsByHierarchyOptions { public GetBlobsByHierarchyOptions() { } public string Delimiter { get { throw null; } set { } } + public string EndBefore { get { throw null; } set { } } public string Prefix { get { throw null; } set { } } public string StartFrom { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } } + public bool UseApacheArrow { get { throw null; } set { } } } public partial class GetBlobsOptions { public GetBlobsOptions() { } + public string EndBefore { get { throw null; } set { } } public string Prefix { get { throw null; } set { } } public string StartFrom { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } } + public bool UseApacheArrow { get { throw null; } set { } } } public partial class GetBlobTagResult { diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index 2e9275801452..53302bc9d7e6 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -1299,18 +1299,22 @@ public partial class GetBlobsByHierarchyOptions { public GetBlobsByHierarchyOptions() { } public string Delimiter { get { throw null; } set { } } + public string EndBefore { get { throw null; } set { } } public string Prefix { get { throw null; } set { } } public string StartFrom { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } } + public bool UseApacheArrow { get { throw null; } set { } } } public partial class GetBlobsOptions { public GetBlobsOptions() { } + public string EndBefore { get { throw null; } set { } } public string Prefix { get { throw null; } set { } } public string StartFrom { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } } + public bool UseApacheArrow { get { throw null; } set { } } } public partial class GetBlobTagResult { diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs index 2e9275801452..53302bc9d7e6 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs @@ -1299,18 +1299,22 @@ public partial class GetBlobsByHierarchyOptions { public GetBlobsByHierarchyOptions() { } public string Delimiter { get { throw null; } set { } } + public string EndBefore { get { throw null; } set { } } public string Prefix { get { throw null; } set { } } public string StartFrom { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } } + public bool UseApacheArrow { get { throw null; } set { } } } public partial class GetBlobsOptions { public GetBlobsOptions() { } + public string EndBefore { get { throw null; } set { } } public string Prefix { get { throw null; } set { } } public string StartFrom { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } } + public bool UseApacheArrow { get { throw null; } set { } } } public partial class GetBlobTagResult { diff --git a/sdk/storage/Azure.Storage.Blobs/assets.json b/sdk/storage/Azure.Storage.Blobs/assets.json index 78e5eeba71c2..8901a970d80f 100644 --- a/sdk/storage/Azure.Storage.Blobs/assets.json +++ b/sdk/storage/Azure.Storage.Blobs/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Blobs", - "Tag": "net/storage/Azure.Storage.Blobs_632dc57c2e" + "Tag": "net/storage/Azure.Storage.Blobs_e7ef41304f" } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index 04eb9344e71b..01402d40ab84 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -38,6 +38,7 @@ + diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs index 4f44b45cb5e7..81828f48576e 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs @@ -8,6 +8,9 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using System.Xml.Linq; +using Apache.Arrow; +using Apache.Arrow.Ipc; using Azure.Core; using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; @@ -2524,10 +2527,12 @@ public virtual Pageable GetBlobs( CancellationToken cancellationToken = default) => new GetBlobsAsyncCollection( this, - options?.Traits ?? BlobTraits.None, - options?.States ?? BlobStates.None, - options?.Prefix, - startFrom: options?.StartFrom) + useApacheArrow: options?.UseApacheArrow ?? false, + traits: options?.Traits ?? BlobTraits.None, + states: options?.States ?? BlobStates.None, + prefix: options?.Prefix, + startFrom: options?.StartFrom, + endBefore: options?.EndBefore) .ToSyncCollection(cancellationToken); /// @@ -2563,10 +2568,12 @@ public virtual AsyncPageable GetBlobsAsync( CancellationToken cancellationToken = default) => new GetBlobsAsyncCollection( this, - options?.Traits ?? BlobTraits.None, - options?.States ?? BlobStates.None, - options?.Prefix, - options?.StartFrom) + useApacheArrow: options?.UseApacheArrow ?? false, + traits: options?.Traits ?? BlobTraits.None, + states: options?.States ?? BlobStates.None, + prefix: options?.Prefix, + startFrom: options?.StartFrom, + endBefore: options?.EndBefore) .ToAsyncCollection(cancellationToken); /// @@ -2612,7 +2619,7 @@ public virtual Pageable GetBlobs( BlobStates states, string prefix, CancellationToken cancellationToken) => - new GetBlobsAsyncCollection(this, traits, states, prefix, startFrom: default).ToSyncCollection(cancellationToken); + new GetBlobsAsyncCollection(this, false, traits, states, prefix, startFrom: default, endBefore: default).ToSyncCollection(cancellationToken); /// /// The @@ -2657,7 +2664,7 @@ public virtual AsyncPageable GetBlobsAsync( BlobStates states, string prefix, CancellationToken cancellationToken) => - new GetBlobsAsyncCollection(this, traits, states, prefix, startFrom: default).ToAsyncCollection(cancellationToken); + new GetBlobsAsyncCollection(this, false, traits, states, prefix, startFrom: default, endBefore: default).ToAsyncCollection(cancellationToken); /// /// The operation returns a @@ -2673,6 +2680,9 @@ public virtual AsyncPageable GetBlobsAsync( /// /// List Blobs. /// + /// + /// Specifies whether to use Apache Arrow to list blobs. + /// /// /// An optional string value that identifies the segment of the list /// of blobs to be returned with the next listing operation. The @@ -2697,6 +2707,11 @@ public virtual AsyncPageable GetBlobsAsync( /// is used to list paths starting from a defined location within prefix’s specified range. /// For non-recursive list, only one entity level is supported. /// + /// + /// Optional. Specifies a fully qualified path within the container, + /// ending the listing when all results before have been returned. + /// This is only supported if is set to true. + /// /// /// Gets or sets a value indicating the size of the page that should be /// requested. @@ -2719,11 +2734,13 @@ public virtual AsyncPageable GetBlobsAsync( /// containing each failure instance. /// internal async Task> GetBlobsInternal( + bool useApacheArrow, string marker, BlobTraits traits, BlobStates states, string prefix, string startFrom, + string endBefore, int? pageSizeHint, bool async, CancellationToken cancellationToken) @@ -2743,35 +2760,95 @@ internal async Task> GetBlobsInternal( try { scope.Start(); - ResponseWithHeaders response; - if (async) + ListBlobsFlatSegmentResponse listblobFlatResponse; + Response rawResponse; + + if (useApacheArrow) { - response = await ContainerRestClient.ListBlobFlatSegmentAsync( - prefix: prefix, - marker: marker, - maxresults: pageSizeHint, - include: BlobExtensions.AsIncludeItems(traits, states), - startFrom: startFrom, - cancellationToken: cancellationToken) - .ConfigureAwait(false); + ResponseWithHeaders arrowResponse; + + if (async) + { + arrowResponse = await ContainerRestClient.ListBlobFlatSegmentApacheArrowAsync( + prefix: prefix, + marker: marker, + maxresults: pageSizeHint, + include: BlobExtensions.AsIncludeItems(traits, states), + startFrom: startFrom, + endBefore: endBefore, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + else + { + arrowResponse = ContainerRestClient.ListBlobFlatSegmentApacheArrow( + prefix: prefix, + marker: marker, + maxresults: pageSizeHint, + include: BlobExtensions.AsIncludeItems(traits, states), + startFrom: startFrom, + endBefore: endBefore, + cancellationToken: cancellationToken); + } + + rawResponse = arrowResponse.GetRawResponse(); + + if (arrowResponse.Headers.ContentType == Constants.Blob.ApacheArrowContentType) + { + // Parse using Apache Arrow + listblobFlatResponse = await ParseArrowListBlobsFlatResponse( + arrowResponse.Value, + prefix, + marker, + pageSizeHint, + async, + cancellationToken).ConfigureAwait(false); + } + else + { + // XML fallback: server returned XML despite requesting Arrow + listblobFlatResponse = default; + var document = XDocument.Load(arrowResponse.Value, LoadOptions.PreserveWhitespace); + if (document.Element("EnumerationResults") is XElement enumerationResultsElement) + { + listblobFlatResponse = ListBlobsFlatSegmentResponse.DeserializeListBlobsFlatSegmentResponse(enumerationResultsElement); + } + } } else { - response = ContainerRestClient.ListBlobFlatSegment( - prefix: prefix, - marker: marker, - maxresults: pageSizeHint, - include: BlobExtensions.AsIncludeItems(traits, states), - startFrom: startFrom, - cancellationToken: cancellationToken); - } + ResponseWithHeaders response; + + if (async) + { + response = await ContainerRestClient.ListBlobFlatSegmentAsync( + prefix: prefix, + marker: marker, + maxresults: pageSizeHint, + include: BlobExtensions.AsIncludeItems(traits, states), + startFrom: startFrom, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + else + { + response = ContainerRestClient.ListBlobFlatSegment( + prefix: prefix, + marker: marker, + maxresults: pageSizeHint, + include: BlobExtensions.AsIncludeItems(traits, states), + startFrom: startFrom, + cancellationToken: cancellationToken); + } - ListBlobsFlatSegmentResponse listblobFlatResponse = response.Value; + listblobFlatResponse = response.Value; + rawResponse = response.GetRawResponse(); + } if ((traits & BlobTraits.Metadata) != BlobTraits.Metadata) { - List blobItemInternals = response.Value.Segment.BlobItems.Select(r => new BlobItemInternal( + List blobItemInternals = listblobFlatResponse.Segment.BlobItems.Select(r => new BlobItemInternal( r.Name, r.Deleted, r.Snapshot, @@ -2787,7 +2864,7 @@ internal async Task> GetBlobsInternal( return Response.FromValue( listblobFlatResponse, - response.GetRawResponse()); + rawResponse); } catch (Exception ex) { @@ -2802,8 +2879,258 @@ internal async Task> GetBlobsInternal( } } } + + private async Task ParseArrowListBlobsFlatResponse( + Stream arrowStream, + string prefix, + string marker, + int? maxResults, + bool async, + CancellationToken cancellationToken) + { + var (nextMarker, blobItems, _) = await ParseArrowBlobsResponse( + arrowStream, + async, + cancellationToken).ConfigureAwait(false); + + return new ListBlobsFlatSegmentResponse( + serviceEndpoint: Uri.GetLeftPart(UriPartial.Authority), + containerName: Name, + prefix: prefix, + marker: marker, + maxResults: maxResults, + segment: new BlobFlatListSegment(blobItems), + nextMarker: nextMarker); + } #endregion GetBlobs + #region ApacheArrowHelpers + private async Task<(string NextMarker, List BlobItems, List BlobPrefixes)> ParseArrowBlobsResponse( + Stream arrowStream, + bool async, + CancellationToken cancellationToken) + { + using var reader = new ArrowStreamReader(arrowStream); + + string nextMarker = null; + if (reader.Schema.Metadata != null) + { + reader.Schema.Metadata.TryGetValue("NextMarker", out nextMarker); + } + + var blobItems = new List(); + var blobPrefixes = new List(); + + while (true) + { + RecordBatch batch = async + ? await reader.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false) + : reader.ReadNextRecordBatch(); + + if (batch == null) + { + break; + } + + // Blob-level columns + StringArray nameCol = GetArrowColumn(batch, "Name") as StringArray; + StringArray resourceTypeCol = GetArrowColumn(batch, "ResourceType") as StringArray; + BooleanArray deletedCol = GetArrowColumn(batch, "Deleted") as BooleanArray; + StringArray snapshotCol = GetArrowColumn(batch, "Snapshot") as StringArray; + StringArray versionIdCol = GetArrowColumn(batch, "VersionId") as StringArray; + BooleanArray isCurrentVersionCol = GetArrowColumn(batch, "IsCurrentVersion") as BooleanArray; + BooleanArray hasVersionsOnlyCol = GetArrowColumn(batch, "HasVersionsOnly") as BooleanArray; + + // Properties columns + TimestampArray creationTimeCol = GetArrowColumn(batch, "Creation-Time") as TimestampArray; + TimestampArray lastModifiedCol = GetArrowColumn(batch, "Last-Modified") as TimestampArray; + StringArray etagCol = GetArrowColumn(batch, "Etag") as StringArray; + UInt64Array contentLengthCol = GetArrowColumn(batch, "Content-Length") as UInt64Array; + StringArray contentTypeCol = GetArrowColumn(batch, "Content-Type") as StringArray; + StringArray contentEncodingCol = GetArrowColumn(batch, "Content-Encoding") as StringArray; + StringArray contentLanguageCol = GetArrowColumn(batch, "Content-Language") as StringArray; + StringArray contentMD5Col = GetArrowColumn(batch, "Content-MD5") as StringArray; + StringArray contentDispositionCol = GetArrowColumn(batch, "Content-Disposition") as StringArray; + StringArray cacheControlCol = GetArrowColumn(batch, "Cache-Control") as StringArray; + UInt64Array blobSequenceNumberCol = GetArrowColumn(batch, "x-ms-blob-sequence-number") as UInt64Array; + StringArray blobTypeCol = GetArrowColumn(batch, "BlobType") as StringArray; + StringArray leaseStatusCol = GetArrowColumn(batch, "LeaseStatus") as StringArray; + StringArray leaseStateCol = GetArrowColumn(batch, "LeaseState") as StringArray; + StringArray leaseDurationCol = GetArrowColumn(batch, "LeaseDuration") as StringArray; + StringArray copyIdCol = GetArrowColumn(batch, "CopyId") as StringArray; + StringArray copyStatusCol = GetArrowColumn(batch, "CopyStatus") as StringArray; + StringArray copySourceCol = GetArrowColumn(batch, "CopySource") as StringArray; + StringArray copyProgressCol = GetArrowColumn(batch, "CopyProgress") as StringArray; + TimestampArray copyCompletionTimeCol = GetArrowColumn(batch, "CopyCompletionTime") as TimestampArray; + StringArray copyStatusDescriptionCol = GetArrowColumn(batch, "CopyStatusDescription") as StringArray; + StringArray destinationSnapshotCol = GetArrowColumn(batch, "CopyDestinationSnapshot") as StringArray; + BooleanArray serverEncryptedCol = GetArrowColumn(batch, "ServerEncrypted") as BooleanArray; + BooleanArray incrementalCopyCol = GetArrowColumn(batch, "IncrementalCopy") as BooleanArray; + TimestampArray deletedTimeCol = GetArrowColumn(batch, "DeletedTime") as TimestampArray; + UInt64Array remainingRetentionDaysCol = GetArrowColumn(batch, "RemainingRetentionDays") as UInt64Array; + StringArray accessTierCol = GetArrowColumn(batch, "AccessTier") as StringArray; + BooleanArray accessTierInferredCol = GetArrowColumn(batch, "AccessTierInferred") as BooleanArray; + StringArray archiveStatusCol = GetArrowColumn(batch, "ArchiveStatus") as StringArray; + StringArray smartAccessTierCol = GetArrowColumn(batch, "SmartAccessTier") as StringArray; + StringArray customerProvidedKeySha256Col = GetArrowColumn(batch, "CustomerProvidedKeySha256") as StringArray; + StringArray encryptionScopeCol = GetArrowColumn(batch, "EncryptionScope") as StringArray; + TimestampArray accessTierChangeTimeCol = GetArrowColumn(batch, "AccessTierChangeTime") as TimestampArray; + UInt64Array tagCountCol = GetArrowColumn(batch, "TagCount") as UInt64Array; + BooleanArray sealedCol = GetArrowColumn(batch, "Sealed") as BooleanArray; + StringArray rehydratePriorityCol = GetArrowColumn(batch, "RehydratePriority") as StringArray; + TimestampArray lastAccessTimeCol = GetArrowColumn(batch, "LastAccessTime") as TimestampArray; + TimestampArray immutabilityPolicyUntilDateCol = GetArrowColumn(batch, "ImmutabilityPolicyUntilDate") as TimestampArray; + StringArray immutabilityPolicyModeCol = GetArrowColumn(batch, "ImmutabilityPolicyMode") as StringArray; + BooleanArray legalHoldCol = GetArrowColumn(batch, "LegalHold") as BooleanArray; + + // Map columns + MapArray tagsCol = GetArrowColumn(batch, "Tags") as MapArray; + MapArray metadataCol = GetArrowColumn(batch, "Metadata") as MapArray; + MapArray orMetadataCol = GetArrowColumn(batch, "OrMetadata") as MapArray; + + for (int i = 0; i < batch.Length; i++) + { + string resourceType = resourceTypeCol?.GetString(i); + + // BlobPrefix rows only have Name populated; all other columns are null + if (string.Equals(resourceType, "blobprefix", StringComparison.InvariantCultureIgnoreCase)) + { + blobPrefixes.Add(new BlobPrefix( + new BlobName(encoded: false, content: nameCol?.GetString(i)))); + continue; + } + + string contentMD5Str = contentMD5Col?.GetString(i); + byte[] contentMD5 = contentMD5Str != null ? Convert.FromBase64String(contentMD5Str) : null; + + var properties = new BlobPropertiesInternal( + creationTime: creationTimeCol?.GetTimestamp(i), + lastModified: lastModifiedCol?.GetTimestamp(i) ?? default, + etag: etagCol?.GetString(i), + contentLength: ReadNullableLong(contentLengthCol, i), + contentType: contentTypeCol?.GetString(i), + contentEncoding: contentEncodingCol?.GetString(i), + contentLanguage: contentLanguageCol?.GetString(i), + contentMD5: contentMD5, + contentDisposition: contentDispositionCol?.GetString(i), + cacheControl: cacheControlCol?.GetString(i), + blobSequenceNumber: ReadNullableLong(blobSequenceNumberCol, i), + blobType: ReadEnum(blobTypeCol, i, s => s.ToBlobType()), + leaseStatus: ReadEnum(leaseStatusCol, i, s => s.ToLeaseStatus()), + leaseState: ReadEnum(leaseStateCol, i, s => s.ToLeaseState()), + leaseDuration: ReadEnum(leaseDurationCol, i, s => s.ToLeaseDurationType()), + copyId: copyIdCol?.GetString(i), + copyStatus: ReadEnum(copyStatusCol, i, s => s.ToCopyStatus()), + copySource: copySourceCol?.GetString(i), + copyProgress: copyProgressCol?.GetString(i), + copyCompletionTime: copyCompletionTimeCol?.GetTimestamp(i), + copyStatusDescription: copyStatusDescriptionCol?.GetString(i), + serverEncrypted: ReadNullableBool(serverEncryptedCol, i), + incrementalCopy: ReadNullableBool(incrementalCopyCol, i), + destinationSnapshot: destinationSnapshotCol?.GetString(i), + deletedTime: deletedTimeCol?.GetTimestamp(i), + remainingRetentionDays: ReadNullableInt(remainingRetentionDaysCol, i), + accessTier: ReadEnum(accessTierCol, i, s => new AccessTier(s)), + accessTierInferred: ReadNullableBool(accessTierInferredCol, i), + archiveStatus: ReadEnum(archiveStatusCol, i, s => s.ToArchiveStatus()), + smartAccessTier: ReadEnum(smartAccessTierCol, i, s => new AccessTier(s)), + customerProvidedKeySha256: customerProvidedKeySha256Col?.GetString(i), + encryptionScope: encryptionScopeCol?.GetString(i), + accessTierChangeTime: accessTierChangeTimeCol?.GetTimestamp(i), + tagCount: ReadNullableInt(tagCountCol, i), + expiresOn: null, + isSealed: ReadNullableBool(sealedCol, i), + rehydratePriority: ReadEnum(rehydratePriorityCol, i, s => s.ToRehydratePriority().Value), + lastAccessedOn: lastAccessTimeCol?.GetTimestamp(i), + immutabilityPolicyExpiresOn: immutabilityPolicyUntilDateCol?.GetTimestamp(i), + immutabilityPolicyMode: ReadEnum(immutabilityPolicyModeCol, i, s => s.ToBlobImmutabilityPolicyMode()), + legalHold: ReadNullableBool(legalHoldCol, i)); + + IReadOnlyDictionary metadata = ReadArrowMap(metadataCol, i); + IReadOnlyDictionary orMetadata = ReadArrowMap(orMetadataCol, i); + + BlobTags blobTags = null; + IReadOnlyDictionary tagsDict = ReadArrowMap(tagsCol, i); + if (tagsDict != null) + { + List tagList = new List(); + foreach (KeyValuePair kvp in tagsDict) + { + tagList.Add(new BlobTag(kvp.Key, kvp.Value)); + } + blobTags = new BlobTags(tagList); + } + + blobItems.Add(new BlobItemInternal( + name: new BlobName(encoded: false, content: nameCol?.GetString(i)), + deleted: ReadNullableBool(deletedCol, i) == true, + snapshot: snapshotCol?.GetString(i) ?? string.Empty, + versionId: versionIdCol?.GetString(i), + isCurrentVersion: ReadNullableBool(isCurrentVersionCol, i), + properties: properties, + metadata: metadata, + blobTags: blobTags, + hasVersionsOnly: ReadNullableBool(hasVersionsOnlyCol, i), + orMetadata: orMetadata)); + } + } + + return (nextMarker, blobItems, blobPrefixes); + } + + private static IArrowArray GetArrowColumn(RecordBatch batch, string name) + { + int index = batch.Schema.GetFieldIndex(name); + return index >= 0 ? batch.Column(index) : null; + } + + private static IReadOnlyDictionary ReadArrowMap(MapArray mapArray, int rowIndex) + { + if (mapArray == null || mapArray.IsNull(rowIndex)) + { + return null; + } + + StringArray keys = mapArray.Keys as StringArray; + StringArray values = mapArray.Values as StringArray; + int start = mapArray.ValueOffsets[rowIndex]; + int length = mapArray.ValueOffsets[rowIndex + 1] - start; + + var dict = new Dictionary(length); + for (int j = start; j < start + length; j++) + { + string key = keys.GetString(j); + string value = values.GetString(j); + if (key != null) + { + dict[key] = value; + } + } + return dict; + } + + private static bool? ReadNullableBool(BooleanArray array, int index) + { + return array != null && !array.IsNull(index) ? (bool?)array.GetValue(index) : null; + } + + private static long? ReadNullableLong(UInt64Array array, int index) + { + return array != null && !array.IsNull(index) ? (long?)array.GetValue(index) : null; + } + + private static int? ReadNullableInt(UInt64Array array, int index) + { + return array != null && !array.IsNull(index) ? (int?)array.GetValue(index) : null; + } + + private static T? ReadEnum(StringArray array, int index, Func parse) where T : struct + { + string value = array?.GetString(index); + return value != null ? parse(value) : null; + } + #endregion ApacheArrowHelpers + #region GetBlobsByHierarchy /// /// The @@ -2840,11 +3167,13 @@ public virtual Pageable GetBlobsByHierarchy( CancellationToken cancellationToken = default) => new GetBlobsByHierarchyAsyncCollection( this, + useApacheArrow: options?.UseApacheArrow ?? false, options?.Delimiter, options?.Traits ?? BlobTraits.None, options?.States ?? BlobStates.None, options?.Prefix, - options?.StartFrom) + options?.StartFrom, + options?.EndBefore) .ToSyncCollection(cancellationToken); /// @@ -2882,11 +3211,13 @@ public virtual AsyncPageable GetBlobsByHierarchyAsync( CancellationToken cancellationToken = default) => new GetBlobsByHierarchyAsyncCollection( this, + useApacheArrow: options?.UseApacheArrow ?? false, options?.Delimiter, options?.Traits ?? BlobTraits.None, options?.States ?? BlobStates.None, options?.Prefix, - options?.StartFrom) + options?.StartFrom, + options?.EndBefore) .ToAsyncCollection(cancellationToken); /// @@ -2952,7 +3283,7 @@ public virtual Pageable GetBlobsByHierarchy( string delimiter, string prefix, CancellationToken cancellationToken = default) => - new GetBlobsByHierarchyAsyncCollection(this, delimiter, traits, states, prefix, startFrom: default).ToSyncCollection(cancellationToken); + new GetBlobsByHierarchyAsyncCollection(this, false, delimiter, traits, states, prefix, startFrom: default, endBefore: default).ToSyncCollection(cancellationToken); /// /// The @@ -3017,7 +3348,7 @@ public virtual AsyncPageable GetBlobsByHierarchyAsync( string delimiter, string prefix, CancellationToken cancellationToken) => - new GetBlobsByHierarchyAsyncCollection(this, delimiter, traits, states, prefix, startFrom: default).ToAsyncCollection(cancellationToken); + new GetBlobsByHierarchyAsyncCollection(this, false, delimiter, traits, states, prefix, startFrom: default, endBefore: default).ToAsyncCollection(cancellationToken); /// /// The operation returns @@ -3035,6 +3366,9 @@ public virtual AsyncPageable GetBlobsByHierarchyAsync( /// /// List Blobs. /// + /// + /// Specifies whether to use Apache Arrow to list blobs. + /// /// /// An optional string value that identifies the segment of the list /// of blobs to be returned with the next listing operation. The @@ -3077,6 +3411,11 @@ public virtual AsyncPageable GetBlobsByHierarchyAsync( /// For non-recursive list, only one entity level is supported. /// For recursive list, multiple entity levels are supported. (Inclusive). /// + /// + /// Optional. Specifies a fully qualified path within the container, + /// ending the listing when all results before have been returned. + /// This is only supported if is set to true. + /// /// /// Gets or sets a value indicating the size of the page that should be /// requested. @@ -3099,12 +3438,14 @@ public virtual AsyncPageable GetBlobsByHierarchyAsync( /// containing each failure instance. /// internal async Task> GetBlobsByHierarchyInternal( + bool useApacheArrow, string marker, string delimiter, BlobTraits traits, BlobStates states, string prefix, string startFrom, + string endBefore, int? pageSizeHint, bool async, CancellationToken cancellationToken) @@ -3125,37 +3466,100 @@ internal async Task> GetBlobsByHiera try { scope.Start(); - ResponseWithHeaders response; - if (async) + ListBlobsHierarchySegmentResponse listblobHierachyResponse; + Response rawResponse; + + if (useApacheArrow) { - response = await ContainerRestClient.ListBlobHierarchySegmentAsync( - delimiter: delimiter, - prefix: prefix, - marker: marker, - maxresults: pageSizeHint, - include: BlobExtensions.AsIncludeItems(traits, states), - startFrom: startFrom, - cancellationToken: cancellationToken) - .ConfigureAwait(false); + ResponseWithHeaders arrowResponse; + + if (async) + { + arrowResponse = await ContainerRestClient.ListBlobHierarchySegmentApacheArrowAsync( + prefix: prefix, + delimiter: delimiter, + marker: marker, + maxresults: pageSizeHint, + include: BlobExtensions.AsIncludeItems(traits, states), + startFrom: startFrom, + endBefore: endBefore, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + else + { + arrowResponse = ContainerRestClient.ListBlobHierarchySegmentApacheArrow( + prefix: prefix, + delimiter: delimiter, + marker: marker, + maxresults: pageSizeHint, + include: BlobExtensions.AsIncludeItems(traits, states), + startFrom: startFrom, + endBefore: endBefore, + cancellationToken: cancellationToken); + } + + rawResponse = arrowResponse.GetRawResponse(); + + if (arrowResponse.Headers.ContentType == Constants.Blob.ApacheArrowContentType) + { + // Parse using Apache Arrow + listblobHierachyResponse = await ParseArrowListBlobsHierarchyResponse( + arrowResponse.Value, + prefix, + marker, + pageSizeHint, + delimiter, + async, + cancellationToken).ConfigureAwait(false); + } + else + { + // XML fallback: server returned XML despite requesting Arrow + listblobHierachyResponse = default; + var document = XDocument.Load(arrowResponse.Value, LoadOptions.PreserveWhitespace); + if (document.Element("EnumerationResults") is XElement enumerationResultsElement) + { + listblobHierachyResponse = ListBlobsHierarchySegmentResponse.DeserializeListBlobsHierarchySegmentResponse(enumerationResultsElement); + } + } } else { - response = ContainerRestClient.ListBlobHierarchySegment( - delimiter: delimiter, - prefix: prefix, - marker: marker, - maxresults: pageSizeHint, - include: BlobExtensions.AsIncludeItems(traits, states), - startFrom: startFrom, - cancellationToken: cancellationToken); - } + ResponseWithHeaders response; - ListBlobsHierarchySegmentResponse listblobHierachyResponse = response.Value; + if (async) + { + response = await ContainerRestClient.ListBlobHierarchySegmentAsync( + delimiter: delimiter, + prefix: prefix, + marker: marker, + maxresults: pageSizeHint, + include: BlobExtensions.AsIncludeItems(traits, states), + startFrom: startFrom, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + else + { + response = ContainerRestClient.ListBlobHierarchySegment( + delimiter: delimiter, + prefix: prefix, + marker: marker, + maxresults: pageSizeHint, + include: BlobExtensions.AsIncludeItems(traits, states), + startFrom: startFrom, + cancellationToken: cancellationToken); + } + + listblobHierachyResponse = response.Value; + rawResponse = response.GetRawResponse(); + } if ((traits & BlobTraits.Metadata) != BlobTraits.Metadata) { - List blobItemInternals = response.Value.Segment.BlobItems.Select(r => new BlobItemInternal( + List blobItemInternals = listblobHierachyResponse.Segment.BlobItems.Select(r => new BlobItemInternal( r.Name, r.Deleted, r.Snapshot, @@ -3171,7 +3575,7 @@ internal async Task> GetBlobsByHiera return Response.FromValue( listblobHierachyResponse, - response.GetRawResponse()); + rawResponse); } catch (Exception ex) { @@ -3186,6 +3590,31 @@ internal async Task> GetBlobsByHiera } } } + + private async Task ParseArrowListBlobsHierarchyResponse( + Stream arrowStream, + string prefix, + string marker, + int? maxResults, + string delimiter, + bool async, + CancellationToken cancellationToken) + { + var (nextMarker, blobItems, blobPrefixes) = await ParseArrowBlobsResponse( + arrowStream, + async, + cancellationToken).ConfigureAwait(false); + + return new ListBlobsHierarchySegmentResponse( + serviceEndpoint: Uri.GetLeftPart(UriPartial.Authority), + containerName: Name, + prefix: prefix, + marker: marker, + maxResults: maxResults, + delimiter: delimiter, + segment: new BlobHierarchyListSegment(blobPrefixes, blobItems), + nextMarker: nextMarker); + } #endregion GetBlobsByHierarchy #region UploadBlob diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerListBlobFlatSegmentApacheArrowHeaders.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerListBlobFlatSegmentApacheArrowHeaders.cs new file mode 100644 index 000000000000..c7eb897f6cb1 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerListBlobFlatSegmentApacheArrowHeaders.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using Azure.Core; + +namespace Azure.Storage.Blobs +{ + internal partial class ContainerListBlobFlatSegmentApacheArrowHeaders + { + private readonly Response _response; + public ContainerListBlobFlatSegmentApacheArrowHeaders(Response response) + { + _response = response; + } + /// The media type of the response body. Either 'application/vnd.apache.arrow.stream' if Arrow is enabled for the account, or 'application/xml' as a fallback. Clients must check this header to determine how to deserialize the response. + public string ContentType => _response.Headers.TryGetValue("Content-Type", out string value) ? value : null; + /// Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above. + public string Version => _response.Headers.TryGetValue("x-ms-version", out string value) ? value : null; + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerListBlobHierarchySegmentApacheArrowHeaders.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerListBlobHierarchySegmentApacheArrowHeaders.cs new file mode 100644 index 000000000000..688d350d44a4 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerListBlobHierarchySegmentApacheArrowHeaders.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using Azure.Core; + +namespace Azure.Storage.Blobs +{ + internal partial class ContainerListBlobHierarchySegmentApacheArrowHeaders + { + private readonly Response _response; + public ContainerListBlobHierarchySegmentApacheArrowHeaders(Response response) + { + _response = response; + } + /// The media type of the body of the response. For List Blobs this is 'application/xml'. + public string ContentType => _response.Headers.TryGetValue("Content-Type", out string value) ? value : null; + /// Indicates the version of the Blob service used to execute the request. This header is returned for requests made against version 2009-09-19 and above. + public string Version => _response.Headers.TryGetValue("x-ms-version", out string value) ? value : null; + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerRestClient.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerRestClient.cs index f067266a5596..a701c7831e4b 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerRestClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerRestClient.cs @@ -1431,6 +1431,101 @@ public ResponseWithHeaders include, int? timeout, string startFrom, string endBefore) + { + var message = _pipeline.CreateMessage(); + var request = message.Request; + request.Method = RequestMethod.Get; + var uri = new RawRequestUriBuilder(); + uri.AppendRaw(_url, false); + uri.AppendQuery("restype", "container", true); + uri.AppendQuery("comp", "list", true); + if (prefix != null) + { + uri.AppendQuery("prefix", prefix, true); + } + if (marker != null) + { + uri.AppendQuery("marker", marker, true); + } + if (maxresults != null) + { + uri.AppendQuery("maxresults", maxresults.Value, true); + } + if (include != null && !(include is Common.ChangeTrackingList changeTrackingList && changeTrackingList.IsUndefined)) + { + uri.AppendQueryDelimited("include", include, ",", true); + } + if (timeout != null) + { + uri.AppendQuery("timeout", timeout.Value, true); + } + if (startFrom != null) + { + uri.AppendQuery("startFrom", startFrom, true); + } + if (endBefore != null) + { + uri.AppendQuery("endBefore", endBefore, true); + } + request.Uri = uri; + request.Headers.Add("Accept", "application/vnd.apache.arrow.stream"); + request.Headers.Add("x-ms-version", _version); + return message; + } + + /// The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache Arrow use case so response is returned as raw to be deserialized by the client. + /// Filters the results to return only containers whose name begins with the specified prefix. + /// A string value that identifies the portion of the list of containers to be returned with the next listing operation. The operation returns the NextMarker value within the response body if the listing operation did not return all containers remaining to be listed with the current page. The NextMarker value can be used as the value for the marker parameter in a subsequent call to request the next page of list items. The marker value is opaque to the client. + /// Specifies the maximum number of containers to return. If the request does not specify maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the listing operation crosses a partition boundary, then the service will return a continuation token for retrieving the remainder of the results. For this reason, it is possible that the service will return fewer results than specified by maxresults, or than the default of 5000. + /// Include this parameter to specify one or more datasets to include in the response. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. + /// Specifies the relative path to list paths from. For non-recursive list, only one entity level is supported; For recursive list, multiple entity levels are supported. (Inclusive). + /// Specifies the relative path to end before list paths. (Exclusive). + /// The cancellation token to use. + public async Task> ListBlobFlatSegmentApacheArrowAsync(string prefix = null, string marker = null, int? maxresults = null, IEnumerable include = null, int? timeout = null, string startFrom = null, string endBefore = null, CancellationToken cancellationToken = default) + { + using var message = CreateListBlobFlatSegmentApacheArrowRequest(prefix, marker, maxresults, include, timeout, startFrom, endBefore); + await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); + var headers = new ContainerListBlobFlatSegmentApacheArrowHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + { + var value = message.ExtractResponseContent(); + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + default: + throw new RequestFailedException(message.Response); + } + } + + /// The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache Arrow use case so response is returned as raw to be deserialized by the client. + /// Filters the results to return only containers whose name begins with the specified prefix. + /// A string value that identifies the portion of the list of containers to be returned with the next listing operation. The operation returns the NextMarker value within the response body if the listing operation did not return all containers remaining to be listed with the current page. The NextMarker value can be used as the value for the marker parameter in a subsequent call to request the next page of list items. The marker value is opaque to the client. + /// Specifies the maximum number of containers to return. If the request does not specify maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the listing operation crosses a partition boundary, then the service will return a continuation token for retrieving the remainder of the results. For this reason, it is possible that the service will return fewer results than specified by maxresults, or than the default of 5000. + /// Include this parameter to specify one or more datasets to include in the response. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. + /// Specifies the relative path to list paths from. For non-recursive list, only one entity level is supported; For recursive list, multiple entity levels are supported. (Inclusive). + /// Specifies the relative path to end before list paths. (Exclusive). + /// The cancellation token to use. + public ResponseWithHeaders ListBlobFlatSegmentApacheArrow(string prefix = null, string marker = null, int? maxresults = null, IEnumerable include = null, int? timeout = null, string startFrom = null, string endBefore = null, CancellationToken cancellationToken = default) + { + using var message = CreateListBlobFlatSegmentApacheArrowRequest(prefix, marker, maxresults, include, timeout, startFrom, endBefore); + _pipeline.Send(message, cancellationToken); + var headers = new ContainerListBlobFlatSegmentApacheArrowHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + { + var value = message.ExtractResponseContent(); + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + default: + throw new RequestFailedException(message.Response); + } + } + internal HttpMessage CreateListBlobHierarchySegmentRequest(string prefix, string delimiter, string marker, int? maxresults, IEnumerable include, string startFrom, int? timeout) { var message = _pipeline.CreateMessage(); @@ -1536,6 +1631,107 @@ public ResponseWithHeaders include, int? timeout, string startFrom, string endBefore) + { + var message = _pipeline.CreateMessage(); + var request = message.Request; + request.Method = RequestMethod.Get; + var uri = new RawRequestUriBuilder(); + uri.AppendRaw(_url, false); + uri.AppendQuery("restype", "container", true); + uri.AppendQuery("comp", "list", true); + if (prefix != null) + { + uri.AppendQuery("prefix", prefix, true); + } + if (delimiter != null) + { + uri.AppendQuery("delimiter", delimiter, true); + } + if (marker != null) + { + uri.AppendQuery("marker", marker, true); + } + if (maxresults != null) + { + uri.AppendQuery("maxresults", maxresults.Value, true); + } + if (include != null && !(include is Common.ChangeTrackingList changeTrackingList && changeTrackingList.IsUndefined)) + { + uri.AppendQueryDelimited("include", include, ",", true); + } + if (timeout != null) + { + uri.AppendQuery("timeout", timeout.Value, true); + } + if (startFrom != null) + { + uri.AppendQuery("startFrom", startFrom, true); + } + if (endBefore != null) + { + uri.AppendQuery("endBefore", endBefore, true); + } + request.Uri = uri; + request.Headers.Add("Accept", "application/vnd.apache.arrow.stream"); + request.Headers.Add("x-ms-version", _version); + return message; + } + + /// [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache Arrow use case so response is returned as raw to be deserialized by the client. + /// Filters the results to return only containers whose name begins with the specified prefix. + /// When the request includes this parameter, the operation returns a BlobPrefix element in the response body that acts as a placeholder for all blobs whose names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string. + /// A string value that identifies the portion of the list of containers to be returned with the next listing operation. The operation returns the NextMarker value within the response body if the listing operation did not return all containers remaining to be listed with the current page. The NextMarker value can be used as the value for the marker parameter in a subsequent call to request the next page of list items. The marker value is opaque to the client. + /// Specifies the maximum number of containers to return. If the request does not specify maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the listing operation crosses a partition boundary, then the service will return a continuation token for retrieving the remainder of the results. For this reason, it is possible that the service will return fewer results than specified by maxresults, or than the default of 5000. + /// Include this parameter to specify one or more datasets to include in the response. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. + /// Specifies the relative path to list paths from. For non-recursive list, only one entity level is supported; For recursive list, multiple entity levels are supported. (Inclusive). + /// Specifies the relative path to end before list paths. (Exclusive). + /// The cancellation token to use. + public async Task> ListBlobHierarchySegmentApacheArrowAsync(string prefix = null, string delimiter = null, string marker = null, int? maxresults = null, IEnumerable include = null, int? timeout = null, string startFrom = null, string endBefore = null, CancellationToken cancellationToken = default) + { + using var message = CreateListBlobHierarchySegmentApacheArrowRequest(prefix, delimiter, marker, maxresults, include, timeout, startFrom, endBefore); + await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); + var headers = new ContainerListBlobHierarchySegmentApacheArrowHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + { + var value = message.ExtractResponseContent(); + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + default: + throw new RequestFailedException(message.Response); + } + } + + /// [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache Arrow use case so response is returned as raw to be deserialized by the client. + /// Filters the results to return only containers whose name begins with the specified prefix. + /// When the request includes this parameter, the operation returns a BlobPrefix element in the response body that acts as a placeholder for all blobs whose names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string. + /// A string value that identifies the portion of the list of containers to be returned with the next listing operation. The operation returns the NextMarker value within the response body if the listing operation did not return all containers remaining to be listed with the current page. The NextMarker value can be used as the value for the marker parameter in a subsequent call to request the next page of list items. The marker value is opaque to the client. + /// Specifies the maximum number of containers to return. If the request does not specify maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the listing operation crosses a partition boundary, then the service will return a continuation token for retrieving the remainder of the results. For this reason, it is possible that the service will return fewer results than specified by maxresults, or than the default of 5000. + /// Include this parameter to specify one or more datasets to include in the response. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. + /// Specifies the relative path to list paths from. For non-recursive list, only one entity level is supported; For recursive list, multiple entity levels are supported. (Inclusive). + /// Specifies the relative path to end before list paths. (Exclusive). + /// The cancellation token to use. + public ResponseWithHeaders ListBlobHierarchySegmentApacheArrow(string prefix = null, string delimiter = null, string marker = null, int? maxresults = null, IEnumerable include = null, int? timeout = null, string startFrom = null, string endBefore = null, CancellationToken cancellationToken = default) + { + using var message = CreateListBlobHierarchySegmentApacheArrowRequest(prefix, delimiter, marker, maxresults, include, timeout, startFrom, endBefore); + _pipeline.Send(message, cancellationToken); + var headers = new ContainerListBlobHierarchySegmentApacheArrowHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + { + var value = message.ExtractResponseContent(); + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + default: + throw new RequestFailedException(message.Response); + } + } + internal HttpMessage CreateGetAccountInfoRequest(int? timeout) { var message = _pipeline.CreateMessage(); @@ -1767,6 +1963,88 @@ public ResponseWithHeaders include, int? timeout, string startFrom, string endBefore) + { + var message = _pipeline.CreateMessage(); + var request = message.Request; + request.Method = RequestMethod.Get; + var uri = new RawRequestUriBuilder(); + uri.AppendRaw(_url, false); + uri.AppendRawNextLink(nextLink, false); + request.Uri = uri; + request.Headers.Add("Accept", "application/vnd.apache.arrow.stream"); + request.Headers.Add("x-ms-version", _version); + return message; + } + + /// [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache Arrow use case so response is returned as raw to be deserialized by the client. + /// The URL to the next page of results. + /// Filters the results to return only containers whose name begins with the specified prefix. + /// When the request includes this parameter, the operation returns a BlobPrefix element in the response body that acts as a placeholder for all blobs whose names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string. + /// A string value that identifies the portion of the list of containers to be returned with the next listing operation. The operation returns the NextMarker value within the response body if the listing operation did not return all containers remaining to be listed with the current page. The NextMarker value can be used as the value for the marker parameter in a subsequent call to request the next page of list items. The marker value is opaque to the client. + /// Specifies the maximum number of containers to return. If the request does not specify maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the listing operation crosses a partition boundary, then the service will return a continuation token for retrieving the remainder of the results. For this reason, it is possible that the service will return fewer results than specified by maxresults, or than the default of 5000. + /// Include this parameter to specify one or more datasets to include in the response. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. + /// Specifies the relative path to list paths from. For non-recursive list, only one entity level is supported; For recursive list, multiple entity levels are supported. (Inclusive). + /// Specifies the relative path to end before list paths. (Exclusive). + /// The cancellation token to use. + /// is null. + public async Task> ListBlobHierarchySegmentApacheArrowNextPageAsync(string nextLink, string prefix = null, string delimiter = null, string marker = null, int? maxresults = null, IEnumerable include = null, int? timeout = null, string startFrom = null, string endBefore = null, CancellationToken cancellationToken = default) + { + if (nextLink == null) + { + throw new ArgumentNullException(nameof(nextLink)); + } + + using var message = CreateListBlobHierarchySegmentApacheArrowNextPageRequest(nextLink, prefix, delimiter, marker, maxresults, include, timeout, startFrom, endBefore); + await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); + var headers = new ContainerListBlobHierarchySegmentApacheArrowHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + { + var value = message.ExtractResponseContent(); + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + default: + throw new RequestFailedException(message.Response); + } + } + + /// [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache Arrow use case so response is returned as raw to be deserialized by the client. + /// The URL to the next page of results. + /// Filters the results to return only containers whose name begins with the specified prefix. + /// When the request includes this parameter, the operation returns a BlobPrefix element in the response body that acts as a placeholder for all blobs whose names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string. + /// A string value that identifies the portion of the list of containers to be returned with the next listing operation. The operation returns the NextMarker value within the response body if the listing operation did not return all containers remaining to be listed with the current page. The NextMarker value can be used as the value for the marker parameter in a subsequent call to request the next page of list items. The marker value is opaque to the client. + /// Specifies the maximum number of containers to return. If the request does not specify maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the listing operation crosses a partition boundary, then the service will return a continuation token for retrieving the remainder of the results. For this reason, it is possible that the service will return fewer results than specified by maxresults, or than the default of 5000. + /// Include this parameter to specify one or more datasets to include in the response. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. + /// Specifies the relative path to list paths from. For non-recursive list, only one entity level is supported; For recursive list, multiple entity levels are supported. (Inclusive). + /// Specifies the relative path to end before list paths. (Exclusive). + /// The cancellation token to use. + /// is null. + public ResponseWithHeaders ListBlobHierarchySegmentApacheArrowNextPage(string nextLink, string prefix = null, string delimiter = null, string marker = null, int? maxresults = null, IEnumerable include = null, int? timeout = null, string startFrom = null, string endBefore = null, CancellationToken cancellationToken = default) + { + if (nextLink == null) + { + throw new ArgumentNullException(nameof(nextLink)); + } + + using var message = CreateListBlobHierarchySegmentApacheArrowNextPageRequest(nextLink, prefix, delimiter, marker, maxresults, include, timeout, startFrom, endBefore); + _pipeline.Send(message, cancellationToken); + var headers = new ContainerListBlobHierarchySegmentApacheArrowHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + { + var value = message.ExtractResponseContent(); + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + default: + throw new RequestFailedException(message.Response); + } + } + private static ResponseClassifier _responseClassifier201; private static ResponseClassifier ResponseClassifier201 => _responseClassifier201 ??= new StatusCodeClassifier(stackalloc ushort[] { 201 }); } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsAsyncCollection.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsAsyncCollection.cs index c2fd83c87452..9c24beb94f01 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsAsyncCollection.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsAsyncCollection.cs @@ -15,23 +15,29 @@ namespace Azure.Storage.Blobs.Models internal class GetBlobsAsyncCollection : StorageCollectionEnumerator { private readonly BlobContainerClient _client; + private readonly bool _useApacheArrow; private readonly BlobTraits _traits; private readonly BlobStates _states; private readonly string _prefix; private readonly string _startFrom; + private readonly string _endBefore; public GetBlobsAsyncCollection( BlobContainerClient client, + bool useApacheArrow, BlobTraits traits, BlobStates states, string prefix, - string startFrom) + string startFrom, + string endBefore) { _client = client; + _useApacheArrow = useApacheArrow; _traits = traits; _states = states; _prefix = prefix; _startFrom = startFrom; + _endBefore = endBefore; } public override async ValueTask> GetNextPageAsync( @@ -45,11 +51,13 @@ public override async ValueTask> GetNextPageAsync( if (async) { response = await _client.GetBlobsInternal( + useApacheArrow: _useApacheArrow, marker: continuationToken, traits: _traits, states: _states, prefix: _prefix, startFrom: _startFrom, + endBefore: _endBefore, pageSizeHint: pageSizeHint, async: async, cancellationToken: cancellationToken) @@ -58,11 +66,13 @@ public override async ValueTask> GetNextPageAsync( else { response = _client.GetBlobsInternal( + useApacheArrow: _useApacheArrow, marker: continuationToken, traits: _traits, states: _states, prefix: _prefix, startFrom: _startFrom, + endBefore: _endBefore, pageSizeHint: pageSizeHint, async: async, cancellationToken: cancellationToken) diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsByHierarchyAsyncCollection.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsByHierarchyAsyncCollection.cs index 12af6fa746cf..95b5f7a972bd 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsByHierarchyAsyncCollection.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsByHierarchyAsyncCollection.cs @@ -13,26 +13,32 @@ namespace Azure.Storage.Blobs.Models internal class GetBlobsByHierarchyAsyncCollection : StorageCollectionEnumerator { private readonly BlobContainerClient _client; + private readonly bool _useApacheArrow; private readonly BlobTraits _traits; private readonly BlobStates _states; private readonly string _delimiter; private readonly string _prefix; private readonly string _startFrom; + private readonly string _endBefore; public GetBlobsByHierarchyAsyncCollection( BlobContainerClient client, + bool useApacheArrow, string delimiter, BlobTraits traits, BlobStates states, string prefix, - string startFrom) + string startFrom, + string endBefore) { _client = client; + _useApacheArrow = useApacheArrow; _delimiter = delimiter; _traits = traits; _states = states; _prefix = prefix; _startFrom = startFrom; + _endBefore = endBefore; } public override async ValueTask> GetNextPageAsync( @@ -46,12 +52,14 @@ public override async ValueTask> GetNextPageAsync( if (async) { response = await _client.GetBlobsByHierarchyInternal( + useApacheArrow: _useApacheArrow, marker: continuationToken, delimiter: _delimiter, traits: _traits, states: _states, prefix: _prefix, startFrom: _startFrom, + endBefore: _endBefore, pageSizeHint: pageSizeHint, async: async, cancellationToken: cancellationToken) @@ -60,12 +68,14 @@ public override async ValueTask> GetNextPageAsync( else { response = _client.GetBlobsByHierarchyInternal( + useApacheArrow: _useApacheArrow, marker: continuationToken, delimiter: _delimiter, traits: _traits, states: _states, prefix: _prefix, startFrom: _startFrom, + endBefore: _endBefore, pageSizeHint: pageSizeHint, async: async, cancellationToken: cancellationToken) diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsByHierarchyOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsByHierarchyOptions.cs index 07de8ee17a6a..841b0a2e64de 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsByHierarchyOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsByHierarchyOptions.cs @@ -50,5 +50,18 @@ public class GetBlobsByHierarchyOptions /// For recursive list, multiple entity levels are supported. (Inclusive). /// public string StartFrom { get; set; } + + /// + /// Optional. Specifies a fully qualified path within the container, + /// ending the listing when all results before have been returned. + /// This is only supported if is set to true. + /// + public string EndBefore { get; set; } + + /// + /// Optional. Specifies whether we are using Apache Arrow, rather than XML, + /// to list blobs. Defaults to false. + /// + public bool UseApacheArrow { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsOptions.cs index e0a9c17562de..9dafe6a06227 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/GetBlobsOptions.cs @@ -31,5 +31,18 @@ public class GetBlobsOptions /// For non-recursive list, only one entity level is supported. /// public string StartFrom { get; set; } + + /// + /// Optional. Specifies a fully qualified path within the container, + /// ending the listing when all results before have been returned. + /// This is only supported if is set to true. + /// + public string EndBefore { get; set; } + + /// + /// Optional. Specifies whether we are using Apache Arrow, rather than XML, + /// to list blobs. Defaults to false. + /// + public bool UseApacheArrow { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/autorest.md b/sdk/storage/Azure.Storage.Blobs/src/autorest.md index e2d508918749..96cafcc2e38c 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/autorest.md +++ b/sdk/storage/Azure.Storage.Blobs/src/autorest.md @@ -4,7 +4,7 @@ Run `dotnet build /t:GenerateCode` to generate code. ``` yaml input-file: - - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/15d7f54a5389d5906ffb4e56bb2f38fe5525c0d3/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-06-06/blob.json + - https://raw.githubusercontent.com/nickliu-msft/azure-rest-api-specs/ab1ec63862fdf4506cfb1cdd4c8105281b5de3f0/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-10-06/blob.json generation1-convenience-client: true # https://github.com/Azure/autorest/issues/4075 skip-semantics-validation: true diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientBuilderExtensions.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientBuilderExtensions.cs index 5a3cb6859e03..b348a0c7d634 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ClientBuilderExtensions.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientBuilderExtensions.cs @@ -70,6 +70,9 @@ public static BlobServiceClient GetServiceClient_Hns(this BlobsClientBuilder cli public static BlobServiceClient GetServiceClient_SoftDelete(this BlobsClientBuilder clientBuilder) => clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigSoftDelete); + public static BlobServiceClient GetServiceClient_SoftDelete_OAuth(this BlobsClientBuilder clientBuilder, TokenCredential tokenCredential) => + clientBuilder.GetServiceClientFromOauthConfig(clientBuilder.Tenants.TestConfigSoftDelete, tokenCredential); + public static async Task GetTestContainerAsync( this BlobsClientBuilder clientBuilder, BlobServiceClient service = default, diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ContainerClientTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ContainerClientTests.cs index b3f057cd5ba5..d1907c429111 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ContainerClientTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ContainerClientTests.cs @@ -2826,6 +2826,525 @@ public async Task ListBlobsFlatSegmentAsync_StartFrom() Assert.AreEqual(3, blobs.Count); } + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + // Act + var blobs = new List(); + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true + }; + await foreach (Page page in test.Container.GetBlobsAsync(options: options).AsPages()) + { + blobs.AddRange(page.Values); + } + + // Assert + Assert.AreEqual(BlobNames.Length, blobs.Count); + + var foundBlobNames = blobs.Select(blob => blob.Name).ToArray(); + + Assert.IsTrue(BlobNames.All(blobName => foundBlobNames.Contains(blobName))); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_Tags() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(); + AppendBlobClient appendBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + IDictionary tags = BuildTags(); + AppendBlobCreateOptions options = new AppendBlobCreateOptions + { + Tags = tags + }; + await appendBlob.CreateAsync(options); + + GetBlobsOptions getBlobsOptions = new GetBlobsOptions + { + UseApacheArrow = true, + Traits = BlobTraits.Tags + }; + + // Act + IList blobItems = await test.Container.GetBlobsAsync(options: getBlobsOptions).ToListAsync(); + + // Assert + AssertDictionaryEquality(tags, blobItems[0].Tags); + Assert.AreEqual(tags.Count, blobItems[0].Properties.TagCount); + } + + [Ignore("Feature not supported in current test environment")] + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + [TestCase(null)] + [TestCase(RehydratePriority.Standard)] + [TestCase(RehydratePriority.High)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_RehydratePriority(RehydratePriority? rehydratePriority) + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(); + + BlockBlobClient blockBlob = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName())); + byte[] data = GetRandomBuffer(Constants.KB); + using Stream stream = new MemoryStream(data); + await blockBlob.UploadAsync(stream); + + if (rehydratePriority.HasValue) + { + await blockBlob.SetAccessTierAsync( + AccessTier.Archive); + + await blockBlob.SetAccessTierAsync( + AccessTier.Hot, + rehydratePriority: rehydratePriority.Value); + } + + // Act + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true + }; + IList blobItems = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(rehydratePriority, blobItems[0].Properties.RehydratePriority); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_MaxResults() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + // Act + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true + }; + + int numPages = 0; + // Act + await foreach (Page page in test.Container.GetBlobsAsync(options: options) + .AsPages(pageSizeHint: 2)) + { + // Assert + Assert.AreEqual(2, page.Values.Count); + ++numPages; + } + + // Assert + Assert.AreEqual(4, numPages); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_Metadata() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + IDictionary metadata = BuildMetadata(); + await blob.CreateIfNotExistsAsync(metadata: metadata); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + Traits = BlobTraits.Metadata + }; + + // Act + IList blobs = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + AssertDictionaryEquality(metadata, blobs.First().Metadata); + } + + [Ignore("Feature not supported in current test environment")] + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_EncryptionScope() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + blob = InstrumentClient(blob.WithEncryptionScope(TestConfigDefault.EncryptionScope)); + + await blob.CreateIfNotExistsAsync(); + + // Act + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true + }; + IList blobs = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(TestConfigDefault.EncryptionScope, blobs.First().Properties.EncryptionScope); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_Deleted() + { + // Arrange + BlobServiceClient blobServiceClient = BlobsClientBuilder.GetServiceClient_SoftDelete_OAuth(TestEnvironment.Credential); + await using DisposingContainer test = await GetTestContainerAsync(blobServiceClient); + string blobName = GetNewBlobName(); + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName)); + await blob.CreateIfNotExistsAsync(); + await blob.DeleteIfExistsAsync(); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + States = BlobStates.Deleted + }; + + // Act + IList blobs = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(blobName, blobs[0].Name); + Assert.IsTrue(blobs[0].Deleted); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_Uncommited() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + var blobName = GetNewBlobName(); + BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(blobName)); + var data = GetRandomBuffer(Constants.KB); + var blockId = ToBase64(GetNewBlockName()); + + using (var stream = new MemoryStream(data)) + { + await blob.StageBlockAsync( + base64BlockId: blockId, + content: stream); + } + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + States = BlobStates.Uncommitted + }; + + // Act + IList blobs = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(1, blobs.Count); + Assert.AreEqual(blobName, blobs.First().Name); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_Snapshot() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + await blob.CreateIfNotExistsAsync(); + Response snapshotResponse = await blob.CreateSnapshotAsync(); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + States = BlobStates.Snapshots + }; + + // Act + IList blobs = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(2, blobs.Count); + Assert.AreEqual(snapshotResponse.Value.Snapshot.ToString(), blobs.First().Snapshot); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_Prefix() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + Prefix = "foo" + }; + + // Act + IList blobs = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(3, blobs.Count); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_Error() + { + // Arrange + BlobServiceClient service = GetServiceClient_SharedKey(); + BlobContainerClient container = InstrumentClient(service.GetBlobContainerClient(GetNewContainerName())); + var id = Recording.Random.NewGuid().ToString(); + + // Act + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true + }; + await TestHelper.AssertExpectedExceptionAsync( + container.GetBlobsAsync(options: options).ToListAsync(), + e => Assert.AreEqual("ContainerNotFound", e.ErrorCode)); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_PreservesWhitespace() + { + await VerifyBlobNameWhitespaceRoundtrips(" prefix"); + await VerifyBlobNameWhitespaceRoundtrips("suffix "); + await VerifyBlobNameWhitespaceRoundtrips(" "); + + async Task VerifyBlobNameWhitespaceRoundtrips(string blobName) + { + await using DisposingContainer test = await GetTestContainerAsync(); + BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(blobName)); + await blob.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("data"))); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true + }; + BlobItem blobItem = await test.Container.GetBlobsAsync(options: options).FirstAsync(); + Assert.AreEqual(blobName, blobItem.Name); + } + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_VersionId() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + Response createResponse = await blob.CreateAsync(); + IDictionary metadata = BuildMetadata(); + Response setMetadataResponse = await blob.SetMetadataAsync(metadata); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + States = BlobStates.Version + }; + + // Act + var blobs = new List(); + await foreach (Page page in test.Container.GetBlobsAsync(options: options).AsPages()) + { + blobs.AddRange(page.Values); + } + + // Assert + Assert.AreEqual(1, blobs.Count); + Assert.IsNull(blobs[0].IsLatestVersion); + Assert.AreEqual(createResponse.Value.VersionId, blobs[0].VersionId); + } + + [Ignore("Feature not supported in current test environment")] + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_ObjectReplication() + { + // TODO: The tests will temporarily use designated account, containers and blobs to check the + // existence of OR Metadata + BlobServiceClient sourceServiceClient = GetServiceClient_SharedKey(); + + // This is a recorded ONLY test with a special container we previously setup, as we can't auto setup policies yet + BlobContainerClient sourceContainer = InstrumentClient(sourceServiceClient.GetBlobContainerClient("test1")); + + // Act + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true + }; + IList blobs = await sourceContainer.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + // Since this is a PLAYBACK ONLY test. We expect all the blobs in this source container/account + // to have OrMetadata + Assert.IsNotNull(blobs.First().ObjectReplicationSourceProperties); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_LastAccessed() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + // Act + var blobs = new List(); + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true + }; + await foreach (Page page in test.Container.GetBlobsAsync(options: options).AsPages()) + { + blobs.AddRange(page.Values); + } + + // Assert + Assert.AreNotEqual(DateTimeOffset.MinValue, blobs.FirstOrDefault().Properties.LastAccessedOn); + } + + [Ignore("Feature not supported in current test environment")] + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_DeletedWithVersions() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(); + + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + Response createResponse = await blob.CreateAsync(); + IDictionary metadata = BuildMetadata(); + Response setMetadataResponse = await blob.SetMetadataAsync(metadata); + await blob.DeleteAsync(); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + States = BlobStates.DeletedWithVersions + }; + + // Act + List blobItems = new List(); + await foreach (BlobItem blobItem in test.Container.GetBlobsAsync(options)) + { + blobItems.Add(blobItem); + } + + // Assert + Assert.AreEqual(1, blobItems.Count); + Assert.AreEqual(blob.Name, blobItems[0].Name); + Assert.IsTrue(blobItems[0].HasVersionsOnly); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_EncodedBlobName() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(); + string blobName = "dir1/dir2/file\uFFFF.blob"; + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName)); + await blob.CreateAsync(); + + // Act + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true + }; + BlobItem blobItem = await test.Container.GetBlobsAsync(options: options).FirstAsync(); + + // Assert + Assert.AreEqual(blobName, blobItem.Name); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_StartFrom() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + StartFrom = "foo" + }; + + // Act + IList blobs = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(3, blobs.Count); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_EndBefore() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + EndBefore = "foo" + }; + + // Act + IList blobs = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(5, blobs.Count); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsFlatSegmentAsync_UseApacheArrow_StartFromEndBefore() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + GetBlobsOptions options = new GetBlobsOptions + { + UseApacheArrow = true, + StartFrom = "foo", + EndBefore = "foo/foo" + }; + + // Act + IList blobs = await test.Container.GetBlobsAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(2, blobs.Count); + } + [RecordedTest] [PlaybackOnly("Service bug - https://github.com/Azure/azure-sdk-for-net/issues/16516")] public async Task ListBlobsHierarchySegmentAsync() @@ -2841,7 +3360,516 @@ public async Task ListBlobsHierarchySegmentAsync() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { - Delimiter = delimiter, + Delimiter = delimiter, + }; + + await foreach (Page page in test.Container.GetBlobsByHierarchyAsync(options: options).AsPages()) + { + blobs.AddRange(page.Values.Where(item => item.IsBlob).Select(item => item.Blob)); + prefixes.AddRange(page.Values.Where(item => item.IsPrefix).Select(item => item.Prefix)); + } + + Assert.AreEqual(3, blobs.Count); + Assert.AreEqual(2, prefixes.Count); + + var foundBlobNames = blobs.Select(blob => blob.Name).ToArray(); + var foundBlobPrefixes = prefixes.ToArray(); + IEnumerable expectedPrefixes = + BlobNames + .Where(blobName => blobName.Contains(delimiter)) + .Select(blobName => blobName.Split(new[] { delimiter[0] })[0] + delimiter) + .Distinct() + ; + + Assert.IsTrue( + BlobNames + .Where(blobName => !blobName.Contains(delimiter)) + .All(blobName => foundBlobNames.Contains(blobName)) + ); + + Assert.IsTrue( + expectedPrefixes + .All(prefix => foundBlobPrefixes.Contains(prefix)) + ); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + public async Task ListBlobsHierarchySegmentAsync_Tags() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(); + AppendBlobClient appendBlob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + IDictionary tags = BuildTags(); + AppendBlobCreateOptions options = new AppendBlobCreateOptions + { + Tags = tags + }; + await appendBlob.CreateAsync(options); + + GetBlobsByHierarchyOptions getBlobsByHierarchyOptions = new GetBlobsByHierarchyOptions + { + Traits = BlobTraits.Tags + }; + + // Act + IList blobHierachyItems = await test.Container.GetBlobsByHierarchyAsync(getBlobsByHierarchyOptions).ToListAsync(); + + // Assert + AssertDictionaryEquality(tags, blobHierachyItems[0].Blob.Tags); + Assert.AreEqual(tags.Count, blobHierachyItems[0].Blob.Properties.TagCount); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [TestCase(null)] + [TestCase(RehydratePriority.Standard)] + [TestCase(RehydratePriority.High)] + public async Task ListBlobsHierarchySegmentAsync_RehydratePriority(RehydratePriority? rehydratePriority) + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(); + + BlockBlobClient blockBlob = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName())); + byte[] data = GetRandomBuffer(Constants.KB); + using Stream stream = new MemoryStream(data); + await blockBlob.UploadAsync(stream); + + if (rehydratePriority.HasValue) + { + await blockBlob.SetAccessTierAsync( + AccessTier.Archive); + + await blockBlob.SetAccessTierAsync( + AccessTier.Hot, + rehydratePriority: rehydratePriority.Value); + } + + // Act + IList blobItems = await test.Container.GetBlobsByHierarchyAsync().ToListAsync(); + + // Assert + Assert.AreEqual(rehydratePriority, blobItems[0].Blob.Properties.RehydratePriority); + } + + [RecordedTest] + [PlaybackOnly("Service bug - https://github.com/Azure/azure-sdk-for-net/issues/16516")] + [AsyncOnly] + public async Task ListBlobsHierarchySegmentAsync_MaxResults() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + Delimiter = "/" + }; + + // Act + Page page = await test.Container.GetBlobsByHierarchyAsync(options: options) + .AsPages(pageSizeHint: 2) + .FirstAsync(); + + // Assert + Assert.AreEqual(2, page.Values.Count); + } + + [RecordedTest] + public async Task ListBlobsHierarchySegmentAsync_Metadata() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + IDictionary metadata = BuildMetadata(); + await blob.CreateIfNotExistsAsync(metadata: metadata); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + Traits = BlobTraits.Metadata, + }; + + // Act + BlobHierarchyItem item = await test.Container.GetBlobsByHierarchyAsync(options: options).FirstAsync(); + + // Assert + AssertDictionaryEquality(metadata, item.Blob.Metadata); + } + + [RecordedTest] + public async Task ListBlobsHierarchySegmentAsync_Metadata_NoMetadata() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + await blob.CreateAsync(); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + Traits = BlobTraits.Metadata + }; + + // Act + BlobHierarchyItem item = await test.Container.GetBlobsByHierarchyAsync(options: options).FirstAsync(); + + // Assert + Assert.IsNotNull(item.Blob.Metadata); + Assert.AreEqual(0, item.Blob.Metadata.Count); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_07_07)] + public async Task ListBlobsHierarchySegmentAsync_EncryptionScope() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + blob = InstrumentClient(blob.WithEncryptionScope(TestConfigDefault.EncryptionScope)); + await blob.CreateIfNotExistsAsync(); + + // Act + BlobHierarchyItem item = await test.Container.GetBlobsByHierarchyAsync().FirstAsync(); + + // Assert + Assert.AreEqual(TestConfigDefault.EncryptionScope, item.Blob.Properties.EncryptionScope); + } + + [RecordedTest] + public async Task ListBlobsHierarchySegmentAsync_Deleted() + { + // Arrange + BlobServiceClient blobServiceClient = BlobsClientBuilder.GetServiceClient_SoftDelete(); + await using DisposingContainer test = await GetTestContainerAsync(blobServiceClient); + string blobName = GetNewBlobName(); + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName)); + await blob.CreateAsync(); + await blob.DeleteAsync(); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + States = BlobStates.Deleted + }; + + // Act + IList blobs = await test.Container.GetBlobsByHierarchyAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(blobName, blobs[0].Blob.Name); + Assert.IsTrue(blobs[0].Blob.Deleted); + } + + [RecordedTest] + public async Task ListBlobsHierarchySegmentAsync_Uncommited() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + var blobName = GetNewBlobName(); + BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(blobName)); + var data = GetRandomBuffer(Constants.KB); + var blockId = ToBase64(GetNewBlockName()); + + using (var stream = new MemoryStream(data)) + { + await blob.StageBlockAsync( + base64BlockId: blockId, + content: stream); + } + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + States = BlobStates.Uncommitted, + }; + + // Act + IList blobs = await test.Container.GetBlobsByHierarchyAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(1, blobs.Count); + Assert.AreEqual(blobName, blobs.First().Blob.Name); + } + + [RecordedTest] + public async Task ListBlobsHierarchySegmentAsync_Snapshot() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + await blob.CreateIfNotExistsAsync(); + Response snapshotResponse = await blob.CreateSnapshotAsync(); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + States = BlobStates.Snapshots, + }; + + // Act + IList blobs = await test.Container.GetBlobsByHierarchyAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(2, blobs.Count); + Assert.AreEqual(snapshotResponse.Value.Snapshot.ToString(), blobs.First().Blob.Snapshot); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + public async Task ListBlobsHierarchySegmentAsync_VersionId() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + Response createResponse = await blob.CreateAsync(); + IDictionary metadata = BuildMetadata(); + Response setMetadataResponse = await blob.SetMetadataAsync(metadata); + + // Act + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + States = BlobStates.Version, + }; + + var blobs = new List(); + await foreach (Page page in test.Container.GetBlobsByHierarchyAsync(options: options).AsPages()) + { + blobs.AddRange(page.Values); + } + + // Assert + Assert.AreEqual(2, blobs.Count); + Assert.IsNull(blobs[0].Blob.IsLatestVersion); + Assert.AreEqual(createResponse.Value.VersionId, blobs[0].Blob.VersionId); + Assert.IsTrue(blobs[1].Blob.IsLatestVersion); + Assert.AreEqual(setMetadataResponse.Value.VersionId, blobs[1].Blob.VersionId); + } + + [RecordedTest] + public async Task ListBlobsHierarchySegmentAsync_Prefix() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + Prefix = "foo", + }; + + // Act + IList blobs = await test.Container.GetBlobsByHierarchyAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(3, blobs.Count); + } + + [RecordedTest] + public async Task ListBlobsHierarchySegmentAsync_Error() + { + // Arrange + BlobServiceClient service = GetServiceClient_SharedKey(); + BlobContainerClient container = InstrumentClient(service.GetBlobContainerClient(GetNewContainerName())); + var id = Recording.Random.NewGuid().ToString(); + + // Act + await TestHelper.AssertExpectedExceptionAsync( + container.GetBlobsByHierarchyAsync().ToListAsync(), + e => Assert.AreEqual("ContainerNotFound", e.ErrorCode)); + } + + [PlaybackOnly("Object Replication policies is only enabled on certain storage accounts")] + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + public async Task ListBlobsHierarchySegmentAsync_ObjectReplication() + { + // TODO: The tests will temporarily use designated account, containers and blobs to check the + // existence of OR Metadata + BlobServiceClient sourceServiceClient = GetServiceClient_SharedKey(); + + // This is a recorded ONLY test with a special container we previously setup, as we can't auto setup policies yet + BlobContainerClient sourceContainer = InstrumentClient(sourceServiceClient.GetBlobContainerClient("test1")); + + // Act + BlobHierarchyItem item = await sourceContainer.GetBlobsByHierarchyAsync().FirstAsync(); + + // Assert + // Since this is a PLAYBACK ONLY test. We expect all the blobs in this source container/account + // to have OrMetadata + Assert.IsNotNull(item.Blob.ObjectReplicationSourceProperties); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_02_10)] + public async Task ListBlobsHierarchySegmentAsync_LastAccessed() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName())); + var data = GetRandomBuffer(Constants.KB); + using Stream stream = new MemoryStream(data); + await blob.UploadAsync(content: stream); + + // Act + BlobHierarchyItem item = await test.Container.GetBlobsByHierarchyAsync().FirstAsync(); + + // Assert + Assert.IsNotNull(item.Blob.Properties.LastAccessedOn); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_10_02)] + public async Task ListBlobsHierarchySegmentAsync_DeletedWithVersions() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(); + + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(GetNewBlobName())); + Response createResponse = await blob.CreateAsync(); + IDictionary metadata = BuildMetadata(); + Response setMetadataResponse = await blob.SetMetadataAsync(metadata); + await blob.DeleteAsync(); + + // Act + List blobHierarchyItems = new List(); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + States = BlobStates.DeletedWithVersions, + }; + + await foreach (BlobHierarchyItem blobItem in test.Container.GetBlobsByHierarchyAsync(options: options)) + { + blobHierarchyItems.Add(blobItem); + } + + // Assert + Assert.AreEqual(1, blobHierarchyItems.Count); + Assert.AreEqual(blob.Name, blobHierarchyItems[0].Blob.Name); + Assert.IsTrue(blobHierarchyItems[0].Blob.HasVersionsOnly); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2021_02_12)] + [TestCase(false)] + [TestCase(true)] + public async Task ListBlobsHierarchySegmentAsync_EncodedBlobName(bool delimiter) + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(); + string blobName = "dir1/dir2/file\uFFFF.blob"; + AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName)); + await blob.CreateAsync(); + + // Act + BlobHierarchyItem item; + if (delimiter) + { + item = await test.Container.GetBlobsByHierarchyAsync().FirstAsync(); + + // Assert + Assert.IsTrue(item.IsBlob); + Assert.AreEqual(blobName, item.Blob.Name); + } + else + { + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + Delimiter = ".b", + }; + + item = await test.Container.GetBlobsByHierarchyAsync( + options: options).FirstAsync(); + + // Assert + Assert.IsTrue(item.IsPrefix); + Assert.AreEqual("dir1/dir2/file\uffff.b", item.Prefix); + } + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2021_12_02)] + public async Task ListBlobsHierarchySegmentAsync_VersionPrefixDelimiter() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(); + await SetUpContainerForListing(test.Container); + + var blobs = new List(); + var prefixes = new List(); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + States = BlobStates.Version, + Delimiter = "/", + Prefix = "baz" + }; + + await foreach (BlobHierarchyItem blobItem in test.Container.GetBlobsByHierarchyAsync( + options: options)) + { + if (blobItem.IsBlob) + { + blobs.Add(blobItem.Blob); + } + else + { + prefixes.Add(blobItem.Prefix); + } + } + + Assert.AreEqual(1, blobs.Count); + Assert.AreEqual(1, prefixes.Count); + + Assert.AreEqual("baz", blobs[0].Name); + Assert.IsNotNull(blobs[0].VersionId); + + Assert.AreEqual("baz/", prefixes[0]); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_02_06)] + public async Task ListBlobsHierarchySegmentAsync_StartFrom() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + StartFrom = "foo" + }; + + // Act + IList blobHierachyItems = await test.Container.GetBlobsByHierarchyAsync(options).ToListAsync(); + + // Assert + Assert.AreEqual(3, blobHierachyItems.Count); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + var blobs = new List(); + var prefixes = new List(); + var delimiter = "/"; + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + UseApacheArrow = true, + Delimiter = delimiter }; await foreach (Page page in test.Container.GetBlobsByHierarchyAsync(options: options).AsPages()) @@ -2875,8 +3903,8 @@ public async Task ListBlobsHierarchySegmentAsync() } [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - public async Task ListBlobsHierarchySegmentAsync_Tags() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_Tags() { // Arrange await using DisposingContainer test = await GetTestContainerAsync(); @@ -2890,23 +3918,25 @@ public async Task ListBlobsHierarchySegmentAsync_Tags() GetBlobsByHierarchyOptions getBlobsByHierarchyOptions = new GetBlobsByHierarchyOptions { + UseApacheArrow = true, Traits = BlobTraits.Tags }; // Act - IList blobHierachyItems = await test.Container.GetBlobsByHierarchyAsync(getBlobsByHierarchyOptions).ToListAsync(); + IList blobHierachyItems = await test.Container.GetBlobsByHierarchyAsync(options: getBlobsByHierarchyOptions).ToListAsync(); // Assert AssertDictionaryEquality(tags, blobHierachyItems[0].Blob.Tags); Assert.AreEqual(tags.Count, blobHierachyItems[0].Blob.Properties.TagCount); } + [Ignore("Feature not supported in current test environment")] [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] [TestCase(null)] [TestCase(RehydratePriority.Standard)] [TestCase(RehydratePriority.High)] - public async Task ListBlobsHierarchySegmentAsync_RehydratePriority(RehydratePriority? rehydratePriority) + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_RehydratePriority(RehydratePriority? rehydratePriority) { // Arrange await using DisposingContainer test = await GetTestContainerAsync(); @@ -2927,16 +3957,19 @@ await blockBlob.SetAccessTierAsync( } // Act - IList blobItems = await test.Container.GetBlobsByHierarchyAsync().ToListAsync(); + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + UseApacheArrow = true + }; + IList blobItems = await test.Container.GetBlobsByHierarchyAsync(options: options).ToListAsync(); // Assert Assert.AreEqual(rehydratePriority, blobItems[0].Blob.Properties.RehydratePriority); } [RecordedTest] - [PlaybackOnly("Service bug - https://github.com/Azure/azure-sdk-for-net/issues/16516")] - [AsyncOnly] - public async Task ListBlobsHierarchySegmentAsync_MaxResults() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_MaxResults() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -2945,20 +3978,26 @@ public async Task ListBlobsHierarchySegmentAsync_MaxResults() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { - Delimiter = "/" + UseApacheArrow = true }; + int numPages = 0; // Act - Page page = await test.Container.GetBlobsByHierarchyAsync(options: options) - .AsPages(pageSizeHint: 2) - .FirstAsync(); + await foreach (Page page in test.Container.GetBlobsByHierarchyAsync(options: options) + .AsPages(pageSizeHint: 2)) + { + // Assert + Assert.AreEqual(2, page.Values.Count); + ++numPages; + } // Assert - Assert.AreEqual(2, page.Values.Count); + Assert.AreEqual(4, numPages); } [RecordedTest] - public async Task ListBlobsHierarchySegmentAsync_Metadata() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_Metadata() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -2969,7 +4008,8 @@ public async Task ListBlobsHierarchySegmentAsync_Metadata() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { - Traits = BlobTraits.Metadata, + UseApacheArrow = true, + Traits = BlobTraits.Metadata }; // Act @@ -2980,7 +4020,8 @@ public async Task ListBlobsHierarchySegmentAsync_Metadata() } [RecordedTest] - public async Task ListBlobsHierarchySegmentAsync_Metadata_NoMetadata() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_Metadata_NoMetadata() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -2990,6 +4031,7 @@ public async Task ListBlobsHierarchySegmentAsync_Metadata_NoMetadata() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { + UseApacheArrow = true, Traits = BlobTraits.Metadata }; @@ -3001,9 +4043,10 @@ public async Task ListBlobsHierarchySegmentAsync_Metadata_NoMetadata() Assert.AreEqual(0, item.Blob.Metadata.Count); } + [Ignore("Feature not supported in current test environment")] [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_07_07)] - public async Task ListBlobsHierarchySegmentAsync_EncryptionScope() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_EncryptionScope() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -3013,17 +4056,22 @@ public async Task ListBlobsHierarchySegmentAsync_EncryptionScope() await blob.CreateIfNotExistsAsync(); // Act - BlobHierarchyItem item = await test.Container.GetBlobsByHierarchyAsync().FirstAsync(); + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + UseApacheArrow = true + }; + BlobHierarchyItem item = await test.Container.GetBlobsByHierarchyAsync(options: options).FirstAsync(); // Assert Assert.AreEqual(TestConfigDefault.EncryptionScope, item.Blob.Properties.EncryptionScope); } [RecordedTest] - public async Task ListBlobsHierarchySegmentAsync_Deleted() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_Deleted() { // Arrange - BlobServiceClient blobServiceClient = BlobsClientBuilder.GetServiceClient_SoftDelete(); + BlobServiceClient blobServiceClient = BlobsClientBuilder.GetServiceClient_SoftDelete_OAuth(TestEnvironment.Credential); await using DisposingContainer test = await GetTestContainerAsync(blobServiceClient); string blobName = GetNewBlobName(); AppendBlobClient blob = InstrumentClient(test.Container.GetAppendBlobClient(blobName)); @@ -3032,6 +4080,7 @@ public async Task ListBlobsHierarchySegmentAsync_Deleted() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { + UseApacheArrow = true, States = BlobStates.Deleted }; @@ -3044,7 +4093,8 @@ public async Task ListBlobsHierarchySegmentAsync_Deleted() } [RecordedTest] - public async Task ListBlobsHierarchySegmentAsync_Uncommited() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_Uncommited() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -3063,7 +4113,8 @@ await blob.StageBlockAsync( GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { - States = BlobStates.Uncommitted, + UseApacheArrow = true, + States = BlobStates.Uncommitted }; // Act @@ -3075,7 +4126,8 @@ await blob.StageBlockAsync( } [RecordedTest] - public async Task ListBlobsHierarchySegmentAsync_Snapshot() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_Snapshot() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -3086,7 +4138,8 @@ public async Task ListBlobsHierarchySegmentAsync_Snapshot() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { - States = BlobStates.Snapshots, + UseApacheArrow = true, + States = BlobStates.Snapshots }; // Act @@ -3098,8 +4151,8 @@ public async Task ListBlobsHierarchySegmentAsync_Snapshot() } [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - public async Task ListBlobsHierarchySegmentAsync_VersionId() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_VersionId() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -3112,7 +4165,8 @@ public async Task ListBlobsHierarchySegmentAsync_VersionId() // Act GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { - States = BlobStates.Version, + UseApacheArrow = true, + States = BlobStates.Version }; var blobs = new List(); @@ -3122,15 +4176,14 @@ public async Task ListBlobsHierarchySegmentAsync_VersionId() } // Assert - Assert.AreEqual(2, blobs.Count); + Assert.AreEqual(1, blobs.Count); Assert.IsNull(blobs[0].Blob.IsLatestVersion); Assert.AreEqual(createResponse.Value.VersionId, blobs[0].Blob.VersionId); - Assert.IsTrue(blobs[1].Blob.IsLatestVersion); - Assert.AreEqual(setMetadataResponse.Value.VersionId, blobs[1].Blob.VersionId); } [RecordedTest] - public async Task ListBlobsHierarchySegmentAsync_Prefix() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_Prefix() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -3139,7 +4192,8 @@ public async Task ListBlobsHierarchySegmentAsync_Prefix() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { - Prefix = "foo", + UseApacheArrow = true, + Prefix = "foo" }; // Act @@ -3150,7 +4204,8 @@ public async Task ListBlobsHierarchySegmentAsync_Prefix() } [RecordedTest] - public async Task ListBlobsHierarchySegmentAsync_Error() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_Error() { // Arrange BlobServiceClient service = GetServiceClient_SharedKey(); @@ -3158,15 +4213,19 @@ public async Task ListBlobsHierarchySegmentAsync_Error() var id = Recording.Random.NewGuid().ToString(); // Act + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + UseApacheArrow = true + }; await TestHelper.AssertExpectedExceptionAsync( - container.GetBlobsByHierarchyAsync().ToListAsync(), + container.GetBlobsByHierarchyAsync(options: options).ToListAsync(), e => Assert.AreEqual("ContainerNotFound", e.ErrorCode)); } - [PlaybackOnly("Object Replication policies is only enabled on certain storage accounts")] + [Ignore("Feature not supported in current test environment")] [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)] - public async Task ListBlobsHierarchySegmentAsync_ObjectReplication() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_ObjectReplication() { // TODO: The tests will temporarily use designated account, containers and blobs to check the // existence of OR Metadata @@ -3176,7 +4235,11 @@ public async Task ListBlobsHierarchySegmentAsync_ObjectReplication() BlobContainerClient sourceContainer = InstrumentClient(sourceServiceClient.GetBlobContainerClient("test1")); // Act - BlobHierarchyItem item = await sourceContainer.GetBlobsByHierarchyAsync().FirstAsync(); + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + UseApacheArrow = true + }; + BlobHierarchyItem item = await sourceContainer.GetBlobsByHierarchyAsync(options: options).FirstAsync(); // Assert // Since this is a PLAYBACK ONLY test. We expect all the blobs in this source container/account @@ -3185,8 +4248,8 @@ public async Task ListBlobsHierarchySegmentAsync_ObjectReplication() } [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_02_10)] - public async Task ListBlobsHierarchySegmentAsync_LastAccessed() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_LastAccessed() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -3197,15 +4260,20 @@ public async Task ListBlobsHierarchySegmentAsync_LastAccessed() await blob.UploadAsync(content: stream); // Act - BlobHierarchyItem item = await test.Container.GetBlobsByHierarchyAsync().FirstAsync(); + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + UseApacheArrow = true + }; + BlobHierarchyItem item = await test.Container.GetBlobsByHierarchyAsync(options: options).FirstAsync(); // Assert Assert.IsNotNull(item.Blob.Properties.LastAccessedOn); } + [Ignore("Feature not supported in current test environment")] [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2020_10_02)] - public async Task ListBlobsHierarchySegmentAsync_DeletedWithVersions() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_DeletedWithVersions() { // Arrange await using DisposingContainer test = await GetTestContainerAsync(); @@ -3221,7 +4289,8 @@ public async Task ListBlobsHierarchySegmentAsync_DeletedWithVersions() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { - States = BlobStates.DeletedWithVersions, + UseApacheArrow = true, + States = BlobStates.DeletedWithVersions }; await foreach (BlobHierarchyItem blobItem in test.Container.GetBlobsByHierarchyAsync(options: options)) @@ -3236,10 +4305,10 @@ public async Task ListBlobsHierarchySegmentAsync_DeletedWithVersions() } [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2021_02_12)] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] [TestCase(false)] [TestCase(true)] - public async Task ListBlobsHierarchySegmentAsync_EncodedBlobName(bool delimiter) + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_EncodedBlobName(bool delimiter) { // Arrange await using DisposingContainer test = await GetTestContainerAsync(); @@ -3251,7 +4320,11 @@ public async Task ListBlobsHierarchySegmentAsync_EncodedBlobName(bool delimiter) BlobHierarchyItem item; if (delimiter) { - item = await test.Container.GetBlobsByHierarchyAsync().FirstAsync(); + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + UseApacheArrow = true + }; + item = await test.Container.GetBlobsByHierarchyAsync(options: options).FirstAsync(); // Assert Assert.IsTrue(item.IsBlob); @@ -3261,7 +4334,8 @@ public async Task ListBlobsHierarchySegmentAsync_EncodedBlobName(bool delimiter) { GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { - Delimiter = ".b", + UseApacheArrow = true, + Delimiter = ".b" }; item = await test.Container.GetBlobsByHierarchyAsync( @@ -3273,9 +4347,10 @@ public async Task ListBlobsHierarchySegmentAsync_EncodedBlobName(bool delimiter) } } + [Ignore("Feature not supported in current test environment")] [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2021_12_02)] - public async Task ListBlobsHierarchySegmentAsync_VersionPrefixDelimiter() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_VersionPrefixDelimiter() { // Arrange await using DisposingContainer test = await GetTestContainerAsync(); @@ -3286,6 +4361,7 @@ public async Task ListBlobsHierarchySegmentAsync_VersionPrefixDelimiter() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { + UseApacheArrow = true, States = BlobStates.Version, Delimiter = "/", Prefix = "baz" @@ -3314,8 +4390,8 @@ public async Task ListBlobsHierarchySegmentAsync_VersionPrefixDelimiter() } [RecordedTest] - [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_02_06)] - public async Task ListBlobsHierarchySegmentAsync_StartFrom() + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_StartFrom() { await using DisposingContainer test = await GetTestContainerAsync(); @@ -3324,16 +4400,62 @@ public async Task ListBlobsHierarchySegmentAsync_StartFrom() GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions { + UseApacheArrow = true, StartFrom = "foo" }; // Act - IList blobHierachyItems = await test.Container.GetBlobsByHierarchyAsync(options).ToListAsync(); + IList blobHierachyItems = await test.Container.GetBlobsByHierarchyAsync(options: options).ToListAsync(); // Assert Assert.AreEqual(3, blobHierachyItems.Count); } + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_EndBefore() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + UseApacheArrow = true, + EndBefore = "foo" + }; + + // Act + IList blobHierachyItems = await test.Container.GetBlobsByHierarchyAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(5, blobHierachyItems.Count); + } + + [RecordedTest] + [ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2026_06_06)] + public async Task ListBlobsHierarchySegmentAsync_UseApacheArrow_StartFromEndBefore() + { + await using DisposingContainer test = await GetTestContainerAsync(); + + // Arrange + await SetUpContainerForListing(test.Container); + + GetBlobsByHierarchyOptions options = new GetBlobsByHierarchyOptions + { + UseApacheArrow = true, + StartFrom = "foo", + EndBefore = "foo/foo" + }; + + // Act + IList blobHierachyItems = await test.Container.GetBlobsByHierarchyAsync(options: options).ToListAsync(); + + // Assert + Assert.AreEqual(2, blobHierachyItems.Count); + } + [RecordedTest] public async Task UploadBlobAsync() { diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs index 62c94d01feb7..b2f942a6b849 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs @@ -235,6 +235,7 @@ internal static class Blob public const int QuickQueryDownloadSize = 4 * Constants.MB; public const string MetadataHeaderPrefix = "x-ms-meta-"; public const string ObjectReplicationRulesHeaderPrefix = "x-ms-or-"; + public const string ApacheArrowContentType = "application/vnd.apache.arrow.stream"; internal static class Append {