Skip to content

Commit f59ab21

Browse files
katmsftkatmsft
authored
Integrate storage retry policy. (#722)
* Integrate storage retry policy. * Removed the logic to make URL empty when '.blob.' is not found in URL.
1 parent 1b351de commit f59ab21

File tree

12 files changed

+137
-61
lines changed

12 files changed

+137
-61
lines changed

‎sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
2525
{
2626
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerOperationPolicies;
2727
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerRetryPolicies;
28+
29+
/**
30+
* @brief Specify the number of retries and other retry-related options.
31+
*/
32+
StorageRetryWithSecondaryOptions RetryOptions;
2833
};
2934

3035
/**
@@ -34,6 +39,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
3439
{
3540
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerOperationPolicies;
3641
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerRetryPolicies;
42+
43+
/**
44+
* @brief Specify the number of retries and other retry-related options.
45+
*/
46+
StorageRetryWithSecondaryOptions RetryOptions;
3747
};
3848

3949
/**
@@ -43,6 +53,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
4353
{
4454
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerOperationPolicies;
4555
std::vector<std::unique_ptr<Azure::Core::Http::HttpPolicy>> PerRetryPolicies;
56+
57+
/**
58+
* @brief Specify the number of retries and other retry-related options.
59+
*/
60+
StorageRetryWithSecondaryOptions RetryOptions;
4661
};
4762

4863
/**

‎sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "azure/storage/common/shared_key_policy.hpp"
1111
#include "azure/storage/common/storage_common.hpp"
1212
#include "azure/storage/common/storage_per_retry_policy.hpp"
13+
#include "azure/storage/common/storage_retry_policy.hpp"
1314
#include "azure/storage/common/storage_version.hpp"
1415
#include "azure/storage/files/datalake/datalake_file_client.hpp"
1516
#include "azure/storage/files/datalake/datalake_utilities.hpp"
@@ -55,12 +56,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
5556
{
5657
policies.emplace_back(p->Clone());
5758
}
58-
policies.emplace_back(
59-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
59+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
60+
dfsRetryOptions.SecondaryHostForRetryReads
61+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
62+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
6063
for (const auto& p : options.PerRetryPolicies)
6164
{
6265
policies.emplace_back(p->Clone());
6366
}
67+
6468
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
6569
policies.emplace_back(std::make_unique<SharedKeyPolicy>(credential));
6670
policies.emplace_back(std::make_unique<Azure::Core::Http::TransportPolicy>(
@@ -82,8 +86,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
8286
{
8387
policies.emplace_back(p->Clone());
8488
}
85-
policies.emplace_back(
86-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
89+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
90+
dfsRetryOptions.SecondaryHostForRetryReads
91+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
92+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
8793
for (const auto& p : options.PerRetryPolicies)
8894
{
8995
policies.emplace_back(p->Clone());
@@ -110,8 +116,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
110116
{
111117
policies.emplace_back(p->Clone());
112118
}
113-
policies.emplace_back(
114-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
119+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
120+
dfsRetryOptions.SecondaryHostForRetryReads
121+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
122+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
115123
for (const auto& p : options.PerRetryPolicies)
116124
{
117125
policies.emplace_back(p->Clone());

‎sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "azure/storage/common/shared_key_policy.hpp"
1111
#include "azure/storage/common/storage_common.hpp"
1212
#include "azure/storage/common/storage_per_retry_policy.hpp"
13+
#include "azure/storage/common/storage_retry_policy.hpp"
1314
#include "azure/storage/common/storage_version.hpp"
1415
#include "azure/storage/files/datalake/datalake_utilities.hpp"
1516

@@ -133,12 +134,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
133134
{
134135
policies.emplace_back(p->Clone());
135136
}
136-
policies.emplace_back(
137-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
137+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
138+
dfsRetryOptions.SecondaryHostForRetryReads
139+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
140+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
138141
for (const auto& p : options.PerRetryPolicies)
139142
{
140143
policies.emplace_back(p->Clone());
141144
}
145+
142146
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
143147
policies.emplace_back(std::make_unique<SharedKeyPolicy>(credential));
144148
policies.emplace_back(std::make_unique<Azure::Core::Http::TransportPolicy>(
@@ -161,12 +165,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
161165
{
162166
policies.emplace_back(p->Clone());
163167
}
164-
policies.emplace_back(
165-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
168+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
169+
dfsRetryOptions.SecondaryHostForRetryReads
170+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
171+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
166172
for (const auto& p : options.PerRetryPolicies)
167173
{
168174
policies.emplace_back(p->Clone());
169175
}
176+
170177
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
171178
policies.emplace_back(
172179
std::make_unique<Core::Credentials::Policy::BearerTokenAuthenticationPolicy>(
@@ -187,12 +194,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
187194
{
188195
policies.emplace_back(p->Clone());
189196
}
190-
policies.emplace_back(
191-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
197+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
198+
dfsRetryOptions.SecondaryHostForRetryReads
199+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
200+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
192201
for (const auto& p : options.PerRetryPolicies)
193202
{
194203
policies.emplace_back(p->Clone());
195204
}
205+
196206
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
197207
policies.emplace_back(std::make_unique<Azure::Core::Http::TransportPolicy>(
198208
std::make_shared<Azure::Core::Http::CurlTransport>()));

‎sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "azure/storage/common/shared_key_policy.hpp"
1212
#include "azure/storage/common/storage_common.hpp"
1313
#include "azure/storage/common/storage_per_retry_policy.hpp"
14+
#include "azure/storage/common/storage_retry_policy.hpp"
1415
#include "azure/storage/common/storage_version.hpp"
1516
#include "azure/storage/files/datalake/datalake_directory_client.hpp"
1617
#include "azure/storage/files/datalake/datalake_file_client.hpp"
@@ -31,6 +32,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
3132
{
3233
blobOptions.PerRetryPolicies.emplace_back(p->Clone());
3334
}
35+
blobOptions.RetryOptions = options.RetryOptions;
36+
blobOptions.RetryOptions.SecondaryHostForRetryReads
37+
= Details::GetBlobUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
3438
return blobOptions;
3539
}
3640
} // namespace
@@ -74,12 +78,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
7478
{
7579
policies.emplace_back(p->Clone());
7680
}
77-
policies.emplace_back(
78-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
81+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
82+
dfsRetryOptions.SecondaryHostForRetryReads
83+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
84+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
7985
for (const auto& p : options.PerRetryPolicies)
8086
{
8187
policies.emplace_back(p->Clone());
8288
}
89+
8390
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
8491
policies.emplace_back(std::make_unique<SharedKeyPolicy>(credential));
8592
policies.emplace_back(std::make_unique<Azure::Core::Http::TransportPolicy>(
@@ -105,12 +112,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
105112
{
106113
policies.emplace_back(p->Clone());
107114
}
108-
policies.emplace_back(
109-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
115+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
116+
dfsRetryOptions.SecondaryHostForRetryReads
117+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
118+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
110119
for (const auto& p : options.PerRetryPolicies)
111120
{
112121
policies.emplace_back(p->Clone());
113122
}
123+
114124
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
115125
policies.emplace_back(
116126
std::make_unique<Core::Credentials::Policy::BearerTokenAuthenticationPolicy>(
@@ -136,12 +146,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
136146
{
137147
policies.emplace_back(p->Clone());
138148
}
139-
policies.emplace_back(
140-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
149+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
150+
dfsRetryOptions.SecondaryHostForRetryReads
151+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
152+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
141153
for (const auto& p : options.PerRetryPolicies)
142154
{
143155
policies.emplace_back(p->Clone());
144156
}
157+
145158
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
146159
policies.emplace_back(std::make_unique<Azure::Core::Http::TransportPolicy>(
147160
std::make_shared<Azure::Core::Http::CurlTransport>()));

‎sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "azure/storage/common/shared_key_policy.hpp"
1111
#include "azure/storage/common/storage_common.hpp"
1212
#include "azure/storage/common/storage_per_retry_policy.hpp"
13+
#include "azure/storage/common/storage_retry_policy.hpp"
1314
#include "azure/storage/common/storage_version.hpp"
1415
#include "azure/storage/files/datalake/datalake_utilities.hpp"
1516

@@ -29,6 +30,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
2930
{
3031
blobOptions.PerRetryPolicies.emplace_back(p->Clone());
3132
}
33+
blobOptions.RetryOptions = options.RetryOptions;
34+
blobOptions.RetryOptions.SecondaryHostForRetryReads
35+
= Details::GetBlobUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
3236
return blobOptions;
3337
}
3438

@@ -114,12 +118,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
114118
{
115119
policies.emplace_back(p->Clone());
116120
}
117-
policies.emplace_back(
118-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
121+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
122+
dfsRetryOptions.SecondaryHostForRetryReads
123+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
124+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
119125
for (const auto& p : options.PerRetryPolicies)
120126
{
121127
policies.emplace_back(p->Clone());
122128
}
129+
123130
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
124131
policies.emplace_back(std::make_unique<SharedKeyPolicy>(credential));
125132
policies.emplace_back(std::make_unique<Azure::Core::Http::TransportPolicy>(
@@ -142,12 +149,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
142149
{
143150
policies.emplace_back(p->Clone());
144151
}
145-
policies.emplace_back(
146-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
152+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
153+
dfsRetryOptions.SecondaryHostForRetryReads
154+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
155+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
147156
for (const auto& p : options.PerRetryPolicies)
148157
{
149158
policies.emplace_back(p->Clone());
150159
}
160+
151161
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
152162
policies.emplace_back(
153163
std::make_unique<Core::Credentials::Policy::BearerTokenAuthenticationPolicy>(
@@ -169,13 +179,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
169179
{
170180
policies.emplace_back(p->Clone());
171181
}
172-
policies.emplace_back(
173-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
182+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
183+
dfsRetryOptions.SecondaryHostForRetryReads
184+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
185+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
186+
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
174187
for (const auto& p : options.PerRetryPolicies)
175188
{
176189
policies.emplace_back(p->Clone());
177190
}
178-
policies.emplace_back(std::make_unique<StoragePerRetryPolicy>());
191+
179192
policies.emplace_back(std::make_unique<Azure::Core::Http::TransportPolicy>(
180193
std::make_shared<Azure::Core::Http::CurlTransport>()));
181194
m_pipeline = std::make_shared<Azure::Core::Http::HttpPipeline>(policies);

‎sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "azure/storage/common/storage_common.hpp"
1212
#include "azure/storage/common/storage_credential.hpp"
1313
#include "azure/storage/common/storage_per_retry_policy.hpp"
14+
#include "azure/storage/common/storage_retry_policy.hpp"
1415
#include "azure/storage/common/storage_version.hpp"
1516
#include "azure/storage/files/datalake/datalake_file_system_client.hpp"
1617
#include "azure/storage/files/datalake/datalake_utilities.hpp"
@@ -28,6 +29,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
2829
{
2930
blobOptions.PerRetryPolicies.emplace_back(p->Clone());
3031
}
32+
blobOptions.RetryOptions = options.RetryOptions;
33+
blobOptions.RetryOptions.SecondaryHostForRetryReads
34+
= Details::GetBlobUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
3135
return blobOptions;
3236
}
3337

@@ -82,8 +86,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
8286
{
8387
policies.emplace_back(p->Clone());
8488
}
85-
policies.emplace_back(
86-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
89+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
90+
dfsRetryOptions.SecondaryHostForRetryReads
91+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
92+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
8793
for (const auto& p : options.PerRetryPolicies)
8894
{
8995
policies.emplace_back(p->Clone());
@@ -112,8 +118,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
112118
{
113119
policies.emplace_back(p->Clone());
114120
}
115-
policies.emplace_back(
116-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
121+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
122+
dfsRetryOptions.SecondaryHostForRetryReads
123+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
124+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
117125
for (const auto& p : options.PerRetryPolicies)
118126
{
119127
policies.emplace_back(p->Clone());
@@ -140,8 +148,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
140148
{
141149
policies.emplace_back(p->Clone());
142150
}
143-
policies.emplace_back(
144-
std::make_unique<Azure::Core::Http::RetryPolicy>(Azure::Core::Http::RetryOptions()));
151+
StorageRetryWithSecondaryOptions dfsRetryOptions = options.RetryOptions;
152+
dfsRetryOptions.SecondaryHostForRetryReads
153+
= Details::GetDfsUriFromUri(options.RetryOptions.SecondaryHostForRetryReads);
154+
policies.emplace_back(std::make_unique<StorageRetryPolicy>(dfsRetryOptions));
145155
for (const auto& p : options.PerRetryPolicies)
146156
{
147157
policies.emplace_back(p->Clone());

‎sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
3030
{
3131
result.replace(pos, c_BlobEndPointIdentifier.size(), c_DfsEndPointIdentifier);
3232
}
33-
// DfsUri will be empty if there is no dfs endpoint.
34-
pos = result.find(c_DfsEndPointIdentifier);
35-
if (pos == std::string::npos)
36-
{
37-
result.clear();
38-
}
3933
return result;
4034
}
4135

0 commit comments

Comments
 (0)