Skip to content

Commit 44c16e2

Browse files
wezhangwezhang
committed
Fix NPE of exploring ADLS Gen2 FS in Spark job conf
1 parent f376245 commit 44c16e2

File tree

6 files changed

+30
-32
lines changed

6 files changed

+30
-32
lines changed

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/sdk/common/ADLSGen2OAuthHttpObservable.java‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424

2525
import static com.microsoft.azure.hdinsight.sdk.storage.adlsgen2.ADLSGen2FSOperation.PERMISSIONS_HEADER;
2626
import static com.microsoft.azure.hdinsight.sdk.storage.adlsgen2.ADLSGen2FSOperation.UMASK_HEADER;
27+
import static java.util.Collections.emptyList;
28+
import static java.util.Optional.ofNullable;
2729

2830
public class ADLSGen2OAuthHttpObservable extends OAuthTokenHttpObservable {
2931
private static final String resource = "https://storage.azure.com/";
@@ -51,15 +53,12 @@ public Observable<CloseableHttpResponse> request(@NotNull final HttpRequestBase
5153
final List<NameValuePair> parameters,
5254
final List<Header> addOrReplaceHeaders) {
5355
// Filter out set permission related headers since they are not supported in request with OAuth
54-
List<Header> filteredHeaders = addOrReplaceHeaders;
55-
if (filteredHeaders != null) {
56-
filteredHeaders =
57-
filteredHeaders.stream()
58-
.filter(header -> !header.getName().equalsIgnoreCase(PERMISSIONS_HEADER)
59-
&& !header.getName().equalsIgnoreCase(UMASK_HEADER))
60-
.collect(Collectors.toList());
61-
}
56+
List<Header> filteredHeaders = ofNullable(addOrReplaceHeaders)
57+
.orElse(emptyList())
58+
.stream()
59+
.filter(header -> !header.getName().equalsIgnoreCase(PERMISSIONS_HEADER) && !header.getName().equalsIgnoreCase(UMASK_HEADER))
60+
.collect(Collectors.toList());
6261

63-
return super.request(httpRequest, entity, parameters, filteredHeaders);
62+
return super.request(httpRequest, entity, ofNullable(parameters).orElse(emptyList()), filteredHeaders);
6463
}
6564
}

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/sdk/common/AzureSparkClusterManager.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import rx.Observable;
2121

2222
import java.io.IOException;
23+
import java.util.Collections;
2324
import java.util.List;
2425

2526
public class AzureSparkClusterManager extends AzureSparkCosmosClusterManager implements ILogger {
@@ -33,10 +34,9 @@ public static AzureSparkClusterManager getInstance() {
3334
return AzureSparkClusterManager.LazyHolder.INSTANCE;
3435
}
3536

36-
@Nullable
3737
@Override
3838
public List<NameValuePair> getAccountFilter() {
39-
return null;
39+
return Collections.emptyList();
4040
}
4141

4242
public Observable<SubscriptionDetail> getSubscriptionDetailByStoreAccountName(String storeAccountName) {

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/sdk/common/HttpObservable.java‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,6 @@ public HttpObservable setHttpClient(@NotNull CloseableHttpClient httpClient) {
182182
return this;
183183
}
184184

185-
@Nullable
186185
public Header[] getDefaultHeaders() throws IOException {
187186
return defaultHeaders.getAllHeaders();
188187
}
@@ -351,8 +350,8 @@ public <T> T convertJsonResponseToObject(@NotNull final HttpResponse resp, @NotN
351350
*/
352351
public Observable<CloseableHttpResponse> request(@NotNull final HttpRequestBase httpRequest,
353352
@Nullable final HttpEntity entity,
354-
@Nullable final List<NameValuePair> parameters,
355-
@Nullable final List<Header> addOrReplaceHeaders) {
353+
final List<NameValuePair> parameters,
354+
final List<Header> addOrReplaceHeaders) {
356355
return Observable.fromCallable(() -> {
357356
URIBuilder builder = new URIBuilder(httpRequest.getURI());
358357

@@ -387,8 +386,8 @@ public Observable<CloseableHttpResponse> request(@NotNull final HttpRequestBase
387386
*/
388387
public Observable<HttpResponse> requestWithHttpResponse(@NotNull final HttpRequestBase httpRequest,
389388
@Nullable final HttpEntity entity,
390-
@Nullable final List<NameValuePair> parameters,
391-
@Nullable final List<Header> addOrReplaceHeaders) {
389+
final List<NameValuePair> parameters,
390+
final List<Header> addOrReplaceHeaders) {
392391
return request(httpRequest, entity, parameters, addOrReplaceHeaders)
393392
.flatMap(HttpObservable::toStringOnlyOkResponse);
394393
}
@@ -400,41 +399,41 @@ public Observable<HttpResponse> head(@NotNull final String uri,
400399
}
401400

402401
public <T> Observable<T> get(@NotNull final String uri,
403-
@Nullable final List<NameValuePair> parameters,
404-
@Nullable final List<Header> addOrReplaceHeaders,
402+
final List<NameValuePair> parameters,
403+
final List<Header> addOrReplaceHeaders,
405404
@NotNull final Class<T> clazz) {
406405
return requestWithHttpResponse(new HttpGet(uri), null, parameters, addOrReplaceHeaders)
407406
.map(resp -> this.convertJsonResponseToObject(resp, clazz));
408407
}
409408

410409
public <T> Observable<T> put(@NotNull final String uri,
411410
@Nullable final HttpEntity entity,
412-
@Nullable final List<NameValuePair> parameters,
413-
@Nullable final List<Header> addOrReplaceHeaders,
411+
final List<NameValuePair> parameters,
412+
final List<Header> addOrReplaceHeaders,
414413
@NotNull final Class<T> clazz) {
415414
return requestWithHttpResponse(new HttpPut(uri), entity, parameters, addOrReplaceHeaders)
416415
.map(resp -> this.convertJsonResponseToObject(resp, clazz));
417416
}
418417

419418
public <T> Observable<T> post(@NotNull final String uri,
420419
@Nullable final HttpEntity entity,
421-
@Nullable final List<NameValuePair> parameters,
422-
@Nullable final List<Header> addOrReplaceHeaders,
420+
final List<NameValuePair> parameters,
421+
final List<Header> addOrReplaceHeaders,
423422
@NotNull final Class<T> clazz) {
424423
return requestWithHttpResponse(new HttpPost(uri), entity, parameters, addOrReplaceHeaders)
425424
.map(resp -> this.convertJsonResponseToObject(resp, clazz));
426425
}
427426

428427
public Observable<HttpResponse> delete(@NotNull final String uri,
429-
@Nullable final List<NameValuePair> parameters,
430-
@Nullable final List<Header> addOrReplaceHeaders) {
428+
final List<NameValuePair> parameters,
429+
final List<Header> addOrReplaceHeaders) {
431430
return requestWithHttpResponse(new HttpDelete(uri), null, parameters, addOrReplaceHeaders);
432431
}
433432

434433
public <T> Observable<T> patch(@NotNull final String uri,
435434
@Nullable final HttpEntity entity,
436-
@Nullable final List<NameValuePair> parameters,
437-
@Nullable final List<Header> addOrReplaceHeaders,
435+
final List<NameValuePair> parameters,
436+
final List<Header> addOrReplaceHeaders,
438437
@NotNull final Class<T> clazz) {
439438
return requestWithHttpResponse(new HttpPatch(uri), entity, parameters, addOrReplaceHeaders)
440439
.map(resp -> this.convertJsonResponseToObject(resp, clazz));

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/sdk/common/OAuthTokenHttpObservable.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ public String getAccessToken() throws IOException {
5050
return accessToken;
5151
}
5252

53-
@Nullable
5453
@Override
5554
public Header[] getDefaultHeaders() throws IOException {
5655
Header[] defaultHeaders = super.getDefaultHeaders();

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/sdk/common/SharedKeyHttpObservable.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222
import java.util.List;
2323
import java.util.UUID;
2424

25+
import static java.util.Collections.emptyList;
26+
import static java.util.Optional.ofNullable;
27+
2528
public class SharedKeyHttpObservable extends HttpObservable {
2629
public static String ApiVersion = "2018-11-09";
2730
private SharedKeyCredential cred;
@@ -63,16 +66,15 @@ public Observable<CloseableHttpResponse> request(final HttpRequestBase httpReque
6366
// cannot be added to default header group in case of duplication.
6467
headerGroup.addHeader(new BasicHeader("Content-Length", String.valueOf(entity.getContentLength())));
6568
}
66-
addOrReplaceHeaders.stream().forEach(header -> headerGroup.addHeader(header));
67-
String key = cred.generateSharedKey(httpRequest, headerGroup, parameters);
69+
ofNullable(addOrReplaceHeaders).orElse(emptyList()).forEach(headerGroup::addHeader);
70+
String key = cred.generateSharedKey(httpRequest, headerGroup, ofNullable(parameters).orElse(emptyList()));
6871

6972
getDefaultHeaderGroup().updateHeader(new BasicHeader("Authorization", key));
7073

71-
return super.request(httpRequest, entity, parameters, addOrReplaceHeaders);
74+
return super.request(httpRequest, entity, ofNullable(parameters).orElse(emptyList()), ofNullable(addOrReplaceHeaders).orElse(emptyList()));
7275
}
7376

7477
@Override
75-
@Nullable
7678
public Header[] getDefaultHeaders() throws IOException {
7779
return defaultHeaders.getAllHeaders();
7880
}

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/sdk/common/azure/serverless/AzureSparkCosmosClusterManager.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,6 @@ public AzureSparkCosmosClusterManager() {
8989
// Getters / setters
9090
//
9191

92-
@Nullable
9392
public List<NameValuePair> getAccountFilter() {
9493
return Collections.singletonList(ODataParam.filter(ACCOUNT_FILTER));
9594
}

0 commit comments

Comments
 (0)