Skip to content

Commit 49d8639

Browse files
t-rufangt-rufang
authored
Merge branch 'develop' into develop
2 parents dcdb17b + 9750b0b commit 49d8639

File tree

9 files changed

+273
-176
lines changed

9 files changed

+273
-176
lines changed

‎PluginsAndFeatures/azure-toolkit-for-intellij/src/com/microsoft/azure/hdinsight/serverexplore/ui/AddNewClusterFrom.form‎

Lines changed: 123 additions & 145 deletions
Large diffs are not rendered by default.

‎PluginsAndFeatures/azure-toolkit-for-intellij/src/com/microsoft/azure/hdinsight/serverexplore/ui/AddNewClusterFrom.java‎

Lines changed: 85 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -27,23 +27,29 @@
2727
import com.intellij.openapi.application.ModalityState;
2828
import com.intellij.openapi.project.Project;
2929
import com.intellij.openapi.ui.DialogWrapper;
30-
import com.intellij.openapi.ui.popup.IconButton;
3130
import com.microsoft.azure.hdinsight.common.ClusterManagerEx;
3231
import com.microsoft.azure.hdinsight.sdk.cluster.HDInsightAdditionalClusterDetail;
33-
import com.microsoft.azure.hdinsight.sdk.common.HDIException;
32+
import com.microsoft.azure.hdinsight.sdk.common.AuthenticationException;
3433
import com.microsoft.azure.hdinsight.sdk.storage.HDStorageAccount;
35-
import com.microsoft.azure.hdinsight.serverexplore.AddHDInsightAdditionalClusterImpl;
3634
import com.microsoft.azure.hdinsight.serverexplore.hdinsightnode.HDInsightRootModule;
35+
import com.microsoft.azure.hdinsight.spark.jobs.JobUtils;
3736
import com.microsoft.azuretools.azurecommons.helpers.AzureCmdException;
3837
import com.microsoft.azuretools.azurecommons.helpers.StringHelper;
3938
import com.microsoft.azuretools.telemetry.AppInsightsClient;
4039
import com.microsoft.intellij.hdinsight.messages.HDInsightBundle;
40+
import com.microsoft.tooling.msservices.helpers.azure.sdk.StorageClientSDKManager;
41+
import com.microsoft.tooling.msservices.model.storage.BlobContainer;
42+
import com.microsoft.tooling.msservices.model.storage.ClientStorageAccount;
43+
import org.apache.commons.lang.StringUtils;
4144
import org.jetbrains.annotations.NotNull;
4245
import org.jetbrains.annotations.Nullable;
4346

4447
import javax.swing.*;
4548
import java.awt.*;
4649
import java.awt.event.ActionEvent;
50+
import java.awt.event.FocusAdapter;
51+
import java.awt.event.FocusEvent;
52+
import java.util.Optional;
4753
import java.util.stream.Stream;
4854

4955
public class AddNewClusterFrom extends DialogWrapper {
@@ -55,6 +61,7 @@ public class AddNewClusterFrom extends DialogWrapper {
5561

5662
private String storageName;
5763
private String storageKey;
64+
private String storageContainer;
5865

5966
private HDStorageAccount storageAccount;
6067

@@ -73,6 +80,8 @@ public class AddNewClusterFrom extends DialogWrapper {
7380
private JLabel storageKeyLabel;
7481
private JLabel userNameLabel;
7582
private JLabel passwordLabel;
83+
private JComboBox<BlobContainer> containersComboBox;
84+
private JLabel storageContainerLabel;
7685

7786
private HDInsightRootModule hdInsightModule;
7887

@@ -93,6 +102,47 @@ public AddNewClusterFrom(final Project project, HDInsightRootModule hdInsightMod
93102
errorMessageField.setBorder(BorderFactory.createEmptyBorder());
94103

95104
this.setModal(true);
105+
106+
storageKeyTextField.addFocusListener(new FocusAdapter() {
107+
@Override
108+
public void focusLost(FocusEvent e) {
109+
super.focusLost(e);
110+
111+
if (StringUtils.isNotBlank(storageNameField.getText()) && StringUtils.isNotBlank(storageKeyTextField.getText())) {
112+
ClientStorageAccount storageAccount = new ClientStorageAccount(storageNameField.getText());
113+
storageAccount.setPrimaryKey(storageKeyTextField.getText());
114+
115+
refreshContainers(storageAccount);
116+
}
117+
}
118+
});
119+
120+
storageNameField.addFocusListener(new FocusAdapter() {
121+
@Override
122+
public void focusLost(FocusEvent e) {
123+
super.focusLost(e);
124+
125+
if (StringUtils.isNotBlank(storageNameField.getText()) && StringUtils.isNotBlank(storageKeyTextField.getText())) {
126+
ClientStorageAccount storageAccount = new ClientStorageAccount(storageNameField.getText());
127+
storageAccount.setPrimaryKey(storageKeyTextField.getText());
128+
129+
refreshContainers(storageAccount);
130+
}
131+
}
132+
});
133+
}
134+
135+
private void refreshContainers(@NotNull ClientStorageAccount storageAccount) {
136+
try {
137+
containersComboBox.removeAllItems();
138+
139+
StorageClientSDKManager.getManager().getBlobContainers(storageAccount.getConnectionString())
140+
.forEach(containersComboBox::addItem);
141+
142+
containersComboBox.setMaximumRowCount(6);
143+
} catch (AzureCmdException e) {
144+
containersComboBox.removeAllItems();
145+
}
96146
}
97147

98148
private class HelpAction extends AbstractAction {
@@ -132,6 +182,7 @@ protected void doOKAction() {
132182
clusterNameLabel,
133183
storageNameLabel,
134184
storageKeyLabel,
185+
storageContainerLabel,
135186
userNameLabel,
136187
passwordLabel);
137188

@@ -157,19 +208,41 @@ protected void doOKAction() {
157208
isCarryOnNextStep = false;
158209
}
159210
}
211+
212+
if (containersComboBox.getSelectedItem() == null) {
213+
errorMessage = "The storage container isn't selected";
214+
isCarryOnNextStep = false;
215+
} else {
216+
storageContainer = ((BlobContainer) containersComboBox.getSelectedItem()).getName();
217+
}
160218
}
161219

162220
if (isCarryOnNextStep) {
163221
getStorageAccount();
164-
}
165222

166-
if (isCarryOnNextStep) {
167-
if (storageAccount != null) {
223+
if (storageAccount == null) {
224+
isCarryOnNextStep = false;
225+
} else {
168226
HDInsightAdditionalClusterDetail hdInsightAdditionalClusterDetail = new HDInsightAdditionalClusterDetail(clusterName, userName, password, storageAccount);
169-
ClusterManagerEx.getInstance().addHDInsightAdditionalCluster(hdInsightAdditionalClusterDetail);
170-
hdInsightModule.refreshWithoutAsync();
227+
try {
228+
JobUtils.authenticate(hdInsightAdditionalClusterDetail);
229+
230+
ClusterManagerEx.getInstance().addHDInsightAdditionalCluster(hdInsightAdditionalClusterDetail);
231+
hdInsightModule.refreshWithoutAsync();
232+
} catch (AuthenticationException authErr) {
233+
isCarryOnNextStep = false;
234+
errorMessage = "Authentication Error: " + Optional.ofNullable(authErr.getMessage())
235+
.filter(msg -> !msg.isEmpty())
236+
.orElse("Wrong username/password") +
237+
" (" + authErr.getErrorCode() + ")";
238+
} catch (Exception ex) {
239+
isCarryOnNextStep = false;
240+
errorMessage = "Authentication Error: " + ex.getMessage();
241+
}
171242
}
243+
}
172244

245+
if (isCarryOnNextStep) {
173246
super.doOKAction();
174247
} else {
175248
errorMessageField.setText(errorMessage);
@@ -190,17 +263,10 @@ private static String getClusterName(String userNameOrUrl) {
190263
private void getStorageAccount() {
191264
addNewClusterPanel.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
192265

193-
ApplicationManager.getApplication().invokeAndWait(new Runnable() {
194-
@Override
195-
public void run() {
196-
try {
197-
storageAccount = AddHDInsightAdditionalClusterImpl.getStorageAccount(clusterName, storageName, storageKey, userName, password);
198-
isCarryOnNextStep = true;
199-
} catch (AzureCmdException | HDIException e) {
200-
isCarryOnNextStep = false;
201-
errorMessage = e.getMessage();
202-
}
203-
}
266+
ApplicationManager.getApplication().invokeAndWait(() -> {
267+
storageAccount = new HDStorageAccount(
268+
null, ClusterManagerEx.getInstance().getBlobFullName(storageName), storageKey, false, storageContainer);
269+
isCarryOnNextStep = true;
204270
}, ModalityState.NON_MODAL);
205271

206272
addNewClusterPanel.setCursor(Cursor.getDefaultCursor());

‎PluginsAndFeatures/azure-toolkit-for-intellij/src/com/microsoft/azure/hdinsight/spark/ui/SparkSubmissionContentPanelConfigurable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ private void refreshClusterListAsync() {
122122
submissionPanel.setClustersListRefreshEnabled(false);
123123

124124
DefaultLoader.getIdeHelper().executeOnPooledThread(() -> {
125-
HDInsightUtil.showInfoOnSubmissionMessageWindow(myProject, "List spark clusters ...", true);
125+
HDInsightUtil.showInfoOnSubmissionMessageWindow(myProject, "List spark clusters ...");
126126
List<IClusterDetail> cachedClusters = ClusterManagerEx.getInstance().getClusterDetailsWithoutAsync(true);
127127

128128
if (!ClusterManagerEx.getInstance().isSelectedSubscriptionExist()) {

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/common/ClusterManagerEx.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,10 +135,11 @@ public Optional<IClusterDetail> getClusterDetailByName(String clusterName) {
135135
.flatMap(cluster -> {
136136
try {
137137
cluster.getConfigurationInfo();
138+
139+
return Optional.of(cluster);
138140
} catch (Exception ignore) {
141+
return Optional.empty();
139142
}
140-
141-
return cluster.isConfigInfoAvailable() ? Optional.of(cluster) : Optional.empty();
142143
});
143144
}
144145

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/common/StreamUtil.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,12 @@
2929
import org.apache.commons.io.FileUtils;
3030
import org.apache.http.HttpEntity;
3131
import org.apache.http.client.methods.CloseableHttpResponse;
32+
import org.apache.http.entity.StringEntity;
3233

3334
import javax.swing.*;
3435
import java.io.*;
3536
import java.net.URL;
36-
import java.util.Date;
37-
import java.util.HashMap;
38-
import java.util.List;
39-
import java.util.UUID;
37+
import java.util.*;
4038

4139
public class StreamUtil {
4240

@@ -56,7 +54,9 @@ public static String getResultFromInputStream(InputStream inputStream) throws IO
5654
public static HttpResponse getResultFromHttpResponse(CloseableHttpResponse response) throws IOException {
5755
int code = response.getStatusLine().getStatusCode();
5856
String reason = response.getStatusLine().getReasonPhrase();
59-
HttpEntity entity = response.getEntity();
57+
// Entity for HEAD is empty
58+
HttpEntity entity = Optional.ofNullable(response.getEntity())
59+
.orElse(new StringEntity(""));
6060
try (InputStream inputStream = entity.getContent()) {
6161
String response_content = getResultFromInputStream(inputStream);
6262
return new HttpResponse(code, response_content, new HashMap<String, List<String>>(), reason);

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/serverexplore/HDInsightRootModuleImpl.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.microsoft.azuretools.azurecommons.helpers.AzureCmdException;
3333
import com.microsoft.tooling.msservices.serviceexplorer.Node;
3434
import com.microsoft.tooling.msservices.serviceexplorer.NodeActionEvent;
35+
import com.microsoft.tooling.msservices.serviceexplorer.RefreshableNode;
3536

3637
import java.util.List;
3738

@@ -84,4 +85,10 @@ public void refreshWithoutAsync() {
8485

8586
}
8687

88+
@Override
89+
protected void onNodeClick(NodeActionEvent e) {
90+
if (!initialized) {
91+
this.load(false);
92+
}
93+
}
8794
}

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/serverexplore/hdinsightnode/ClusterNode.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ protected void actionPerformed(NodeActionEvent e) {
134134

135135
@Override
136136
protected void refreshItems() {
137+
this.load(false);
138+
137139
if(!clusterDetail.isEmulator()) {
138140
JobViewManager.registerJovViewNode(clusterDetail.getName(), clusterDetail);
139141
JobViewNode jobViewNode = new JobViewNode(this, clusterDetail.getName());

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/spark/common/SparkBatchSubmission.java‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,16 @@
2727
import org.apache.http.auth.AuthScope;
2828
import org.apache.http.auth.UsernamePasswordCredentials;
2929
import org.apache.http.client.CredentialsProvider;
30-
import org.apache.http.client.methods.CloseableHttpResponse;
31-
import org.apache.http.client.methods.HttpDelete;
32-
import org.apache.http.client.methods.HttpGet;
33-
import org.apache.http.client.methods.HttpPost;
30+
import org.apache.http.client.methods.*;
3431
import org.apache.http.entity.StringEntity;
3532
import org.apache.http.impl.client.BasicCredentialsProvider;
3633
import org.apache.http.impl.client.CloseableHttpClient;
3734
import org.apache.http.impl.client.HttpClients;
35+
import rx.Observable;
36+
import rx.schedulers.Schedulers;
3837

3938
import java.io.IOException;
39+
import java.util.concurrent.TimeUnit;
4040

4141
public class SparkBatchSubmission {
4242

@@ -91,6 +91,30 @@ public HttpResponse getHttpResponseViaGet(String connectUrl) throws IOException
9191
}
9292
}
9393

94+
public HttpResponse getHttpResponseViaHead(String connectUrl) throws IOException {
95+
CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(getCredentialsProvider())
96+
.build();
97+
98+
HttpHead httpHead = new HttpHead(connectUrl);
99+
httpHead.addHeader("Content-Type", "application/json");
100+
httpHead.addHeader("User-Agent", getUserAgentName());
101+
httpHead.addHeader("X-Requested-By", "ambari");
102+
103+
// WORKAROUND: https://github.com/Microsoft/azure-tools-for-java/issues/1358
104+
// The Ambari local account will cause Kerberos authentication initializing infinitely.
105+
// Set a timer here to cancel the progress.
106+
Observable.timer(3, TimeUnit.SECONDS, Schedulers.io())
107+
.take(1)
108+
.subscribe(i -> httpHead.abort());
109+
110+
try(CloseableHttpResponse response = httpclient.execute(httpHead)) {
111+
return StreamUtil.getResultFromHttpResponse(response);
112+
}
113+
}
114+
115+
public String getUserAgentName() {
116+
return userAgentName;
117+
}
94118

95119
/**
96120
* get all batches spark jobs

‎Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/spark/jobs/JobUtils.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import com.microsoft.azure.hdinsight.common.StreamUtil;
3636
import com.microsoft.azure.hdinsight.sdk.cluster.EmulatorClusterDetail;
3737
import com.microsoft.azure.hdinsight.sdk.cluster.IClusterDetail;
38+
import com.microsoft.azure.hdinsight.sdk.common.AuthenticationException;
3839
import com.microsoft.azure.hdinsight.sdk.common.HDIException;
3940
import com.microsoft.azure.hdinsight.sdk.rest.yarn.rm.App;
4041
import com.microsoft.azure.hdinsight.sdk.rest.yarn.rm.ApplicationMasterLogs;
@@ -589,4 +590,22 @@ public static Single<SimpleImmutableEntry<IClusterDetail, String>> deployArtifac
589590
public static Cache getGlobalCache() {
590591
return globalCache;
591592
}
593+
594+
public static AbstractMap.SimpleImmutableEntry<Integer, Map<String, List<String>>>
595+
authenticate(IClusterDetail clusterDetail) throws HDIException, IOException {
596+
SparkBatchSubmission submission = SparkBatchSubmission.getInstance();
597+
submission.setCredentialsProvider(clusterDetail.getHttpUserName(), clusterDetail.getHttpPassword());
598+
String livyUrl = URI.create(ClusterManagerEx.getInstance().getClusterConnectionString(clusterDetail.getName()))
599+
.resolve("/livy/")
600+
.toString();
601+
com.microsoft.azure.hdinsight.sdk.common.HttpResponse response = submission.getHttpResponseViaHead(livyUrl);
602+
603+
int statusCode = response.getCode();
604+
605+
if (statusCode >= 200 && statusCode < 300) {
606+
return new AbstractMap.SimpleImmutableEntry<>(statusCode, response.getHeaders());
607+
}
608+
609+
throw new AuthenticationException("Authentication failed", statusCode);
610+
}
592611
}

0 commit comments

Comments
 (0)