Skip to content

Commit 6f82f15

Browse files
suhstevesuhsteve
authored
Fix build pipeline due to Ubuntu 1604 Image deprecation and bintray no longer available as spark-package repository (#1007)
1 parent b0a3068 commit 6f82f15

File tree

3 files changed

+49
-29
lines changed

3 files changed

+49
-29
lines changed

‎azure-pipelines-e2e-tests-template.yml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,21 @@ stages:
1818
jobs:
1919
- ${{ each option in test.jobOptions }}:
2020
- job: Run_${{ replace(option.pool, ' ', '_') }}
21-
pool: ${{ option.pool }}
21+
${{ if eq(lower(option.pool), 'windows') }}:
22+
pool: Hosted VS2017
23+
${{ else }}:
24+
pool:
25+
name: NetCore1ESPool-Public
26+
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
2227

2328
steps:
29+
- task: UseDotNet@2
30+
displayName: 'Use .NET Core sdk'
31+
inputs:
32+
packageType: sdk
33+
version: 3.1.x
34+
installationPath: $(Agent.ToolsDirectory)/dotnet
35+
2436
- task: DownloadBuildArtifacts@0
2537
displayName: Download Build Artifacts
2638
inputs:
@@ -72,7 +84,7 @@ stages:
7284
script: |
7385
echo "Download Hadoop utils for Windows."
7486
curl -k -L -o hadoop.zip https://github.com/steveloughran/winutils/releases/download/tag_2017-08-29-hadoop-2.8.1-native/hadoop-2.8.1.zip
75-
unzip hadoop.zip
87+
Expand-Archive -Path hadoop.zip -Destination .
7688
New-Item -ItemType Directory -Force -Path hadoop\bin
7789
cp hadoop-2.8.1\winutils.exe hadoop\bin
7890

‎azure-pipelines.yml‎

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -203,131 +203,131 @@ stages:
203203
tests:
204204
- version: '2.4.0'
205205
jobOptions:
206-
- pool: 'Hosted VS2017'
206+
- pool: 'Windows'
207207
testOptions: ""
208208
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
209209
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
210-
- pool: 'Hosted Ubuntu 1604'
210+
- pool: 'Linux'
211211
testOptions: ""
212212
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
213213
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
214214
- version: '2.4.1'
215215
jobOptions:
216-
- pool: 'Hosted VS2017'
216+
- pool: 'Windows'
217217
testOptions: ""
218218
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
219219
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
220-
- pool: 'Hosted Ubuntu 1604'
220+
- pool: 'Linux'
221221
testOptions: ""
222222
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
223223
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
224224
- version: '2.4.3'
225225
jobOptions:
226-
- pool: 'Hosted VS2017'
226+
- pool: 'Windows'
227227
testOptions: ""
228228
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
229229
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
230-
- pool: 'Hosted Ubuntu 1604'
230+
- pool: 'Linux'
231231
testOptions: ""
232232
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
233233
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
234234
- version: '2.4.4'
235235
jobOptions:
236-
- pool: 'Hosted VS2017'
236+
- pool: 'Windows'
237237
testOptions: ""
238238
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
239239
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
240-
- pool: 'Hosted Ubuntu 1604'
240+
- pool: 'Linux'
241241
testOptions: ""
242242
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
243243
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
244244
- version: '2.4.5'
245245
jobOptions:
246-
- pool: 'Hosted VS2017'
246+
- pool: 'Windows'
247247
testOptions: ""
248248
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
249249
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
250-
- pool: 'Hosted Ubuntu 1604'
250+
- pool: 'Linux'
251251
testOptions: ""
252252
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
253253
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
254254
- version: '2.4.6'
255255
jobOptions:
256-
- pool: 'Hosted VS2017'
256+
- pool: 'Windows'
257257
testOptions: ""
258258
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
259259
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
260-
- pool: 'Hosted Ubuntu 1604'
260+
- pool: 'Linux'
261261
testOptions: ""
262262
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
263263
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
264264
- version: '2.4.7'
265265
jobOptions:
266-
- pool: 'Hosted VS2017'
266+
- pool: 'Windows'
267267
testOptions: ""
268268
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
269269
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
270-
- pool: 'Hosted Ubuntu 1604'
270+
- pool: 'Linux'
271271
testOptions: ""
272272
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
273273
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
274274
- version: '2.4.8'
275275
jobOptions:
276-
- pool: 'Hosted VS2017'
276+
- pool: 'Windows'
277277
testOptions: ""
278278
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
279279
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
280-
- pool: 'Hosted Ubuntu 1604'
280+
- pool: 'Linux'
281281
testOptions: ""
282282
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
283283
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
284284
- version: '3.0.0'
285285
jobOptions:
286-
- pool: 'Hosted VS2017'
286+
- pool: 'Windows'
287287
testOptions: ""
288288
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0)
289289
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0)
290-
- pool: 'Hosted Ubuntu 1604'
290+
- pool: 'Linux'
291291
testOptions: ""
292292
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
293293
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
294294
- version: '3.0.1'
295295
jobOptions:
296-
- pool: 'Hosted VS2017'
296+
- pool: 'Windows'
297297
testOptions: ""
298298
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0)
299299
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0)
300-
- pool: 'Hosted Ubuntu 1604'
300+
- pool: 'Linux'
301301
testOptions: ""
302302
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
303303
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
304304
- version: '3.0.2'
305305
jobOptions:
306-
- pool: 'Hosted VS2017'
306+
- pool: 'Windows'
307307
testOptions: ""
308308
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0)
309309
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0)
310-
- pool: 'Hosted Ubuntu 1604'
310+
- pool: 'Linux'
311311
testOptions: ""
312312
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
313313
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
314314
- version: '3.1.1'
315315
jobOptions:
316-
- pool: 'Hosted VS2017'
316+
- pool: 'Windows'
317317
testOptions: ""
318318
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
319319
forwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
320-
- pool: 'Hosted Ubuntu 1604'
320+
- pool: 'Linux'
321321
testOptions: ""
322322
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1)
323323
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1)
324324
- version: '3.1.2'
325325
jobOptions:
326-
- pool: 'Hosted VS2017'
326+
- pool: 'Windows'
327327
testOptions: ""
328328
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
329329
forwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
330-
- pool: 'Hosted Ubuntu 1604'
330+
- pool: 'Linux'
331331
testOptions: ""
332332
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1)
333333
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1)

‎src/csharp/Microsoft.Spark.E2ETest/SparkFixture.cs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ public SparkFixture()
8484
};
8585

8686
_process.Start();
87+
_process.BeginErrorReadLine();
8788
_process.BeginOutputReadLine();
8889

8990
bool processExited = false;
@@ -192,6 +193,12 @@ private void BuildSparkCmd(out string filename, out string args)
192193
Path.Combine(_tempDirectory.Path, "spark-warehouse")).AbsoluteUri;
193194
string warehouseDir = $"--conf spark.sql.warehouse.dir={warehouseUri}";
194195

196+
// Spark24 < 2.4.8, Spark30 < 3.0.3 and Spark31 < 3.1.2 use bintray as the repository
197+
// service for spark-packages. As of May 1st, 2021 bintray has been sunset and is no
198+
// longer available. Specify additional remote repositories to search for the maven
199+
// coordinates given with --packages.
200+
string repositories = "--repositories https://repos.spark-packages.org/";
201+
195202
string extraArgs = Environment.GetEnvironmentVariable(
196203
EnvironmentVariableNames.ExtraSparkSubmitArgs) ?? "";
197204

@@ -205,7 +212,8 @@ private void BuildSparkCmd(out string filename, out string args)
205212
string logOption = "--conf spark.driver.extraJavaOptions=-Dlog4j.configuration=" +
206213
$"{resourceUri}/log4j.properties";
207214

208-
args = $"{logOption} {warehouseDir} {AddPackages(extraArgs)} {classArg} --master local {jar} debug";
215+
args = $"{logOption} {warehouseDir} {AddPackages(extraArgs)} {repositories} {classArg} " +
216+
$"--master local {jar} debug";
209217
}
210218

211219
private string GetJarPrefix()

0 commit comments

Comments
 (0)