Skip to content

Commit eceb8f2

Browse files
cosminbascacosminbasca
committed
ci.jsonnet: refactor common part
- create mx project
1 parent 964ab64 commit eceb8f2

File tree

6 files changed

+304
-290
lines changed

6 files changed

+304
-290
lines changed

‎ci.jsonnet

Lines changed: 17 additions & 284 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
local utils = import 'ci_common/utils.libsonnet';
2+
local const = import 'ci_common/constants.libsonnet';
3+
local builder = import 'ci_common/builder.libsonnet';
4+
15
{
26
overlay: "a02d502769517eff6324561f77f7cc73ded73284",
37

@@ -10,307 +14,36 @@
1014
//
1115
// ======================================================================================================
1216

13-
// ======================================================================================================
14-
//
15-
// locals (will not appear in the generated json)
16-
//
17-
// ======================================================================================================
18-
19-
// timelimit
20-
local TIME_LIMIT = {
21-
"30m": "00:30:00",
22-
"1h": "1:00:00",
23-
"2h": "2:00:00",
24-
"3h": "3:00:00",
25-
"4h": "4:00:00",
26-
"8h": "8:00:00",
27-
"10h": "10:00:00",
28-
"16h": "16:00:00",
29-
"20h": "20:00:00",
30-
},
31-
TIME_LIMIT: TIME_LIMIT,
32-
33-
// targets
34-
local TARGET = {
35-
onDemand: ["bench"],
36-
postMerge: ["post-merge"],
37-
weekly: ["weekly"],
38-
gate: ["gate"],
39-
},
40-
TARGET: TARGET,
41-
42-
// ------------------------------------------------------------------------------------------------------
43-
//
44-
// utility funcs
45-
//
46-
// ------------------------------------------------------------------------------------------------------
47-
local utils = {
48-
download: function(name, version, platformSpecific = true)
49-
{name: name, version: version, platformspecific: platformSpecific},
50-
51-
getValue: function(object, field)
52-
if (!std.objectHas(object, field)) then
53-
error "unknown field: "+field+" in "+object+", valid choices are: "+std.objectFields(object)
54-
else
55-
object[field],
56-
57-
graalOption: function(name, value)
58-
["--Ja", "@-Dgraal."+name+"="+value],
59-
60-
contains: function(array, value)
61-
std.count(array, value) > 0,
62-
},
63-
64-
// ------------------------------------------------------------------------------------------------------
65-
//
66-
// configurations
67-
//
68-
// ------------------------------------------------------------------------------------------------------
69-
local JVM = {
70-
server: "server",
71-
},
72-
JVM: JVM,
73-
74-
local JVM_CONFIG = {
75-
core: "graal-core",
76-
enterprise: "graal-enterprise",
77-
native: "native",
78-
hostspot: "default",
79-
},
80-
JVM_CONFIG: JVM_CONFIG,
81-
82-
// ------------------------------------------------------------------------------------------------------
83-
//
84-
// platform mixins
85-
//
86-
// ------------------------------------------------------------------------------------------------------
87-
local linuxMixin = {
88-
capabilities +: ["linux", "amd64"],
89-
packages +: {
90-
"maven": "==3.3.9",
91-
"git": ">=1.8.3",
92-
"mercurial": ">=3.2.4",
93-
"gcc": "==4.9.1",
94-
"llvm": "==4.0.1",
95-
"python": "==3.6.5",
96-
"libffi": ">=3.2.1",
97-
"bzip2": ">=1.0.6",
98-
},
99-
downloads +: {
100-
LIBGMP: utils.download("libgmp", "6.1.0"),
101-
},
102-
},
103-
linuxMixin: linuxMixin,
104-
105-
local linuxBenchMixin = linuxMixin + {
106-
capabilities +: ["no_frequency_scaling", "tmpfs25g", "x52"],
107-
},
108-
linuxBenchMixin: linuxBenchMixin,
109-
110-
local darwinMixin = {
111-
capabilities +: ["darwin_sierra", "amd64"],
112-
timelimit: TIME_LIMIT["1h"],
113-
packages +: {
114-
"pip:astroid": "==1.1.0",
115-
"pip:pylint": "==1.1.0",
116-
"llvm": "==4.0.1",
117-
},
118-
},
119-
120-
local getPlatform = function(platform)
121-
local PLATFORMS = {
122-
"linux": linuxMixin,
123-
"linuxBench": linuxBenchMixin,
124-
"darwin": darwinMixin,
125-
};
126-
utils.getValue(PLATFORMS, platform),
127-
128-
// ------------------------------------------------------------------------------------------------------
129-
//
130-
// mixins
131-
//
132-
// ------------------------------------------------------------------------------------------------------
133-
local pypyMixin = {
134-
downloads +: {
135-
PYPY_HOME: utils.download("pypy3", "7.1.0.beta"),
136-
},
137-
},
138-
pypyMixin: pypyMixin,
139-
140-
local eclipseMixin = {
141-
downloads +: {
142-
ECLIPSE: utils.download("eclipse", "4.5.2.1"),
143-
JDT: utils.download("ecj", "4.6.1", false),
144-
},
145-
environment +: {
146-
ECLIPSE_EXE: "$ECLIPSE/eclipse",
147-
},
148-
},
149-
150-
local labsjdk8Mixin = {
151-
downloads +: {
152-
JAVA_HOME: utils.download("oraclejdk", "8u212-jvmci-20-b01"),
153-
EXTRA_JAVA_HOMES : { pathlist: [utils.download("oraclejdk", "11+28")] },
154-
},
155-
environment +: {
156-
CI: "true",
157-
GRAALVM_CHECK_EXPERIMENTAL_OPTIONS: "true",
158-
PATH: "$JAVA_HOME/bin:$PATH",
159-
},
160-
},
161-
labsjdk8Mixin: labsjdk8Mixin,
162-
163-
local graalMixin = labsjdk8Mixin + {
164-
environment +: {
165-
HOST_VM: JVM.server,
166-
},
167-
},
168-
169-
local graalCoreMixin = graalMixin + {
170-
environment +: {
171-
HOST_VM_CONFIG: JVM_CONFIG.core,
172-
},
173-
},
174-
graalCoreMixin: graalCoreMixin,
175-
176-
local sulongMixin = labsjdk8Mixin + {
177-
environment +: {
178-
CPPFLAGS: "-I$LIBGMP/include",
179-
LD_LIBRARY_PATH: "$LIBGMP/lib:$LLVM/lib:$LD_LIBRARY_PATH",
180-
}
181-
},
182-
sulongMixin: sulongMixin,
183-
184-
// ------------------------------------------------------------------------------------------------------
185-
//
186-
// the build templates
187-
//
188-
// ------------------------------------------------------------------------------------------------------
189-
local baseBuilder = {
190-
downloads: {},
191-
environment: {},
192-
setup: [],
193-
logs: ["dumps/*/*"],
194-
timelimit: TIME_LIMIT["30m"],
195-
packages: {},
196-
capabilities: [],
197-
name: null,
198-
targets: [],
199-
run: [],
200-
},
201-
202-
local commonBuilder = baseBuilder + labsjdk8Mixin + {
203-
dynamicImports:: "/sulong,/compiler",
204-
205-
setup +: [
206-
["mx", "sforceimports"],
207-
["mx", "--dynamicimports", self.dynamicImports, "build"],
208-
]
209-
},
210-
commonBuilder: commonBuilder,
211-
212-
// ------------------------------------------------------------------------------------------------------
213-
//
214-
// the gate templates
215-
//
216-
// ------------------------------------------------------------------------------------------------------
217-
local baseGate = commonBuilder + {
218-
tags: "tags must be defined",
219-
220-
// local truffleDebugFlags = utils.graalOption("TraceTruffleCompilation", "true"),
221-
// local truffleDebugFlags = utils.graalOption("TraceTruffleCompilationDetails", "true"),
222-
local truffleDebugFlags = [],
223-
targets: TARGET.gate,
224-
run +: [
225-
["mx"] + truffleDebugFlags + ["--strict-compliance", "--dynamicimports", super.dynamicImports, "--primary", "gate", "--tags", self.tags, "-B=--force-deprecation-as-warning-for-dependencies"],
226-
]
227-
},
228-
229-
local baseGraalGate = baseGate + sulongMixin + graalCoreMixin,
230-
baseGraalGate: baseGraalGate,
231-
232-
// specific gates
233-
local testGate = function(type, platform)
234-
baseGraalGate + {tags:: "python-"+type} + getPlatform(platform) + {name: "python-"+ type +"-"+platform},
235-
236-
local testGateTime = function(type, platform, timelimit)
237-
baseGraalGate + {tags:: "python-"+type} + getPlatform(platform) + {name: "python-"+ type +"-"+platform} + {timelimit: timelimit},
238-
239-
local styleGate = baseGraalGate + eclipseMixin + linuxMixin + {
240-
tags:: "style,fullbuild,python-license",
241-
name: "python-style",
242-
243-
timelimit: TIME_LIMIT["1h"],
244-
},
245-
246-
local graalVmGate = baseGraalGate + linuxMixin {
247-
tags:: "python-graalvm",
248-
name: "python-graalvm",
249-
250-
timelimit: TIME_LIMIT["1h"],
251-
},
252-
253-
// ------------------------------------------------------------------------------------------------------
254-
//
255-
// the deploy templates
256-
//
257-
// ------------------------------------------------------------------------------------------------------
258-
local deployGate = function(platform)
259-
baseBuilder + graalCoreMixin + sulongMixin + getPlatform(platform) + {
260-
targets: TARGET.postMerge,
261-
setup +: [
262-
["mx", "sversions"],
263-
["mx", "build", "--force-javac"],
264-
],
265-
run +: [
266-
["mx", "deploy-binary-if-master", "python-public-snapshots"],
267-
],
268-
name: "deploy-binaries-"+platform,
269-
},
270-
271-
local coverageGate = commonBuilder + {
272-
targets: TARGET.weekly,
273-
timelimit: TIME_LIMIT["4h"],
274-
run +: [
275-
// cannot run with excluded "GeneratedBy" since that would lead to "command line too long"
276-
// ['mx', '--jacoco-whitelist-package', 'com.oracle.graal.python', '--jacoco-exclude-annotation', '@GeneratedBy', '--strict-compliance', "--dynamicimports", super.dynamicImports, "--primary", 'gate', '-B=--force-deprecation-as-warning-for-dependencies', '--strict-mode', '--tags', "python-junit", '--jacocout', 'html'],
277-
// ['mx', '--jacoco-whitelist-package', 'com.oracle.graal.python', '--jacoco-exclude-annotation', '@GeneratedBy', 'sonarqube-upload', "-Dsonar.host.url=$SONAR_HOST_URL", "-Dsonar.projectKey=com.oracle.graalvm.python", "-Dsonar.projectName=GraalVM - Python", '--exclude-generated'],
278-
['mx', '--jacoco-whitelist-package', 'com.oracle.graal.python', '--strict-compliance', "--dynamicimports", super.dynamicImports, "--primary", 'gate', '-B=--force-deprecation-as-warning-for-dependencies', '--strict-mode', '--tags', "python-unittest,python-tagged-unittest,python-junit", '--jacocout', 'html'],
279-
['mx', '--jacoco-whitelist-package', 'com.oracle.graal.python', 'sonarqube-upload', "-Dsonar.host.url=$SONAR_HOST_URL", "-Dsonar.projectKey=com.oracle.graalvm.python", "-Dsonar.projectName=GraalVM - Python", '--exclude-generated'],
280-
],
281-
name: "python-coverage"
282-
} + getPlatform(platform="linux"),
283-
28417
// ------------------------------------------------------------------------------------------------------
28518
//
28619
// the gates
28720
//
28821
// ------------------------------------------------------------------------------------------------------
28922
local gates = [
29023
// unittests
291-
testGate(type="unittest", platform="linux"),
292-
testGate(type="unittest", platform="darwin"),
293-
testGateTime(type="tagged-unittest", platform="linux", timelimit=TIME_LIMIT["2h"]),
294-
testGateTime(type="tagged-unittest", platform="darwin", timelimit=TIME_LIMIT["2h"]),
295-
testGate(type="svm-unittest", platform="linux"),
296-
testGate(type="svm-unittest", platform="darwin"),
24+
builder.testGate(type="unittest", platform="linux"),
25+
builder.testGate(type="unittest", platform="darwin"),
26+
builder.testGateTime(type="tagged-unittest", platform="linux", timelimit=const.TIME_LIMIT["2h"]),
27+
builder.testGateTime(type="tagged-unittest", platform="darwin", timelimit=const.TIME_LIMIT["2h"]),
28+
builder.testGate(type="svm-unittest", platform="linux"),
29+
builder.testGate(type="svm-unittest", platform="darwin"),
29730

29831
// junit
299-
testGate(type="junit", platform="linux"),
300-
testGate(type="junit", platform="darwin"),
32+
builder.testGate(type="junit", platform="linux"),
33+
builder.testGate(type="junit", platform="darwin"),
30134

30235
// style
303-
styleGate,
36+
builder.styleGate,
30437

30538
// coverage
306-
coverageGate,
39+
builder.coverageGate,
30740

30841
// graalvm gates
309-
graalVmGate,
42+
builder.graalVmGate,
31043

31144
// deploy binaries
312-
deployGate(platform="linux"),
313-
deployGate(platform="darwin"),
45+
builder.deployGate(platform="linux"),
46+
builder.deployGate(platform="darwin"),
31447
],
31548

31649
// ======================================================================================================

0 commit comments

Comments
 (0)