Skip to content

Commit a9fe01b

Browse files
timfeltimfel
committed
Report GraalPy compat from more sources
1 parent b4a6ae3 commit a9fe01b

File tree

3 files changed

+229
-35
lines changed

3 files changed

+229
-35
lines changed

‎docs/site/03-compatibility.md

Lines changed: 139 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,95 @@ permalink: compatibility/
1010
const dbs = {};
1111
var module_query = '';
1212
const load_db = function (graalpyVersion) {
13-
return new Promise(function (resolve, reject) {
13+
var graalvmVersion = graalpyVersion.replace(/^v/, "").replace(/^(\d\d)/, "$1.");
14+
var wheelbuilder_scripts = new Promise(function (resolve, reject) {
15+
const xhr = new XMLHttpRequest();
16+
const url = `https://api.github.com/repos/oracle/graalpython/contents/scripts/wheelbuilder/linux?ref=release/graal-vm/${graalvmVersion}`;
17+
xhr.open('GET', url);
18+
xhr.overrideMimeType('text/plain');
19+
xhr.onload = function () {
20+
if (this.status === 200) {
21+
const contents = JSON.parse(this.responseText);
22+
const packages = [];
23+
for (const item of contents) {
24+
const parts = item.name.split('.');
25+
const package_name = parts[0];
26+
const version = parts.slice(1, -1).join('.') || "any";
27+
packages.push(`${package_name},${version},0,GraalPy provides a script to build this package from <a href='${item.html_url}'>source</a>.`);
28+
}
29+
resolve(packages.join("\n"));
30+
} else {
31+
reject(this.statusText);
32+
}
33+
};
34+
xhr.onerror = function () {
35+
reject(url);
36+
};
37+
xhr.send();
38+
});
39+
var patch_metadata = new Promise(function (resolve, reject) {
40+
const xhr = new XMLHttpRequest();
41+
const url = `https://raw.githubusercontent.com/oracle/graalpython/refs/heads/release/graal-vm/${graalvmVersion}/graalpython/lib-graalpython/patches/metadata.toml`;
42+
xhr.open('GET', url);
43+
xhr.overrideMimeType('text/plain');
44+
xhr.onload = function () {
45+
if (this.status === 200) {
46+
const patches = [];
47+
const lines = this.responseText.trim().split('\n');
48+
var currentPatch = null;
49+
for (let i = 0; i < lines.length; i++) {
50+
const line = lines[i].trim();
51+
if (line.startsWith('[[')) {
52+
if (currentPatch != null) {
53+
patches.push(
54+
[currentPatch.name,
55+
currentPatch.version,
56+
0,
57+
currentPatch.comment || "GraalPy automatically applies a patch to run this package."].join(",")
58+
)
59+
}
60+
let pkgName = line.substring(2, line.indexOf(".")).trim();
61+
currentPatch = {name: pkgName, version: "any"};
62+
} else if (line.startsWith('#')) {
63+
if (!currentPatch.comment) {
64+
currentPatch.comment = line.substring(1).trim();
65+
} else {
66+
currentPatch.comment += ' ' + line.substring(1).trim();
67+
}
68+
} else if (line.startsWith('version =')) {
69+
currentPatch.version = line.substring('version ='.length).trim().replace(/'|"/g, '').replace(/^== ?/, "").replace(/, ?/g, " and ");
70+
} else if (line.startsWith('install-priority =')) {
71+
if (parseInt(line.substring('install-priority ='.length).trim(), 10) <= 0) {
72+
if (currentPatch.comment) {
73+
if (!currentPatch.comment.endsWith(".")) {
74+
currentPatch.comment += ".";
75+
}
76+
currentPatch.comment += " This version works, but another should be chosen if possible.";
77+
} else {
78+
currentPatch.comment = "GraalPy provides a patch for this version, but it is recommended to use another if possible.";
79+
}
80+
}
81+
}
82+
}
83+
if (currentPatch != null) {
84+
patches.push(
85+
[currentPatch.name,
86+
currentPatch.version,
87+
0,
88+
currentPatch.comment || "GraalPy provides a patch to make this package work."].join(",")
89+
)
90+
}
91+
resolve(patches.join("\n"));
92+
} else {
93+
reject(this.statusText);
94+
}
95+
};
96+
xhr.onerror = function () {
97+
reject(url);
98+
};
99+
xhr.send();
100+
});
101+
var module_testing_csv = new Promise(function (resolve, reject) {
14102
const xhr = new XMLHttpRequest();
15103
const url = `/python/module_results/python-module-testing-${graalpyVersion}.csv`;
16104
xhr.open('GET', url);
@@ -27,6 +115,36 @@ permalink: compatibility/
27115
};
28116
xhr.send();
29117
});
118+
var wheels_csv = new Promise(function (resolve, reject) {
119+
const xhr = new XMLHttpRequest();
120+
const url = `/python/wheels/${graalpyVersion}.csv`;
121+
xhr.open('GET', url);
122+
xhr.overrideMimeType('text/plain');
123+
xhr.onload = function () {
124+
if (this.status === 200) {
125+
resolve(this.responseText);
126+
} else {
127+
reject(this.statusText);
128+
}
129+
};
130+
xhr.onerror = function () {
131+
reject(url);
132+
};
133+
xhr.send();
134+
});
135+
return new Promise(function (resolve, reject) {
136+
Promise.allSettled([module_testing_csv, patch_metadata, wheelbuilder_scripts, wheels_csv]).then(function (results) {
137+
resolve(results.map(function (result) {
138+
if (result.status === 'fulfilled') {
139+
return result.value;
140+
} else {
141+
return null;
142+
}
143+
}).filter((entry) => !!entry).join("\n"));
144+
}).catch(function (err) {
145+
reject(err);
146+
});
147+
});
30148
};
31149
let pageNumber = 0;
32150
let database;
@@ -115,8 +233,15 @@ permalink: compatibility/
115233
let countNotSupported = 0;
116234
$('#dataTable tbody').empty();
117235
for (let package in database.db) {
118-
const versions = database.db[package]
119-
versions_loop: for (let version in versions) {
236+
const versions = database.db[package];
237+
const version_keys = Object.keys(versions).sort((a, b) => {
238+
const versionA = a.replace(/[~><=! ]/g, '');
239+
const versionB = b.replace(/[~><=! ]/g, '');
240+
if (versionA < versionB) return -1;
241+
if (versionA > versionB) return 1;
242+
return 0;
243+
});
244+
versions_loop: for (const version of version_keys) {
120245
if (version.startsWith('~')) {
121246
continue;
122247
}
@@ -135,16 +260,14 @@ permalink: compatibility/
135260
countNotSupported++;
136261
break;
137262
default:
138-
console.log(`Unknown test_status: ${info.test_status} for package ${info.name}`);
139263
continue versions_loop;
140264
}
141265
const styling = count++ < rowsPerPage ? '' : ' style="display: none;"';
142266
$('#dataTable tbody').append(`
143267
<tr${styling}>
144268
<td class="dataTable-name">${info.name}</td>
145269
<td>${info.version}</td>
146-
<td>${info.pass_percentage}</td>
147-
<td><a href="https://pypi.org/project/${info.name}/" target="_blank">${info.name} on pypi.org</td>
270+
<td>${info.notes}</td>
148271
</tr>`);
149272
}
150273
}
@@ -271,9 +394,13 @@ You can extend it with Python code or leverage packages from the Python ecosyste
271394
<div class="wrapper">
272395
<div class="compatibility">
273396
<div class="container">
274-
<h5 class="compatibility-text">To ensure GraalPy is compatible with common Python packages, the GraalPy team conducts
275-
compatibility testing to verify the presence and correct functioning of
276-
the top 500 packages on PyPI plus some more that are of special interest to us, including
397+
<h5 class="compatibility-text">Python PackagesGraalPy is compatible with many packages, including packages like
398+
PyTorch, NumPy, Huggingface Transformers and many more that are used for Data Science and
399+
Machine Learning. If you want to try a package, first just pick any version and only if you
400+
run into problems, consult the table below to see if there is a version that may work better.</h5>
401+
<h5 class="compatibility-text">To ensure GraalPy is compatible with common Python packages,
402+
the GraalPy team conducts compatibility testing and creates scripts to build and patch many
403+
of the top packages on PyPI plus some more that are of special interest to us, including
277404
libraries and frameworks such as NumPy, Pandas, and Django.</h5>
278405
<h5 class="compatibility-text">Compatibility testing ensures that
279406
developers can leverage GraalPy's powerful capabilities in their existing applications.
@@ -282,7 +409,7 @@ You can extend it with Python code or leverage packages from the Python ecosyste
282409
toolsets.</h5>
283410
<h5 class="compatibility-text">Many more Python packages than are on this list work on GraalPy.
284411
If there is a package you are interested in that you cannot find here, chances are that it
285-
might just work.</h5>
412+
might just work. If it does not, please reach out to us on [Github](https://github.com/oracle/graalpython/issues)</h5>
286413
</div>
287414
</div>
288415
</div>
@@ -326,7 +453,7 @@ You can extend it with Python code or leverage packages from the Python ecosyste
326453
<div class="wrapper">
327454
<div class="compatibility">
328455
<div class="container">
329-
<h3 class="pypage__title-02">Python Packages</h3>
456+
<h3 class="langpage__title-02">Python Packages</h3>
330457
<div class="package__row">
331458
<div class="package__search">
332459
<input type="text" id="compatibility_page__search-field" placeholder="Comma-separated list of packages">
@@ -338,10 +465,6 @@ You can extend it with Python code or leverage packages from the Python ecosyste
338465
</label>
339466
<input type="file" id="add-requirements-btn" accept=".txt">
340467
</div>
341-
<div>
342-
<a href="{{ '/module_results/python-module-testing-v231.csv' | relative_url }}"
343-
target="_blank"><button class="download-data-btn">Download data</button></a>
344-
</div>
345468
</div>
346469
</div>
347470
<div class="compattable-container">
@@ -351,8 +474,7 @@ You can extend it with Python code or leverage packages from the Python ecosyste
351474
<tr class="add-radius">
352475
<th scope="col" title="Name">Name</th>
353476
<th scope="col" title="Version">Version</th>
354-
<th scope="col" title="Test Level">Test Level %</th>
355-
<th scope="col" title="Package URL">Package URL</th>
477+
<th scope="col" title="Notes">Notes</th>
356478
</tr>
357479
</thead>
358480
<tbody></tbody>

‎docs/site/_plugins/wheel_repo_plugin.rb

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# frozen_string_literal: true
22

3+
require "set"
4+
35
# Plugin that generates a Python repository index for GraalPy wheels at
46
# /python/wheels. The input is the graalpy_wheels.txt file next to this file.
57
# The wheels themselves are hosted in GDS.
@@ -50,6 +52,38 @@ def generate(site)
5052
package_page.content = render package_links
5153
site.pages << package_page
5254
end
55+
56+
graalpy_versions = all_wheels.group_by do |wheel|
57+
wheel.filename =~ /graalpy(\d\d\d?)_/
58+
$1
59+
end
60+
graalpy_versions.each do |version, wheels|
61+
package_csv = Jekyll::PageWithoutAFile.new(site, "", "/wheels/", "v#{version}.csv")
62+
package_csv.content = wheels.group_by(&:name).map do |name, wheels|
63+
wheels.group_by { |w| w.filename.split("-")[1] }.map do |version, wheels|
64+
comment = "The GraalPy team provides binary wheels of this package for "
65+
platforms = wheels.map do |wheel|
66+
parts = wheel.filename.split("-")
67+
if parts.any? { |p| p.include? "linux" }
68+
"Linux"
69+
elsif parts.any? { |p| p.include? "win" }
70+
"Windows"
71+
elsif parts.any? { |p| p.include? "macos" }
72+
"macOS"
73+
end
74+
end.sort.compact
75+
if platforms.length == 1
76+
comment += "#{platforms.first}."
77+
elsif platforms.length == 2
78+
comment += "#{platforms.first} and #{platforms.last}."
79+
else
80+
comment += "#{platforms[0]}, #{platforms[1]}, and #{platforms[2]}."
81+
end
82+
"#{name},#{version},0,#{comment}"
83+
end
84+
end.flatten.join("\n")
85+
site.pages << package_csv
86+
end
5387
end
5488

5589
def render(links)

‎docs/site/assets/js/check_compatibility_helpers.js

Lines changed: 56 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,44 @@ class ModuleListing {
1313
}
1414

1515
class DBEntry {
16-
constructor([library_name, library_version, test_status, pass_percentage]) {
16+
constructor(library_name, library_version, test_status, notes) {
1717
this.name = library_name;
1818
this.version = Utilities.normalize_version(library_version);
1919
this.test_status = parseInt(test_status);
20-
this.pass_percentage = pass_percentage;
20+
this._notes = notes;
21+
}
22+
23+
is_test_percentage() {
24+
return !!this._notes.match(/^\d+\.\d+$/)
25+
}
26+
27+
has_no_test_results() {
28+
return !!this._notes.match(/^0+\.0+$/)
29+
}
30+
31+
set notes(value) {
32+
this._notes = value;
33+
}
34+
35+
get notes() {
36+
let notes = this._notes;
37+
if (this.is_test_percentage()) {
38+
if (this.has_no_test_results()) {
39+
if (this.test_status < 2) {
40+
notes = "The package installs, but the test suite was not set up for GraalPy.";
41+
} else if (this.test_status == 2) {
42+
notes = "The package fails to build or install.";
43+
} else {
44+
notes = "The package is unsupported.";
45+
}
46+
} else {
47+
notes = notes + "% of the tests are passing on GraalPy.";
48+
}
49+
}
50+
if (!notes.endsWith(".")) {
51+
notes += ".";
52+
}
53+
return notes;
2154
}
2255
}
2356

@@ -29,14 +62,28 @@ class DB {
2962
const lines = db_contents.split('\n');
3063

3164
for (let l in lines) {
32-
const entry = new DBEntry(lines[l].split(','));
65+
if (!lines[l]) {
66+
continue;
67+
}
68+
let [name, version, test_status, ...notes] = lines[l].split(',');
69+
const entry = new DBEntry(name, version, test_status, notes.join(','));
3370

3471
if (!(entry.name in this.db)) {
3572
this.db[entry.name] = {};
3673
}
3774

38-
this.db[entry.name][entry.version] = entry;
39-
this.db[entry.name][Utilities.approximate_recommendation(entry.version)] = entry;
75+
for (const v of [entry.version, Utilities.approximate_recommendation(entry.version)]) {
76+
let previous_entry = this.db[entry.name][v];
77+
if (previous_entry && !previous_entry.notes.includes(entry.notes)) {
78+
if (previous_entry.is_test_percentage() && previous_entry.has_no_test_results()) {
79+
previous_entry.notes = entry.notes;
80+
} else {
81+
previous_entry.notes = entry.notes + " " + previous_entry.notes;
82+
}
83+
} else {
84+
this.db[entry.name][v] = entry;
85+
}
86+
}
4087
}
4188
}
4289

@@ -49,18 +96,18 @@ class DB {
4996
for (let version in versions) {
5097
if (!version.startsWith('~')) {
5198
const entry = versions[version];
52-
ret.push([entry.name, version, entry.test_status, entry.pass_percentage]);
99+
ret.push([entry.name, version, entry.test_status, entry.notes]);
53100
}
54101
}
55102
} else {
56103
if (requested_version in this.db[requested_name]) {
57104
const entry = this.db[requested_name][requested_version];
58-
ret.push([entry.name, entry.version, entry.test_status, entry.pass_percentage]);
105+
ret.push([entry.name, entry.version, entry.test_status, entry.notes]);
59106
} else {
60107
const semver_match = Utilities.approximate_recommendation(requested_version);
61108
if (semver_match in this.db[requested_name]) {
62109
const entry = this.db[requested_name][semver_match];
63-
ret.push([entry.name, entry.version, entry.test_status, entry.pass_percentage]);
110+
ret.push([entry.name, entry.version, entry.test_status, entry.notes]);
64111
} else {
65112
ret.push([requested_name, requested_version, 'unknown', undefined]);
66113
}
@@ -87,20 +134,11 @@ class DB {
87134
}
88135

89136
class Utilities {
90-
static pretty_name(language) {
91-
switch(language) {
92-
case 'js': return 'GraalVM JavaScript';
93-
case 'r': return 'GraalVM R';
94-
case 'ruby': return 'GraalVM Ruby';
95-
case 'python': return 'GraalVM Python';
96-
}
97-
}
98-
99137
static normalize_version(version) {
100138
if (version === undefined) {
101139
return undefined;
102140
} else {
103-
return version.replace(/^v(\d.*)/, '$1');
141+
return version.replace(/^v(\d.*)/, '$1').replace(/ and /g, ", ");
104142
}
105143
}
106144

0 commit comments

Comments
 (0)