Skip to content

Commit 7eac6a4

Browse files
teonbrooksteonbrooks
committed
wip
1 parent 1ac09da commit 7eac6a4

File tree

8 files changed

+334
-197
lines changed

8 files changed

+334
-197
lines changed

‎internals/scripts/InstallMNE.mjs‎

Lines changed: 173 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -1,103 +1,197 @@
11
#!/usr/bin/env node
22
/**
3-
* Downloads MNE-Python and its pure-Python dependencies from PyPI as wheel
4-
* files for offline use with Pyodide. Binary dependencies (numpy, scipy,
5-
* matplotlib, pandas) are already included via the `pyodide` npm package and
6-
* do NOT need to be downloaded here.
3+
* Downloads everything MNE-Python needs to run offline inside Pyodide:
74
*
8-
* Downloaded wheels are saved to:
9-
* src/renderer/utils/pyodide/src/packages/
5+
* Part 1 — Pyodide binary packages (from the Pyodide CDN)
6+
* Reads pyodide-lock.json that was extracted by InstallPyodide.mjs,
7+
* recursively resolves all dependencies of numpy / scipy / matplotlib /
8+
* pandas, and downloads each .whl (or .zip) into the same /pyodide/
9+
* directory as the runtime. loadPackage() will find them locally and
10+
* will not need to reach the CDN at runtime.
1011
*
11-
* A manifest.json is written there so the web worker knows which filenames
12-
* to pass to micropip.install() at startup.
12+
* Part 2 — Pure-Python packages (from PyPI)
13+
* MNE itself and its pure-Python dependencies (pooch, tqdm, platformdirs)
14+
* are not bundled with Pyodide. These are downloaded as py3-none-any
15+
* wheels into src/renderer/utils/pyodide/src/packages/ and installed via
16+
* micropip at worker startup. A manifest.json is written there so the
17+
* worker knows the exact filenames.
1318
*
1419
* Usage: node internals/scripts/InstallMNE.mjs
20+
* Runs automatically via the postinstall npm hook.
1521
*/
1622

1723
import fs from 'fs';
1824
import https from 'https';
1925
import path from 'path';
2026
import chalk from 'chalk';
2127

22-
const PACKAGES_DIR = path.resolve(
23-
'src/renderer/utils/pyodide/src/packages'
24-
);
28+
// ---------------------------------------------------------------------------
29+
// Paths
30+
// ---------------------------------------------------------------------------
31+
32+
const PYODIDE_DIR = path.resolve('src/renderer/utils/pyodide/src/pyodide');
33+
const LOCK_FILE = path.join(PYODIDE_DIR, 'pyodide-lock.json');
34+
35+
const PACKAGES_DIR = path.resolve('src/renderer/utils/pyodide/src/packages');
2536
const MANIFEST_FILE = path.join(PACKAGES_DIR, 'manifest.json');
2637

27-
/**
28-
* Pure-Python packages required by MNE that are not bundled with Pyodide.
29-
* Each entry is resolved against the PyPI JSON API to find the latest
30-
* pure-Python wheel (py3-none-any or py2.py3-none-any).
31-
*/
32-
const PACKAGES_TO_DOWNLOAD = [
33-
'mne',
34-
'pooch',
35-
'tqdm',
36-
'platformdirs',
37-
];
38+
// ---------------------------------------------------------------------------
39+
// Root packages whose full transitive dependency tree we need from Pyodide CDN
40+
// ---------------------------------------------------------------------------
41+
42+
const PYODIDE_ROOT_PACKAGES = ['numpy', 'scipy', 'matplotlib', 'pandas'];
43+
44+
// ---------------------------------------------------------------------------
45+
// Pure-Python packages to download from PyPI (not bundled with Pyodide)
46+
// ---------------------------------------------------------------------------
47+
48+
const PYPI_PACKAGES = ['mne', 'pooch', 'tqdm', 'platformdirs'];
3849

3950
// ---------------------------------------------------------------------------
40-
// Network helpers
51+
// Shared network helpers
4152
// ---------------------------------------------------------------------------
4253

43-
function httpsGet(url) {
54+
function downloadBinary(url, dest) {
4455
return new Promise((resolve, reject) => {
45-
const req = https.get(url, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => {
46-
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
47-
resolve(httpsGet(res.headers.location));
48-
return;
49-
}
50-
if (res.statusCode !== 200) {
51-
reject(new Error(`HTTP ${res.statusCode} for ${url}`));
52-
return;
53-
}
54-
let body = '';
55-
res.setEncoding('utf8');
56-
res.on('data', (chunk) => { body += chunk; });
57-
res.on('end', () => resolve(body));
58-
res.on('error', reject);
59-
});
60-
req.on('error', reject);
56+
const doGet = (reqUrl) => {
57+
https
58+
.get(reqUrl, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => {
59+
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
60+
doGet(res.headers.location);
61+
return;
62+
}
63+
if (res.statusCode !== 200) {
64+
reject(new Error(`HTTP ${res.statusCode} for ${reqUrl}`));
65+
return;
66+
}
67+
const file = fs.createWriteStream(dest);
68+
res.pipe(file);
69+
file.on('finish', () => file.close(resolve));
70+
file.on('error', (err) => { fs.unlink(dest, () => {}); reject(err); });
71+
})
72+
.on('error', (err) => { fs.unlink(dest, () => {}); reject(err); });
73+
};
74+
doGet(url);
6175
});
6276
}
6377

64-
function downloadBinary(url, dest) {
78+
function httpsGetText(url) {
6579
return new Promise((resolve, reject) => {
66-
const doGet = (reqUrl) => {
67-
https.get(reqUrl, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => {
80+
https
81+
.get(url, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => {
6882
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
69-
doGet(res.headers.location);
83+
resolve(httpsGetText(res.headers.location));
7084
return;
7185
}
7286
if (res.statusCode !== 200) {
73-
reject(new Error(`HTTP ${res.statusCode} for ${reqUrl}`));
87+
reject(new Error(`HTTP ${res.statusCode} for ${url}`));
7488
return;
7589
}
76-
const file = fs.createWriteStream(dest);
77-
res.pipe(file);
78-
file.on('finish', () => file.close(resolve));
79-
file.on('error', (err) => { fs.unlink(dest, () => {}); reject(err); });
80-
}).on('error', (err) => { fs.unlink(dest, () => {}); reject(err); });
81-
};
82-
doGet(url);
90+
let body = '';
91+
res.setEncoding('utf8');
92+
res.on('data', (c) => { body += c; });
93+
res.on('end', () => resolve(body));
94+
res.on('error', reject);
95+
})
96+
.on('error', reject);
8397
});
8498
}
8599

86100
// ---------------------------------------------------------------------------
87-
// PyPI helpers
101+
// Part 1 — Pyodide binary packages
102+
// ---------------------------------------------------------------------------
103+
104+
/**
105+
* Recursively walks the `depends` graph in the lock file and returns every
106+
* package entry (including root packages) needed to satisfy the given roots.
107+
* Package name matching is case-insensitive.
108+
*/
109+
function resolveAllDeps(lockPackages, rootNames) {
110+
// Build a lowercase → original-key index for case-insensitive lookup.
111+
const index = {};
112+
for (const key of Object.keys(lockPackages)) {
113+
index[key.toLowerCase()] = key;
114+
}
115+
116+
const resolved = new Set();
117+
const queue = rootNames.map((n) => n.toLowerCase());
118+
119+
while (queue.length) {
120+
const lower = queue.shift();
121+
const key = index[lower];
122+
if (!key || resolved.has(key)) continue;
123+
resolved.add(key);
124+
for (const dep of lockPackages[key].depends ?? []) {
125+
queue.push(dep.toLowerCase());
126+
}
127+
}
128+
129+
return [...resolved].map((key) => lockPackages[key]);
130+
}
131+
132+
async function downloadPyodidePackages() {
133+
if (!fs.existsSync(LOCK_FILE)) {
134+
console.warn(
135+
chalk.yellow(
136+
' ⚠ pyodide-lock.json not found — run `npm install` first to ' +
137+
'extract the Pyodide runtime, then re-run this script.'
138+
)
139+
);
140+
return;
141+
}
142+
143+
const lockData = JSON.parse(fs.readFileSync(LOCK_FILE, 'utf8'));
144+
145+
// The lock file's info.version may be an internal dev label (e.g. "0.28.0.dev0").
146+
// Always derive the CDN URL from the installed npm package version instead.
147+
const npmPkgPath = path.resolve('node_modules/pyodide/package.json');
148+
const cdnVersion = fs.existsSync(npmPkgPath)
149+
? JSON.parse(fs.readFileSync(npmPkgPath, 'utf8')).version
150+
: lockData.info.version;
151+
const cdnBase = `https://cdn.jsdelivr.net/pyodide/v${cdnVersion}/full/`;
152+
153+
const allPkgs = resolveAllDeps(lockData.packages, PYODIDE_ROOT_PACKAGES);
154+
155+
console.log(
156+
chalk.blue.bold(
157+
`Downloading ${allPkgs.length} Pyodide packages from CDN (v${cdnVersion})…`
158+
)
159+
);
160+
161+
for (const pkg of allPkgs) {
162+
process.stdout.write(chalk.blue(` ${pkg.name ?? pkg.file_name}: `));
163+
164+
const dest = path.join(PYODIDE_DIR, pkg.file_name);
165+
if (fs.existsSync(dest)) {
166+
console.log(chalk.gray('already present, skipping'));
167+
continue;
168+
}
169+
170+
const url = cdnBase + pkg.file_name;
171+
try {
172+
await downloadBinary(url, dest);
173+
console.log(chalk.green('downloaded'));
174+
} catch (err) {
175+
console.log(chalk.red(`FAILED — ${err.message}`));
176+
if (fs.existsSync(dest)) fs.unlinkSync(dest);
177+
}
178+
}
179+
}
180+
181+
// ---------------------------------------------------------------------------
182+
// Part 2 — Pure-Python packages from PyPI
88183
// ---------------------------------------------------------------------------
89184

90185
/**
91-
* Returns the best pure-Python wheel for the latest release of `packageName`.
186+
* Queries the PyPI JSON API for `packageName` and returns the best
187+
* pure-Python wheel for the latest release.
92188
* Preference: py3-none-any > py2.py3-none-any > *-none-any
93189
*/
94190
async function resolvePureWheel(packageName) {
95-
const raw = await httpsGet(`https://pypi.org/pypi/${packageName}/json`);
191+
const raw = await httpsGetText(`https://pypi.org/pypi/${packageName}/json`);
96192
const data = JSON.parse(raw);
97193
const version = data.info.version;
98-
const urls = data.urls; // files for the latest release
99-
100-
const wheels = urls.filter((f) => f.filename.endsWith('.whl'));
194+
const wheels = data.urls.filter((f) => f.filename.endsWith('.whl'));
101195

102196
const ranked = [
103197
wheels.find((f) => f.filename.endsWith('-py3-none-any.whl')),
@@ -107,19 +201,15 @@ async function resolvePureWheel(packageName) {
107201

108202
if (ranked.length === 0) {
109203
throw new Error(
110-
`No pure-Python wheel found for ${packageName} ${version}. ` +
111-
`Binary packages must come from the Pyodide npm bundle.`
204+
`No pure-Python wheel found for ${packageName} ${version}. ` +
205+
`Binary packages must come from the Pyodide CDN.`
112206
);
113207
}
114208

115209
return { version, wheel: ranked[0] };
116210
}
117211

118-
// ---------------------------------------------------------------------------
119-
// Main
120-
// ---------------------------------------------------------------------------
121-
122-
async function installPackage(packageName, manifest) {
212+
async function installPyPIPackage(packageName, manifest) {
123213
process.stdout.write(chalk.blue(` ${packageName}: `));
124214

125215
let version, wheel;
@@ -131,7 +221,6 @@ async function installPackage(packageName, manifest) {
131221
}
132222

133223
const dest = path.join(PACKAGES_DIR, wheel.filename);
134-
135224
if (fs.existsSync(dest)) {
136225
console.log(chalk.gray(`${version} already present, skipping`));
137226
manifest[packageName] = { version, filename: wheel.filename };
@@ -148,27 +237,33 @@ async function installPackage(packageName, manifest) {
148237
}
149238
}
150239

151-
async function main() {
240+
async function downloadPyPIPackages() {
152241
fs.mkdirSync(PACKAGES_DIR, { recursive: true });
153242

154-
// Preserve any previously downloaded packages in the manifest.
243+
// Preserve previously downloaded packages already recorded in the manifest.
155244
let manifest = {};
156245
if (fs.existsSync(MANIFEST_FILE)) {
157-
try {
158-
manifest = JSON.parse(fs.readFileSync(MANIFEST_FILE, 'utf8'));
159-
} catch {
160-
manifest = {};
161-
}
246+
try { manifest = JSON.parse(fs.readFileSync(MANIFEST_FILE, 'utf8')); }
247+
catch { manifest = {}; }
162248
}
163249

164-
console.log(chalk.blue.bold('Downloading MNE-Python wheels from PyPI…'));
165-
for (const pkg of PACKAGES_TO_DOWNLOAD) {
166-
await installPackage(pkg, manifest);
250+
console.log(chalk.blue.bold('\nDownloading MNE-Python wheels from PyPI…'));
251+
for (const pkg of PYPI_PACKAGES) {
252+
await installPyPIPackage(pkg, manifest);
167253
}
168254

169255
fs.writeFileSync(MANIFEST_FILE, JSON.stringify(manifest, null, 2));
170-
console.log(chalk.green.bold('\nAll MNE wheels ready.'));
171-
console.log(chalk.gray(`Manifest → ${MANIFEST_FILE}`));
256+
console.log(chalk.gray(` Manifest → ${MANIFEST_FILE}`));
257+
}
258+
259+
// ---------------------------------------------------------------------------
260+
// Entry point
261+
// ---------------------------------------------------------------------------
262+
263+
async function main() {
264+
await downloadPyodidePackages();
265+
await downloadPyPIPackages();
266+
console.log(chalk.green.bold('\nAll packages ready.'));
172267
}
173268

174269
main().catch((err) => {

0 commit comments

Comments
 (0)