Skip to content

Commit d0ab9e6

Browse files
hoodmanehoodmane
authored
Python: Add compat flag to allow using external Python sdk (#5281)
1 parent 915f56b commit d0ab9e6

File tree

9 files changed

+97
-45
lines changed

9 files changed

+97
-45
lines changed

‎build/python_metadata.bzl‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,11 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
134134
"abi": "3.12",
135135
"sha256": "5e6e21dbeda7c1eaadb99e6e52aa2ce45325b51e9a417198701e68e0cfd12a4c",
136136
},
137+
{
138+
"name": "python-workers-runtime-sdk",
139+
"abi": None,
140+
"sha256": "fc4fb50f73973c257277155b3cb113aa2cf68e9da8ef424ecb049b41bc463183",
141+
},
137142
],
138143
},
139144
{
@@ -165,6 +170,11 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
165170
"abi": "3.13",
166171
"sha256": "955091f1bd2eb33255ff2633df990bedc96e2f6294e78f2b416078777394f942",
167172
},
173+
{
174+
"name": "python-workers-runtime-sdk",
175+
"abi": None,
176+
"sha256": "fc4fb50f73973c257277155b3cb113aa2cf68e9da8ef424ecb049b41bc463183",
177+
},
168178
],
169179
},
170180
{

‎src/pyodide/internal/metadata.ts‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,15 @@ export type CompatibilityFlags = MetadataReader.CompatibilityFlags;
5050
export const COMPATIBILITY_FLAGS: MetadataReader.CompatibilityFlags =
5151
MetadataReader.getCompatibilityFlags();
5252
export const WORKFLOWS_ENABLED: boolean =
53-
COMPATIBILITY_FLAGS.python_workflows ?? false;
54-
export const LEGACY_GLOBAL_HANDLERS: boolean = !(
55-
COMPATIBILITY_FLAGS.python_no_global_handlers ?? false
56-
);
57-
export const LEGACY_VENDOR_PATH: boolean = !(
58-
COMPATIBILITY_FLAGS.python_workers_force_new_vendor_path ?? false
59-
);
53+
!!COMPATIBILITY_FLAGS.python_workflows;
54+
const NO_GLOBAL_HANDLERS: boolean =
55+
!!COMPATIBILITY_FLAGS.python_no_global_handlers;
56+
const FORCE_NEW_VENDOR_PATH: boolean =
57+
!!COMPATIBILITY_FLAGS.python_workers_force_new_vendor_path;
6058
export const IS_DEDICATED_SNAPSHOT_ENABLED: boolean =
61-
COMPATIBILITY_FLAGS.python_dedicated_snapshot ?? false;
59+
!!COMPATIBILITY_FLAGS.python_dedicated_snapshot;
60+
const EXTERNAL_SDK = !!COMPATIBILITY_FLAGS.enable_python_external_sdk;
61+
62+
export const LEGACY_GLOBAL_HANDLERS = !NO_GLOBAL_HANDLERS;
63+
export const LEGACY_VENDOR_PATH = !FORCE_NEW_VENDOR_PATH;
64+
export const LEGACY_INCLUDE_SDK = !EXTERNAL_SDK;

‎src/pyodide/python-entrypoint-helper.ts‎

Lines changed: 40 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
SHOULD_SNAPSHOT_TO_DISK,
1515
WORKFLOWS_ENABLED,
1616
LEGACY_GLOBAL_HANDLERS,
17+
LEGACY_INCLUDE_SDK,
1718
} from 'pyodide-internal:metadata';
1819
import { default as Limiter } from 'pyodide-internal:limiter';
1920
import {
@@ -162,55 +163,58 @@ async function applyPatch(pyodide: Pyodide, patchName: string): Promise<void> {
162163
pyodide.pyimport(patchName + '_patch');
163164
}
164165

166+
async function injectWorkersApi(pyodide: Pyodide): Promise<void> {
167+
const sitePackages = pyodide.FS.sitePackages;
168+
if (pyodide.version === '0.26.0a2') {
169+
// Inject at cloudflare.workers for backwards compatibility
170+
pyodide.FS.mkdirTree(`${sitePackages}/cloudflare/workers`);
171+
await injectSitePackagesModule(
172+
pyodide,
173+
'workers-api/src/workers/__init__',
174+
'cloudflare/workers/__init__'
175+
);
176+
await injectSitePackagesModule(
177+
pyodide,
178+
'workers-api/src/workers/_workers',
179+
'cloudflare/workers/_workers'
180+
);
181+
}
182+
// The SDK was moved from `cloudflare.workers` to just `workers`.
183+
// Create workers package structure with workflows submodule
184+
pyodide.FS.mkdir(`${sitePackages}/workers`);
185+
await injectSitePackagesModule(
186+
pyodide,
187+
'workers-api/src/workers/__init__',
188+
'workers/__init__'
189+
);
190+
await injectSitePackagesModule(
191+
pyodide,
192+
'workers-api/src/workers/_workers',
193+
'workers/_workers'
194+
);
195+
await injectSitePackagesModule(
196+
pyodide,
197+
'workers-api/src/workers/workflows',
198+
'workers/workflows'
199+
);
200+
await injectSitePackagesModule(pyodide, 'workers-api/src/asgi', 'asgi');
201+
}
202+
165203
async function setupPatches(pyodide: Pyodide): Promise<void> {
166204
await enterJaegerSpan('setup_patches', async () => {
167205
patchLoadPackage(pyodide);
168206

169207
// install any extra packages into the site-packages directory
170-
const sitePackages = pyodide.FS.sitePackages;
171208
// Expose the doAnImport function and global modules to Python globals
172209
pyodide.registerJsModule(
173210
'_pyodide_entrypoint_helper',
174211
get_pyodide_entrypoint_helper()
175212
);
176213

177214
// Inject modules that enable JS features to be used idiomatically from Python.
178-
//
179-
// NOTE: setupPatches is called after memorySnapshotDoImports, so any modules injected here
180-
// shouldn't be part of the snapshot and should filtered out in filterPythonScriptImports.
181-
if (pyodide.version === '0.26.0a2') {
182-
// Inject at cloudflare.workers for backwards compatibility
183-
pyodide.FS.mkdirTree(`${sitePackages}/cloudflare/workers`);
184-
await injectSitePackagesModule(
185-
pyodide,
186-
'workers-api/src/workers/__init__',
187-
'cloudflare/workers/__init__'
188-
);
189-
await injectSitePackagesModule(
190-
pyodide,
191-
'workers-api/src/workers/_workers',
192-
'cloudflare/workers/_workers'
193-
);
215+
if (LEGACY_INCLUDE_SDK) {
216+
await injectWorkersApi(pyodide);
194217
}
195-
// The SDK was moved from `cloudflare.workers` to just `workers`.
196-
// Create workers package structure with workflows submodule
197-
pyodide.FS.mkdir(`${sitePackages}/workers`);
198-
await injectSitePackagesModule(
199-
pyodide,
200-
'workers-api/src/workers/__init__',
201-
'workers/__init__'
202-
);
203-
await injectSitePackagesModule(
204-
pyodide,
205-
'workers-api/src/workers/_workers',
206-
'workers/_workers'
207-
);
208-
await injectSitePackagesModule(
209-
pyodide,
210-
'workers-api/src/workers/workflows',
211-
'workers/workflows'
212-
);
213-
await injectSitePackagesModule(pyodide, 'workers-api/src/asgi', 'asgi');
214218

215219
// Install patches as needed
216220
if (TRANSITIVE_REQUIREMENTS.has('aiohttp')) {

‎src/pyodide/types/runtime-generated/metadata.d.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ declare namespace MetadataReader {
44
python_no_global_handlers?: boolean;
55
python_workers_force_new_vendor_path?: boolean;
66
python_dedicated_snapshot?: boolean;
7+
enable_python_external_sdk?: boolean;
78
}
89

910
const isWorkerd: () => boolean;

‎src/workerd/io/compatibility-date.capnp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1187,4 +1187,9 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
11871187
enableCtxExports @139 :Bool
11881188
$compatEnableFlag("enable_ctx_exports");
11891189
# Enable the ctx.exports API.
1190+
pythonExternalSDK @140 :Bool
1191+
$compatEnableFlag("enable_python_external_sdk")
1192+
$compatDisableFlag("disable_python_external_sdk")
1193+
$experimental;
1194+
# Don't include the Python sdk from the runtime, use a vendored copy.
11901195
}

‎src/workerd/server/tests/python/vendor_pkg_tests/BUILD‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,7 @@ load(":vendor_test.bzl", "vendored_py_wd_test")
33
vendored_py_wd_test("fastapi")
44

55
vendored_py_wd_test("beautifulsoup4")
6+
7+
vendored_py_wd_test(
8+
"python-workers-runtime-sdk",
9+
)

‎src/workerd/server/tests/python/vendor_pkg_tests/python-workers-runtime-sdk.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# The python-workers-runtime-sdk vendored files include the sdk in tree from workerd
2+
# 9ca3b8e36e86e34f16ecd9680334ba0e0261549c plus one extra function. If we don't at least have the
3+
# things imported in introspection.py we'll crash on startup.
4+
from workers import extra_function
5+
6+
7+
async def test():
8+
assert extra_function() == 42
9+
print("workers-sdk imported successfully!")

‎src/workerd/server/tests/python/vendor_pkg_tests/python-workers-runtime-sdk_vendor.wd-test‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
using Workerd = import "/workerd/workerd.capnp";
2+
3+
const unitTests :Workerd.Config = (
4+
services = [
5+
( name = "workers-sdk-vendor-test",
6+
worker = (
7+
modules = [
8+
(name = "main.py", pythonModule = embed "python-workers-runtime-sdk.py"),
9+
%PYTHON_VENDORED_MODULES%
10+
],
11+
compatibilityDate = "2024-01-15",
12+
compatibilityFlags = [%PYTHON_FEATURE_FLAGS, "enable_python_external_sdk"],
13+
)
14+
),
15+
],
16+
);

‎src/workerd/server/tests/python/vendor_pkg_tests/vendor_test.bzl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,5 +85,5 @@ def vendored_py_wd_test(name, test_template = None, main_py_file = None, vendore
8585

8686
for info in BUNDLE_VERSION_INFO.values():
8787
if name not in info["vendored_packages_for_tests"]:
88-
continue
88+
fail("Not found", name, "in", info["vendored_packages_for_tests"])
8989
_vendored_py_wd_test(bzl_name, info["name"], test_template, main_py_file, vendored_srcs_target_prefix, **kwds)

0 commit comments

Comments
 (0)