Merge pull request #3082 from cloudflare/hoodmane/always-use-external-bundle

hoodmane · web-flow · commit d1badf9c48f3 · 2024-11-19T16:05:21.000+01:00
Remove code for Pyodide builtin bundle
diff --git a/samples/pyodide-fastapi/config.capnp b/samples/pyodide-fastapi/config.capnp
@@ -28,7 +28,7 @@ const mainWorker :Workerd.Worker = (
     ),
   ],
   compatibilityDate = "2023-12-18",
-  compatibilityFlags = ["python_workers", "python_external_bundle"],
+  compatibilityFlags = ["python_workers"],
   # Learn more about compatibility dates at:
   # https://developers.cloudflare.com/workers/platform/compatibility-dates/
 );
diff --git a/samples/pyodide-langchain/config.capnp b/samples/pyodide-langchain/config.capnp
@@ -22,7 +22,7 @@ const mainWorker :Workerd.Worker = (
     (name = "langchain_openai", pythonRequirement = ""),
   ],
   compatibilityDate = "2023-12-18",
-  compatibilityFlags = ["python_workers", "python_external_bundle"],
+  compatibilityFlags = ["python_workers"],
   # Learn more about compatibility dates at:
   # https://developers.cloudflare.com/workers/platform/compatibility-dates/
 );
diff --git a/samples/pyodide-secret/config.capnp b/samples/pyodide-secret/config.capnp
@@ -25,7 +25,7 @@ const mainWorker :Workerd.Worker = (
     (name = "worker.py", pythonModule = embed "./worker.py"),
   ],
   compatibilityDate = "2023-12-18",
-  compatibilityFlags = ["python_workers", "python_external_bundle"],
+  compatibilityFlags = ["python_workers"],
   bindings = [
     (
       name = "secret",
diff --git a/samples/pyodide/config.capnp b/samples/pyodide/config.capnp
@@ -20,7 +20,7 @@ const mainWorker :Workerd.Worker = (
     (name = "worker.py", pythonModule = embed "./worker.py"),
   ],
   compatibilityDate = "2023-12-18",
-  compatibilityFlags = ["python_workers", "python_external_bundle"],
+  compatibilityFlags = ["python_workers"],
   # Learn more about compatibility dates at:
   # https://developers.cloudflare.com/workers/platform/compatibility-dates/
 );
diff --git a/samples/repl-server-python/config.capnp b/samples/repl-server-python/config.capnp
@@ -20,7 +20,7 @@ const mainWorker :Workerd.Worker = (
     (name = "worker.py", pythonModule = embed "./worker.py"),
   ],
   compatibilityDate = "2023-12-18",
-  compatibilityFlags = ["python_workers", "python_external_bundle"],
+  compatibilityFlags = ["python_workers"],
   # Learn more about compatibility dates at:
   # https://developers.cloudflare.com/workers/platform/compatibility-dates/
 );
diff --git a/src/cloudflare/internal/test/ai/BUILD.bazel b/src/cloudflare/internal/test/ai/BUILD.bazel
@@ -1,12 +1,13 @@
 load("//:build/wd_test.bzl", "wd_test")
+load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test")
 
 wd_test(
     src = "ai-api-test.wd-test",
     args = ["--experimental"],
     data = glob(["*.js"]),
 )
 
-wd_test(
+py_wd_test(
     size = "large",
     src = "python-ai-api-test.wd-test",
     args = ["--experimental"],
diff --git a/src/cloudflare/internal/test/ai/python-ai-api-test.wd-test b/src/cloudflare/internal/test/ai/python-ai-api-test.wd-test
@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "ai-api-test.py")
         ],
         compatibilityDate = "2023-01-15",
-        compatibilityFlags = ["nodejs_compat", "python_workers"],
+        compatibilityFlags = ["nodejs_compat", "python_workers_development"],
         bindings = [
         (
           name = "ai",
diff --git a/src/cloudflare/internal/test/d1/BUILD.bazel b/src/cloudflare/internal/test/d1/BUILD.bazel
@@ -1,4 +1,5 @@
 load("//:build/wd_test.bzl", "wd_test")
+load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test")
 
 wd_test(
     src = "d1-api-test.wd-test",
@@ -15,7 +16,7 @@ wd_test(
     data = glob(["*.js"]),
 )
 
-wd_test(
+py_wd_test(
     size = "large",
     src = "python-d1-api-test.wd-test",
     args = ["--experimental"],
diff --git a/src/cloudflare/internal/test/d1/python-d1-api-test.wd-test b/src/cloudflare/internal/test/d1/python-d1-api-test.wd-test
@@ -9,7 +9,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "d1-api-test.py")
         ],
         compatibilityDate = "2023-01-15",
-        compatibilityFlags = ["nodejs_compat", "python_workers"],
+        compatibilityFlags = ["nodejs_compat", "python_workers_development"],
         bindings = [
         (
           name = "d1",
diff --git a/src/cloudflare/internal/test/vectorize/BUILD.bazel b/src/cloudflare/internal/test/vectorize/BUILD.bazel
@@ -1,11 +1,12 @@
 load("//:build/wd_test.bzl", "wd_test")
+load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test")
 
 wd_test(
     src = "vectorize-api-test.wd-test",
     data = glob(["*.js"]),
 )
 
-wd_test(
+py_wd_test(
     size = "large",
     src = "python-vectorize-api-test.wd-test",
     data = glob([
diff --git a/src/cloudflare/internal/test/vectorize/python-vectorize-api-test.wd-test b/src/cloudflare/internal/test/vectorize/python-vectorize-api-test.wd-test
@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           ( name = "worker.py", pythonModule = embed "vectorize-api-test.py" )
         ],
         compatibilityDate = "2023-11-21",
-        compatibilityFlags = ["nodejs_compat", "python_workers"],
+        compatibilityFlags = ["nodejs_compat", "python_workers_development"],
         bindings = [
           ( name = "vectorSearch",
             wrapped = (
diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel
@@ -134,7 +134,6 @@ wd_cc_library(
     implementation_deps = ["//src/workerd/util:string-buffer"],
     visibility = ["//visibility:public"],
     deps = [
-        "//src/pyodide",
         "//src/pyodide:pyodide_extra_capnp",
         "//src/workerd/io",
         "//src/workerd/jsg",
diff --git a/src/workerd/api/pyodide/pyodide.h b/src/workerd/api/pyodide/pyodide.h
@@ -13,7 +13,6 @@
 #include <workerd/util/autogate.h>
 
 #include <pyodide/generated/pyodide_extra.capnp.h>
-#include <pyodide/pyodide.capnp.h>
 
 #include <capnp/serialize.h>
 #include <kj/array.h>
@@ -425,31 +424,19 @@ bool hasPythonModules(capnp::List<server::config::Worker::Module>::Reader module
 template <class Registry>
 void registerPyodideModules(Registry& registry, auto featureFlags) {
   // We add `pyodide:` packages here including python-entrypoint-helper.js.
-  if (!featureFlags.getPythonExternalBundle() &&
-      !util::Autogate::isEnabled(util::AutogateKey::PYTHON_EXTERNAL_BUNDLE)) {
-    registry.addBuiltinBundle(PYODIDE_BUNDLE, kj::none);
-  }
   registry.template addBuiltinModule<PackagesTarReader>(
       "pyodide-internal:packages_tar_reader", workerd::jsg::ModuleRegistry::Type::INTERNAL);
 }
 
 kj::Own<jsg::modules::ModuleBundle> getInternalPyodideModuleBundle(auto featureFlags) {
   jsg::modules::ModuleBundle::BuiltinBuilder builder(
       jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY);
-  if (!featureFlags.getPythonExternalBundle() &&
-      !util::Autogate::isEnabled(util::AutogateKey::PYTHON_EXTERNAL_BUNDLE)) {
-    jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builder, PYODIDE_BUNDLE);
-  }
   return builder.finish();
 }
 
 kj::Own<jsg::modules::ModuleBundle> getExternalPyodideModuleBundle(auto featureFlags) {
   jsg::modules::ModuleBundle::BuiltinBuilder builder(
       jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN);
-  if (!featureFlags.getPythonExternalBundle() &&
-      !util::Autogate::isEnabled(util::AutogateKey::PYTHON_EXTERNAL_BUNDLE)) {
-    jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builder, PYODIDE_BUNDLE);
-  }
   return builder.finish();
 }
 
diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp
@@ -622,13 +622,8 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
   # compatibility flag we arrange to have such promise continuations scheduled to run
   # in the correct IoContext if it is still alive, or dropped on the floor with a warning
   # if the correct IoContext is not still alive.
-  pythonExternalBundle @63 :Bool
-      $compatEnableFlag("python_external_bundle")
+  obsolete63 @63 :Bool
       $experimental;
-  # Temporary flag to load Python from external capnproto bundle loaded at runtime.
-  # We plan to turn this on always quite soon. It would be an autogate but we need to test
-  # our logic both at upload time and at runtime, and this seemed like the easiest way to
-  # make sure we keep things in sync.
 
   setToStringTag @64 :Bool
       $compatEnableFlag("set_tostring_tag")
diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel
@@ -11,6 +11,7 @@ copy_file(
     name = "pyodide_dev.capnp.bin@rule",
     src = "//src/pyodide:pyodide.capnp.bin",
     out = "pyodide-bundle-cache/pyodide_dev.capnp.bin",
+    visibility = ["//visibility:public"],
 )
 
 py_wd_test(
diff --git a/src/workerd/server/tests/python/env-param/env.wd-test b/src/workerd/server/tests/python/env-param/env.wd-test
@@ -14,7 +14,7 @@ const unitTests :Workerd.Config = (
           ),
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers_development", "python_external_bundle"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],
diff --git a/src/workerd/server/tests/python/hello/hello.wd-test b/src/workerd/server/tests/python/hello/hello.wd-test
@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "worker.py")
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers_development", "python_external_bundle"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],
diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl
@@ -22,7 +22,7 @@ const unitTests :Workerd.Config = (
           (name = "{}", pythonRequirement = ""),
         ],
         compatibilityDate = "2024-05-02",
-        compatibilityFlags = ["python_workers_development", "python_external_bundle"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ]
diff --git a/src/workerd/server/tests/python/py_wd_test.bzl b/src/workerd/server/tests/python/py_wd_test.bzl
@@ -8,8 +8,8 @@ def py_wd_test(
         size = "enormous",
         tags = [],
         **kwargs):
-    data += ["pyodide_dev.capnp.bin@rule"]
-    args += ["--pyodide-bundle-disk-cache-dir", "$(location pyodide_dev.capnp.bin@rule)/.."]
+    data += ["//src/workerd/server/tests/python:pyodide_dev.capnp.bin@rule"]
+    args = args + ["--pyodide-bundle-disk-cache-dir", "$(location //src/workerd/server/tests/python:pyodide_dev.capnp.bin@rule)/..", "--experimental"]
 
     wd_test(
         src = src,
diff --git a/src/workerd/server/tests/python/random/random.wd-test b/src/workerd/server/tests/python/random/random.wd-test
@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (
           (name = "worker.py", pythonModule = embed "worker.py")
         ],
         compatibilityDate = "2024-01-15",
-        compatibilityFlags = ["python_workers_development", "python_external_bundle"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],
diff --git a/src/workerd/server/tests/python/sdk/sdk.wd-test b/src/workerd/server/tests/python/sdk/sdk.wd-test
@@ -8,15 +8,15 @@ const python :Workerd.Worker = (
     ( name = "SELF", service = "python-sdk" ),
   ],
   compatibilityDate = "2024-10-01",
-  compatibilityFlags = ["python_workers_development", "python_external_bundle"],
+  compatibilityFlags = ["python_workers_development"],
 );
 
 const server :Workerd.Worker = (
   modules = [
     (name = "server.py", pythonModule = embed "server.py")
   ],
   compatibilityDate = "2024-10-01",
-  compatibilityFlags = ["python_workers_development", "python_external_bundle"],
+  compatibilityFlags = ["python_workers_development"],
 );
 
 const unitTests :Workerd.Config = (
diff --git a/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test b/src/workerd/server/tests/python/subdirectory/subdirectory.wd-test
@@ -10,7 +10,7 @@ const unitTests :Workerd.Config = (
           (name = "subdir/a.py", pythonModule = embed "./subdir/a.py"),
         ],
         compatibilityDate = "2023-12-18",
-        compatibilityFlags = ["python_workers_development", "python_external_bundle"],
+        compatibilityFlags = ["python_workers_development"],
       )
     ),
   ],
diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++
@@ -537,14 +537,11 @@ void WorkerdApi::compileModules(jsg::Lock& lockParam,
       KJ_REQUIRE(featureFlags.getPythonWorkers(),
           "The python_workers compatibility flag is required to use Python.");
       // Inject Pyodide bundle
-      if (featureFlags.getPythonExternalBundle() ||
-          util::Autogate::isEnabled(util::AutogateKey::PYTHON_EXTERNAL_BUNDLE)) {
-        auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags));
-        auto version = getPythonBundleName(pythonRelease);
-        auto bundle = KJ_ASSERT_NONNULL(
-            fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle");
-        modules->addBuiltinBundle(bundle, kj::none);
-      }
+      auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags));
+      auto version = getPythonBundleName(pythonRelease);
+      auto bundle = KJ_ASSERT_NONNULL(
+          fetchPyodideBundle(impl->pythonConfig, version), "Failed to get Pyodide bundle");
+      modules->addBuiltinBundle(bundle, kj::none);
       // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?)
       {
         auto mainModule = confModules.begin();
diff --git a/src/workerd/util/autogate.c++ b/src/workerd/util/autogate.c++
@@ -17,8 +17,6 @@ kj::StringPtr KJ_STRINGIFY(AutogateKey key) {
   switch (key) {
     case AutogateKey::TEST_WORKERD:
       return "test-workerd"_kj;
-    case AutogateKey::PYTHON_EXTERNAL_BUNDLE:
-      return "python-external-bundle"_kj;
     case AutogateKey::COMPILE_CACHE_FOR_BUILTINS:
       return "compile-cache-for-builtins"_kj;
     case AutogateKey::NumOfKeys:
diff --git a/src/workerd/util/autogate.h b/src/workerd/util/autogate.h
@@ -14,7 +14,6 @@ namespace workerd::util {
 // Workerd-specific list of autogate keys (can also be used in internal repo).
 enum class AutogateKey {
   TEST_WORKERD,
-  PYTHON_EXTERNAL_BUNDLE,
   COMPILE_CACHE_FOR_BUILTINS,
   NumOfKeys  // Reserved for iteration.
 };

Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,7 @@ const mainWorker :Workerd.Worker = (`
`25`	`25`	`(name = "worker.py", pythonModule = embed "./worker.py"),`
`26`	`26`	`],`
`27`	`27`	`compatibilityDate = "2023-12-18",`
`28`		`- compatibilityFlags = ["python_workers", "python_external_bundle"],`
	`28`	`+ compatibilityFlags = ["python_workers"],`
`29`	`29`	`bindings = [`
`30`	`30`	`(`
`31`	`31`	`name = "secret",`
Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@ const unitTests :Workerd.Config = (`
`8`	`8`	`(name = "worker.py", pythonModule = embed "ai-api-test.py")`
`9`	`9`	`],`
`10`	`10`	`compatibilityDate = "2023-01-15",`
`11`		`- compatibilityFlags = ["nodejs_compat", "python_workers"],`
	`11`	`+ compatibilityFlags = ["nodejs_compat", "python_workers_development"],`
`12`	`12`	`bindings = [`
`13`	`13`	`(`
`14`	`14`	`name = "ai",`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ const unitTests :Workerd.Config = (`
`9`	`9`	`(name = "worker.py", pythonModule = embed "d1-api-test.py")`
`10`	`10`	`],`
`11`	`11`	`compatibilityDate = "2023-01-15",`
`12`		`- compatibilityFlags = ["nodejs_compat", "python_workers"],`
	`12`	`+ compatibilityFlags = ["nodejs_compat", "python_workers_development"],`
`13`	`13`	`bindings = [`
`14`	`14`	`(`
`15`	`15`	`name = "d1",`