Skip to content

Commit 3d2d361

Browse files
garrettgu10garrettgu10
committed
first commit
0 parents  commit 3d2d361

19 files changed

+773
-0
lines changed

‎.github/workflows/build-packages.yaml

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Build & Publish Pyodide Package Bundle
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
tag:
6+
description: Version Tag (Usually YYYYMMDD)
7+
required: true
8+
default: ""
9+
emsdk_version:
10+
description: Version of emsdk to use
11+
required: true
12+
default: "3.1.52"
13+
pyodide_version:
14+
description: Version of pyodide to use
15+
required: true
16+
default: "0.26.0a3"
17+
18+
jobs:
19+
pyodide-packages:
20+
name: Build & Publish Pyodide Package Bundle
21+
runs-on: ubuntu-latest
22+
environment: r2
23+
steps:
24+
- name: Checkout
25+
uses: actions/checkout@v4
26+
27+
- name: Set up Python
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: 3.12
31+
32+
- name: Install dependencies
33+
env:
34+
EMSDK_VERSION: ${{ inputs.emsdk_version }}
35+
PYODIDE_VERSION: ${{ inputs.pyodide_version }}
36+
run: |
37+
cd packages
38+
bash ./setup.sh
39+
40+
- name: Build and Upload Pyodide Package Bundle
41+
env:
42+
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
43+
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
44+
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
45+
run: |
46+
cd packages
47+
source .venv/bin/activate
48+
source emsdk/emsdk_env.sh
49+
python3 script.py ${{ inputs.tag }}
50+
51+
- name: Create GitHub Release for pyodide_bucket.bzl
52+
uses: ncipollo/release-action@v1
53+
with:
54+
artifacts: "packages/pyodide_bucket.bzl"
55+
token: ${{ secrets.MY_GITHUB_TOKEN }}
56+
tag: ${{ inputs.tag }}
57+
body: "Generated Bazel config file for release ${{ inputs.tag }}. Replace this file in workerd/build to update the Pyodide package bundle."
58+
draft: false
59+
prerelease: false

‎.github/workflows/link-pyodide.yaml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: Link & Release Pyodide for workerd (0.26.0a2) # To change the version, update pyodide/Makefile
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
tag:
6+
description: Version Tag
7+
required: true
8+
default: "0.26.a3"
9+
python_version:
10+
description: Python Version
11+
required: true
12+
default: "3.12.1"
13+
emscripten_version:
14+
description: Emscripten Version
15+
required: true
16+
default: "3.1.52"
17+
18+
jobs:
19+
pyodide-link:
20+
name: Link & Release Pyodide for workerd
21+
runs-on: ubuntu-latest
22+
environment: r2
23+
steps:
24+
- name: Checkout
25+
uses: actions/checkout@v4
26+
27+
- name: Link & Release Pyodide
28+
env:
29+
PYVERSION: ${{ inputs.python_version }}
30+
PYODIDE_EMSCRIPTEN_VERSION: ${{ inputs.emscripten_version }}
31+
run: |
32+
cd pyodide
33+
make -C emsdk
34+
source emsdk/emsdk/emsdk_env.sh
35+
make
36+
mv dist pyodide
37+
tar cjf pyodide-asm.tar.bz2 pyodide/
38+
39+
- name: Create GitHub Release for Linked Pyodide
40+
uses: ncipollo/release-action@v1
41+
with:
42+
artifacts: "pyodide/pyodide-asm.tar.bz2"
43+
token: ${{ secrets.MY_GITHUB_TOKEN }}
44+
name: Pyodide ${{ inputs.tag }}
45+
tag: ${{ inputs.tag }}
46+
body: "TODO write something here"
47+
draft: false
48+
prerelease: false

‎packages/required_packages.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
langchain_openai
2+
langchain
3+
fastapi
4+
aiohttp

‎packages/script.py

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
import os
2+
import json
3+
from pathlib import Path
4+
import tempfile
5+
import zipfile
6+
import tarfile
7+
import uuid
8+
import boto3
9+
import re
10+
import sys
11+
import hashlib
12+
from datetime import datetime
13+
14+
def normalize(name):
15+
return re.sub(r"[-_.]+", "-", name).lower()
16+
17+
# See setup.sh
18+
# prerequisite: emsdk, pyodide, packages -> pyodide/packages
19+
20+
def gen_bzl_config(tag, dist):
21+
bucket_url = "https://pub-45d734c4145d4285b343833ee450ef38.r2.dev/" + tag + "/"
22+
lock_bytes = (dist / "pyodide-lock.json").read_bytes()
23+
lock_hash = hashlib.sha256(lock_bytes).hexdigest()
24+
zip_bytes = (dist / "pyodide_packages.tar.zip").read_bytes()
25+
zip_hash = hashlib.sha256(zip_bytes).hexdigest()
26+
27+
with open("pyodide_bucket.bzl", "w") as f:
28+
f.write("# Do not edit this file by hand. See docs/pyodide.md for info on how to generate it.\n")
29+
f.write("# These variables are factored out here because they are being shared by the WORKSPACE files in\n")
30+
f.write("# both edgeworker and workerd, as well as src/pyodide/BUILD.bazel\n")
31+
f.write("PYODIDE_PACKAGE_BUCKET_URL = \"" + bucket_url + "\"\n")
32+
f.write("PYODIDE_LOCK_SHA256 = \"" + lock_hash + "\"\n")
33+
f.write("PYODIDE_PACKAGES_TAR_ZIP_SHA256 = \"" + zip_hash + "\"\n")
34+
35+
# creates a package bundle .tar.zip file to be bundled in with edgeworker
36+
# the resulting bundle is written to dist/pyodide_packages.tar.zip
37+
def make_bundle(tag, dist = Path("dist")):
38+
with open(dist / "pyodide-lock.json", "r") as file:
39+
lock = json.load(file)
40+
with tempfile.TemporaryDirectory(delete=False) as t:
41+
tempdir = Path(t)
42+
print("making bundle in " + str(tempdir))
43+
# copy pyodide-lock.json into tempdir
44+
with open(tempdir / "pyodide-lock.json", "w") as file:
45+
json.dump(lock, file)
46+
for package in lock["packages"].values():
47+
name = normalize(package["name"])
48+
if name.endswith("-tests") or name == "test":
49+
os.mkdir(tempdir / name)
50+
continue
51+
file = dist / package["file_name"]
52+
with zipfile.ZipFile(file, "r") as zip:
53+
zip.extractall(tempdir / name)
54+
# create temp tarfile from tempdir
55+
with tarfile.open(tempdir / "pyodide_packages.tar", "w") as tar:
56+
tar.add(tempdir, arcname="./")
57+
# create zip file in dist/ from tarfile
58+
with zipfile.ZipFile(dist / "pyodide_packages.tar.zip", "w", compression=zipfile.ZIP_DEFLATED) as zip:
59+
zip.write(tempdir / "pyodide_packages.tar", "pyodide_packages.tar")
60+
61+
gen_bzl_config(tag, dist)
62+
63+
# uploads everything in dist to python-package-bucket at tag/...
64+
def upload_to_r2(tag, dist = Path("dist")):
65+
# upload to r2
66+
s3 = boto3.client("s3",
67+
endpoint_url = "https://" + os.environ.get("R2_ACCOUNT_ID") + ".r2.cloudflarestorage.com",
68+
aws_access_key_id = os.environ.get("R2_ACCESS_KEY_ID"),
69+
aws_secret_access_key = os.environ.get("R2_SECRET_ACCESS_KEY"),
70+
region_name="auto")
71+
72+
# upload entire dist directory to r2
73+
for root, dirs, files in os.walk(dist):
74+
for file in files:
75+
path = Path(root) / file
76+
key = tag + "/" + str(path.relative_to(dist))
77+
print(f"uploading {path} to {key}")
78+
s3.upload_file(str(path), "python-package-bucket", key)
79+
80+
if __name__ == "__main__":
81+
with open("required_packages.txt", "r") as file:
82+
required_packages = file.read().split("\n")
83+
status = os.system(f"pyodide build-recipes --install {' '.join(required_packages)}")
84+
if status != 0:
85+
raise Exception("Failed to build recipes")
86+
87+
# get the current date in YYYYMMDD
88+
tag = datetime.today().strftime("%Y%m%d")
89+
if len(sys.argv) > 1:
90+
tag = sys.argv[1]
91+
92+
make_bundle(tag)
93+
upload_to_r2(tag)

‎packages/setup.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
git clone https://github.com/emscripten-core/emsdk
4+
cd emsdk
5+
./emsdk install $EMSDK_VERSION
6+
./emsdk activate $EMSDK_VERSION
7+
source ./emsdk_env.sh
8+
cd ..
9+
10+
git clone https://github.com/garrettgu10/pyodide.git
11+
(cd pyodide; git checkout langchain)
12+
ln -s pyodide/packages packages
13+
14+
# rust is required for building some wheels
15+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
16+
source "$HOME/.cargo/env"
17+
18+
sudo apt install -y pkg-config
19+
20+
sudo apt install gfortran sqlite3 f2c swig libreadline-dev
21+
22+
#todo: setup python3.12
23+
24+
python3.12 -m venv .venv
25+
source .venv/bin/activate
26+
pip install pyodide-build==$PYODIDE_VERSION
27+
28+
pip install boto3 cython

‎pyodide/.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
*.o
2+
dist
3+
download
4+
emsdk/emsdk
5+
emsdk/.complete

‎pyodide/Makefile

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
include Makefile.envs
2+
3+
PYTHON_INSTALL=download/static-libraries/python-$(PYVERSION)
4+
PYTHON_INCLUDE=$(PYTHON_INSTALL)/include/python$(PYMAJOR).$(PYMINOR)/
5+
LIB_DIR=$(PYTHON_INSTALL)/lib
6+
7+
OPTFLAGS=-O2
8+
CFLAGS=\
9+
$(OPTFLAGS) \
10+
$(DBGFLAGS) \
11+
-fPIC \
12+
-Wall \
13+
-Wno-warn-absolute-paths \
14+
-Werror=unused-variable \
15+
-Werror=sometimes-uninitialized \
16+
-Werror=int-conversion \
17+
-Werror=incompatible-pointer-types \
18+
-Werror=unused-result \
19+
-mreference-types \
20+
-I$(PYTHON_INCLUDE) \
21+
-I$(PYTHON_INCLUDE)/.. \
22+
-s EXCEPTION_CATCHING_ALLOWED=['we only want to allow exception handling in side modules'] \
23+
$(EXTRA_CFLAGS)
24+
25+
LDFLAGS_BASE=\
26+
$(OPTFLAGS) \
27+
$(DBGFLAGS) \
28+
-s WASM_BIGINT \
29+
$(EXTRA_LDFLAGS)
30+
31+
LDFLAGS_MODULE=\
32+
-s MAIN_MODULE=1 \
33+
-s USE_ES6_IMPORT_META \
34+
-s EXPORT_ES6 \
35+
-s EXPORT_ALL=1 \
36+
-s EXPORTED_RUNTIME_METHODS='wasmTable,ERRNO_CODES' \
37+
-s ENVIRONMENT=web \
38+
39+
40+
LDFLAGS_MEM_SIZE= \
41+
-s TOTAL_MEMORY=8mb \
42+
-s STACK_SIZE=3mb \
43+
-s ALLOW_MEMORY_GROWTH=1 \
44+
45+
LDFLAGS_EH= \
46+
-s EXPORT_EXCEPTION_HANDLING_HELPERS \
47+
-s EXCEPTION_CATCHING_ALLOWED=['we only want to allow exception handling in side modules'] \
48+
-s DEMANGLE_SUPPORT=1 \
49+
50+
LDFLAGS_EMSCRIPTEN_LIBS= \
51+
-s AUTO_JS_LIBRARIES=0 \
52+
-s AUTO_NATIVE_LIBRARIES=0 \
53+
-s LZ4=1 \
54+
-s USE_ZLIB \
55+
-s USE_BZIP2 \
56+
-s FORCE_FILESYSTEM=1 \
57+
58+
59+
LDFLAGS= $(LDFLAGS_BASE) \
60+
$(LDFLAGS_MODULE) \
61+
$(LDFLAGS_MEM_SIZE) \
62+
$(LDFLAGS_EH) \
63+
$(LDFLAGS_EMSCRIPTEN_LIBS) \
64+
--extern-pre-js prelude.js
65+
66+
67+
LIBS= \
68+
-L$(LIB_DIR) \
69+
-lpython$(PYMAJOR).$(PYMINOR)$(CPYTHON_ABI_FLAGS) \
70+
-lffi \
71+
-lhiwire \
72+
-lpyodide \
73+
74+
75+
76+
LIB_URL=https://github.com/pyodide/pyodide/releases/download/0.26.0a3/static-libraries-0.26.0a3.tar.bz2
77+
LIB_SHA256=b59e9b03410c2acf5b7335a74902991a0f0f51ada5fcd1372b50db4a7fb3c306
78+
LIB_TARBALL=download/static-libraries-0.26.0a3.tar.bz2
79+
LIB_INSTALL_MARK=$(PYTHON_INSTALL)/.installed-pyodide
80+
81+
PYODIDE_CORE_URL=https://github.com/pyodide/pyodide/releases/download/0.26.0a3/pyodide-core-0.26.0a3.tar.bz2
82+
PYODIDE_CORE_SHA256=f21e9922f070787b149f290e575c673cda3e963fef908a362542ad10826b279a
83+
PYODIDE_TARBALL=download/pyodide-core-0.26.0a3.tar.bz2
84+
PYODIDE_INSTALL=download/pyodide
85+
PYODIDE_INSTALL_MARK=$(PYODIDE_INSTALL)/.installed-pyodide
86+
87+
88+
all: dist/pyodide.asm.js dist/python_stdlib.zip
89+
90+
emsdk/emsdk/.complete:
91+
@date +"[%F %T] Building emsdk..."
92+
make -C emsdk
93+
@date +"[%F %T] done building emsdk."
94+
95+
$(LIB_TARBALL):
96+
mkdir -p download
97+
wget -O $@ $(LIB_URL)
98+
@GOT_SHASUM=`shasum --algorithm 256 $@ | cut -f1 -d' '` \
99+
&& (echo $$GOT_SHASUM | grep -q $(LIB_SHA256)) \
100+
|| (\
101+
rm $@ \
102+
&& echo "Got unexpected shasum $$GOT_SHASUM" \
103+
&& echo "If you are updating, set LIB_SHA256 to this." \
104+
&& exit 1 \
105+
)
106+
107+
$(LIB_INSTALL_MARK): $(LIB_TARBALL)
108+
tar -xf $(LIB_TARBALL) -C download
109+
touch $@
110+
111+
$(PYODIDE_TARBALL):
112+
mkdir -p download
113+
wget -O $@ $(PYODIDE_CORE_URL)
114+
@GOT_SHASUM=`shasum --algorithm 256 $@ | cut -f1 -d' '` \
115+
&& (echo $$GOT_SHASUM | grep -q $(PYODIDE_CORE_SHA256)) \
116+
|| (\
117+
rm $@ \
118+
&& echo "Got unexpected shasum $$GOT_SHASUM" \
119+
&& echo "If you are updating, set PYODIDE_CORE_SHA256 to this." \
120+
&& exit 1 \
121+
)
122+
123+
$(PYODIDE_INSTALL_MARK): $(PYODIDE_TARBALL)
124+
tar -xf $(PYODIDE_TARBALL) -C download
125+
touch $@
126+
127+
$(PYODIDE_INSTALL)/python_stdlib.zip: $(PYODIDE_INSTALL_MARK)
128+
129+
dist/python_stdlib.zip: $(PYODIDE_INSTALL)/python_stdlib.zip
130+
cp $< $@
131+
132+
src/main.o : src/main.c emsdk/emsdk/.complete $(LIB_INSTALL_MARK)
133+
emcc -c src/main.c -o src/main.o $(CFLAGS)
134+
135+
dist/pyodide.asm.js : src/main.o $(LIB_INSTALL_MARK)
136+
mkdir -p dist
137+
emcc src/main.o -o dist/pyodide.asm.js $(LDFLAGS) $(LIBS)
138+
sed -f sed.txt -i dist/pyodide.asm.js
139+
140+
clean:
141+
rm src/main.o
142+
rm -rf dist
143+
rm -rf download

0 commit comments

Comments
 (0)