diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5fcb80a..4bba881 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -142,6 +142,7 @@ jobs: git config --global url."ssh://git@github.com-http-handler/platformatic/http-handler.git".insteadOf "ssh://git@github.com/platformatic/http-handler.git" git config --global url."ssh://git@github.com-http-rewriter/platformatic/http-rewriter.git".insteadOf "ssh://git@github.com/platformatic/http-rewriter.git" + npm run build:wasm ${{ matrix.settings.build }} - name: Build run: ${{ matrix.settings.build }} @@ -155,6 +156,8 @@ jobs: ${{ env.APP_NAME }}.*.node index.d.ts index.js + fix-python-soname.wasm + fix-python-soname.js if-no-files-found: error test-macOS-windows-binding: @@ -239,6 +242,9 @@ jobs: cache: pnpm - name: Install dependencies run: pnpm install + - name: Fix soname + run: pnpm run build:fix + shell: bash - uses: actions/download-artifact@v4 with: name: bindings-${{ matrix.settings.target }} @@ -332,79 +338,9 @@ jobs: apt-get update -y apt-get install -y python3 python3-dev patchelf - echo "=== Starting test setup ===" - echo "Current directory: $(pwd)" - echo "Python version: $(python3 --version)" - echo "Patchelf version: $(patchelf --version)" - echo "Using combined approach: SONAME patching + programmatic RTLD_GLOBAL" - echo "CI environment: CI=$CI, GITHUB_ACTIONS=$GITHUB_ACTIONS" - - # Check what .node files exist - echo "=== Available .node files ===" - ls -la *.node || echo "No .node files found" - - # Check what .node files exist and patch Python dependencies - echo "=== Checking .node file Python dependencies ===" - for file in *.node; do - if [ -f "$file" ]; then - case "$file" in - *linux*) - echo "Checking $file..." - echo "Python dependencies before patching:" - ldd "$file" 2>/dev/null | grep python || echo "No Python dependencies found" - - # Check if we need to patch SONAME - current_python_lib=$(ldd "$file" 2>/dev/null | grep "libpython" | head -1 | awk '{print $1}') - if [ -n "$current_python_lib" ]; then - echo "Current Python library: $current_python_lib" - - # Find the actual Python library on the system - system_python_lib=$(find /usr/lib* -name "libpython3*.so.*" -type f 2>/dev/null | head -1) - if [ -n "$system_python_lib" ]; then - system_python_soname=$(basename "$system_python_lib") - echo "System Python library: $system_python_soname" - - # Only patch if they're different - if [ "$current_python_lib" != "$system_python_soname" ]; then - echo "Patching SONAME from $current_python_lib to $system_python_soname" - patchelf --replace-needed "$current_python_lib" "$system_python_soname" "$file" - echo "SONAME patching completed" - else - echo "SONAME already matches system Python" - fi - else - echo "Warning: Could not find system Python library" - fi - else - echo "No Python library dependency found" - fi - - echo "Python dependencies after patching:" - ldd "$file" 2>/dev/null | grep python || echo "No Python dependencies found" - echo "---" - ;; - *) - echo "Skipping non-Linux file: $file" - ;; - esac - fi - done - - # Install pnpm and run tests - echo "=== Installing pnpm ===" corepack disable npm i -gf pnpm - - echo "=== Running pnpm install ===" - # Should be non-interactive in CI environment pnpm install --prefer-offline - - echo "=== Setting up Python library path ===" - export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH - echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" - - - echo "=== Running tests ===" pnpm test publish: @@ -430,6 +366,27 @@ jobs: path: artifacts - name: Move artifacts run: pnpm artifacts + - name: Copy fix-python-soname files to Linux packages + run: | + # Find the WASM and JS files from Linux artifacts + WASM_FILE=$(find artifacts -name "fix-python-soname.wasm" | head -n 1) + JS_FILE=$(find artifacts -name "fix-python-soname.js" | head -n 1) + + if [ -n "$WASM_FILE" ] && [ -n "$JS_FILE" ]; then + echo "Found WASM file: $WASM_FILE" + echo "Found JS file: $JS_FILE" + + # Copy to all Linux npm directories + for dir in npm/*/; do + if [[ "$dir" == *"linux"* ]]; then + echo "Copying files to $dir" + cp "$WASM_FILE" "$dir" + cp "$JS_FILE" "$dir" + fi + done + else + echo "Warning: fix-python-soname files not found in artifacts" + fi - name: List packages run: ls -R ./npm shell: bash diff --git a/.gitignore b/.gitignore index 49ba801..c055161 100644 --- a/.gitignore +++ b/.gitignore @@ -199,5 +199,6 @@ $RECYCLE.BIN/ # Build/Install files /target /*.node +/*.wasm /index.d.ts /index.js diff --git a/Cargo.lock b/Cargo.lock index abfb5fe..c8c82a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,69 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.6.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.59.0", +] + +[[package]] +name = "arwen" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f24bcf6fb87a8db3f69b7050bd1ca7a55d91bf2f07caf7c54f27c97af87fb67" +dependencies = [ + "clap", + "goblin", + "object", + "scroll", + "thiserror", +] + [[package]] name = "async-trait" version = "0.1.88" @@ -76,6 +139,52 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +[[package]] +name = "clap" +version = "4.5.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "convert_case" version = "0.8.0" @@ -85,6 +194,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "ctor" version = "0.4.2" @@ -116,12 +234,41 @@ version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7454e41ff9012c00d53cf7f475c5e3afa3b91b7c90568495495e8d9bf47a1055" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fix-python-soname" +version = "0.1.0" +dependencies = [ + "arwen", +] + +[[package]] +name = "flate2" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "futures" version = "0.3.31" @@ -217,6 +364,26 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "goblin" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daa0a64d21a7eb230583b4c5f4e23b7e4e57974f96620f42a7e75e08ae66d745" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +dependencies = [ + "foldhash", +] + [[package]] name = "heck" version = "0.5.0" @@ -261,6 +428,16 @@ dependencies = [ "regex", ] +[[package]] +name = "indexmap" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indoc" version = "2.0.6" @@ -278,6 +455,12 @@ dependencies = [ "libc", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itoa" version = "1.0.15" @@ -310,6 +493,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + [[package]] name = "memchr" version = "2.7.5" @@ -414,7 +603,12 @@ version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ + "crc32fast", + "flate2", + "hashbrown", + "indexmap", "memchr", + "ruzstd", ] [[package]] @@ -423,6 +617,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "parking_lot" version = "0.12.4" @@ -458,6 +658,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "portable-atomic" version = "1.11.1" @@ -624,12 +830,41 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "ruzstd" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad02996bfc73da3e301efe90b1837be9ed8f4a462b6ed410aa35d00381de89f" +dependencies = [ + "twox-hash", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scroll" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1783eabc414609e28a5ba76aee5ddd52199f7107a0b24c2e9746a1ecc34a683d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "semver" version = "1.0.26" @@ -667,6 +902,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "2.0.104" @@ -735,6 +982,16 @@ dependencies = [ "syn", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + [[package]] name = "unicode-ident" version = "1.0.18" @@ -753,6 +1010,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index fefc2f0..66910a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,7 @@ +[workspace] +members = [".", "fix-python-soname"] +resolver = "2" + [package] name = "python-node" version = "1.0.0" diff --git a/fix-python-soname.js b/fix-python-soname.js new file mode 100755 index 0000000..55ecdb0 --- /dev/null +++ b/fix-python-soname.js @@ -0,0 +1,139 @@ +#!/usr/bin/env node + +/** + * Post-install script to fix Python library soname on Linux + * + * This script automatically detects the system-installed Python version + * and updates the .node binary to link against the correct libpython.so. + * + * Only runs on Linux - other platforms don't need soname fixing. + * Uses a WASM version of the arwen ELF patcher for cross-platform compatibility. + */ + +const { WASI } = require('wasi') +const fs = require('fs') +const path = require('path') +const os = require('os') + +const platform = os.platform() +const arch = os.arch() + +// Function to detect if this is a development install vs dependency install +function isDevInstall() { + const env = process.env + + // Method 1: Check if INIT_CWD and PWD are the same (local dev install) + if (env.INIT_CWD && env.PWD) { + if (env.INIT_CWD === env.PWD || env.INIT_CWD.indexOf(env.PWD) === 0) { + return true + } + } + + // Method 2: Check for .git folder existence (dev environment) + if (fs.existsSync(path.join(__dirname, '.git'))) { + return true + } + + // Method 3: Check if we're in production mode + if (env.NODE_ENV === 'production' || env.npm_config_production) { + return false + } + + return false +} + +// Function to find the correct .node file for this platform +function findNodeFile() { + // Only run on Linux - other platforms don't need soname fixing + if (platform !== 'linux') return + + // Map Node.js arch to napi-rs target + const archMap = { + 'x64': 'x86_64-unknown-linux-gnu', + 'arm64': 'aarch64-unknown-linux-gnu', + } + + const target = archMap[arch] + if (!target) return + + // Try to find the .node file with various naming patterns + const possiblePaths = [ + // Specific platform builds + path.join(__dirname, `python-node.${target}.node`), + path.join(__dirname, `index.${target}.node`), + path.join(__dirname, `npm/${target}/python-node.${target}.node`), + // Generic .node files (common during testing) + path.join(__dirname, 'python-node.node'), + path.join(__dirname, 'index.node'), + // Look for any .node file in current directory + ...fs.readdirSync(__dirname) + .filter(f => f.endsWith('.node') && !f.includes('node_modules')) + .map(f => path.join(__dirname, f)) + ] + + for (const nodePath of possiblePaths) { + if (fs.existsSync(nodePath)) { + return nodePath + } + } + + // Return undefined if no .node file found - this is expected during development + return undefined +} + +// Get the node file path +const nodeFilePath = findNodeFile() +if (!nodeFilePath) { + if (isDevInstall()) { + // No .node file found during dev install - this is expected, skip silently + console.log('No .node file found during development install, skipping soname fix') + process.exit(0) + } else { + // No .node file found when installed as dependency - this is an error + console.error('Error: Could not find *.node file to fix soname') + process.exit(1) + } +} + +// Check if WASM file exists +const wasmPath = path.join(__dirname, 'fix-python-soname.wasm') +if (!fs.existsSync(wasmPath)) { + if (isDevInstall()) { + // WASM file not found during dev install - this is expected, skip with warning + console.log('WASM file not found during development install, skipping soname fix') + process.exit(0) + } else { + // WASM file not found when installed as dependency - this is an error + console.error('Error: fix-python-soname.wasm not found') + process.exit(1) + } +} + +console.log(`Running soname fix on ${nodeFilePath}`) + +// Create a WASI instance +const wasi = new WASI({ + version: 'preview1', + args: ['fix-python-soname', nodeFilePath], + env: process.env, + preopens: { + '/': '/', + } +}) + +async function runSonameFixer() { + try { + const wasm = fs.readFileSync(wasmPath); + const { instance } = await WebAssembly.instantiate(wasm, { + wasi_snapshot_preview1: wasi.wasiImport + }); + + // Run the WASI module + process.exit(wasi.start(instance)) + } catch (error) { + console.error('Error: Failed to run soname fixer:', error.message) + process.exit(1) // Fail hard when installed as dependency + } +} + +runSonameFixer() diff --git a/fix-python-soname/.gitignore b/fix-python-soname/.gitignore new file mode 100644 index 0000000..0172434 --- /dev/null +++ b/fix-python-soname/.gitignore @@ -0,0 +1,7 @@ +/target/ +/pkg/ +/wasm-pack.log +**/*.rs.bk +Cargo.lock +node_modules/ +.DS_Store \ No newline at end of file diff --git a/fix-python-soname/Cargo.toml b/fix-python-soname/Cargo.toml new file mode 100644 index 0000000..347dcde --- /dev/null +++ b/fix-python-soname/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "fix-python-soname" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "fix-python-soname" +path = "src/main.rs" + +[dependencies] +arwen = "0.0.2" diff --git a/fix-python-soname/src/main.rs b/fix-python-soname/src/main.rs new file mode 100644 index 0000000..3a90fd3 --- /dev/null +++ b/fix-python-soname/src/main.rs @@ -0,0 +1,361 @@ +use arwen::elf::ElfContainer; +use std::{ + collections::HashMap, + env, + fs::{self, File}, + path::Path, +}; + +fn find_python_library() -> Result { + // Generate Python versions from 3.20 down to 3.8 + let mut python_versions = Vec::new(); + for major in (8..=20).rev() { + // Standard versioned libraries + python_versions.push(format!("libpython3.{major}.so.1.0")); + python_versions.push(format!("libpython3.{major}.so.1")); + python_versions.push(format!("libpython3.{major}.so")); + // Some distributions include 'm' suffix + python_versions.push(format!("libpython3.{major}m.so.1.0")); + python_versions.push(format!("libpython3.{major}m.so.1")); + python_versions.push(format!("libpython3.{major}m.so")); + } + + eprintln!( + "fix-python-soname: Looking for versions: {:?}", + &python_versions[0..6] + ); + + // Get system architecture + let arch = std::env::consts::ARCH; + let arch_triplet = match arch { + "x86_64" => "x86_64-linux-gnu", + "x86" => "i386-linux-gnu", + "aarch64" => "aarch64-linux-gnu", + "arm" => "arm-linux-gnueabihf", + "powerpc64" => "powerpc64le-linux-gnu", + "s390x" => "s390x-linux-gnu", + _ => "", + }; + + // Comprehensive list of library paths + let mut lib_paths = vec![ + // Standard system paths (most common first) + "/usr/lib", + "/usr/lib64", + "/usr/local/lib", + "/usr/local/lib64", + "/lib", + "/lib64", + "/lib32", + // Debian/Ubuntu multiarch paths + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/i386-linux-gnu", + "/usr/lib/aarch64-linux-gnu", + "/usr/lib/arm-linux-gnueabihf", + // RedHat/CentOS/Fedora Software Collections + "/opt/rh/rh-python38/root/usr/lib64", + "/opt/rh/rh-python39/root/usr/lib64", + "/opt/rh/rh-python310/root/usr/lib64", + "/opt/rh/rh-python311/root/usr/lib64", + // Python built from source + "/usr/local/python/lib", + "/usr/local/python3/lib", + "/opt/python/lib", + "/opt/python3/lib", + "/opt/python-3.11/lib", + "/opt/python-3.12/lib", + // Container/Docker common paths + "/usr/lib/python3", + "/usr/local/lib/python3", + "/opt/lib", + // Snap packages + "/snap/core18/current/usr/lib", + "/snap/core20/current/usr/lib", + "/snap/core22/current/usr/lib", + "/snap/python38/current/usr/lib", + "/snap/python39/current/usr/lib", + "/snap/python310/current/usr/lib", + // Flatpak runtime paths + "/var/lib/flatpak/runtime/org.freedesktop.Platform/x86_64/21.08/active/files/lib", + "/var/lib/flatpak/runtime/org.freedesktop.Platform/x86_64/22.08/active/files/lib", + "/var/lib/flatpak/runtime/org.freedesktop.Platform/x86_64/23.08/active/files/lib", + // Alpine Linux (musl) + "/usr/lib/apk/db", + // Homebrew on Linux + "/home/linuxbrew/.linuxbrew/lib", + "/opt/homebrew/lib", + // macOS paths (for cross-platform support) + "/System/Library/Frameworks/Python.framework/Versions/3.11/lib", + "/System/Library/Frameworks/Python.framework/Versions/3.12/lib", + "/Library/Frameworks/Python.framework/Versions/3.11/lib", + "/Library/Frameworks/Python.framework/Versions/3.12/lib", + // Nix/NixOS paths + "/nix/var/nix/profiles/default/lib", + "/run/current-system/sw/lib", + // Gentoo + "/usr/lib/python-exec/python3.11", + "/usr/lib/python-exec/python3.12", + // System Python config directories + "/usr/lib/python3.8/config-3.8-x86_64-linux-gnu", + "/usr/lib/python3.9/config-3.9-x86_64-linux-gnu", + "/usr/lib/python3.10/config-3.10-x86_64-linux-gnu", + "/usr/lib/python3.11/config-3.11-x86_64-linux-gnu", + "/usr/lib/python3.12/config-3.12-x86_64-linux-gnu", + ]; + + // Add architecture-specific paths if we detected the architecture + if !arch_triplet.is_empty() { + lib_paths.insert( + 0, + Box::leak(format!("/usr/lib/{}", arch_triplet).into_boxed_str()), + ); + lib_paths.insert( + 1, + Box::leak(format!("/usr/local/lib/{}", arch_triplet).into_boxed_str()), + ); + lib_paths.insert( + 2, + Box::leak(format!("/lib/{}", arch_triplet).into_boxed_str()), + ); + } + + // Add conda/anaconda paths from common locations + let conda_paths = vec![ + "/opt/conda/lib", + "/opt/anaconda/lib", + "/opt/anaconda3/lib", + "/opt/miniconda/lib", + "/opt/miniconda3/lib", + ]; + lib_paths.extend(conda_paths); + + // Check for active virtual environments first + if let Ok(venv) = env::var("VIRTUAL_ENV") { + lib_paths.insert(0, Box::leak(format!("{}/lib", venv).into_boxed_str())); + lib_paths.insert(0, Box::leak(format!("{}/lib64", venv).into_boxed_str())); + } + + // Check for active conda environment + if let Ok(conda_prefix) = env::var("CONDA_PREFIX") { + lib_paths.insert( + 0, + Box::leak(format!("{}/lib", conda_prefix).into_boxed_str()), + ); + } + + // Add user-specific paths + if let Ok(home) = env::var("HOME") { + // Conda/Mamba installations + lib_paths.push(Box::leak( + format!("{}/anaconda3/lib", home).into_boxed_str(), + )); + lib_paths.push(Box::leak( + format!("{}/miniconda3/lib", home).into_boxed_str(), + )); + lib_paths.push(Box::leak( + format!("{}/miniforge3/lib", home).into_boxed_str(), + )); + lib_paths.push(Box::leak( + format!("{}/mambaforge/lib", home).into_boxed_str(), + )); + lib_paths.push(Box::leak( + format!("{}/.conda/envs/base/lib", home).into_boxed_str(), + )); + + // Pyenv - search all installed versions + let pyenv_root = format!("{}/.pyenv/versions", home); + if let Ok(entries) = fs::read_dir(&pyenv_root) { + for entry in entries.flatten() { + if entry.file_type().map(|t| t.is_dir()).unwrap_or(false) { + let version_lib = format!("{}/lib", entry.path().display()); + lib_paths.push(Box::leak(version_lib.into_boxed_str())); + } + } + } + + // Local installations + lib_paths.push(Box::leak(format!("{}/.local/lib", home).into_boxed_str())); + lib_paths.push(Box::leak(format!("{}/.local/lib64", home).into_boxed_str())); + + // asdf version manager + let asdf_python = format!("{}/.asdf/installs/python", home); + if let Ok(entries) = fs::read_dir(&asdf_python) { + for entry in entries.flatten() { + if entry.file_type().map(|t| t.is_dir()).unwrap_or(false) { + let version_lib = format!("{}/lib", entry.path().display()); + lib_paths.push(Box::leak(version_lib.into_boxed_str())); + } + } + } + } + + // Add paths from environment variables + if let Ok(ld_library_path) = env::var("LD_LIBRARY_PATH") { + for path in ld_library_path.split(':') { + if !path.is_empty() { + lib_paths.push(Box::leak(path.to_string().into_boxed_str())); + } + } + } + + eprintln!( + "fix-python-soname: Searching in {} directories...", + lib_paths.len() + ); + eprintln!( + "fix-python-soname: First 5 paths: {:?}", + &lib_paths[0..5.min(lib_paths.len())] + ); + + // First try exact version matches + for lib_name in &python_versions { + for lib_path in &lib_paths { + let full_path = format!("{}/{}", lib_path, lib_name); + if Path::new(&full_path).exists() { + eprintln!( + "fix-python-soname: Found Python library: {} at {}", + lib_name, full_path + ); + return Ok(lib_name.to_string()); + } + } + } + + eprintln!("fix-python-soname: No exact match found, searching for any libpython*.so files..."); + + // If no exact match found, search directories for any libpython*.so + for lib_path in &lib_paths { + if let Ok(entries) = fs::read_dir(lib_path) { + let mut found_libs: Vec<(String, u32, u32)> = Vec::new(); + + for entry in entries.flatten() { + if let Some(name) = entry.file_name().to_str() { + if name.starts_with("libpython") && name.contains(".so") { + // Try to extract version numbers + if let Some(version_start) = name.find("3.") { + let version_part = &name[version_start + 2..]; + if let Some(major_end) = version_part.find(|c: char| !c.is_numeric()) { + if let Ok(minor) = version_part[..major_end].parse::() { + found_libs.push((name.to_string(), 3, minor)); + } + } + } + } + } + } + + // Sort by version (newest first) + found_libs.sort_by(|a, b| b.2.cmp(&a.2).then(b.1.cmp(&a.1))); + + if let Some((lib_name, _, _)) = found_libs.first() { + eprintln!( + "fix-python-soname: Found Python library: {} in {}", + lib_name, lib_path + ); + return Ok(lib_name.clone()); + } + } + } + + Err( + "No Python library found on the system. Searched in:\n".to_string() + + &lib_paths[..10].join("\n ") + + "\n ... and more", + ) +} + +fn main() -> Result<(), Box> { + eprintln!("fix-python-soname: Starting soname patcher..."); + + let args: Vec = env::args().collect(); + eprintln!("fix-python-soname: Arguments: {:?}", args); + + if args.len() != 2 { + return Err(format!("Usage: {} ", args[0]).into()); + } + + let node_file_path = &args[1]; + eprintln!("fix-python-soname: Processing file: {}", node_file_path); + + // Find the local Python library + let new_python_lib = find_python_library()?; + + // Read the file + eprintln!("fix-python-soname: Reading ELF file..."); + let file_contents = + fs::read(node_file_path).map_err(|error| format!("Failed to read file: {error}"))?; + eprintln!( + "fix-python-soname: ELF file size: {} bytes", + file_contents.len() + ); + + // Parse the ELF file + eprintln!("fix-python-soname: Parsing ELF file..."); + let mut elf = + ElfContainer::parse(&file_contents).map_err(|error| format!("Failed to parse ELF: {error}"))?; + + // Get the list of needed libraries + eprintln!("fix-python-soname: Getting needed libraries..."); + let needed_libs: Vec = elf + .inner + .elf_needed() + .map(|lib| String::from_utf8_lossy(lib).to_string()) + .collect(); + + eprintln!("fix-python-soname: Needed libraries: {:?}", needed_libs); + + // Find the existing Python dependency + let python_lib = needed_libs + .iter() + .find(|lib| lib.starts_with("libpython") && lib.contains(".so")) + .ok_or("No Python library dependency found in the binary")?; + + eprintln!( + "fix-python-soname: Current Python dependency: {}", + python_lib + ); + + // Check if already pointing to the correct library + if python_lib == &new_python_lib { + eprintln!("fix-python-soname: Already using the correct Python library"); + return Ok(()); + } + + eprintln!("fix-python-soname: Replacing with: {}", new_python_lib); + + // Create a map for replacement + let mut replacements = HashMap::new(); + replacements.insert(python_lib.clone(), new_python_lib); + + // Replace the needed dependency + eprintln!("fix-python-soname: Replacing dependency..."); + elf + .replace_needed(&replacements) + .map_err(|error| format!("Failed to replace needed dependency: {error}"))?; + + // Create backup + let file_path = Path::new(node_file_path); + let backup_path = file_path.with_extension("node.bak"); + eprintln!( + "fix-python-soname: Creating backup at: {}", + backup_path.display() + ); + fs::copy(file_path, &backup_path).map_err(|error| format!("Failed to create backup: {error}"))?; + eprintln!("fix-python-soname: Backup created successfully"); + + // Write the modified file + eprintln!("fix-python-soname: Writing modified ELF file..."); + let output_file = File::create(node_file_path) + .map_err(|error| format!("Failed to create output file: {error}"))?; + + elf + .write(&output_file) + .map_err(|error| format!("Failed to write ELF: {error}"))?; + + eprintln!( + "fix-python-soname: Successfully updated: {}", + node_file_path + ); + + Ok(()) +} diff --git a/npm/linux-x64-gnu/package.json b/npm/linux-x64-gnu/package.json index 1a80bf8..5769137 100644 --- a/npm/linux-x64-gnu/package.json +++ b/npm/linux-x64-gnu/package.json @@ -9,8 +9,13 @@ ], "main": "python-node.linux-x64-gnu.node", "files": [ - "python-node.linux-x64-gnu.node" + "python-node.linux-x64-gnu.node", + "fix-python-soname.js", + "fix-python-soname.wasm" ], + "scripts": { + "postinstall": "node fix-python-soname.js" + }, "license": "MIT", "engines": { "node": ">= 10" diff --git a/package.json b/package.json index 322de33..0d7add1 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,10 @@ "artifacts": "napi artifacts", "build": "npm run build:debug -- --release", "build:debug": "napi build --platform --features napi-support", - "prepublishOnly": "napi prepublish -t npm", + "build:fix": "node fix-python-soname.js", + "build:wasm": "rustup target add wasm32-wasip1 && cargo build --package fix-python-soname --target wasm32-wasip1 --release && cp target/wasm32-wasip1/release/fix-python-soname.wasm .", + "prepublishOnly": "npm run build:wasm && napi prepublish -t npm", + "postinstall": "npm run build:fix", "lint": "oxlint", "test": "node --test test/**.test.mjs", "universal": "napi universal", diff --git a/src/asgi/mod.rs b/src/asgi/mod.rs index 06064b4..59d83ae 100644 --- a/src/asgi/mod.rs +++ b/src/asgi/mod.rs @@ -22,6 +22,11 @@ use tokio::sync::oneshot; use crate::{HandlerError, PythonHandlerTarget}; +/// HTTP response tuple: (status_code, headers, body) +type HttpResponse = (u16, Vec<(String, String)>, Vec); +/// Result type for HTTP response operations +type HttpResponseResult = Result; + /// Global runtime for when no tokio runtime is available static FALLBACK_RUNTIME: OnceLock = OnceLock::new(); @@ -413,7 +418,7 @@ fn start_python_event_loop_thread(event_loop: PyObject) { /// Collect ASGI response messages async fn collect_response_messages( mut tx_receiver: tokio::sync::mpsc::UnboundedReceiver>, - response_tx: oneshot::Sender, Vec), HandlerError>>, + response_tx: oneshot::Sender, ) { let mut status = 500u16; let mut headers = Vec::new();