From 92e190ebd408ef87828f5e4a2070bebe1c956b57 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Aug 2024 11:28:53 -0400 Subject: [PATCH 1/5] feat: wip fixing plv8 on darwin aarch64 --- flake.nix | 9 ++++++++- nix/ext/plv8.nix | 15 +++++++++++++-- nix/tests/prime.sql | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 3e60780bc..33b7ff6ea 100644 --- a/flake.nix +++ b/flake.nix @@ -417,7 +417,14 @@ fi done createdb -p 5432 -h localhost testing - psql -p 5432 -h localhost -d testing -Xaf ${./nix/tests/prime.sql} + if ! psql -p 5432 -h localhost -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql} > $TMPDIR/psql_output.log 2>&1; then + echo "Error executing SQL file. Error output:" + cat $TMPDIR/psql_output.log + echo "PostgreSQL log content:" + cat $TMPDIR/logfile/postgresql.log + pg_ctl -D "$PGDATA" stop + exit 1 + fi pg_prove -p 5432 -h localhost -d testing ${sqlTests}/*.sql diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 29b611b08..c8792d808 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -9,6 +9,8 @@ , coreutils , gnugrep , gcc +, patchelf +, xcbuild }: stdenv.mkDerivation (finalAttrs: { @@ -35,10 +37,13 @@ stdenv.mkDerivation (finalAttrs: { perl ] ++ lib.optionals stdenv.isDarwin [ gcc + xcbuild ]; buildInputs = [ - v8 + (v8.overrideAttrs (oldAttrs: { + version = "9.7.106.18"; + })) postgresql ]; @@ -47,7 +52,7 @@ stdenv.mkDerivation (finalAttrs: { makeFlags = [ # Nixpkgs build a v8 monolith instead of separate v8_libplatform. "USE_SYSTEM_V8=1" - "SHLIB_LINK=-lv8" + "SHLIB_LINK=-L${v8}/lib -lv8_monolith -Wl,--no-as-needed" "V8_OUTDIR=${v8}/lib" ]; NIX_LDFLAGS = lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) @@ -78,6 +83,12 @@ stdenv.mkDerivation (finalAttrs: { # so changing them does not cause issues. mv "$out/nix/store"/*/* "$out" rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" + + # Use install_name_tool for macOS + ${lib.optionalString stdenv.isDarwin '' + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + ''} ''; passthru = { diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 23101262b..a1ca84ce5 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -67,7 +67,7 @@ TODO: PLs can not be enabled on M1 Macs locally ERROR: could not load library "/nix/store/..../lib/plv8-3.1.5.so symbol not found in flat namespace */ --- create extension plv8; +create extension plv8; -- create extension plcoffee; -- create extension plls; From 25428b6d44d6ebe59ecd58ccd5a4e72856ea81e2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Aug 2024 14:30:44 -0400 Subject: [PATCH 2/5] fix: build plv8 and success in enabling on darwin --- nix/ext/plv8.nix | 63 +++++++++++++++++++++++++++++++-------------- nix/tests/prime.sql | 4 +-- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index c8792d808..4f52fbaf8 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -1,3 +1,5 @@ + + { stdenv , lib , fetchFromGitHub @@ -8,16 +10,14 @@ , runCommand , coreutils , gnugrep -, gcc +, clang , patchelf , xcbuild +, darwin }: stdenv.mkDerivation (finalAttrs: { pname = "plv8"; - # plv8 latest is https://github.com/plv8/plv8/releases/tag/v3.2.2 - # FIXME we need to increment this build toward 3.2.2 - # 3.1.7 is the highest version that can be built with pg 16 version = "3.1.5"; src = fetchFromGitHub { @@ -28,15 +28,13 @@ stdenv.mkDerivation (finalAttrs: { }; patches = [ - # Allow building with system v8. - # https://github.com/plv8/plv8/pull/505 (rejected) ./0001-build-Allow-using-V8-from-system.patch ]; nativeBuildInputs = [ perl ] ++ lib.optionals stdenv.isDarwin [ - gcc + clang xcbuild ]; @@ -45,49 +43,76 @@ stdenv.mkDerivation (finalAttrs: { version = "9.7.106.18"; })) postgresql + ] ++ lib.optionals stdenv.isDarwin [ + darwin.apple_sdk.frameworks.CoreFoundation + darwin.apple_sdk.frameworks.Kerberos ]; buildFlags = [ "all" ]; makeFlags = [ - # Nixpkgs build a v8 monolith instead of separate v8_libplatform. "USE_SYSTEM_V8=1" - "SHLIB_LINK=-L${v8}/lib -lv8_monolith -Wl,--no-as-needed" "V8_OUTDIR=${v8}/lib" + "PG_CONFIG=${postgresql}/bin/pg_config" + ] ++ lib.optionals stdenv.isDarwin [ + "CC=${clang}/bin/clang" + "CXX=${clang}/bin/clang++" + "SHLIB_LINK=-L${v8}/lib -lv8_monolith -Wl,-rpath,${v8}/lib" ]; - NIX_LDFLAGS = lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) - "-undefined dynamic_lookup"; - NIX_CFLAGS_COMPILE = lib.optionals (stdenv.isDarwin && stdenv.isAarch64) [ + NIX_LDFLAGS = lib.optionals stdenv.isDarwin (lib.concatStringsSep " " [ + "-L${postgresql}/lib" + "-L${v8}/lib" + "-lv8_monolith" + "-lpq" + "-lpgcommon" + "-lpgport" + "-F${darwin.apple_sdk.frameworks.CoreFoundation}/Library/Frameworks" + "-framework CoreFoundation" + "-F${darwin.apple_sdk.frameworks.Kerberos}/Library/Frameworks" + "-framework Kerberos" + "-undefined dynamic_lookup" + "-flat_namespace" + ]); + + NIX_CFLAGS_COMPILE = lib.optionals stdenv.isDarwin [ "-I${v8}/include" "-I${postgresql}/include" + "-I${postgresql}/include/server" + "-I${postgresql}/include/internal" ]; + installFlags = [ - # PGXS only supports installing to postgresql prefix so we need to redirect this "DESTDIR=${placeholder "out"}" ]; - # No configure script. dontConfigure = true; postPatch = '' patchShebangs ./generate_upgrade.sh - # https://github.com/plv8/plv8/pull/506 substituteInPlace generate_upgrade.sh \ --replace " 2.3.10 " " 2.3.10 2.3.11 2.3.12 2.3.13 2.3.14 2.3.15 " + + ${lib.optionalString stdenv.isDarwin '' + # Replace g++ with clang++ in Makefile + sed -i 's/g++/clang++/g' Makefile + ''} + ''; + + preBuild = lib.optionalString stdenv.isDarwin '' + export CC=${clang}/bin/clang + export CXX=${clang}/bin/clang++ ''; postInstall = '' - # Move the redirected to proper directory. - # There appear to be no references to the install directories - # so changing them does not cause issues. mv "$out/nix/store"/*/* "$out" rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" - # Use install_name_tool for macOS ${lib.optionalString stdenv.isDarwin '' install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}.so install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}.so ''} ''; diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index a1ca84ce5..0c16d3ef0 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -68,8 +68,8 @@ ERROR: could not load library "/nix/store/..../lib/plv8-3.1.5.so symbol not found in flat namespace */ create extension plv8; --- create extension plcoffee; --- create extension plls; +create extension plcoffee; +create extension plls; create extension postgis; create extension postgis_raster; From 6c50c09d6a0d5924568c1f832aec85e21de34ec2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Aug 2024 18:39:46 +0000 Subject: [PATCH 3/5] fix: now builds correct and enable on aarch64-linux --- nix/ext/plv8.nix | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 4f52fbaf8..ef476f0ab 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -1,5 +1,3 @@ - - { stdenv , lib , fetchFromGitHub @@ -58,9 +56,11 @@ stdenv.mkDerivation (finalAttrs: { "CC=${clang}/bin/clang" "CXX=${clang}/bin/clang++" "SHLIB_LINK=-L${v8}/lib -lv8_monolith -Wl,-rpath,${v8}/lib" + ] ++ lib.optionals (!stdenv.isDarwin) [ + "SHLIB_LINK=-L${v8}/lib -lv8_monolith -Wl,-rpath,${v8}/lib" ]; - NIX_LDFLAGS = lib.optionals stdenv.isDarwin (lib.concatStringsSep " " [ + NIX_LDFLAGS = (lib.optionals stdenv.isDarwin [ "-L${postgresql}/lib" "-L${v8}/lib" "-lv8_monolith" @@ -68,14 +68,21 @@ stdenv.mkDerivation (finalAttrs: { "-lpgcommon" "-lpgport" "-F${darwin.apple_sdk.frameworks.CoreFoundation}/Library/Frameworks" - "-framework CoreFoundation" + "-framework" "CoreFoundation" "-F${darwin.apple_sdk.frameworks.Kerberos}/Library/Frameworks" - "-framework Kerberos" - "-undefined dynamic_lookup" + "-framework" "Kerberos" + "-undefined" "dynamic_lookup" "-flat_namespace" + ]) ++ (lib.optionals (!stdenv.isDarwin) [ + "-L${postgresql}/lib" + "-L${v8}/lib" + "-lv8_monolith" + "-lpq" + "-lpgcommon" + "-lpgport" ]); - NIX_CFLAGS_COMPILE = lib.optionals stdenv.isDarwin [ + NIX_CFLAGS_COMPILE = [ "-I${v8}/include" "-I${postgresql}/include" "-I${postgresql}/include/server" @@ -114,6 +121,10 @@ stdenv.mkDerivation (finalAttrs: { install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}.so ''} + + ${lib.optionalString (!stdenv.isDarwin) '' + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + ''} ''; passthru = { From d72b79d3e0e08b7c766e751a1d08c7d068b72acd Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 13 Aug 2024 14:52:36 -0400 Subject: [PATCH 4/5] fix: pg_regression updated to include plv8 extensions again --- .../expected/extensions_sql_interface.out | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 2f4c8385e..70d153fc3 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -20,13 +20,10 @@ where installed_version is null order by name asc; - name ----------- + name +--------- pg_cron - plcoffee - plls - plv8 -(4 rows) +(1 row) /* @@ -98,8 +95,11 @@ order by pgsodium | f pgstattuple | t pgtap | t + plcoffee | f + plls | f plpgsql | f plpgsql_check | f + plv8 | f postgis | f postgis_raster | f postgis_sfcgal | t @@ -121,7 +121,7 @@ order by vector | t wrappers | f xml2 | f -(78 rows) +(81 rows) /* @@ -3030,6 +3030,12 @@ order by pgtap | public | volatility_is | name, name[], text, text | text pgtap | public | volatility_is | name, text | text pgtap | public | volatility_is | name, text, text | text + plcoffee | pg_catalog | plcoffee_call_handler | | language_handler + plcoffee | pg_catalog | plcoffee_call_validator | oid | void + plcoffee | pg_catalog | plcoffee_inline_handler | internal | void + plls | pg_catalog | plls_call_handler | | language_handler + plls | pg_catalog | plls_call_validator | oid | void + plls | pg_catalog | plls_inline_handler | internal | void plpgsql | pg_catalog | plpgsql_call_handler | | language_handler plpgsql | pg_catalog | plpgsql_inline_handler | internal | void plpgsql | pg_catalog | plpgsql_validator | oid | void @@ -3055,6 +3061,12 @@ order by plpgsql_check | public | plpgsql_profiler_reset_all | | void plpgsql_check | public | plpgsql_show_dependency_tb | fnname text, relid regclass | TABLE(type text, oid oid, schema text, name text, params text) plpgsql_check | public | plpgsql_show_dependency_tb | funcoid regprocedure, relid regclass | TABLE(type text, oid oid, schema text, name text, params text) + plv8 | pg_catalog | plv8_call_handler | | language_handler + plv8 | pg_catalog | plv8_call_validator | oid | void + plv8 | pg_catalog | plv8_info | | json + plv8 | pg_catalog | plv8_inline_handler | internal | void + plv8 | pg_catalog | plv8_reset | | void + plv8 | pg_catalog | plv8_version | | text postgis | public | _postgis_deprecate | oldname text, newname text, version text | void postgis | public | _postgis_index_extent | tbl regclass, col text | box2d postgis | public | _postgis_join_selectivity | regclass, text, regclass, text, text | double precision @@ -5092,7 +5104,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(4941 rows) +(4953 rows) /* From 34c9bc25326d8db9fa6cc1eccd878b10a9ea7555 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Aug 2024 18:56:26 +0000 Subject: [PATCH 5/5] fix: print logs to stdout/stderr instead of logfile --- flake.nix | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 33b7ff6ea..34e757212 100644 --- a/flake.nix +++ b/flake.nix @@ -417,15 +417,12 @@ fi done createdb -p 5432 -h localhost testing - if ! psql -p 5432 -h localhost -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql} > $TMPDIR/psql_output.log 2>&1; then - echo "Error executing SQL file. Error output:" - cat $TMPDIR/psql_output.log - echo "PostgreSQL log content:" + if ! psql -p 5432 -h localhost -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then + echo "Error executing SQL file. PostgreSQL log content:" cat $TMPDIR/logfile/postgresql.log pg_ctl -D "$PGDATA" stop exit 1 fi - pg_prove -p 5432 -h localhost -d testing ${sqlTests}/*.sql mkdir -p $out/regression_output