wip

Author: mxcl

.github/Dockerfile

@@ -3,7 +3,7 @@ RUN apt-get update && apt-get install -y curl
 RUN curl https://pkgx.sh/$(uname)/$(uname -m).tgz | tar xz -C /usr/local/bin
 
 ENV PKGX_DIST_URL="https://dist.pkgx.dev/v2"
-RUN pkgx +llvm.org +patchelf +deno^2 +gnu.org/gcc/libstdcxx^14
+RUN pkgx +llvm.org^19 +patchelf +deno^2 +gnu.org/gcc/libstdcxx^14
 COPY ./brewkit /work/brewkit
 COPY ./.github /work/.github
 COPY ./deno.jsonc /work/deno.jsonc

README.md

@@ -1,60 +1,54 @@
 # `pantry^2`
 
-- Configuration is hardcoded to /etc
-- We use vendor-built binaries where possible
-  - We assume the vendor knows how to build their product better than us
-  - Means issues are almost always upstream and not ours
-  - Notably excepting when the prebuilt binaries vendor libraries we provide
-- windows/x86-64 support from day 1
-- Relocatable without environment fixes for all base packages
-- If we package a package manager and it provides a tool (in a unpainful way)
-  then pkgx will invoke that other tool to get the thing
-  - eg. `cargo-binstall`, `npx`, `uvx` etc. will be used instead of us
-    packaging them
-  - these other mechanisms are generally _preferred_ by the developer and
-    end-user, we'll just know how to get you the end-product with a consistent
-    CLI
-  - this means we can focus on ensuring the base is as good as possible
-    without distraction from a massive pkg list
-  - We will still index everything and show it at [pkgx.dev/pkgs]
-- Minimizing deps on macOS
-  - we added deps because Linux needed them in v1 due to laziness and urgency
+## Goals
+
+- Better optimized packages
+- Much more careful dependency selection
+  - We have reviewed everything that is transitive thoroughly
+- Leaner, more streamlined packages
+  - Python reduced from 280MB to 78MB
+  - LLVM pruned to just what is needed and split out into multiple packages
+    reduing its size from 1.5GB to 300MB
+- More consistent namespacing
+- More reliable builds
+- More robust bottles
+  - eg. `rpath` fixes are now much more thorough and careful
+- Better configured and more consistently configured packages
+  - In general we set configuration to be `/etc` with user config carve outs
+- Across the board dependency on OpenSSL^3 rather ^1.1.1
+- Use of macOS system libraries where possible
+- Windows native support
+- Faster build infrastructure
+- More minimal pre-requisite footprint on Linux
+- 100% relocatable packages without environment fixes for all base packages
+- Use vendor built binaries where possible and sensible
+  - Provided the vendor does not statically link in libraries this is a
+    better choice since we assume the vendor knows how to build their product
+    better than us
 - No magic
-  - git no longer looks for `git-foo` if you type `git foo`
-  - etc.
-  - we would consider adding these back, but not as _part_ of the package
-    itself. The package itself should solely focus on our other goals and
-    otherwise be vanilla.
-- `pkgx^1,^2` will use pantry^1, `pkgx^3` will use pantry^2
-- building with as minimal images as is possible to ensure we are sure about
-  what goes into our packages
+  - eg. `git` stub that looks to `pkgx` for `git-foo`
+- More flexible build infrastructure
+- Less unwise hacks
+  - eg. symlinking versioned include directories, instead we now set `CPATH`
+    in the package.yaml runtime env
+  - Separating out eg. `gem` from Ruby for purity reasons was a mistake
+- No more packaging for projects without versioning or with careless
+  versioning. Eg. `llama.cpp` and `vim`.
+  - Users still want these so we can instead make them build from source on
+    demand.
+  - It is gross currently since these projects release multiple times *a day*
+
+## Future Goals
+
 - First class `pkgm` support
   - everything should install to `/usr/local` and just work
-- No weird handling for calver
-- no (or much less) pre-reqs on Linux
-  - we only need glibc and libgcc
-  - strictly we don’t need the user to provide libgcc, but us building and
-    using our own is more or a pita than we can handle RN
-- variable deps
-  - eg. a major version requires new deps
-  - eg. heaven forbid, a minor version changes the dep constraint
-- sources tarball locations can vary by version
-- program lists can vary by version
-- no deversioning of inner directories (for `pkgm`)
-- no support for dollar prefixed moustaches in `package.yml`
-- independently versioned things must be independent
-  - generally we are already good at this
-- things without established versioning and/or programs are not valid to be
-  packaged. They go in `pkgo`
+- Zero pre-requisties on Linux
+- Dependencies can vary by version
+- Program lists can vary by version
 - standardize pkgs
   - use XDG and standard dirs on other platforms
   - configure things that install things to install them to
     `${INSTALL_ROOT:-$HOME/.local}` by default
-- more consistent project names
-  - no foo.github.io, just github.com/foo etc.
-  - no strict adherance to homepages, it's more about namespacing
-
-[pkgx.dev/pkgs]: https://pkgx.dev/pkgs
 
 ## Usage
 
@@ -94,18 +88,6 @@ $ ls bin
 
 - glibc, libgcc and their `-dev` pkgs are still required at this time
 
-## Wins
-
-- Python from 280MB to 78MB
-- Cleaner rpath handling across the board
-  - linux rpath fixes were actually broken with `pantry^1`
-- Less env pollution by carefully using `pkgx` during builds rather than
-  importing dep-envs before builds, meaning more reliable builds with less
-  unexpected deps
-- Carefuly pruning of deps and build options for all base deps
-- much tighter python venv handling
-- more XDG adherance for base packages
-
 ### Build Infra Wins
 
 - a pkg depending on itself now works without conflict since destination
@@ -129,34 +111,6 @@ complexity mounts and mounts. You may as well just use a real language.
 
 Also we needed a cross platform language to support Windows. Bash is not that.
 
-### Criteria for Inclusion
-
-- We require that packages are versioned.
-- We require that the project bew licensed such that we are permitted to
-  redistribute it.
-- Stuff that is so new that its build instructions are likely to change a lot
-in future may be rejected due to our inability to reliably maintain that.
-- Things that can have other general executors (eg. npx) should be run that
-  way using the `providers` system.
-- Things that do not respect reasonable release schedules may be rejected
-  (eg. we have seen packages release 10+ times a day, every day).
-
-### Requirements
-
-For local builds we need some stuff set up:
-
-* macOS
-  * The Xcode Command Line Tools are expected to be installed
-* Windows
-  * Visual Studio Community Edition with C++
-  * Some things expect `nmake` and thus you will need to run a
-    “developer prompt”
-* Linux
-  * glibc libs and headers
-  * stdc++
-
-Mostly, otherwise we source build tools from `pkgx`.
-
 ### Boostrapping New Platforms
 
 You will need preinstalled versions of the following:

bin/pkg-build

@@ -60,19 +60,19 @@ async function prep_toolchain(tmpdir: Path, srcroot: Path) {
     const llvm_bin = new Path(await backticks_quiet`pkgx -q +llvm.org -- which clang`).parent();
     const ln = (tool: string, llvm_tool: string) =>
       bin.join(tool).ln('s', { target: llvm_bin.join(llvm_tool) });
-    ln('cc', 'clang');
-    ln('gcc', 'clang');
-    ln('clang', 'clang');
-    ln('c++', 'clang++');
-    ln('g++', 'clang++');
-    ln('clang++', 'clang++');
-    ln('cpp', 'clang-cpp');
-    ln('ld', 'ld.lld');
-    ln('ar', 'llvm-ar');
-    ln('as', 'llvm-as');
-    ln('nm', 'llvm-nm');
-    ln('strings', 'llvm-strings');
-    ln('ranlib', 'llvm-ranlib');
+    // ln('cc', 'clang');
+    // ln('gcc', 'clang');
+    // ln('clang', 'clang');
+    // ln('c++', 'clang++');
+    // ln('g++', 'clang++');
+    // ln('clang++', 'clang++');
+    // ln('cpp', 'clang-cpp');
+    // ln('ld', 'ld.lld');
+    // ln('ar', 'llvm-ar');
+    // ln('as', 'llvm-as');
+    // ln('nm', 'llvm-nm');
+    // ln('strings', 'llvm-strings');
+    // ln('ranlib', 'llvm-ranlib');
 
     env["PATH"] = `${bin}:/usr/bin:/bin:/usr/sbin:/sbin`;
     // stops ./configure from erroring in our container

brewkit/run.ts

@@ -4,7 +4,15 @@ function run_f(args: string[]) {
   }
   const cmd = args.shift()!;
 
-  console.error("%c+", "color:yellow", cmd, ...args);
+  const pretty_args = args.map((arg) => {
+    if (arg.includes(" ")) {
+      return arg.replaceAll(/\s+/g, "\\ ");
+    } else {
+      return arg;
+    }
+  });
+
+  console.error("%c+", "color:yellow", cmd, ...pretty_args);
 
   const { success, code } = new Deno.Command(cmd, {
     args,
@@ -79,7 +87,7 @@ function splitArgs(input: string): string[] {
   input = input.replace(/#.*$/gm, "");
 
   while ((match = regex.exec(input)) !== null) {
-    args.push(match[1] || match[2] || match[3]);
+    args.push((match[1] || match[2] || match[3]).trim());
   }
 
   return args;

projects/gnu.org/gcc/build.ts

@@ -1,24 +1,37 @@
 import { env_include, BuildOptions, Path, run, unarchive } from "brewkit";
 import { expandGlob } from "jsr:@std/fs@1/expand-glob";
 
-export default async function build({ prefix, version }: BuildOptions) {
+export default async function build({ prefix, version, deps }: BuildOptions) {
   await unarchive(`https://ftpmirror.gnu.org/gnu/gcc/gcc-${version}/gcc-${version}.tar.gz`);
 
   let extra = "";
 
+  Object.entries(Deno.env.toObject()).forEach(([k,v]) => {
+    console.error(`${k}=${v}`);
+  });
+
   if (Deno.build.os == "linux") {
-    const old = Deno.env.get("PKGX_DIST_URL");
-    Deno.env.delete("PKGX_DIST_URL");
-    Deno.env.delete("PKGX_PANTRY_DIR");
-    env_include("gnu.org/gcc^14");
-    Deno.env.set("PKGX_DIST_URL", "https://dist.pkgx.dev/v2");
-    Deno.env.set("PKGX_PANTRY_DIR", old!);
+    Deno.env.delete("LD_LIBRARY_PATH");
+    // const old = Deno.env.get("PKGX_PANTRY_DIR");
+    // Deno.env.delete("PKGX_DIST_URL");
+    // Deno.env.delete("PKGX_PANTRY_DIR");
+    // env_include("gnu.org/gcc^14");
+    // Deno.env.set("LD_LIBRARY_PATH", [
+    //   deps['gnu.org/mpfr'].prefix.join("lib").string,
+    //   deps['gnu.org/mpc'].prefix.join("lib").string,
+    //   deps['gnu.org/gmp'].prefix.join("lib").string
+    // ].join(":"));  // temporary to stop illegal instructions
+    // Deno.env.set("PKGX_DIST_URL", "https://dist.pkgx.dev/v2");
+    // Deno.env.set("PKGX_PANTRY_DIR", old!);
 
     extra = `
       --disable-multilib
-      --enable-default-pie
-      --enable-pie-tools
-      --enable-host-pie
+      --build=x86_64-pc-linux-gnu  # or compile fails due to dupe pid_t
+      --with-arch=x86-64   # or c++ crashes with illegal instructions inside docker
+      --with-tune=generic  # ^^ same
+      #--enable-default-pie
+      #--enable-pie-tools
+      #--enable-host-pie
       `;
   }
 
@@ -28,16 +41,16 @@ export default async function build({ prefix, version }: BuildOptions) {
         --prefix=${prefix}
         --enable-languages=c,c++
         --disable-multilib
-        --disable-bootstrap
-        --with-bugurl="https://github.com/pkgxdev/manifests/issues"
-      # "--with-boot-ldflags=-static-libgcc -static-libstdc++"
+        --enable-bootstrap
+        --with-bugurl=https://github.com/pkgxdev/manifests/issues
+        "--with-boot-ldflags=-static-libgcc -static-libstdc++"
         --disable-nls
         --enable-shared
       # --with-system-zlib  #TODO
         "--with-pkgversion=pkgx GCC ${version}"
         ${extra}
         `;
-  run`make --jobs ${navigator.hardwareConcurrency}`;
+  run`make`; //--jobs ${navigator.hardwareConcurrency}`;
   run`make install-strip`; // cannot install in parallel
 
   const lib64 = prefix.join("lib64");

projects/gnu.org/gcc/libstdcxx/build.ts

@@ -3,12 +3,12 @@ import { env_include, BuildOptions, Path, run, unarchive } from "brewkit";
 export default async function build({ prefix, version }: BuildOptions) {
   await unarchive(`https://ftpmirror.gnu.org/gnu/gcc/gcc-${version}/gcc-${version}.tar.gz`);
 
-  const old = Deno.env.get("PKGX_DIST_URL");
-  Deno.env.delete("PKGX_DIST_URL");
-  Deno.env.delete("PKGX_PANTRY_DIR");
+//   const old = Deno.env.get("PKGX_DIST_URL");
+//   Deno.env.delete("PKGX_DIST_URL");
+//   Deno.env.delete("PKGX_PANTRY_DIR");
   env_include("gnu.org/gcc^14");
-  Deno.env.set("PKGX_DIST_URL", "https://dist.pkgx.dev/v2");
-  Deno.env.set("PKGX_PANTRY_DIR", old!);
+//   Deno.env.set("PKGX_DIST_URL", "https://dist.pkgx.dev/v2");
+//   Deno.env.set("PKGX_PANTRY_DIR", old!);
 
   Path.cwd().join("build").mkdir().cd();
 
@@ -24,8 +24,8 @@ export default async function build({ prefix, version }: BuildOptions) {
         `;
   run`make
         --jobs ${navigator.hardwareConcurrency}
-        all-target-libstdc++-v3
-        all-target-libgcc`;
+        # all-target-libstdc++-v3
+        # all-target-libgcc`;
   run`make
         install-strip-target-libstdc++-v3
         install-strip-target-libgcc`;

projects/gnu.org/gcc/package.yml

@@ -28,12 +28,12 @@ programs:
 platforms:
   - linux/x86-64
 
-dependencies:
-  gnu.org/mpc: ^1.3
+# dependencies:
+#   gnu.org/mpc: ^1.3
 
-linux:
-  dependencies:
-    gnu.org/gcc/libstdcxx: ^14
+# linux:
+#   dependencies:
+#     gnu.org/gcc/libstdcxx: ^14
 
 # we don’t provide a shared libgcc or libstdc++
 # we probs should provide a shared libgcc in the future but a share dlibstdc++

projects/google.com/highway/build.ts

@@ -1,14 +1,14 @@
-import { BuildOptions, unarchive, run, Path } from "brewkit";
+import { BuildOptions, unarchive, run } from "brewkit";
 
 export default async function ({ prefix, version }: BuildOptions) {
   await unarchive(`https://github.com/google/highway/archive/refs/tags/${version}.tar.gz`);
   run`cmake
         -B bld
+        -G "Unix Makefiles"
         -DBUILD_SHARED_LIBS=ON
         -DCMAKE_INSTALL_PREFIX=${prefix}
         -DHWY_ENABLE_TESTS=OFF
         -DHWY_ENABLE_EXAMPLES=OFF
-        -DCMAKE_BUILD_TYPE=Release
         `;
   run`cmake --build bld --target install`;
 

projects/jpeg.org/jpegxl/build.ts

@@ -17,24 +17,20 @@ export default async function ({ prefix, version, tag }: BuildOptions) {
 
   if (Deno.build.os == "linux") {
     // ld.lld: error: undefined reference: __extendhfsf2
-    Deno.env.set("LDFLAGS", "-Wl,--allow-shlib-undefined");
+    // Deno.env.set("LDFLAGS", "-Wl,--allow-shlib-undefined");
   }
 
   run`cmake
+        -G "Unix Makefiles"
         -B bld
+        -DCMAKE_VERBOSE_MAKEFILE=ON
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_INSTALL_PREFIX=${prefix}
         -DBUILD_TESTING=OFF
         -DJPEGXL_ENABLE_SKCMS=OFF
         -DJPEGXL_ENABLE_BENCHMARK=OFF
         -DJPEGXL_VERSION=${version}
         `;
-  run`cmake --build bld --target install`;
-
-//   linux/x86-64:
-//     ARGS:
-//       - '-DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined,-lstdc++fs'
-//   linux/aarch64:
-//     ARGS:
-//       - '-DCMAKE_EXE_LINKER_FLAGS=-Wl,-lstdc++fs'
-}
+  // run`cmake --build bld --target install`;
+  run`make -C bld`;
+}