Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion libs/native/libraries/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,11 @@ dependencies {
libs "org.elasticsearch:zstd:${zstdVersion}:linux-aarch64"
libs "org.elasticsearch:zstd:${zstdVersion}:linux-x86-64"
libs "org.elasticsearch:zstd:${zstdVersion}:windows-x86-64"
libs "org.elasticsearch:vec:${vecVersion}@zip" // temporarily comment this out, if testing a locally built native lib
// if testing locally, its assumed that the custom compiled binaries are copied to the correct directory
// See `simdvec/native/build.gradle` for details on how to compile locally
if (System.getenv("LOCAL_VEC_BINARY_OS") == null){
libs "org.elasticsearch:vec:${vecVersion}@zip"
}
}

def extractLibs = tasks.register('extractLibs', Copy) {
Expand Down
27 changes: 24 additions & 3 deletions libs/simdvec/native/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ var os = org.gradle.internal.os.OperatingSystem.current()
// docker run 9c9f36564c148b275aeecc42749e7b4580ded79dcf51ff6ccc008c8861e7a979 > build/libs/vec/shared/$arch/libvec.so
//
// To run tests and benchmarks on a locally built libvec,
// 1. Temporarily comment out the download in libs/native/library/build.gradle
// libs "org.elasticsearch:vec:${vecVersion}@zip"
// 2. Copy your locally built libvec binary, e.g.
// 1. Set the LOCAL_VEC_BINARY_OS environment variable to any value to skip downloading the binary, e.g.
// export LOCAL_VEC_BINARY_OS=darwin
// 2. Copy your locally built libvec binary (assumes from the repo root), e.g.
// cp libs/simdvec/native/build/libs/vec/shared/aarch64/libvec.dylib libs/native/libraries/build/platform/darwin-aarch64/libvec.dylib
//
// Look at the disassemble:
Expand Down Expand Up @@ -129,3 +129,24 @@ tasks.register('buildSharedLibrary') {
throw new GradleException("Unsupported platform: " + platformName)
}
}

tasks.register('buildSharedLibraryAndCopy') {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mark-vieira is there a more canonical way to wire this for local building over to libs/native?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The more idiomatic way would probably be to use composite builds here. If we configured this project to publish the zip as an artifact, configure it with the same maven coordinates as the binary artifact, then in theory we could just use --include-build libs/simdvec/native to get the same behavior.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know what composite builds are :/

My end goal is to provide simplified mechanisms not only for local testing but to unlock us writing up some buildkite jobs to build and benchmark native code changes (will be in subsequent PRs).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this unblocks you for now, seems fine to me. We can always improve this. @elastic/es-delivery should have some eyes on this though.

description = 'Assembles native shared library for the host architecture and copies to libs/native/libraries/build/platform/'
// check if `LOCAL_VEC_BINARY_OS` is set, if not throw an error to prevent accidental overwrites
if (System.getenv("LOCAL_VEC_BINARY_OS") == null){
throw new GradleException("LOCAL_VEC_BINARY_OS is set, skipping copy to prevent overwriting local binary.")
}
dependsOn "buildSharedLibrary"
doLast {
// output dir is lowercased System.getenv("LOCAL_VEC_BINARY_OS") and arch
// we must translate platformName to match expected arch naming
def copyArch = platformName.equals("amd64") ? "x64" : platformName
def outputDir = file("../../native/libraries/build/platform/" + System.getenv("LOCAL_VEC_BINARY_OS").toLowerCase() + "-" + copyArch)
copy {
from layout.buildDirectory.dir('output')
into outputDir
duplicatesStrategy = 'INCLUDE'
}
println "Copied built libvec to " + outputDir.toString()
}
}