build_pg_ext.py

import distutils.sysconfig as sysconfig
import json
import os
import sys
import platform
import requests

"""
Usage: python3 scripts/build_pg_ext.py debug                               #Debug build
Usage: python3 scripts/build_pg_ext.py dev                                 #Develop build
Usage: python3 scripts/build_pg_ext.py prod                                #Release build
Usage: python3 scripts/build_pg_ext.py debug --deeplake-shared             #Debug build with shared deeplake_api linking
Usage: python3 scripts/build_pg_ext.py debug --deeplake-static             #Debug build with static deeplake_api linking (force)
Usage: python3 scripts/build_pg_ext.py dev --pg-versions 17,18             #Build for PostgreSQL 17 and 18
Usage: python3 scripts/build_pg_ext.py dev --pg-versions 17                #Build for PostgreSQL 17 only
Usage: python3 scripts/build_pg_ext.py prod --pg-versions all              #Build for all supported PostgreSQL versions
"""

def get_pinned_version():
    """
    Read the pinned deeplake API version from DEEPLAKE_API_VERSION file.
    """
    # Look for version file in repo root (one level up from scripts/)
    script_dir = os.path.dirname(os.path.abspath(__file__))
    repo_root = os.path.dirname(script_dir)
    version_file = os.path.join(repo_root, "DEEPLAKE_API_VERSION")

    if not os.path.exists(version_file):
        raise Exception(f"Version file not found: {version_file}")

    with open(version_file, 'r') as f:
        version = f.read().strip()

    if not version:
        raise Exception(f"Version file is empty: {version_file}")

    return version


def download_api_lib(api_root_dir, overwrite=True):
    """
    Download and extract the full deeplake API library including:
    - include/ headers
    - lib/ libraries
    - cmake configurations
    - pkg-config files
    """
    # create directory for the deeplake api library if it does not exist
    os.makedirs(api_root_dir, exist_ok=True)

    machine = platform.machine()

    # Get pinned version from DEEPLAKE_API_VERSION file
    version = get_pinned_version()
    print(f"Using pinned deeplake API version: {version}")

    # Check if library already exists
    lib_dir = os.path.join(api_root_dir, "lib")
    version_marker = os.path.join(api_root_dir, ".version")

    if os.path.exists(version_marker) and not overwrite:
        with open(version_marker, 'r') as f:
            existing_version = f.read().strip()
        if existing_version == version:
            print(f"Library version {version} already exists. Skipping download.")
            return
        else:
            print(f"Found existing version {existing_version}, but pinned version is {version}. Downloading...")

    # Construct asset name based on platform
    archive_name = f"deeplake-api-{version}-linux-{machine}"
    zip_archive_name = f"{archive_name}.zip"
    tar_archive_name = f"{archive_name}.tar.gz"

    # Construct download URL directly (GitHub releases follow a predictable URL pattern)
    asset_url = f"https://github.com/activeloopai/deeplake/releases/download/v{version}/{zip_archive_name}"

    print(f"Downloading prebuilt api libraries from {asset_url} ...")

    response = requests.get(asset_url)
    if response.status_code != 200:
        raise Exception(f"Failed to download api libraries from {asset_url}. Status code: {response.status_code}")

    with open(f"{zip_archive_name}", "wb") as f:
        f.write(response.content)

    print("Extracting archives...")
    err = os.system(f"unzip -o {zip_archive_name} -d /tmp && rm {zip_archive_name}")
    if err:
        raise Exception(f"Failed to extract api libraries. Command exited with code {err}.")

    err = os.system(f"tar -xzf /tmp/{tar_archive_name} -C /tmp && rm /tmp/{tar_archive_name}")
    if err:
        raise Exception(f"Failed to extract api libraries. Command exited with code {err}.")

    extracted_path = f"/tmp/{archive_name}"

    # Remove existing library directory if overwrite is enabled
    if overwrite:
        print(f"Removing existing library at {api_root_dir}...")
        err = os.system(f"rm -rf {api_root_dir}/*")
        if err:
            raise Exception(f"Failed to remove existing library. Command exited with code {err}.")

    # Copy entire library structure (include/, lib/, cmake configs, pkg-config, etc.)
    print(f"Installing full library structure to {api_root_dir}...")
    err = os.system(f"cp -r {extracted_path}/* {api_root_dir}/ && rm -rf {extracted_path}")
    if err:
        raise Exception(f"Failed to copy api library structure. Command exited with code {err}.")

    # Write version marker
    with open(version_marker, 'w') as f:
        f.write(version)

    print(f"Successfully installed deeplake API library version {version}")

def run(mode: str, incremental: bool, deeplake_link_type: str = None, pg_versions: list = None):
    modes = ["debug", "dev", "prod"]
    if mode not in modes:
        raise Exception(f"Invalid mode - '{mode}'. Possible values - {', '.join(modes)}")

    os.chdir("cpp")

    # include path to specific verison from manylinux docker container in the path, so vcpkg_find_acquire_package can find it
    os.putenv("PATH", os.getenv("PATH")+f":/opt/_internal/cpython-{sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]}/bin/")

    preset = f"--preset=deeplake-pg-{mode}-windows" if platform.system() == "Windows" else f"--preset=deeplake-pg-{mode}"

    try:
        if not incremental:
            # Download DeepLake API library before CMake configuration
            # (CMake's find_package needs it to exist during configuration)
            download_api_lib(".ext/deeplake_api")

            cmake_cmd = (f"cmake "
                         f"{preset} ")

            architectures = os.environ.get("CMAKE_OSX_ARCHITECTURES", "")
            if architectures:
                cmake_cmd += f"-D CMAKE_OSX_ARCHITECTURES={architectures} "

            # Add PostgreSQL version options if specified
            if pg_versions is not None:
                supported_versions = {17, 18}
                # Set all versions to OFF by default
                for ver in supported_versions:
                    if ver in pg_versions:
                        cmake_cmd += f"-D BUILD_PG_{ver}=ON "
                    else:
                        cmake_cmd += f"-D BUILD_PG_{ver}=OFF "

            # Add deeplake linking type option if specified
            if deeplake_link_type == "shared":
                cmake_cmd += "-D USE_DEEPLAKE_SHARED=ON "
            elif deeplake_link_type == "static":
                cmake_cmd += "-D USE_DEEPLAKE_SHARED=OFF "
            # If None, let CMake auto-detect based on file existence

            err = os.system(cmake_cmd)
            if err:
                raise Exception(
                    f"Cmake command failed with exit code {err}. Full command: `{cmake_cmd}`"
                )

        make_cmd = f"cmake --build {preset}"
        err = os.system(make_cmd)
        if err:
            raise Exception(f"Command `{make_cmd}` failed with exit code {err}.")

        # Install the built extension to PostgreSQL directories
        preset_name = f"deeplake-pg-{mode}-windows" if platform.system() == "Windows" else f"deeplake-pg-{mode}"
        install_dir = f"../builds/{preset_name}"
        install_cmd = f"cmake --install {install_dir}"
        print(f"Installing extension to PostgreSQL directories...")
        err = os.system(install_cmd)
        if err:
            raise Exception(f"Command `{install_cmd}` failed with exit code {err}.")

    finally:
        os.chdir("..")
        write_mode(mode)

def read_mode():
    try:
        with open('.buildinfo') as f:
            try:
                data = json.load(f)
            except Exception as e:
                raise Exception("No previous mode found for incremental build. Please run full build.")

        if "mode" in data["deeplake-pg"]:
            return data["deeplake-pg"]["mode"]
    except Exception as e:
        raise Exception("No previous mode found for incremental build. Please run full build.")


def write_mode(mode: str):
    data = dict()
    try:
        with open('.buildinfo', 'r') as f:
            try:
                data = json.load(f)
            except Exception as e:
                pass
    except Exception as e:
        pass

    if not "deeplake-pg" in data.keys():
        data["deeplake-pg"] = dict()

    data["deeplake-pg"]["mode"] = mode

    with open('.buildinfo', 'w') as f:
        f.write(json.dumps(data))


if __name__ == "__main__":
    if len(sys.argv) == 1:
        run(mode=read_mode(), incremental=True)
    else:
        mode = sys.argv[1]
        deeplake_link_type = None
        pg_versions = None

        # Parse optional flags
        i = 2
        while i < len(sys.argv):
            arg = sys.argv[i]
            if arg == "--deeplake-shared":
                deeplake_link_type = "shared"
                i += 1
            elif arg == "--deeplake-static":
                deeplake_link_type = "static"
                i += 1
            elif arg == "--pg-versions":
                if i + 1 >= len(sys.argv):
                    raise Exception("--pg-versions requires a value (e.g., '17,18' or 'all')")
                versions_str = sys.argv[i + 1]
                if versions_str == "all":
                    pg_versions = [17, 18]
                else:
                    try:
                        pg_versions = [int(v.strip()) for v in versions_str.split(',')]
                        # Validate versions
                        supported = {17, 18}
                        invalid = set(pg_versions) - supported
                        if invalid:
                            raise Exception(f"Invalid PostgreSQL versions: {invalid}. Supported: {supported}")
                    except ValueError:
                        raise Exception(f"Invalid --pg-versions format: '{versions_str}'. Use comma-separated numbers (e.g., '17,18') or 'all'")
                i += 2
            else:
                raise Exception(f"Invalid option '{arg}'. Use --deeplake-shared, --deeplake-static, or --pg-versions")

        run(mode=mode, incremental=False, deeplake_link_type=deeplake_link_type, pg_versions=pg_versions)