diff --git a/build_docker.sh b/build_docker.sh index c3446c1c1..b8c7d96d4 100755 --- a/build_docker.sh +++ b/build_docker.sh @@ -16,18 +16,53 @@ BUILD_ARGUMENTS=() mapfile -t BUILD_ARGUMENTS < <(compgen -e | sed -nE '/^(.*)(_proxy|_PROXY)$/{s/^/--build-arg=/;p}') BUILD_ARGUMENTS+=( "--progress=plain" "--network=host" "--build-arg" "IMAGE_CACHE_REGISTRY=${IMAGE_CACHE_REGISTRY}" ) -docker buildx build "${BUILD_ARGUMENTS[@]}" -t "${IMAGE_REGISTRY}/mcm/sdk:${IMAGE_TAG}" -f "${SCRIPT_DIR}/sdk/Dockerfile" "$@" "${SCRIPT_DIR}" +MCM_SDK="$(echo docker buildx build "${BUILD_ARGUMENTS[@]}" -t "${IMAGE_REGISTRY}/mcm/sdk:${IMAGE_TAG}" -f "${SCRIPT_DIR}/sdk/Dockerfile" "$@" "${SCRIPT_DIR}")" +MCM_MEDIA_PROXY="$(echo docker buildx build "${BUILD_ARGUMENTS[@]}" -t "${IMAGE_REGISTRY}/mcm/media-proxy:${IMAGE_TAG}" --target media-proxy -f "${SCRIPT_DIR}/media-proxy/Dockerfile" "$@" "${SCRIPT_DIR}")" +MCM_MTL_MANAGER="$(echo docker buildx build "${BUILD_ARGUMENTS[@]}" -t "${IMAGE_REGISTRY}/mcm/mtl-manager:${IMAGE_TAG}" --target mtl-manager -f "${SCRIPT_DIR}/media-proxy/Dockerfile" "$@" "${SCRIPT_DIR}")" +MCM_MESH_AGENT="$(echo docker buildx build "${BUILD_ARGUMENTS[@]}" -t "${IMAGE_REGISTRY}/mcm/mesh-agent:${IMAGE_TAG}" --target mesh-agent -f "${SCRIPT_DIR}/media-proxy/Dockerfile" "$@" "${SCRIPT_DIR}")" -docker buildx build "${BUILD_ARGUMENTS[@]}" -t "${IMAGE_REGISTRY}/mcm/media-proxy:${IMAGE_TAG}" --target media-proxy -f "${SCRIPT_DIR}/media-proxy/Dockerfile" "$@" "${SCRIPT_DIR}" -docker buildx build "${BUILD_ARGUMENTS[@]}" -t "${IMAGE_REGISTRY}/mcm/mtl-manager:${IMAGE_TAG}" --target mtl-manager -f "${SCRIPT_DIR}/media-proxy/Dockerfile" "$@" "${SCRIPT_DIR}" -docker buildx build "${BUILD_ARGUMENTS[@]}" -t "${IMAGE_REGISTRY}/mcm/mesh-agent:${IMAGE_TAG}" --target mesh-agent -f "${SCRIPT_DIR}/media-proxy/Dockerfile" "$@" "${SCRIPT_DIR}" +FFMPEG_7_0="$(echo docker buildx build "${BUILD_ARGUMENTS[@]}" --build-arg FFMPEG_VER="7.0" -t "${IMAGE_REGISTRY}/mcm/ffmpeg:${IMAGE_TAG}" -f "${SCRIPT_DIR}/ffmpeg-plugin/Dockerfile" "$@" "${SCRIPT_DIR}")" +FFMPEG_6_1="$(echo docker buildx build "${BUILD_ARGUMENTS[@]}" --build-arg FFMPEG_VER="6.1" -t "${IMAGE_REGISTRY}/mcm/ffmpeg:6.1-${IMAGE_TAG}" -f "${SCRIPT_DIR}/ffmpeg-plugin/Dockerfile" "$@" "${SCRIPT_DIR}")" -docker buildx build "${BUILD_ARGUMENTS[@]}" --build-arg FFMPEG_VER="7.0" -t "${IMAGE_REGISTRY}/mcm/ffmpeg:${IMAGE_TAG}" -f "${SCRIPT_DIR}/ffmpeg-plugin/Dockerfile" "$@" "${SCRIPT_DIR}" -docker buildx build "${BUILD_ARGUMENTS[@]}" --build-arg FFMPEG_VER="6.1" -t "${IMAGE_REGISTRY}/mcm/ffmpeg:6.1-${IMAGE_TAG}" -f "${SCRIPT_DIR}/ffmpeg-plugin/Dockerfile" "$@" "${SCRIPT_DIR}" +export MCM_SDK +export MCM_MEDIA_PROXY +export MCM_MTL_MANAGER +export MCM_MESH_AGENT +export FFMPEG_7_0 +export FFMPEG_6_1 -docker tag "${IMAGE_REGISTRY}/mcm/sdk:${IMAGE_TAG}" "mcm/sample-app:${IMAGE_TAG}" -docker tag "${IMAGE_REGISTRY}/mcm/media-proxy:${IMAGE_TAG}" "mcm/media-proxy:${IMAGE_TAG}" -docker tag "${IMAGE_REGISTRY}/mcm/mtl-manager:${IMAGE_TAG}" "mcm/mtl-manager:${IMAGE_TAG}" -docker tag "${IMAGE_REGISTRY}/mcm/mesh-agent:${IMAGE_TAG}" "mcm/mesh-agent:${IMAGE_TAG}" -docker tag "${IMAGE_REGISTRY}/mcm/ffmpeg:${IMAGE_TAG}" "mcm/ffmpeg:${IMAGE_TAG}" -docker tag "${IMAGE_REGISTRY}/mcm/ffmpeg:6.1-${IMAGE_TAG}" "mcm/ffmpeg:6.1-${IMAGE_TAG}" +function build_and_tag_all_dockerfiles() { + $MCM_SDK + $MCM_MEDIA_PROXY + $MCM_MTL_MANAGER + $MCM_MESH_AGENT + $FFMPEG_7_0 + $FFMPEG_6_1 + + docker tag "${IMAGE_REGISTRY}/mcm/sdk:${IMAGE_TAG}" "mcm/sample-app:${IMAGE_TAG}" + docker tag "${IMAGE_REGISTRY}/mcm/media-proxy:${IMAGE_TAG}" "mcm/media-proxy:${IMAGE_TAG}" + docker tag "${IMAGE_REGISTRY}/mcm/mtl-manager:${IMAGE_TAG}" "mcm/mtl-manager:${IMAGE_TAG}" + docker tag "${IMAGE_REGISTRY}/mcm/mesh-agent:${IMAGE_TAG}" "mcm/mesh-agent:${IMAGE_TAG}" + docker tag "${IMAGE_REGISTRY}/mcm/ffmpeg:${IMAGE_TAG}" "mcm/ffmpeg:${IMAGE_TAG}" + docker tag "${IMAGE_REGISTRY}/mcm/ffmpeg:6.1-${IMAGE_TAG}" "mcm/ffmpeg:6.1-${IMAGE_TAG}" +} +function print_build_help_dockerfiles() { + echo -e MCM_SDK="${MCM_SDK}\n" + echo -e MCM_MEDIA_PROXY="${MCM_MEDIA_PROXY}\n" + echo -e MCM_MTL_MANAGER="${MCM_MTL_MANAGER}\n" + echo -e MCM_MESH_AGENT="${MCM_MESH_AGENT}\n" + echo -e FFMPEG_7_0="${FFMPEG_7_0}\n" + echo -e FFMPEG_6_1="${FFMPEG_6_1}\n" +} + +# Allow sourcing of the script. Run only when asked +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then +# ======= RUNTIME EXEC START + build_and_tag_all_dockerfiles + print_build_help_dockerfiles +# ======= RUNTIME EXEC STOP +else +# ======= SOURCED SCRIPT START + return 0 +# ======= RUNTIME SCRIPT STOP +fi diff --git a/media-proxy/Dockerfile b/media-proxy/Dockerfile index 0ceb773a9..8768c98bf 100644 --- a/media-proxy/Dockerfile +++ b/media-proxy/Dockerfile @@ -8,30 +8,27 @@ ARG IMAGE_NAME=library/ubuntu:22.04@sha256:a6d2b38300ce017add71440577d5b0a90460d FROM ${IMAGE_CACHE_REGISTRY}/${IMAGE_NAME} AS builder +ARG MCM_DIR="/opt/mcm" ARG VERSIONS_FILE="versions.env" -COPY "${VERSIONS_FILE}" /opt/versions.env -ENV BASH_ENV=/opt/versions.env +ARG GITHUB_CREDENTIALS="" +COPY . "${MCM_DIR}" +COPY "${VERSIONS_FILE}" "${MCM_DIR}/versions.env" -ARG MTL_VER="" -ARG DPDK_VER="" -ARG GPRC_VER="" -ARG JPEGXS_VER="" -ARG LIBFABRIC_VER="" +SHELL ["/bin/bash", "-ex", "-o", "pipefail", "-c"] +ENV BASH_ENV="${MCM_DIR}/versions.env" -ARG MCM_DIR="/opt/mcm" -ARG MTL_DIR="/opt/mtl" -ARG DPDK_DIR="/opt/dpdk" -ARG XDP_DIR="/opt/xdp" -ARG GRPC_DIR="/opt/grpc" -ARG JPEGXS_DIR="/opt/jpegxs" -ARG LIBFABRIC_DIR="/opt/libfabric" +ENV PREFIX_DIR="/install" +ENV REPO_DIR="${MCM_DIR}" +ENV BUILD_DIR="${REPO_DIR}/_build}" +ENV GRPC_DIR="${BUILD_DIR}/grpc" +ENV DPDK_DIR="${BUILD_DIR}/dpdk" +ENV XDP_DIR="${BUILD_DIR}/xdp" +ENV JPEGXS_DIR="${BUILD_DIR}/jpegxs" +ENV LIBFABRIC_DIR="${BUILD_DIR}/libfabric" -ARG GITHUB_CREDENTIALS="" -ARG PREFIX_DIR="/install" ENV PATH="${PREFIX_DIR}/bin:/root/.local/bin:/root/bin:/root/usr/bin:$PATH" ENV PKG_CONFIG_PATH=/usr/lib/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib/x86_64-linux-gnu/pkgconfig -SHELL ["/bin/bash", "-ex", "-o", "pipefail", "-c"] # Install package dependencies RUN apt-get update --fix-missing && \ apt-get full-upgrade -y && \ @@ -56,73 +53,11 @@ RUN curl -Lf "https://go.dev/dl/go${GOLANG_GO_VER}.linux-amd64.tar.gz" -o "/usr/ go install "${GOLANG_PROTOBUF_GEN}" && \ go install "${GOLANG_GRPC_GEN}" -# Clone MTL, DPDK and xdp-tools repo -WORKDIR ${DPDK_DIR} -RUN git clone --branch main https://${GITHUB_CREDENTIALS}github.com/OpenVisualCloud/Media-Transport-Library "${MTL_DIR}" && \ - git -C "${MTL_DIR}" checkout ${MTL_VER} && \ - git clone --branch v${DPDK_VER} --depth 1 https://github.com/DPDK/dpdk.git "${DPDK_DIR}" && \ - git clone --recurse-submodules https://github.com/xdp-project/xdp-tools.git "${XDP_DIR}" && \ - git clone --depth 1 --branch "${LIBFABRIC_VER}" https://github.com/ofiwg/libfabric "${LIBFABRIC_DIR}" && \ - git clone --branch ${GPRC_VER} --recurse-submodules --depth 1 --shallow-submodules https://github.com/grpc/grpc "${GRPC_DIR}" && \ - git clone https://github.com/OpenVisualCloud/SVT-JPEG-XS "${JPEGXS_DIR}" && \ - git -C ${JPEGXS_DIR} checkout ${JPEGXS_VER} && \ - patch -p1 -i <(cat "${MTL_DIR}/patches/dpdk/${DPDK_VER}/"*.patch) -RUN patch -d "${DPDK_DIR}" -p1 < "${MTL_DIR}/patches/dpdk/${DPDK_VER}/hdr_split/"*.patch || true -RUN patch -d "${DPDK_DIR}" -p1 < "${MTL_DIR}/patches/dpdk/${DPDK_VER}/tsn/"*.patch || true - -# Build the xdp-tools project -WORKDIR ${XDP_DIR} -RUN ./configure && \ - make && \ - make install && \ - DESTDIR="${PREFIX_DIR}" make install && \ - make -C "${XDP_DIR}/lib/libbpf/src" && \ - make -C "${XDP_DIR}/lib/libbpf/src" install && \ - DESTDIR="${PREFIX_DIR}" make -C "${XDP_DIR}/lib/libbpf/src" install - -WORKDIR ${LIBFABRIC_DIR} -RUN ./autogen.sh && \ - ./configure --enable-verbs && \ - make "-j$(nproc)" && \ - make install && \ - DESTDIR="${PREFIX_DIR}" make install - -# Build the DPDK -WORKDIR ${DPDK_DIR} -RUN meson setup build && \ - ninja -C build && \ - meson install -C build && \ - DESTDIR="${PREFIX_DIR}" meson install -C build - -# Build MTL -WORKDIR ${MTL_DIR} -RUN ./build.sh && \ - DESTDIR="${PREFIX_DIR}" meson install -C build - -# gRPC -WORKDIR ${GRPC_DIR}/cmake/build -RUN cmake -DgRPC_BUILD_TESTS=OFF -DgRPC_INSTALL=ON ../.. && \ - make "-j$(nproc)" && \ - make install && \ - cmake -DgRPC_BUILD_TESTS=ON -DgRPC_INSTALL=ON ../.. && \ - make "-j$(nproc)" grpc_cli && \ - cp grpc_cli "${PREFIX_DIR}/usr/local/bin/" - -# Build and install JPEG XS -WORKDIR ${JPEGXS_DIR}/Build/linux -RUN ./build.sh release install && \ - ./build.sh release --prefix "${PREFIX_DIR}/usr/local" install - -# Build and install JPEG XS imtl plugin -WORKDIR ${JPEGXS_DIR}/imtl-plugin -RUN ./build.sh build-only && \ - DESTDIR="${PREFIX_DIR}" meson install --no-rebuild -C build && \ - mkdir -p /install/usr/local/etc/ && \ - cp ${JPEGXS_DIR}/imtl-plugin/kahawai.json /install/usr/local/etc/jpegxs.json +WORKDIR "${MCM_DIR}" +RUN scripts/setup_build_env.sh # Build MCM WORKDIR ${MCM_DIR} -COPY . ${MCM_DIR} RUN ./build.sh "${PREFIX_DIR}" && \ cp "${MTL_DIR}/build/manager/MtlManager" "${PREFIX_DIR}/usr/local/bin/" && \ cp -fR "/usr/local/lib/bpf/"* "${PREFIX_DIR}/usr/local/lib/bpf/" && \ @@ -137,7 +72,7 @@ RUN ./build.sh "${PREFIX_DIR}" && \ ARG IMAGE_CACHE_REGISTRY ARG IMAGE_NAME -FROM ${IMAGE_CACHE_REGISTRY}/${IMAGE_NAME} as media-proxy +FROM ${IMAGE_CACHE_REGISTRY}/${IMAGE_NAME} AS media-proxy LABEL maintainer="milosz.linkiewicz@intel.com,konstantin.ilichev@intel.com" LABEL org.opencontainers.image.title="Media Communications Mesh Media Proxy" @@ -184,7 +119,7 @@ SHELL ["/bin/bash", "-c"] ENTRYPOINT ["/usr/local/bin/media_proxy"] HEALTHCHECK --interval=30s --timeout=5s CMD ps aux | grep "media_proxy" || exit 1 -FROM media-proxy as mtl-manager +FROM media-proxy AS mtl-manager LABEL maintainer="milosz.linkiewicz@intel.com,konstantin.ilichev@intel.com" LABEL org.opencontainers.image.title="Media Communications Mesh Mtl Manager" @@ -198,7 +133,7 @@ SHELL ["/bin/bash", "-c"] ENTRYPOINT ["/usr/local/bin/MtlManager"] HEALTHCHECK --interval=30s --timeout=5s CMD if(( $(ps aux | grep "MtlManager" | wc -l) != 2 )); then exit 1; fi -FROM media-proxy as mesh-agent +FROM media-proxy AS mesh-agent LABEL maintainer="milosz.linkiewicz@intel.com,konstantin.ilichev@intel.com" LABEL org.opencontainers.image.title="Media Communications Mesh Mesh Agent" diff --git a/scripts/common.sh b/scripts/common.sh index bc88aae70..9a514d926 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,52 +1,22 @@ #!/bin/bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2024 Intel Corporation +# Copyright 2025 Intel Corporation REPO_DIR="$(readlink -f "$(dirname -- "${BASH_SOURCE[0]}")/..")" -export REPO_DIR -export BUILD_DIR="${BUILD_DIR:-${REPO_DIR}/_build}" -export DRIVERS_DIR="${DRIVERS_DIR:-/opt/intel/drivers}" -export PREFIX_DIR="${PREFIX_DIR:-${REPO_DIR}/_install}" - -MCM_VERSIONS_FILE_PATH="$(readlink -f "${MCM_VERSIONS_FILE:-${REPO_DIR}/versions.env}")" -export MCM_VERSIONS_FILE_PATH +MCM_VERSIONS_FILE_PATH="$(readlink -f "${MCM_VERSIONS_FILE_PATH:-${REPO_DIR}/versions.env}")" +if [[ -f "${MCM_VERSIONS_FILE_PATH}" ]]; then # shellcheck source="versions.env" -. "${MCM_VERSIONS_FILE_PATH}" - -export MTL_DIR="${BUILD_DIR}/mtl" -MTL_VERSIONS_FILE_PATH="${MTL_VERSIONS_FILE:-${MTL_DIR}/versions.env}" - -if [[ -f "${MTL_VERSIONS_FILE_PATH}" ]]; then - MTL_VERSIONS_FILE_PATH="$(readlink -f "${MTL_VERSIONS_FILE_PATH}")" - export MTL_VERSIONS_FILE_PATH -# shellcheck source=/dev/null - . "${MTL_VERSIONS_FILE_PATH}" + . "${MCM_VERSIONS_FILE_PATH}" fi -export DPDK_DIR="${BUILD_DIR}/dpdk" -export XDP_DIR="${BUILD_DIR}/xdp" -export BPF_DIR="${XDP_DIR}/lib/libbpf" -export GRPC_DIR="${BUILD_DIR}/grpc" -export JPEGXS_DIR="${BUILD_DIR}/jpegxs" -export LIBFABRIC_DIR="${BUILD_DIR}/libfabric" -export LIBFDT_DIR="${BUILD_DIR}/libfdt" -export JSONC_DIR="${BUILD_DIR}/json-c" -export CMAKE_DIR="${BUILD_DIR}/cmake" -export NASM_DIR="${BUILD_DIR}/nasm" - -export ICE_DIR="${DRIVERS_DIR}/ice/${ICE_VER}" -export IRDMA_DIR="${DRIVERS_DIR}/irdma/${IRDMA_VER}" - PM="${PM:-apt-get}" -KERNEL_VERSION="${KERNEL_VERSION:-$(uname -r)}" -export TZ="${TZ:-Europe/Warsaw}" -export NPROC="${NPROC:-$(nproc)}" +NPROC="${NPROC:-$(nproc)}" if ! grep "/root/.local/bin" <<< "${PATH}" > /dev/null 2>&1; then - export PATH="/root/.local/bin:/root/bin:/root/usr/bin:${PATH}" - export PKG_CONFIG_PATH="/usr/lib/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib/x86_64-linux-gnu/pkgconfig:${PKG_CONFIG_PATH}" + export PATH="/root/.local/bin:/root/bin:/root/usr/bin:${PATH}" + export PKG_CONFIG_PATH="/usr/lib/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib/x86_64-linux-gnu/pkgconfig:${PKG_CONFIG_PATH}" fi BOLD="${BOLD:-\e[1;}" diff --git a/scripts/setup_build_env.sh b/scripts/setup_build_env.sh index a5f21bc7a..c4fc16a61 100755 --- a/scripts/setup_build_env.sh +++ b/scripts/setup_build_env.sh @@ -3,12 +3,6 @@ SCRIPT_DIR="$(readlink -f "$(dirname -- "${BASH_SOURCE[0]}")")" . "${SCRIPT_DIR}/common.sh" -export PM="${PM:-apt-get}" -export DEBIAN_FRONTEND="noninteractive" - -export KERNEL_VERSION="${KERNEL_VERSION:-$(uname -r)}" -export INSTALL_USE_CUSTOM_PATH="${INSTALL_USE_CUSTOM_PATH:-false}" - SYS_PKG_CONFIG_PATH="${SYS_PKG_CONFIG_PATH:-/usr/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/local/lib/x86_64-linux-gnu/pkgconfig}" if [[ "${PKG_CONFIG_PATH}" != *"${SYS_PKG_CONFIG_PATH}"* ]]; then export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${SYS_PKG_CONFIG_PATH}" diff --git a/scripts/setup_ice_irdma.sh b/scripts/setup_ice_irdma.sh index a88140943..78bc6cf20 100755 --- a/scripts/setup_ice_irdma.sh +++ b/scripts/setup_ice_irdma.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash SCRIPT_DIR="$(readlink -f "$(dirname -- "${BASH_SOURCE[0]}")")" -export WORKING_DIR="${BUILD_DIR:-${REPO_DIR}/build/rdma}" -export PERF_DIR="${DRIVERS_DIR}/perftest" - . "${SCRIPT_DIR}/setup_build_env.sh" function print_usage() @@ -15,11 +12,13 @@ function print_usage() log_info "" log_info "\tall" log_info "\t\t get, build, and install whole stack." - log_info "\tget-ice" - log_info "\t\t get, build, and install ice, rdma, and irdma drivers stack." - log_info "\tget-irdma" - log_info "\t\t setup and pre-configure rdma, irdma and libfabrics ." - log_info "\tget-perftest" + log_info "\tice" + log_info "\t\t get, build, and install ICE (cvl) drivers stack." + log_info "\tiavf" + log_info "\t\t get, build, and install IAVF (virtual functions) drivers stack." + log_info "\tirdma" + log_info "\t\t setup and pre-configure iRDMA and libfabrics." + log_info "\tperftest" log_info "\t\t download and install perftest and dependencies." log_info "\tcheck-mtu" log_info "\t\t fast-check basics in environment." @@ -27,7 +26,7 @@ function print_usage() log_info "\t\t temporarily set MTU to 9000 for given interface." log_info "\trun-perftest " log_info "\t\t execute installed perftests." - log_info "" + log_info "" log_info "\tintel" log_info "\t\t animation in bash" } @@ -79,52 +78,80 @@ function install_os_dependencies() return 0 } -function get_irdma_driver_tgz() { - echo "https://downloadmirror.intel.com/${IRDMA_DMID}/irdma-${IRDMA_VER}.tgz" -} - function get_and_patch_intel_drivers() { log_info "Intel drivers: Starting download and patching actions." - if [[ ! -d "${MTL_DIR}" ]]; then - git_download_strip_unpack "OpenVisualCloud/Media-Transport-Library" "${MTL_VER}" "${MTL_DIR}" - fi + if [[ ! -d "${MTL_DIR}" ]]; then git_download_strip_unpack "OpenVisualCloud/Media-Transport-Library" "${MTL_VER}" "${MTL_DIR}"; fi + if [[ -d "${ICE_DIR}" ]]; then rm -rf "${ICE_DIR}"; fi + + git_download_strip_unpack "intel/ethernet-linux-ice" "refs/tags/v${ICE_VER}" "${ICE_DIR}" + if [ ! -d "${MTL_DIR}/patches/ice_drv/${ICE_VER}/" ]; then - log_error "MTL patch for ICE=v${ICE_VER} could not be found: ${MTL_DIR}/patches/ice_drv/${ICE_VER}" + log_error "No Intel ICE (cvl) patches for ICE=v${ICE_VER} found at ${MTL_DIR}/patches/ice_drv/${ICE_VER}" + log_error "version supported: $(ls "${MTL_DIR}/patches/ice_drv/")" return 1 fi - if [[ -d "${ICE_DIR}" ]]; then - rm -rf "${ICE_DIR}" - fi - git_download_strip_unpack "intel/ethernet-linux-ice" "refs/tags/v${ICE_VER}" "${ICE_DIR}" - pushd "${ICE_DIR}" && \ - patch -p1 -i <(cat "${MTL_DIR}/patches/ice_drv/${ICE_VER}/"*.patch) && \ + patch -p1 -i <(cat "${MTL_DIR}/patches/ice_drv/${ICE_VER}/"*.patch) && \ popd && \ - { log_success "Intel drivers: Finished download and patching actions." && return 0; } || + { log_success "Intel drivers: Finished download and patching actions." && return 0; } || \ { log_error "Intel drivers: Failed to download or patch." && return 1; } } -function build_install_and_config_intel_drivers() +function build_install_and_config_ice_driver() { - log_info "Intel ICE: Driver starting the build and install workflow." - if ! as_root make "-j${NPROC}" -C "${ICE_DIR}/src" install; then - log_error "Intel ICE: Failed to build and install drivers" - exit 5 - fi - as_root rmmod irdma 2>&1 || true - as_root rmmod ice - as_root modprobe ice - log_success "Intel ICE: Drivers finished install process." + log_info "Intel ICE: Driver starting the build and install workflow." + get_and_patch_intel_drivers - return 0 + as_root make "-j${NPROC}" -C "${ICE_DIR}/src" clean || true + if ! as_root make "-j${NPROC}" -C "${ICE_DIR}/src" ; then + log_error "Intel ICE: Failed to build and install drivers" + exit 5 + fi + if ! as_root make "-j${NPROC}" -C "${ICE_DIR}/src" install; then + log_error "Intel ICE: Failed to build and install drivers" + exit 5 + fi + as_root rmmod irdma 2>/dev/null || true + as_root rmmod ice 2>/dev/null || true + sleep 1 && \ + as_root modprobe ice && \ + log_success "Intel ICE: Drivers finished install process." + return 0 +} + +function build_install_and_config_intel_driver() +{ + build_install_and_config_ice_driver + return $? } -function build_install_and_config_irdma_drivers() +function build_install_and_config_iavf_driver() { - IRDMA_REPO="$(get_irdma_driver_tgz)" + log_info "Intel IAVF: Driver starting the build and install workflow." + git_download_strip_unpack "intel/ethernet-linux-iavf" "refs/tags/v${IAVF_VER}" "${IAVF_DIR}" + + as_root make "-j${NPROC}" -C "${IAVF_DIR}/src" clean || true + if ! as_root make "-j${NPROC}" -C "${IAVF_DIR}/src" ; then + log_error "Intel IAVF: Failed to build and install drivers" + exit 5 + fi + if ! as_root make "-j${NPROC}" -C "${IAVF_DIR}/src" install; then + log_error "Intel IAVF: Failed to build and install drivers" + exit 6 + fi + as_root rmmod iavf || true + sleep 1 && \ + as_root modprobe iavf || true + log_success "Intel IAVF: Drivers finished install process." + return 0 +} + +function build_install_and_config_irdma_driver() +{ + IRDMA_REPO="${IRDMA_REPO:-"https://downloadmirror.intel.com/${IRDMA_DMID}/irdma-${IRDMA_VER}.tgz"}" wget_download_strip_unpack "${IRDMA_REPO}" "${IRDMA_DIR}" - + if pushd "${IRDMA_DIR}"; then "${IRDMA_DIR}/build_core.sh" -y || exit 2 as_root "${IRDMA_DIR}/install_core.sh" || exit 3 @@ -253,14 +280,14 @@ function run_perftest() network_mask="${3:-24}" ip addr show dev "${interface_name}" | grep "${network_address}" || \ ip addr add "${network_address}/${network_mask}" dev "${interface_name}" - taskset -c 1 ib_write_bw --qp=4 --report_gbit -D 60 --tos 96 -R &> "${WORKING_DIR}/perftest_server.log" & + taskset -c 1 ib_write_bw --qp=4 --report_gbit -D 60 --tos 96 -R &> "${PERF_RUN_DIR}/perftest_server.log" & server_pid=$! - ib_write_bw "${network_address}" --qp=4 --report_gbit -D 60 --tos 96 -R &> "${WORKING_DIR}/perftest_client.log" & + ib_write_bw "${network_address}" --qp=4 --report_gbit -D 60 --tos 96 -R &> "${PERF_RUN_DIR}/perftest_client.log" & client_pid=$! log_info "perftest is running. Waiting 60 s..." wait $server_pid wait $client_pid - log_info "perftest completed. See results in $WORKING_DIR/perftest_server.log and ${WORKING_DIR}/perftest_client.log" + log_info "perftest completed. See results in $PERF_RUN_DIR/perftest_server.log and ${PERF_RUN_DIR}/perftest_client.log" ip addr del "${network_address}/${network_mask}" dev "${interface_name}" } @@ -273,38 +300,45 @@ then exit 1 fi - if [[ -f "${WORKING_DIR}" ]]; then - log_error "Can't create rdma directory because of the rdma file ${WORKING_DIR}" + if [[ -f "${PERF_RUN_DIR}" ]]; then + log_error "Can't create rdma directory because of the rdma file ${PERF_RUN_DIR}" exit 1 fi - rm -f "./${WORKING_DIR}/*" - mkdir -p "${WORKING_DIR}" "${PERF_DIR}" - - set -eEo pipefail - - if [[ "${1}" == "get-ice" || "${1}" == "all" ]]; then - install_os_dependencies && \ - get_and_patch_intel_drivers && \ - build_install_and_config_intel_drivers + rm -f "./${PERF_RUN_DIR}/*" + mkdir -p "${PERF_RUN_DIR}" "${PERF_DIR}" + + set -eEo pipefail + + if [[ "${1}" == *"ice" || "${1}" == *"iavf" || "${1}" == *"irdma" || "${1}" == "all" ]]; then + install_os_dependencies + fi + if [[ "${1}" == *"ice" || "${1}" == "all" ]]; then + + build_install_and_config_ice_driver return_code="$?" if [[ "${return_code}" == "0" ]]; then - log_success "Finished: Build and install and configuration of Intel drivers."; + log_success "Finished: Intel ICE (cvl) driver build, install and configuration."; else - log_error "Intel drivers configuration/installation failed." + log_error "Intel ICE drivers configuration/installation failed." fi fi - if [[ "${1}" == "get-irdma" || "${1}" == "all" ]]; then - if [[ "${1}" == "get-irdma" ]]; then - install_os_dependencies && \ - lib_install_fabrics + if [[ "${1}" == *"iavf" || "${1}" == "all" ]]; then + build_install_and_config_iavf_driver + return_code="$?" + if [[ "${return_code}" == "0" ]]; then + log_success "Finished: Intel IAVF driver build, install and configuration."; + else + log_error "Intel IAVF drivers configuration/installation failed." fi - build_install_and_config_irdma_drivers && \ + fi + if [[ "${1}" == *"irdma" || "${1}" == "all" ]]; then + build_install_and_config_irdma_driver && \ config_intel_rdma_driver return_code="$?" if [[ "${return_code}" == "0" ]]; then - log_success "Finished: irdma driver configuration for Intel hardware backend." + log_success "Finished: Intel IRDMA driver build, install and configuration." else - log_error "Intel irdma configuration failed." + log_error "Intel IRDMA drivers configuration/installation failed." exit "${return_code}" fi fi diff --git a/sdk/Dockerfile b/sdk/Dockerfile index bf748d4f8..02b46e31c 100644 --- a/sdk/Dockerfile +++ b/sdk/Dockerfile @@ -8,15 +8,18 @@ ARG IMAGE_NAME=library/ubuntu:22.04@sha256:a6d2b38300ce017add71440577d5b0a90460d FROM ${IMAGE_CACHE_REGISTRY}/${IMAGE_NAME} AS builder +ARG MCM_DIR="/opt/mcm" ARG VERSIONS_FILE="versions.env" COPY "${VERSIONS_FILE}" /opt/versions.env + +SHELL ["/bin/bash", "-exo", "pipefail", "-c"] ENV BASH_ENV=/opt/versions.env -ARG MCM_DIR=/opt/mcm -ARG PREFIX_DIR="/install" -ARG GRPC_DIR="/opt/grpc" +ENV PREFIX_DIR="/install" +ENV REPO_DIR="${MCM_DIR}" +ENV BUILD_DIR="${REPO_DIR}/_build}" +ENV GRPC_DIR="${BUILD_DIR}/grpc" -SHELL ["/bin/bash", "-exo", "pipefail", "-c"] RUN apt-get update && \ apt-get full-upgrade -y && \ apt-get install --no-install-recommends -y \ @@ -48,8 +51,8 @@ LABEL org.opencontainers.image.version="25.03.0" LABEL org.opencontainers.image.vendor="Intel Corporation" LABEL org.opencontainers.image.licenses="BSD 3-Clause License" -ARG MCM_DIR="/opt/mcm" -ARG PREFIX_DIR="/install" +ENV MCM_DIR="/opt/mcm" +ENV PREFIX_DIR="/install" ENV HOME="${MCM_DIR}" ENV DEBIAN_FRONTEND=noninteractive diff --git a/versions.env b/versions.env old mode 100644 new mode 100755 index a341b44cc..9e3452172 --- a/versions.env +++ b/versions.env @@ -1,62 +1,77 @@ +#!/bin/bash + +# shellcheck disable=SC2034 # Versioning file - code is irrelevant +PM="${PM:-apt-get}" +TZ="${TZ:-Europe/Warsaw}" + +DEBIAN_FRONTEND=noninteractive +KERNEL_VERSION="${KERNEL_VERSION:-$(uname -r)}" + +INSTALL_USE_CUSTOM_PATH="${INSTALL_USE_CUSTOM_PATH:-false}" +REPO_DIR="$(readlink -f "$(dirname -- "${BASH_SOURCE[0]}")")" +REPO_DIR="${REPO_DIR:-/opt/intel/mcm}" + +BUILD_DIR="${BUILD_DIR:-${REPO_DIR}/_build}" +DRIVERS_DIR="${DRIVERS_DIR:-/opt/intel/drivers}" +PREFIX_DIR="${PREFIX_DIR:-${REPO_DIR}/_install}" + BPF_VER=42065ea6627ff6e1ab4c65e51042a70fbf30ff7c CMAKE_VER=v3.31.8 -DPDK_VER=23.11 +DPDK_VER=25.03 GPRC_VER=v1.68.2 -ICE_VER=1.14.9 -ICE_DMID=822519 -IRDMA_VER=1.16.10 -IRDMA_DMID=843933 +ICE_VER=2.2.8 +ICE_DMID=859252 +IAVF_VER=4.13.16 +IRDMA_VER=2.0.21 +IRDMA_DMID=859258 JPEGXS_VER=e1030c6c8ee2fb05b76c3fa14cccf8346db7a1fa JSON_C_VER=0.18-20240915 LIBFABRIC_VER=v2.2.0 LIBFDT_VER=v1.7.2 LIBSDL2_VER=2.30.9 -MTL_VER=v25.02 +LIBRIST_COMMIT_VER=9f09a3defd6e59839aae3e3b7b5411213fa04b8a +MTL_VER=v25.09-nightly NASM_RPM_VER=2.15.03-7 +ONE_API_GPU_VER=1.22.3 PERFTEST_VER=24.07.0-0.44 +RUST_HOOK_CARGO_VER=0.5.5 XDP_VER=d7edea3590052581c5fda5f8cfa40ae7be94f05c -IRDMA_REPO=https://downloadmirror.intel.com/${IRDMA_DMID}/irdma-${IRDMA_VER}.tgz +GOLANG_GO_VER=1.23.4 +GOLANG_GRPC_GEN=google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1 +GOLANG_PROTOBUF_GEN=google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.0 + +MTL_DIR="${BUILD_DIR}/mtl" +DPDK_DIR="${BUILD_DIR}/dpdk" +XDP_DIR="${BUILD_DIR}/xdp" +BPF_DIR="${XDP_DIR}/lib/libbpf" +GRPC_DIR="${BUILD_DIR}/grpc" +JPEGXS_DIR="${BUILD_DIR}/jpegxs" +LIBFABRIC_DIR="${BUILD_DIR}/libfabric" +LIBFDT_DIR="${BUILD_DIR}/libfdt" +JSONC_DIR="${BUILD_DIR}/json-c" +CMAKE_DIR="${BUILD_DIR}/cmake" +NASM_DIR="${BUILD_DIR}/nasm" + +MTL_VERSIONS_FILE_PATH="${MTL_VERSIONS_FILE_PATH:-${MTL_DIR}/versions.env}" +if [[ ! -z "${MTL_VERSIONS_FILE_PATH}" && -f "${MTL_VERSIONS_FILE_PATH}" ]]; then + MTL_VERSIONS_FILE_PATH="$(readlink -f "${MTL_VERSIONS_FILE_PATH}")" +# shellcheck source=/dev/null + . "${MTL_VERSIONS_FILE_PATH}" +fi + +ICE_DIR="${DRIVERS_DIR}/ice/${ICE_VER}" +IAVF_DIR="${DRIVERS_DIR}/iavf/${IAVF_VER}" +IRDMA_DIR="${DRIVERS_DIR}/irdma/${IRDMA_VER}" +PERF_DIR="${DRIVERS_DIR}/perftest" +PERF_RUN_DIR="${DRIVERS_DIR}/perftest_run" + ICE_REPO=https://downloadmirror.intel.com/${ICE_DMID}/ice-${ICE_VER}.tar.gz +IRDMA_REPO=https://downloadmirror.intel.com/${IRDMA_DMID}/irdma-${IRDMA_VER}.tgz +IAVF_REPO=https://github.com/intel/ethernet-linux-iavf/releases/download/v${IAVF_VER}/iavf-${IAVF_VER}.tar.gz JSONC_REPO=https://github.com/json-c/json-c/archive/refs/tags/json-c-${JSON_C_VER}.tar.gz LIBFDT_REPO=https://github.com/dgibson/dtc/archive/refs/tags/${LIBFDT_VER}.tar.gz NASM_RPM_LINK=https://rpmfind.net/linux/centos-stream/9-stream/CRB/x86_64/os/Packages/nasm-${NASM_RPM_VER}.el9.x86_64.rpm PERF_REPO=https://github.com/linux-rdma/perftest/releases/download/${PERFTEST_VER}/perftest-${PERFTEST_VER}.g57725f2.tar.gz - -GOLANG_GO_VER=1.23.4 -GOLANG_GRPC_GEN=google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1 -GOLANG_PROTOBUF_GEN=google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.0 - -DEBIAN_FRONTEND=noninteractive -TZ=Europe/Warsaw - -# ICE E810 Driver v1.16.3 -ICE_VER_1_16_3=1.16.3 -ICE_DMID_1_16_3=843928 - -# ICE E810 Driver v1.14.9 -ICE_VER_1_14_9=1.14.9 -ICE_DMID_1_14_9=822519 - -ICE_VER_1_15_5=1.15.5 -ICE_VER_1_15_4=1.15.4 -ICE_VER_1_14_13=1.14.13 -ICE_VER_1_14_11=1.14.11 -ICE_VER_1_14_9_2=1.14.9.2 -ICE_VER_1_13_7=1.13.7 - -# iRDMA v1.14.17 (ICE-14-9) -IRDMA_DMID_1_14_17=822708 -IRDMA_VER_1_14_17=1.14.17 - -# iRDMA v1.15.11 -IRDMA_DMID_1_15_11=832291 -IRDMA_VER_1_15_11=1.15.11 - -# iRDMA v1.15.15 -IRDMA_DMID_1_15_15=842428 -IRDMA_VER_1_15_15=1.15.15 - -# iRDMA v1.16.10 -IRDMA_DMID_1_16_10=843933 -IRDMA_VER_1_16_10=1.16.10 +DPDK_REPO=https://github.com/dpdk/dpdk/archive/refs/tags/v${DPDK_VER}.tar.gz +ONE_API_REPO=https://github.com/oneapi-src/level-zero/archive/refs/tags/v${ONE_API_GPU_VER}.tar.gz