diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index b3a8883a..7a8d09b2 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -14,7 +14,11 @@ on: jobs: python: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v2 diff --git a/ci-common/Dockerfile b/ci-common/Dockerfile index 06b1710e..4c27a9d0 100644 --- a/ci-common/Dockerfile +++ b/ci-common/Dockerfile @@ -121,7 +121,7 @@ COPY ../scripts/common/install_conan.sh /tmp/ RUN --mount=type=cache,sharing=private,target=/tmp/downloads \ /tmp/install_conan.sh -COPY ../packages/conan/settings /opt/conan_home/.conan +COPY ../packages/conan/settings_linux /opt/conan_home/.conan ENV CONAN_USER_HOME=/opt/conan_home diff --git a/ci-common/image.yaml b/ci-common/image.yaml index 264cca49..3c647d04 100644 --- a/ci-common/image.yaml +++ b/ci-common/image.yaml @@ -53,5 +53,5 @@ docker_commands: | RUN --mount=type=cache,sharing=private,target=/tmp/downloads \ /tmp/install_conan.sh - COPY ../packages/conan/settings /opt/conan_home/.conan + COPY ../packages/conan/settings_linux /opt/conan_home/.conan ENV CONAN_USER_HOME=/opt/conan_home diff --git a/packages/conan/.gitignore b/packages/conan/.gitignore index 48756ad1..f4a97956 100644 --- a/packages/conan/.gitignore +++ b/packages/conan/.gitignore @@ -1,5 +1,5 @@ data/ -settings/.conan.db -settings/cacert.pem -settings/artifacts.properties +settings_*/.conan.db +settings_*/cacert.pem +settings_*/artifacts.properties ccache/ \ No newline at end of file diff --git a/packages/conan/recipes/clang/conanfile.py b/packages/conan/recipes/clang/conanfile.py index 1e9a0072..46d81223 100644 --- a/packages/conan/recipes/clang/conanfile.py +++ b/packages/conan/recipes/clang/conanfile.py @@ -1,6 +1,6 @@ from conans import ConanFile, CMake, tools -import os.path +import platform import os @@ -34,7 +34,7 @@ def _source_subfolder(self): return "source" def build_requirements(self): - if tools.Version(self.version) > "11": + if tools.Version(self.version) > "11" and platform.system() == "Linux": self.build_requires(f"python/3.9.7@{self.user}/vfx2022") def configure(self): @@ -53,7 +53,8 @@ def configure(self): def _configure_cmake(self): cmake = CMake(self) - cmake.definitions["GCC_INSTALL_PREFIX"] = os.environ["GCC_INSTALL_PREFIX"] + if platform.system() == "Linux": + cmake.definitions["GCC_INSTALL_PREFIX"] = os.environ["GCC_INSTALL_PREFIX"] cmake.definitions["LLVM_BUILD_LLVM_DYLIB"] = True cmake.definitions["CLANG_INCLUDE_DOCS"] = False cmake.definitions["LIBCXX_INCLUDE_DOCS"] = False diff --git a/packages/conan/recipes/python/conanfile.py b/packages/conan/recipes/python/conanfile.py index 3c770e67..bbd19386 100644 --- a/packages/conan/recipes/python/conanfile.py +++ b/packages/conan/recipes/python/conanfile.py @@ -1,3 +1,4 @@ +from platform import system from conans import AutoToolsBuildEnvironment, ConanFile, tools from contextlib import contextmanager import os @@ -47,23 +48,23 @@ def export_sources(self): def _build_context(self): if self.settings.compiler == "Visual Studio": with tools.vcvars(self.settings): - env = { - "AR": "{} lib".format( - tools.unix_path(self.deps_user_info["automake"].ar_lib) - ), - "CC": "{} cl -nologo".format( - tools.unix_path(self.deps_user_info["automake"].compile) - ), - "CXX": "{} cl -nologo".format( - tools.unix_path(self.deps_user_info["automake"].compile) - ), - "NM": "dumpbin -symbols", - "OBJDUMP": ":", - "RANLIB": ":", - "STRIP": ":", - } - with tools.environment_append(env): - yield + # env = { + # "AR": "{} lib".format( + # tools.unix_path(self.deps_user_info["automake"].ar_lib) + # ), + # "CC": "{} cl -nologo".format( + # tools.unix_path(self.deps_user_info["automake"].compile) + # ), + # "CXX": "{} cl -nologo".format( + # tools.unix_path(self.deps_user_info["automake"].compile) + # ), + # "NM": "dumpbin -symbols", + # "OBJDUMP": ":", + # "RANLIB": ":", + # "STRIP": ":", + # } + # with tools.environment_append(env): + yield else: yield @@ -92,19 +93,26 @@ def _configure_autotools(self): return self._autotools def build(self): - with self._build_context(): - autotools = self._configure_autotools() - autotools.make() + if system() == "Linux": + with self._build_context(): + autotools = self._configure_autotools() + autotools.make() + else: + self.run( + os.path.join(self._source_subfolder, "PCbuild", "build.bat"), + run_environment=True, + ) def package(self): self.copy("COPYING", src=self._source_subfolder, dst="licenses") - self.copy("yum", dst="bin") - self.copy("run-with-system-python", dst="bin") + if system() == "Linux": + self.copy("yum", dst="bin") + self.copy("run-with-system-python", dst="bin") - with self._build_context(): - autotools = self._configure_autotools() - autotools.install() + with self._build_context(): + autotools = self._configure_autotools() + autotools.install() python_version = tools.Version(self.version) if python_version.major == "3": diff --git a/packages/conan/settings/conan.conf b/packages/conan/settings_linux/conan.conf similarity index 100% rename from packages/conan/settings/conan.conf rename to packages/conan/settings_linux/conan.conf diff --git a/packages/conan/settings/hooks/attribute_checker.py b/packages/conan/settings_linux/hooks/attribute_checker.py similarity index 100% rename from packages/conan/settings/hooks/attribute_checker.py rename to packages/conan/settings_linux/hooks/attribute_checker.py diff --git a/packages/conan/settings/profiles/ci_common1 b/packages/conan/settings_linux/profiles/ci_common1 similarity index 100% rename from packages/conan/settings/profiles/ci_common1 rename to packages/conan/settings_linux/profiles/ci_common1 diff --git a/packages/conan/settings/profiles/ci_common2 b/packages/conan/settings_linux/profiles/ci_common2 similarity index 100% rename from packages/conan/settings/profiles/ci_common2 rename to packages/conan/settings_linux/profiles/ci_common2 diff --git a/packages/conan/settings/profiles/vfx2019 b/packages/conan/settings_linux/profiles/vfx2019 similarity index 100% rename from packages/conan/settings/profiles/vfx2019 rename to packages/conan/settings_linux/profiles/vfx2019 diff --git a/packages/conan/settings/profiles/vfx2020 b/packages/conan/settings_linux/profiles/vfx2020 similarity index 100% rename from packages/conan/settings/profiles/vfx2020 rename to packages/conan/settings_linux/profiles/vfx2020 diff --git a/packages/conan/settings/profiles/vfx2021 b/packages/conan/settings_linux/profiles/vfx2021 similarity index 100% rename from packages/conan/settings/profiles/vfx2021 rename to packages/conan/settings_linux/profiles/vfx2021 diff --git a/packages/conan/settings/profiles/vfx2022 b/packages/conan/settings_linux/profiles/vfx2022 similarity index 100% rename from packages/conan/settings/profiles/vfx2022 rename to packages/conan/settings_linux/profiles/vfx2022 diff --git a/packages/conan/settings/remotes.json b/packages/conan/settings_linux/remotes.json similarity index 100% rename from packages/conan/settings/remotes.json rename to packages/conan/settings_linux/remotes.json diff --git a/packages/conan/settings/settings.yml b/packages/conan/settings_linux/settings.yml similarity index 100% rename from packages/conan/settings/settings.yml rename to packages/conan/settings_linux/settings.yml diff --git a/packages/conan/settings/version.txt b/packages/conan/settings_linux/version.txt similarity index 100% rename from packages/conan/settings/version.txt rename to packages/conan/settings_linux/version.txt diff --git a/packages/conan/settings_windows/conan.conf b/packages/conan/settings_windows/conan.conf new file mode 100644 index 00000000..5ce8e3ed --- /dev/null +++ b/packages/conan/settings_windows/conan.conf @@ -0,0 +1,23 @@ +[log] +run_to_output = True # environment CONAN_LOG_RUN_TO_OUTPUT +run_to_file = False # environment CONAN_LOG_RUN_TO_FILE +level = critical # environment CONAN_LOGGING_LEVEL +print_run_commands = False # environment CONAN_PRINT_RUN_COMMANDS + +[general] +revisions_enabled = 1 +default_profile = vfx2022 +compression_level = 9 # environment CONAN_COMPRESSION_LEVEL +sysrequires_sudo = True # environment CONAN_SYSREQUIRES_SUDO +request_timeout = 60 # environment CONAN_REQUEST_TIMEOUT (seconds) +default_package_id_mode = semver_direct_mode # environment CONAN_DEFAULT_PACKAGE_ID_MODE + +[storage] +path = ../d +download_cache = /tmp/downloads + +[proxies] + +[hooks] +attribute_checker + diff --git a/packages/conan/settings_windows/hooks/attribute_checker.py b/packages/conan/settings_windows/hooks/attribute_checker.py new file mode 100644 index 00000000..0cf5ff1f --- /dev/null +++ b/packages/conan/settings_windows/hooks/attribute_checker.py @@ -0,0 +1,9 @@ +def pre_export(output, conanfile, conanfile_path, reference, **kwargs): + # Check basic meta-data + for field in ["url", "license", "description"]: + field_value = getattr(conanfile, field, None) + if not field_value: + output.warn( + "Conanfile doesn't have '%s'. It is recommended to add it as attribute" + % field + ) diff --git a/packages/conan/settings_windows/profiles/ci_common1 b/packages/conan/settings_windows/profiles/ci_common1 new file mode 100644 index 00000000..f18ce08b --- /dev/null +++ b/packages/conan/settings_windows/profiles/ci_common1 @@ -0,0 +1,13 @@ +[settings] +os=Windows +os_build=Windows +arch=x86_64 +arch_build=x86_64 +compiler=visual_studio +compiler.version=15 +compiler.toolset=v141 +compiler.cppstd=14 +build_type=Release +[options] +[build_requires] +[env] diff --git a/packages/conan/settings_windows/profiles/ci_common2 b/packages/conan/settings_windows/profiles/ci_common2 new file mode 100644 index 00000000..9c01971a --- /dev/null +++ b/packages/conan/settings_windows/profiles/ci_common2 @@ -0,0 +1,13 @@ +[settings] +os=Windows +os_build=Windows +arch=x86_64 +arch_build=x86_64 +compiler=Visual Studio +compiler.version=15 +compiler.toolset=v141 +compiler.cppstd=14 +build_type=Release +[options] +[build_requires] +[env] diff --git a/packages/conan/settings_windows/profiles/vfx2019 b/packages/conan/settings_windows/profiles/vfx2019 new file mode 100644 index 00000000..15be9864 --- /dev/null +++ b/packages/conan/settings_windows/profiles/vfx2019 @@ -0,0 +1,7 @@ +include(ci_common1) + +[settings] +python=2.7 +[options] +[build_requires] +[env] diff --git a/packages/conan/settings_windows/profiles/vfx2020 b/packages/conan/settings_windows/profiles/vfx2020 new file mode 100644 index 00000000..2eecaca0 --- /dev/null +++ b/packages/conan/settings_windows/profiles/vfx2020 @@ -0,0 +1,7 @@ +include(ci_common1) + +[settings] +python=3.7 +[options] +[build_requires] +[env] diff --git a/packages/conan/settings_windows/profiles/vfx2021 b/packages/conan/settings_windows/profiles/vfx2021 new file mode 100644 index 00000000..779dba71 --- /dev/null +++ b/packages/conan/settings_windows/profiles/vfx2021 @@ -0,0 +1,7 @@ +include(ci_common2) + +[settings] +python=3.7 +[options] +[build_requires] +[env] diff --git a/packages/conan/settings_windows/profiles/vfx2022 b/packages/conan/settings_windows/profiles/vfx2022 new file mode 100644 index 00000000..b52de271 --- /dev/null +++ b/packages/conan/settings_windows/profiles/vfx2022 @@ -0,0 +1,7 @@ +include(ci_common2) + +[settings] +python=3.9 +[options] +[build_requires] +[env] diff --git a/packages/conan/settings_windows/remotes.json b/packages/conan/settings_windows/remotes.json new file mode 100644 index 00000000..d8775ca8 --- /dev/null +++ b/packages/conan/settings_windows/remotes.json @@ -0,0 +1,19 @@ +{ + "remotes": [ + { + "name": "aswf", + "url": "https://linuxfoundation.jfrog.io/artifactory/api/conan/aswf-conan", + "verify_ssl": true + }, + { + "name": "conan-central", + "url": "https://center.conan.io", + "verify_ssl": true + }, + { + "name": "aswftesting", + "url": "https://linuxfoundation.jfrog.io/artifactory/api/conan/aswf-conan-dev", + "verify_ssl": true + } + ] +} \ No newline at end of file diff --git a/packages/conan/settings_windows/settings.yml b/packages/conan/settings_windows/settings.yml new file mode 100644 index 00000000..1c1fa3ab --- /dev/null +++ b/packages/conan/settings_windows/settings.yml @@ -0,0 +1,138 @@ + +# Only for cross building, 'os_build/arch_build' is the system that runs Conan +os_build: [Windows, WindowsStore, Linux, Macos, FreeBSD, SunOS, AIX] +arch_build: [x86, x86_64, ppc32be, ppc32, ppc64le, ppc64, armv5el, armv5hf, armv6, armv7, armv7hf, armv7s, armv7k, armv8, armv8_32, armv8.3, sparc, sparcv9, mips, mips64, avr, s390, s390x, sh4le, e2k-v2, e2k-v3, e2k-v4, e2k-v5, e2k-v6, e2k-v7] + +# Only for building cross compilation tools, 'os_target/arch_target' is the system for +# which the tools generate code +os_target: [Windows, Linux, Macos, Android, iOS, watchOS, tvOS, FreeBSD, SunOS, AIX, Arduino, Neutrino] +arch_target: [x86, x86_64, ppc32be, ppc32, ppc64le, ppc64, armv5el, armv5hf, armv6, armv7, armv7hf, armv7s, armv7k, armv8, armv8_32, armv8.3, sparc, sparcv9, mips, mips64, avr, s390, s390x, asm.js, wasm, sh4le, e2k-v2, e2k-v3, e2k-v4, e2k-v5, e2k-v6, e2k-v7, xtensalx6, xtensalx106] + +# Rest of the settings are "host" settings: +# - For native building/cross building: Where the library/program will run. +# - For building cross compilation tools: Where the cross compiler will run. +os: + Windows: + subsystem: [None, cygwin, msys, msys2, wsl] + WindowsStore: + version: ["8.1", "10.0"] + WindowsCE: + platform: ANY + version: ["5.0", "6.0", "7.0", "8.0"] + Linux: + version: [None, rhel7, rhel8] + Macos: + version: [None, "10.6", "10.7", "10.8", "10.9", "10.10", "10.11", "10.12", "10.13", "10.14", "10.15", "11.0", "12.0", "13.0"] + sdk: [None, "macosx"] + subsystem: [None, catalyst] + Android: + api_level: ANY + iOS: + version: ["7.0", "7.1", "8.0", "8.1", "8.2", "8.3", "9.0", "9.1", "9.2", "9.3", "10.0", "10.1", "10.2", "10.3", + "11.0", "11.1", "11.2", "11.3", "11.4", "12.0", "12.1", "12.2", "12.3", "12.4", + "13.0", "13.1", "13.2", "13.3", "13.4", "13.5", "13.6", "13.7", + "14.0", "14.1", "14.2", "14.3", "14.4", "14.5", "14.6", "14.7", "14.8", "15.0", "15.1"] + sdk: [None, "iphoneos", "iphonesimulator"] + watchOS: + version: ["4.0", "4.1", "4.2", "4.3", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1", "6.2", + "7.0", "7.1", "7.2", "7.3", "7.4", "7.5", "7.6", "8.0", "8.1"] + sdk: [None, "watchos", "watchsimulator"] + tvOS: + version: ["11.0", "11.1", "11.2", "11.3", "11.4", "12.0", "12.1", "12.2", "12.3", "12.4", + "13.0", "13.2", "13.3", "13.4", "14.0", "14.2", "14.3", "14.4", "14.5", "14.6", "14.7", + "15.0", "15.1"] + sdk: [None, "appletvos", "appletvsimulator"] + FreeBSD: + SunOS: + AIX: + Arduino: + board: ANY + Emscripten: + Neutrino: + version: ["6.4", "6.5", "6.6", "7.0", "7.1"] +arch: [x86, x86_64, ppc32be, ppc32, ppc64le, ppc64, armv4, armv4i, armv5el, armv5hf, armv6, armv7, armv7hf, armv7s, armv7k, armv8, armv8_32, armv8.3, sparc, sparcv9, mips, mips64, avr, s390, s390x, asm.js, wasm, sh4le, e2k-v2, e2k-v3, e2k-v4, e2k-v5, e2k-v6, e2k-v7, xtensalx6, xtensalx106] +compiler: + sun-cc: + version: ["5.10", "5.11", "5.12", "5.13", "5.14", "5.15"] + threads: [None, posix] + libcxx: [libCstd, libstdcxx, libstlport, libstdc++] + gcc: &gcc + version: ["4.1", "4.4", "4.5", "4.6", "4.7", "4.8", "4.9", + "5", "5.1", "5.2", "5.3", "5.4", "5.5", + "6", "6.1", "6.2", "6.3", "6.4", "6.5", + "7", "7.1", "7.2", "7.3", "7.4", "7.5", + "8", "8.1", "8.2", "8.3", "8.4", + "9", "9.1", "9.2", "9.3", + "10", "10.1", "10.2", "10.3", + "11", "11.1"] + libcxx: [libstdc++, libstdc++11] + threads: [None, posix, win32] # Windows MinGW + exception: [None, dwarf2, sjlj, seh] # Windows MinGW + cppstd: [None, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20, 23, gnu23] + Visual Studio: &visual_studio + runtime: [MD, MT, MTd, MDd] + version: ["8", "9", "10", "11", "12", "14", "15", "16", "17"] + toolset: [None, v90, v100, v110, v110_xp, v120, v120_xp, + v140, v140_xp, v140_clang_c2, LLVM-vs2012, LLVM-vs2012_xp, + LLVM-vs2013, LLVM-vs2013_xp, LLVM-vs2014, LLVM-vs2014_xp, + LLVM-vs2017, LLVM-vs2017_xp, v141, v141_xp, v141_clang_c2, v142, + llvm, ClangCL, v143] + cppstd: [None, 14, 17, 20] + msvc: + version: ["19.0", + "19.1", "19.10", "19.11", "19.12", "19.13", "19.14", "19.15", "19.16", + "19.2", "19.20", "19.21", "19.22", "19.23", "19.24", "19.25", "19.26", "19.27", "19.28", "19.29", + "19.3", "19.30"] + runtime: [static, dynamic] + runtime_type: [Debug, Release] + cppstd: [14, 17, 20, 23] + clang: + version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0", + "5.0", "6.0", "7.0", "7.1", + "8", "9", "10", "11", "12", "13"] + libcxx: [None, libstdc++, libstdc++11, libc++, c++_shared, c++_static] + cppstd: [None, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20, 23, gnu23] + runtime: [None, MD, MT, MTd, MDd] + apple-clang: &apple_clang + version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1", "9.0", "9.1", "10.0", "11.0", "12.0", "13.0"] + libcxx: [libstdc++, libc++] + cppstd: [None, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20] + intel: + version: ["11", "12", "13", "14", "15", "16", "17", "18", "19", "19.1"] + update: [None, ANY] + base: + gcc: + <<: *gcc + threads: [None] + exception: [None] + Visual Studio: + <<: *visual_studio + apple-clang: + <<: *apple_clang + intel-cc: + version: ["2021.1", "2021.2", "2021.3"] + update: [None, ANY] + mode: ["icx", "classic", "dpcpp"] + libcxx: [None, libstdc++, libstdc++11, libc++] + cppstd: [None, 98, gnu98, 03, gnu03, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20, 23, gnu23] + runtime: [None, static, dynamic] + runtime_type: [None, Debug, Release] + qcc: + version: ["4.4", "5.4", "8.3"] + libcxx: [cxx, gpp, cpp, cpp-ne, accp, acpp-ne, ecpp, ecpp-ne] + cppstd: [None, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17] + mcst-lcc: + version: ["1.19", "1.20", "1.21", "1.22", "1.23", "1.24", "1.25"] + base: + gcc: + <<: *gcc + threads: [None] + exceptions: [None] + +build_type: [None, Debug, Release, RelWithDebInfo, MinSizeRel] + +# ASWF-specific settings +python: [None, "2.7", "3.7", "3.9"] + + +cppstd: [None, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20, 23, gnu23] # Deprecated, use compiler.cppstd diff --git a/packages/conan/settings_windows/version.txt b/packages/conan/settings_windows/version.txt new file mode 100644 index 00000000..1933dfa1 --- /dev/null +++ b/packages/conan/settings_windows/version.txt @@ -0,0 +1 @@ +1.42.0 \ No newline at end of file diff --git a/packages/windows/Dockerfile b/packages/windows/Dockerfile new file mode 100644 index 00000000..a9c2f55a --- /dev/null +++ b/packages/windows/Dockerfile @@ -0,0 +1,35 @@ + +# https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container advises tht dotnet is needed for build tools +FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019 AS ci-windows-conan + +SHELL ["powershell.exe", "-ExecutionPolicy", "Bypass", "-Command"] + +ARG ASWF_CONAN_VERSION +ARG ASWF_CONAN_PYTHON_VERSION +ENV ASWF_CONAN_VERSION=${ASWF_CONAN_VERSION} \ + ASWF_CONAN_PYTHON_VERSION=${ASWF_CONAN_PYTHON_VERSION} + +ENV chocolateyUseWindowsCompression=false \ + PYTHONIOENCODING=UTF-8 + + +RUN iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')); \ + $env:Path += '";C:\tools\python3;C:\tools\python3\Scripts"'; \ + choco install --no-progress --yes python3 --version=${Env:ASWF_CONAN_PYTHON_VERSION} --params '"/InstallDir:C:\tools\python3"' + +RUN choco install --no-progress --yes visualstudio2017buildtools --version=15.9.41.0 +RUN choco install --no-progress --yes visualstudio2017-workload-vctools --version=1.3.1 +RUN choco install --no-progress --yes --execution-timeout=0 visualstudio2017-workload-manageddesktop --version=1.2.1 + +RUN python -m pip install --quiet --upgrade pip; \ + python -m pip install conan==${Env:ASWF_CONAN_VERSION} --quiet --upgrade --force-reinstall --no-cache + # ; \ +# python -m pip install win-unicode-console --quiet --upgrade --force-reinstall --no-cache; \ +# python -m pip install conan_package_tools --quiet --upgrade --force-reinstall --no-cache + +RUN choco install --no-progress --yes cmake --installargs 'ADD_CMAKE_TO_PATH=System' + +SHELL ["cmd", "/S", "/C"] +WORKDIR "C:/Users/ContainerAdministrator" +# ENTRYPOINT ["cmd.exe", "C:\\Program Files (x86)\\Microsoft Visual C++ Build Tools\\vcbuildtools_msbuild.bat"] + diff --git a/python/aswfdocker/aswfinfo.py b/python/aswfdocker/aswfinfo.py index aa73e22d..4b28585a 100644 --- a/python/aswfdocker/aswfinfo.py +++ b/python/aswfdocker/aswfinfo.py @@ -3,6 +3,8 @@ """ ASWF repository information """ +import platform + from aswfdocker import utils, constants @@ -15,6 +17,7 @@ def __init__( source_branch: str = "", aswf_version: str = "", repo_root: str = "", + target_os: str = "", ): self.aswf_version = aswf_version self.repo_uri = repo_uri @@ -31,6 +34,10 @@ def __init__( else: self.vcs_ref = constants.DEV_BUILD_DATE self.build_date = constants.DEV_BUILD_DATE + if target_os == "": + self.target_os = platform.system() + else: + self.target_os = target_os def set_org(self, org): self.docker_org = org diff --git a/python/aswfdocker/builder.py b/python/aswfdocker/builder.py index 7912d8d9..3a515b72 100644 --- a/python/aswfdocker/builder.py +++ b/python/aswfdocker/builder.py @@ -7,6 +7,7 @@ import subprocess import json import os +import platform import tempfile import typing @@ -56,14 +57,19 @@ def make_bake_dict(self) -> typing.Dict[str, dict]: version = version_info.ci_common_version major_version = utils.get_major_version(version) versions_to_bake.add(version) + if platform.system() == "Linux": + target = constants.LINUX_CONAN_DOCKER_TARGET + docker_file = "packages/common/Dockerfile" + else: + target = constants.WINDOWS_CONAN_DOCKER_TARGET + docker_file = "packages\\windows\\Dockerfile" tags = list( map( lambda tag: f"{constants.DOCKER_REGISTRY}/{self.build_info.docker_org}" - + f"/ci-centos7-gl-conan:{tag}", + + f"/{target}:{tag}", [version, major_version], ) ) - docker_file = "packages/common/Dockerfile" else: tags = version_info.get_tags( version, @@ -105,7 +111,10 @@ def make_bake_dict(self) -> typing.Dict[str, dict]: } if self.group_info.type == constants.ImageType.PACKAGE: if use_conan: - target_dict["target"] = "ci-centos7-gl-conan" + if platform.system() == "Linux": + target_dict["target"] = constants.LINUX_CONAN_DOCKER_TARGET + else: + target_dict["target"] = constants.WINDOWS_CONAN_DOCKER_TARGET else: target_dict["target"] = image root["target"][f"{image}-{major_version}"] = target_dict @@ -140,12 +149,21 @@ def _run_in_docker(self, base_cmd, cmd, dry_run): dry_run=dry_run, ) + def _get_conan_home(self): + if platform.system() == "Linux": + return constants.LINUX_CONAN_USER_HOME + elif platform.system() == "Windows": + return constants.WINDOWS_CONAN_USER_HOME + else: + raise NotImplementedError("OS not supported!") + def _get_conan_env_vars(self, version_info): envs = { - "CONAN_USER_HOME": constants.CONAN_USER_HOME, - "CCACHE_DIR": "/tmp/ccache", + "CONAN_USER_HOME": self._get_conan_home(), "CONAN_NON_INTERACTIVE": "1", } + if platform.system() == "Linux": + envs["CCACHE_DIR"] = "/tmp/ccache" if "CONAN_LOGIN_USERNAME" in os.environ: envs["CONAN_LOGIN_USERNAME"] = os.environ["CONAN_PASSWORD"] if "ARTIFACTORY_USER" in os.environ: @@ -161,17 +179,22 @@ def _get_conan_env_vars(self, version_info): def _get_conan_vols(self): conan_base = os.path.join(utils.get_git_top_level(), "packages", "conan") vols = { - os.path.join(conan_base, "settings"): os.path.join( - constants.CONAN_USER_HOME, ".conan" - ), - os.path.join(conan_base, "data"): os.path.join( - constants.CONAN_USER_HOME, "d" - ), + os.path.join( + conan_base, "settings_" + platform.system().lower() + ): os.path.join(self._get_conan_home(), ".conan"), os.path.join(conan_base, "recipes"): os.path.join( - constants.CONAN_USER_HOME, "recipes" + self._get_conan_home(), "recipes" ), - os.path.join(conan_base, "ccache"): "/tmp/ccache", } + if platform.system() == "Linux": + vols[os.path.join(conan_base, "ccache")] = "/tmp/ccache" + # Build failures happen when data is mounted on windows unfortunately + vols[os.path.join(conan_base, "data")] = ( + os.path.join(self._get_conan_home(), "d"), + ) + for path in vols: + if not os.path.exists(path): + os.makedirs(path) return vols def _get_conan_base_cmd(self, version_info): @@ -182,9 +205,13 @@ def _get_conan_base_cmd(self, version_info): for name, value in self._get_conan_vols().items(): base_cmd.append("-v") base_cmd.append(f"{name}:{value}") + if platform.system() == "Linux": + target = constants.LINUX_CONAN_DOCKER_TARGET + else: + target = constants.WINDOWS_CONAN_DOCKER_TARGET tag = ( f"{constants.DOCKER_REGISTRY}/{self.build_info.docker_org}" - + f"/ci-centos7-gl-conan:{version_info.ci_common_version}" + + f"/{target}:{version_info.ci_common_version}" ) base_cmd.append(tag) return base_cmd @@ -234,7 +261,7 @@ def _build_conan_package( build_cmd = [ "conan", "create", - os.path.join(constants.CONAN_USER_HOME, "recipes", image), + os.path.join(self._get_conan_home(), "recipes", image), conan_version, ] if keep_source: @@ -306,16 +333,35 @@ def build( ): logger.warning("Skipping %s as it is a conan-only package!", image) continue + logger.debug("Found %s:%s", image, version) images_and_versions.append((image, version)) if not images_and_versions: + logger.warning("Nothing to build.") return - path = self.make_bake_jsonfile() - if path: - self._run( - f"docker buildx bake -f {path} --progress {progress}", dry_run=dry_run - ) + if platform.system() == "Linux": + path = self.make_bake_jsonfile() + if path: + self._run( + f"docker buildx bake -f {path} --progress {progress}", + dry_run=dry_run, + ) + else: + # Unfortunately no buildx available on windows for windows containers... + d = self.make_bake_dict() + if d["group"]["default"]["targets"]: + target = d["target"][d["group"]["default"]["targets"][0]] + docker_file = target["dockerfile"] + context = target["context"] + args = " ".join( + map(lambda i: f"--build-arg {i[0]}={i[1]}", target["args"].items()) + ) + tags = " ".join(map(lambda t: f"-t {t}", target["tags"])) + self._run( + f"docker build -f {docker_file} {args} {tags} {context}", + dry_run=dry_run, + ) if not self.use_conan or self.group_info.type == constants.ImageType.IMAGE: return diff --git a/python/aswfdocker/constants.py b/python/aswfdocker/constants.py index de3b5ad0..5ce4239c 100644 --- a/python/aswfdocker/constants.py +++ b/python/aswfdocker/constants.py @@ -32,4 +32,12 @@ class ImageType(enum.Enum): ALL = "all" -CONAN_USER_HOME = "/tmp/c" +# OS specific variables +# Cross-building on windows hosts with linux docker should be possible so we keep both +# variables available + +LINUX_CONAN_USER_HOME = "/tmp/c" +WINDOWS_CONAN_USER_HOME = "C:\\tmp\\c" + +LINUX_CONAN_DOCKER_TARGET = "ci-centos7-gl-conan" +WINDOWS_CONAN_DOCKER_TARGET = "ci-windows-conan" diff --git a/python/aswfdocker/groupinfo.py b/python/aswfdocker/groupinfo.py index 7d06d85a..8845cf54 100644 --- a/python/aswfdocker/groupinfo.py +++ b/python/aswfdocker/groupinfo.py @@ -58,10 +58,11 @@ def iter_images_versions(self, get_image=False): all_versions = list(self.index.iter_versions(self.type, image)) major_versions = [utils.get_major_version(v) for v in all_versions] logger.debug( - "iter image=%s ci_image=%s all_versions=%s", + "iter image=%s ci_image=%s all_versions=%s major_versions=%s", image, ci_image, all_versions, + major_versions, ) for version in [v for v in self.versions if v in major_versions]: logger.debug("iter version=%s", version) diff --git a/python/aswfdocker/tests/test_builder.py b/python/aswfdocker/tests/test_builder.py index f754437b..8e9bdc4b 100644 --- a/python/aswfdocker/tests/test_builder.py +++ b/python/aswfdocker/tests/test_builder.py @@ -8,6 +8,7 @@ import unittest import logging import tempfile +import platform from click.testing import CliRunner @@ -66,11 +67,17 @@ def test_package_baseqt_2019_dict_conan(self): ) baked = b.make_bake_dict() self.assertIn("ASWF_QT_VERSION", baked["target"]["ci-package-qt-1"]["args"]) + if self.build_info.target_os == "Linux": + target = constants.LINUX_CONAN_DOCKER_TARGET + folder = "common" + else: + target = constants.WINDOWS_CONAN_DOCKER_TARGET + folder = "windows" self.assertEqual( baked["target"]["ci-package-qt-1"]["tags"], [ - f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-centos7-gl-conan:1", - f"{constants.DOCKER_REGISTRY}/aswflocaltesting/ci-centos7-gl-conan:1", + f"{constants.DOCKER_REGISTRY}/aswflocaltesting/{target}:1", + f"{constants.DOCKER_REGISTRY}/aswflocaltesting/{target}:1", ], ) self.assertEqual( @@ -78,7 +85,7 @@ def test_package_baseqt_2019_dict_conan(self): ) self.assertEqual( baked["target"]["ci-package-qt-1"]["dockerfile"], - "packages/common/Dockerfile", + os.path.join("packages", folder, "Dockerfile"), ) def test_image_base_2019_dict(self): @@ -336,6 +343,10 @@ def setUp(self): self._log_handlers = logging.getLogger("").handlers logging.getLogger("").handlers = [] self._i = 0 + if platform.system() == "Linux": + self._docker_home = constants.LINUX_CONAN_USER_HOME + else: + self._docker_home = constants.WINDOWS_CONAN_USER_HOME def tearDown(self): logging.getLogger("").handlers = self._log_handlers @@ -356,14 +367,71 @@ def test_builder_cli(self): ], ) self.assertFalse(result.exception) - bake_path = os.path.join( - tempfile.gettempdir(), "docker-bake-PACKAGE-vfx1-2019.json" - ) - cmd = f"docker buildx bake -f {bake_path} --progress auto" - self.assertEqual( - result.output, - f"INFO:aswfdocker.builder:Would run: '{cmd}'\n", - ) + if platform.system() == "Linux": + bake_path = os.path.join( + tempfile.gettempdir(), "docker-bake-PACKAGE-vfx1-2019.json" + ) + cmd = f"docker buildx bake -f {bake_path} --progress auto" + self.assertEqual( + result.output, + f"INFO:aswfdocker.builder:Would run: '{cmd}'\n", + ) + else: + cmd = ( + "docker build -f packages/vfx1/Dockerfile " + "--build-arg ASWF_ORG=aswftesting " + "--build-arg ASWF_PKG_ORG=aswftesting " + "--build-arg ASWF_VERSION=2019.2 " + "--build-arg CI_COMMON_VERSION=1 " + "--build-arg ASWF_CONAN_CHANNEL=vfx2019 " + "--build-arg ASWF_ALEMBIC_VERSION=1.7.11 " + "--build-arg ASWF_BLOSC_VERSION=1.5.0 " + "--build-arg ASWF_BOOST_VERSION=1.66.0 " + "--build-arg ASWF_CMAKE_VERSION=3.12.4 " + "--build-arg ASWF_CPPUNIT_VERSION=1.14.0 " + "--build-arg ASWF_GLEW_VERSION=2.1.0 " + "--build-arg ASWF_GLFW_VERSION=3.1.2 " + "--build-arg ASWF_GTEST_VERSION=1.8.1 " + "--build-arg ASWF_HDF5_VERSION=1.8.21 " + "--build-arg ASWF_IMATH_VERSION=2.3.0 " + "--build-arg ASWF_LOG4CPLUS_VERSION=1.1.2 " + "--build-arg ASWF_NUMPY_VERSION=1.14 " + "--build-arg ASWF_OCIO_CONFIGS_VERSION=1.0_r2 " + "--build-arg ASWF_OCIO_VERSION=1.1.0 " + "--build-arg ASWF_OIIO_VERSION=2.0.8 " + "--build-arg ASWF_OPENEXR_VERSION=2.3.0 " + "--build-arg ASWF_OPENSUBDIV_VERSION=3_3_3 " + "--build-arg ASWF_OPENVDB_VERSION=6.2.1 " + "--build-arg ASWF_OSL_VERSION=1.10.9 " + "--build-arg ASWF_OTIO_VERSION=0.12.1 " + "--build-arg ASWF_PARTIO_VERSION=1.10.1 " + "--build-arg ASWF_PTEX_VERSION=2.1.33 " + "--build-arg ASWF_PYBIND11_VERSION=2.4.3 " + "--build-arg ASWF_PYSIDE_VERSION=5.12.6 " + "--build-arg ASWF_PYTHON_VERSION=2.7.15 " + "--build-arg ASWF_PYTHON_MAJOR_MINOR_VERSION=2.7 " + "--build-arg ASWF_QT_VERSION=5.12.6 " + "--build-arg ASWF_TBB_VERSION=2018 " + "--build-arg ASWF_USD_VERSION=19.11 " + "--build-arg ASWF_VFXPLATFORM_VERSION=2019 " + "--build-arg ASWF_DTS_VERSION=6 " + "--build-arg ASWF_CUDA_VERSION=10.2 " + "--build-arg ASWF_CCACHE_VERSION=4.0 " + "--build-arg ASWF_CONAN_VERSION=1.42.0 " + "--build-arg ASWF_CONAN_PYTHON_VERSION=3.9.5 " + "--build-arg ASWF_NINJA_VERSION=1.10.1 " + "--build-arg ASWF_SONAR_VERSION=4.6.2.2472 " + "--build-arg ASWF_CLANG_MAJOR_VERSION=7 " + "--build-arg ASWF_CLANG_VERSION=7.1.0 " + "-t docker.io/aswftesting/ci-package-openexr:2019 " + "-t docker.io/aswftesting/ci-package-openexr:2019.2 " + "-t docker.io/aswftesting/ci-package-openexr:latest " + "-t docker.io/aswftesting/ci-package-openexr:2019-2.3.0 ." + ) + self.assertEqual( + result.output, + f"INFO:aswfdocker.builder:Would run: '{cmd}'\n", + ) self.assertEqual(result.exit_code, 0) def test_builder_cli_conan(self): @@ -388,14 +456,22 @@ def test_builder_cli_conan(self): ) cmds = result.output.strip().splitlines() self.assertEqual(len(cmds), 4) - self.assertEqual( - cmds[0], - f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} --progress auto'", - ) + if platform.system() == "Linux": + self.assertEqual( + cmds[0], + f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} --progress auto'", + ) + else: + self.assertTrue( + cmds[0].startswith( + "INFO:aswfdocker.builder:Would run: 'docker build -f packages" + ), + msg=cmds[0], + ) self.assertTrue( cmds[1].startswith( "INFO:aswfdocker.builder:Would run: 'docker run -e CONAN_USER_HOME=" - + constants.CONAN_USER_HOME + + self._docker_home ), msg=cmds[1], ) @@ -405,7 +481,7 @@ def test_builder_cli_conan(self): ) self.assertTrue( cmds[2].endswith( - f"conan create {constants.CONAN_USER_HOME}/recipes/openexr openexr/2.3.0@aswftesting/vfx2019'" + f"conan create {os.path.join(self._docker_home, 'recipes', 'openexr')} openexr/2.3.0@aswftesting/vfx2019'" ), msg=cmds[2], ) @@ -418,14 +494,22 @@ def test_builder_cli_fromtag(self): ["build", "--full-name", "aswftesting/ci-common:1-clang7", "--dry-run"], ) self.assertFalse(result.exception) - bake_path = os.path.join( - tempfile.gettempdir(), "docker-bake-IMAGE-common-1-clang7.json" - ) - cmd = f"docker buildx bake -f {bake_path} --progress auto" - self.assertEqual( - result.output, - f"INFO:aswfdocker.builder:Would run: '{cmd}'\n", - ) + if platform.system() == "Linux": + bake_path = os.path.join( + tempfile.gettempdir(), "docker-bake-IMAGE-common-1-clang7.json" + ) + cmd = f"docker buildx bake -f {bake_path} --progress auto" + self.assertEqual( + result.output, + f"INFO:aswfdocker.builder:Would run: '{cmd}'\n", + ) + else: + self.assertTrue( + result.output.startswith( + "INFO:aswfdocker.builder:Would run: 'docker build -f ci-common/Dockerfile" + ), + msg=result.output, + ) self.assertEqual(result.exit_code, 0) def test_builderlist_cli(self): @@ -449,11 +533,20 @@ def test_builderlist_cli(self): bake_path = os.path.join( tempfile.gettempdir(), "docker-bake-PACKAGE-vfx1-2019-2020.json" ) - cmd = f"docker buildx bake -f {bake_path} --progress auto" - self.assertEqual( - result.output, - f"INFO:aswfdocker.builder:Would run: '{cmd}'\n", - ) + self.assertEqual(result.exit_code, 0) + if platform.system() == "Linux": + cmd = f"docker buildx bake -f {bake_path} --progress auto" + self.assertEqual( + result.output, + f"INFO:aswfdocker.builder:Would run: '{cmd}'\n", + ) + else: + self.assertTrue( + result.output.startswith( + "INFO:aswfdocker.builder:Would run: 'docker build -f packages/vfx1/Dockerfile" + ), + msg=result.output, + ) self.assertEqual(result.exit_code, 0) def _assertEndsWith(self, cmds, expected): @@ -485,20 +578,28 @@ def test_builderlist_cli_conan(self): ], ) self.assertFalse(result.exception, msg=result.output) - bake_path = os.path.join( - tempfile.gettempdir(), "docker-bake-PACKAGE-vfx1-2019-2020.json" - ) cmds = result.output.strip().splitlines() self.assertEqual(len(cmds), 13) - self.assertEqual( - cmds[self._i], - f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} --progress auto'", - ) + if platform.system() == "Linux": + bake_path = os.path.join( + tempfile.gettempdir(), "docker-bake-PACKAGE-vfx1-2019-2020.json" + ) + self.assertEqual( + cmds[self._i], + f"INFO:aswfdocker.builder:Would run: 'docker buildx bake -f {bake_path} --progress auto'", + ) + else: + self.assertTrue( + cmds[self._i].startswith( + "INFO:aswfdocker.builder:Would run: 'docker build -f packages" + ), + msg=cmds[self._i], + ) self._i += 1 self.assertTrue( cmds[self._i].startswith( "INFO:aswfdocker.builder:Would run: 'docker run -e CONAN_USER_HOME=" - + constants.CONAN_USER_HOME + + self._docker_home ), msg=cmds[self._i], ) @@ -506,7 +607,7 @@ def test_builderlist_cli_conan(self): self._assertEndsWith(cmds, "conan config set general.default_profile=vfx2019'") self._assertEndsWith( cmds, - f"conan create {constants.CONAN_USER_HOME}/recipes/openexr" + f"conan create {os.path.join(self._docker_home, 'recipes', 'openexr')}" " openexr/2.3.0@aswftesting/vfx2019" " --keep-source --keep-build --build=missing'", ) @@ -527,7 +628,7 @@ def test_builderlist_cli_conan(self): self._assertEndsWith(cmds, "conan config set general.default_profile=vfx2020'") self._assertEndsWith( cmds, - f"conan create {constants.CONAN_USER_HOME}/recipes/openexr" + f"conan create {os.path.join(self._docker_home, 'recipes', 'openexr')}" " openexr/2.4.0@aswftesting/vfx2020" " --keep-source --keep-build --build=missing'", ) @@ -540,14 +641,22 @@ def test_builder_cli_allversions(self): ["build", "--group", "common", "--version", "all", "--dry-run"], ) self.assertFalse(result.exception) - bake_path = os.path.join( - tempfile.gettempdir(), - "docker-bake-IMAGE-common-1-clang10-1-clang6-1-clang7-1" - "-clang8-1-clang9-2-clang10-2-clang11-2-clang12-2-clang13.json", - ) - cmd = f"docker buildx bake -f {bake_path} --progress auto" - self.assertEqual( - result.output, - f"INFO:aswfdocker.builder:Would run: '{cmd}'\n", - ) + if platform.system() == "Linux": + bake_path = os.path.join( + tempfile.gettempdir(), + "docker-bake-IMAGE-common-1-clang10-1-clang6-1-clang7-1" + "-clang8-1-clang9-2-clang10-2-clang11-2-clang12-2-clang13.json", + ) + cmd = f"docker buildx bake -f {bake_path} --progress auto" + self.assertEqual( + result.output, + f"INFO:aswfdocker.builder:Would run: '{cmd}'\n", + ) + else: + self.assertTrue( + result.output.startswith( + "INFO:aswfdocker.builder:Would run: 'docker build -f ci-common/Dockerfile" + ), + msg=result.output, + ) self.assertEqual(result.exit_code, 0) diff --git a/python/aswfdocker/tests/test_utils.py b/python/aswfdocker/tests/test_utils.py index 6f15519a..064b4d72 100644 --- a/python/aswfdocker/tests/test_utils.py +++ b/python/aswfdocker/tests/test_utils.py @@ -8,6 +8,9 @@ import logging import tempfile import os +import platform + +import pytest from click.testing import CliRunner @@ -127,6 +130,9 @@ def test_cli_getdockerpush(self): self.assertEqual(result.exit_code, 0) self.assertEqual(result.output, "false") + @pytest.mark.skipif( + platform.system() != "Linux", reason="This utility only makes sense on linux" + ) def test_cli_download(self): runner = CliRunner() with tempfile.TemporaryDirectory() as tmpdirname: diff --git a/sonar-project.properties b/sonar-project.properties index b486b6ce..21444d85 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -7,3 +7,4 @@ sonar.python.xunit.reportPath=test-pytest-results.xml sonar.c.file.suffixes=- sonar.cpp.file.suffixes=- sonar.objc.file.suffixes=- +sonar.exclusions=packages/conan/recipes/**