Skip to content

Commit 9357425

Browse files
aktechaktech
authored
Merge pull request #227 from czgdp1807/de-dock-ci
Add conda/mamba CI workflows
2 parents 41be499 + 5993068 commit 9357425

File tree

10 files changed

+207
-31
lines changed

10 files changed

+207
-31
lines changed

‎.bazelrc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,6 @@ build --protocopt=--experimental_allow_proto3_optional
2323
# parameter 'user_link_flags' is deprecated and will be removed soon.
2424
# It may be temporarily re-enabled by setting --incompatible_require_linker_input_cc_api=false
2525
build --incompatible_require_linker_input_cc_api=false
26+
2627
build:macos --apple_platform_type=macos
2728
build:macos_arm64 --cpu=darwin_arm64

‎.github/workflows/build.yml‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -25,29 +25,3 @@ jobs:
2525
uses: ./.github/reusable-build
2626
with:
2727
python-version: ${{ matrix.python-version }}
28-
29-
upload_to_pypi:
30-
name: Upload to PyPI
31-
runs-on: ubuntu-latest
32-
if: (github.event_name == 'release' && startsWith(github.ref, 'refs/tags')) || (github.event_name == 'workflow_dispatch')
33-
needs: [build]
34-
environment:
35-
name: pypi
36-
url: https://pypi.org/p/ml-metadata/
37-
permissions:
38-
id-token: write
39-
steps:
40-
- name: Retrieve wheels
41-
uses: actions/[email protected]
42-
with:
43-
merge-multiple: true
44-
path: wheels
45-
46-
- name: List the build artifacts
47-
run: |
48-
ls -lAs wheels/
49-
50-
- name: Upload to PyPI
51-
uses: pypa/gh-action-pypi-publish@release/v1.9
52-
with:
53-
packages_dir: wheels/

‎.github/workflows/conda-build.yml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
name: Build ml-metadata with Conda
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
branches:
9+
- master
10+
workflow_dispatch:
11+
12+
jobs:
13+
build:
14+
runs-on: ${{ matrix.os }}
15+
strategy:
16+
fail-fast: false
17+
matrix:
18+
os: [ubuntu-latest]
19+
python-version: ["3.9", "3.10", "3.11"]
20+
21+
steps:
22+
- name: Checkout
23+
uses: actions/checkout@v4
24+
25+
- name: Setup Micromamba
26+
uses: mamba-org/setup-micromamba@v1
27+
with:
28+
environment-file: ci/environment.yml
29+
cache-environment: true
30+
create-args: >-
31+
python=${{ matrix.python-version }}
32+
33+
- name: Display environment info
34+
shell: bash -l {0}
35+
run: |
36+
micromamba info
37+
micromamba list
38+
39+
- name: Install Bazel
40+
shell: bash -l {0}
41+
run: |
42+
# Install Bazelisk (manages Bazel versions)
43+
if [ "$RUNNER_OS" == "Linux" ]; then
44+
curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64
45+
elif [ "$RUNNER_OS" == "macOS" ]; then
46+
curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-darwin-amd64
47+
fi
48+
chmod +x /tmp/bazelisk
49+
sudo mv /tmp/bazelisk /usr/local/bin/bazel
50+
echo "USE_BAZEL_VERSION=6.5.0" >> $GITHUB_ENV
51+
bazel --version
52+
53+
- name: Install build tooling
54+
shell: bash -l {0}
55+
run: |
56+
python -m pip install --upgrade pip build wheel "setuptools<69.3"
57+
58+
- name: Build the package
59+
shell: bash -l {0}
60+
run: |
61+
python -m build --wheel --no-isolation
62+
63+
- name: Repair wheel for manylinux
64+
if: runner.os == 'Linux'
65+
shell: bash -l {0}
66+
run: |
67+
python -m pip install auditwheel
68+
WHEEL_PATH="$(ls dist/*.whl)"
69+
auditwheel repair --plat auto -w dist "${WHEEL_PATH}"
70+
rm "${WHEEL_PATH}"
71+
72+
- name: Upload wheel artifact
73+
uses: actions/[email protected]
74+
with:
75+
name: ml-metadata-wheel-${{ matrix.os }}-py${{ matrix.python-version }}
76+
path: dist/*.whl
77+
78+
upload_to_pypi:
79+
name: Upload to PyPI
80+
runs-on: ubuntu-latest
81+
if: (github.event_name == 'release' && startsWith(github.ref, 'refs/tags')) || (github.event_name == 'workflow_dispatch')
82+
needs: [build]
83+
environment:
84+
name: pypi
85+
url: https://pypi.org/p/ml-metadata/
86+
permissions:
87+
id-token: write
88+
steps:
89+
- name: Retrieve wheels
90+
uses: actions/[email protected]
91+
with:
92+
merge-multiple: true
93+
path: wheels
94+
95+
- name: List the build artifacts
96+
run: |
97+
ls -lAs wheels/
98+
99+
- name: Upload to PyPI
100+
uses: pypa/gh-action-pypi-publish@release/v1.9
101+
with:
102+
packages_dir: wheels/

‎.github/workflows/conda-test.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: Test ml-metadata with Conda
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
branches:
9+
- master
10+
workflow_dispatch:
11+
12+
jobs:
13+
test:
14+
runs-on: ${{ matrix.os }}
15+
strategy:
16+
fail-fast: false
17+
matrix:
18+
os: [ubuntu-latest]
19+
python-version: ["3.9", "3.10", "3.11"]
20+
21+
steps:
22+
- name: Checkout
23+
uses: actions/checkout@v4
24+
25+
- name: Setup Micromamba
26+
uses: mamba-org/setup-micromamba@v1
27+
with:
28+
environment-file: ci/environment.yml
29+
cache-environment: true
30+
create-args: >-
31+
python=${{ matrix.python-version }}
32+
33+
- name: Display environment info
34+
shell: bash -l {0}
35+
run: |
36+
micromamba info
37+
micromamba list
38+
39+
- name: Install Bazel
40+
shell: bash -l {0}
41+
run: |
42+
# Install Bazelisk (manages Bazel versions)
43+
if [ "$RUNNER_OS" == "Linux" ]; then
44+
curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64
45+
elif [ "$RUNNER_OS" == "macOS" ]; then
46+
curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-darwin-amd64
47+
fi
48+
chmod +x /tmp/bazelisk
49+
sudo mv /tmp/bazelisk /usr/local/bin/bazel
50+
echo "USE_BAZEL_VERSION=6.5.0" >> $GITHUB_ENV
51+
bazel --version
52+
53+
- name: Build the package
54+
shell: bash -l {0}
55+
run: |
56+
python setup.py bdist_wheel
57+
58+
- name: Install built wheel (Linux/macOS)
59+
shell: bash -l {0}
60+
run: |
61+
pip install dist/*.whl
62+
63+
- name: Run tests
64+
shell: bash -l {0}
65+
run: |
66+
# cleanup (interferes with tests)
67+
rm -rf bazel-*
68+
# run tests
69+
pytest -vv

‎WORKSPACE‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,4 +282,4 @@ ml_metadata_workspace()
282282

283283
# Specify the minimum required bazel version.
284284
load("@bazel_skylib//lib:versions.bzl", "versions")
285-
versions.check("6.1.0")
285+
versions.check("6.5.0")

‎ci/environment.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Conda environment for building and testing ml-metadata on Linux
2+
name: mlmd-dev
3+
channels:
4+
- conda-forge
5+
dependencies:
6+
# Note: Bazel is installed separately via official installer (conda package is unreliable)
7+
- setuptools<69.3 # Pin to avoid Metadata-Version 2.4 (PyPI only supports up to 2.3)
8+
- wheel
9+
- pip
10+
- numpy>=1.23,<2.0
11+
- pytest
12+
- pytest-cov
13+
- patchelf # For wheel repair on Linux
14+
- cmake=3.29
15+
16+
# C/C++ compilers - GCC 8.x to match manylinux2014 devtoolset-8
17+
- gcc_linux-64=8.5.0
18+
- gxx_linux-64=8.5.0
19+
- sysroot_linux-64=2.17 # CentOS 7/manylinux2014 compatible glibc headers
20+
21+
- pip:
22+
- auditwheel # For manylinux wheel compliance

‎ml_metadata/.bazelversion‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,4 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14-
6.1.0
14+
6.5.0

‎ml_metadata/tools/docker_build/Dockerfile.manylinux2010‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@
1515
# Dockerfile for building a manylinux2010 MLMD wheel.
1616

1717
# This docker image is essentially pypa/manylinux2010 + bazel.
18-
FROM gcr.io/tfx-oss-public/manylinux2014-bazel:bazel-6.1.0
18+
FROM gcr.io/tfx-oss-public/manylinux2014-bazel:bazel-6.5.0
19+
20+
# Install CMake 3.24 for compatibility with older CMake projects (libmysqlclient)
21+
RUN yum remove -y cmake cmake3 && \
22+
curl -L https://github.com/Kitware/CMake/releases/download/v3.24.4/cmake-3.24.4-linux-x86_64.tar.gz | tar -xz -C /opt && \
23+
ln -sf /opt/cmake-3.24.4-linux-x86_64/bin/cmake /usr/local/bin/cmake && \
24+
ln -sf /opt/cmake-3.24.4-linux-x86_64/bin/ctest /usr/local/bin/ctest && \
25+
ln -sf /opt/cmake-3.24.4-linux-x86_64/bin/cpack /usr/local/bin/cpack
26+
1927
WORKDIR /build
2028
CMD ["ml_metadata/tools/docker_build/build_manylinux.sh"]

‎ml_metadata/tools/docker_build/build_manylinux.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
WORKING_DIR=$PWD
2626

2727
function setup_environment() {
28-
source scl_source enable devtoolset-8
28+
source scl_source enable devtoolset-10
2929
source scl_source enable rh-python38
3030
if [[ -z "${PYTHON_VERSION}" ]]; then
3131
echo "Must set PYTHON_VERSION env to 39|310|311|"; exit 1;

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ requires = [
1717
"setuptools",
1818
"wheel",
1919
# Required for using org_tensorflow bazel repository.
20-
"numpy~=1.22.0",
20+
"numpy>=1.23,<2.0",
2121
]
2222

2323
[tool.pytest.ini_options]

0 commit comments

Comments
 (0)