Skip to content

Commit 34dfadd

Browse files
alanchiaotensorflower-gardener
alanchiao
authored and
tensorflower-gardener
committed
Setup continuous integration build scripts for opensource.
Verified running TFMOT unit tests with ci/kokoro/build.sh against TF 2.0.X. PiperOrigin-RevId: 322877301
1 parent 9b12529 commit 34dfadd

File tree

6 files changed

+182
-0
lines changed

6 files changed

+182
-0
lines changed

‎google3/third_party/tensorflow_model_optimization/opensource/ci/kokoro/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
The entrypoint build script is gcp_ubuntu/build.sh.

‎google3/third_party/tensorflow_model_optimization/opensource/ci/kokoro/build.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
# Copyright 2020 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# Fail on any error.
18+
set -e
19+
20+
# Display commands being run.
21+
# WARNING: please only enable 'set -x' if necessary for debugging, and be very
22+
# careful if you handle credentials (e.g. from Keystore) with 'set -x':
23+
# statements like "export VAR=$(cat /tmp/keystore/credentials)" will result in
24+
# the credentials being printed in build logs.
25+
# Additionally, recursive invocation with credentials as command-line
26+
# parameters, will print the full command, with credentials, in the build logs.
27+
# set -x
28+
29+
pip3 install --requirement "requirements.txt"
30+
31+
bazel test --python_path=$PYTHON_BIN //tensorflow_model_optimization/python/core/...

‎google3/third_party/tensorflow_model_optimization/opensource/ci/kokoro/gcp_ubuntu/Dockerfile‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Copyright 2020 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# An image for building prerequisites for testing TensorFlow Model Optimization
16+
# on ubuntu.
17+
#
18+
# TODO(tfmot): generalize to different versions of TensorFlow to
19+
# run CI against.
20+
#
21+
# Build as follows:
22+
# docker build --tag={TAG} /path/to/directory/with/this/file
23+
#
24+
# Run interactively as follows, where WORKDIR is the root directory
25+
# of the tensorflow model optimization Github repository.
26+
#
27+
# docker run -it --volume "${WORKDIR}:/usr/src/tfmot" {TAG}
28+
29+
FROM ubuntu:18.04
30+
WORKDIR /usr/src/tfmot
31+
32+
RUN apt-get update -y
33+
34+
# Install Python 3.
35+
#
36+
# Using Python 3.7 since 3.8 is not compatible with TensorFlow
37+
# versions <= 2.1.
38+
RUN apt-get install -y software-properties-common
39+
RUN add-apt-repository ppa:deadsnakes/ppa
40+
RUN apt-get install -y python3.7
41+
ENV PYTHON_BIN /usr/bin/python3.7
42+
43+
# Install Bazel.
44+
# https://docs.bazel.build/versions/master/install-ubuntu.html
45+
ARG BAZEL_VERSION=3.4.1
46+
RUN apt-get install -y unzip zip wget \
47+
&& wget "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION?}/bazel-${BAZEL_VERSION?}-installer-linux-x86_64.sh" \
48+
&& chmod +x "bazel-${BAZEL_VERSION?}-installer-linux-x86_64.sh" \
49+
&& "./bazel-${BAZEL_VERSION?}-installer-linux-x86_64.sh" --user \
50+
&& rm "bazel-${BAZEL_VERSION?}-installer-linux-x86_64.sh"
51+
ENV PATH "/root/bin:${PATH}"
52+
53+
# Install clang, which is required by bazel during building TFMOT.
54+
RUN apt-get install -y clang
55+
ENV CC clang
56+
57+
# Install pip.
58+
RUN apt-get install -y python3-pip
59+
# Upgrade pip3 to be able to detect additional TensorFlow versions.
60+
RUN pip3 install --upgrade pip
61+
62+
# Install latest TensorFlow 2.0.X.
63+
RUN pip3 install tensorflow==2.0.0

‎google3/third_party/tensorflow_model_optimization/opensource/ci/kokoro/gcp_ubuntu/build.sh‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/bin/bash
2+
3+
# Copyright 2020 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
# This script targets building for continuous integration. It can
18+
# be used to reproduce errors locally by modifying WORKDIR to be
19+
# the top-level directory of the checked out TFMOT Github repository.
20+
21+
# TODO(tfmot): switch to prebuilt Docker image to speed this up.
22+
23+
# Fail on any error.
24+
set -e
25+
26+
# Display commands being run.
27+
# WARNING: please only enable 'set -x' if necessary for debugging, and be very
28+
# careful if you handle credentials (e.g. from Keystore) with 'set -x':
29+
# statements like "export VAR=$(cat /tmp/keystore/credentials)" will result in
30+
# the credentials being printed in build logs.
31+
# Additionally, recursive invocation with credentials as command-line
32+
# parameters, will print the full command, with credentials, in the build logs.
33+
# set -x
34+
35+
# Code under repo is checked out to
36+
# ${KOKORO_ARTIFACTS_DIR}/github/tensorflow_model_optimization.
37+
WORKDIR="${KOKORO_ARTIFACTS_DIR}/github/tensorflow_model_optimization"
38+
39+
docker build --tag tfmot \
40+
$WORKDIR/ci/kokoro/gcp_ubuntu
41+
42+
# Mount the checked out repository, make that the working directory and run
43+
# ci/kokoro/build.sh from the repository, which runs all the unit tests.
44+
docker run \
45+
--volume "${WORKDIR?}:${WORKDIR?}" \
46+
--workdir="${WORKDIR?}" \
47+
--rm \
48+
tfmot:latest \
49+
ci/kokoro/build.sh
50+
51+
# Kokoro will rsync this entire directory back to the executor orchestrating the
52+
# build which takes forever and is totally useless.
53+
sudo rm -rf "${KOKORO_ARTIFACTS_DIR?}"/*

‎google3/third_party/tensorflow_model_optimization/opensource/ci/kokoro/gcp_ubuntu/continuous.cfg‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto
2+
3+
# Copyright 2020 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
build_file: "tensorflow_model_optimization/ci/kokoro/gcp_ubuntu/build.sh"

‎google3/third_party/tensorflow_model_optimization/opensource/ci/kokoro/gcp_ubuntu/presubmit.cfg‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto
2+
3+
# Copyright 2020 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
build_file: "tensorflow_model_optimization/ci/kokoro/gcp_ubuntu/build.sh"

0 commit comments

Comments
 (0)