Skip to content
This repository was archived by the owner on May 21, 2024. It is now read-only.

Commit 9d60ea8

Browse files
msteffenmsteffen
authored
Pull out protobuf build changes (#262)
* Copy pachyderm's etc/testing directory, which starts up Pachyderm in python-pachyderm's CI, into python-pachyderm (as it's removed in github.com/pachyderm/pachyderm's main repo in 2.0 * Update proto build process (`make` argument and `proto/pachyderm/Dockerfile`). Then rebuild pachyderm protos for 1.13.1 * Update pachyderm submodule to 1.13.1 * Update PPS mixin API to support new 1.13.1 arguments
1 parent b15b02e commit 9d60ea8

27 files changed

+17349
-143
lines changed

‎Makefile‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@ docker-build-proto:
1919

2020
src/python_pachyderm/proto: docker-build-proto
2121
@echo "Building with pachyderm core v$(PACHYDERM_VERSION)"
22+
rm -rf src/python_pachyderm/proto
2223
cd proto/pachyderm && \
2324
git fetch --all && \
2425
git checkout v$(PACHYDERM_VERSION)
2526
find ./proto/pachyderm/src/client -regex ".*\.proto" \
27+
| grep -v 'internal' \
2628
| xargs tar cf - \
2729
| docker run -i pachyderm_python_proto \
2830
| tar xf -
29-
test -d src/python_pachyderm/proto && rm -rf src/python_pachyderm/proto
3031
mv src/python_pachyderm/client src/python_pachyderm/proto
3132
find src/python_pachyderm/proto -type d -exec touch {}/__init__.py \;
3233

@@ -36,19 +37,18 @@ init:
3637
ci-install:
3738
sudo apt-get update
3839
sudo apt-get install jq socat
39-
cd proto/pachyderm && \
40-
sudo etc/testing/travis_cache.sh && \
41-
sudo etc/testing/travis_install.sh && \
42-
curl -o /tmp/pachctl.deb -L https://github.com/pachyderm/pachyderm/releases/download/v$(PACHYDERM_VERSION)/pachctl_$(PACHYDERM_VERSION)_amd64.deb && \
43-
sudo dpkg -i /tmp/pachctl.deb
40+
sudo etc/testing/travis_cache.sh
41+
sudo etc/testing/travis_install.sh
42+
curl -o /tmp/pachctl.deb -L https://github.com/pachyderm/pachyderm/releases/download/v$(PACHYDERM_VERSION)/pachctl_$(PACHYDERM_VERSION)_amd64.deb
43+
sudo dpkg -i /tmp/pachctl.deb
4444
pip install tox tox-travis
4545

4646
ci-setup:
4747
docker version
4848
which pachctl
49-
cd proto/pachyderm && make launch-kube
50-
pachctl deploy local
51-
until timeout 1s ./proto/pachyderm/etc/kube/check_ready.sh app=pachd; do sleep 1; done
49+
etc/kube/start-minikube.sh
50+
echo 'y' | pachctl deploy local
51+
until timeout 1s ./etc/kube/check_ready.sh app=pachd; do sleep 1; done
5252
pachctl version
5353

5454
release:

‎etc/kube/check_ready.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
3+
if [ -z "$1" ]; then
4+
echo "Usage $0 label=value <optional_namespace>"
5+
exit 1
6+
fi
7+
8+
namespace=default
9+
10+
if [ -n "$2" ]; then
11+
namespace=$2
12+
fi
13+
14+
results=$(kubectl get pods \
15+
-l "$1" \
16+
--namespace="$namespace" \
17+
-o jsonpath='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' \
18+
| tr ';' "\n")
19+
20+
if [ -z "$results" ]; then
21+
echo "Empty result"
22+
echo "$results"
23+
exit 1
24+
fi
25+
26+
27+
readyPods=$(echo "$results" | tr ' ' "\n" | grep -c "Ready=True")
28+
allPods=$(echo "$results" | tr ' ' "\n" | grep -c "Ready=")
29+
30+
if [ "$allPods" -eq 0 ]; then
31+
echo "No pods found yet"
32+
exit 1
33+
fi
34+
35+
if [ "$readyPods" -ne "$allPods" ]; then
36+
echo "${readyPods}/${allPods} ready"
37+
exit 1
38+
fi
39+
40+
echo "All pods are ready."
41+
exit 0

‎etc/kube/internal.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
set -Ee
3+
4+
jq '.spec.containers[1].command |= . + [ "--storage-backend=etcd2" ]' </etc/kubernetes/manifests/master.json >/etc/kubernetes/manifests/master2.json
5+
mv /etc/kubernetes/manifests/master2.json /etc/kubernetes/manifests/master.json
6+
/hyperkube kubelet \
7+
--containerized \
8+
--hostname-override="127.0.0.1" \
9+
--address="0.0.0.0" \
10+
--api-servers=http://localhost:8080 \
11+
--cluster_dns=10.0.0.10 \
12+
--cluster_domain=cluster.local \
13+
--pod-manifest-path=/etc/kubernetes/manifests \
14+
--allow-privileged=true \
15+
--feature-gates="Accelerators=true"

‎etc/kube/start-minikube.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
3+
set -Eex
4+
5+
# Parse flags
6+
VERSION=v1.13.0
7+
minikube_args=(
8+
"--vm-driver=none"
9+
"--kubernetes-version=${VERSION}"
10+
)
11+
while getopts ":v" opt; do
12+
case "${opt}" in
13+
v)
14+
VERSION="v${OPTARG}"
15+
;;
16+
\?)
17+
echo "Invalid argument: ${opt}"
18+
exit 1
19+
;;
20+
esac
21+
done
22+
23+
if [[ -n "${TRAVIS}" ]]; then
24+
minikube_args+=("--bootstrapper=kubeadm")
25+
fi
26+
27+
28+
# Note that we update the 'PATH' to include '~/cached-deps' in '.travis.yml',
29+
# but this doesn't update the PATH for calls using 'sudo'. To make a 'sudo' run
30+
# a binary in '~/cached-deps', you need to explicitly set the path like so:
31+
# sudo env "PATH=$PATH" minikube foo
32+
sudo env "PATH=$PATH" "CHANGE_MINIKUBE_NONE_USER=true" \
33+
minikube start "${minikube_args[@]}"
34+
35+
# Try to connect for three minutes
36+
for _ in $(seq 36); do
37+
if kubectl version &>/dev/null; then
38+
exit 0
39+
fi
40+
sleep 5
41+
done
42+
43+
# Give up--kubernetes isn't coming up
44+
minikube delete
45+
sleep 30 # Wait for minikube to go completely down
46+
exit 1

‎etc/testing/kube_debug.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
echo "=== TEST FAILED OR TIMED OUT, DUMPING DEBUG INFO ==="
4+
5+
# TODO: Extend this to show kubectl describe output for failed pods, this will
6+
# probably show why things are hanging.
7+
8+
# SC2016 blocks variables in single-quoted strings, but these are 'eval'ed below
9+
# shellcheck disable=SC2016
10+
cmds=(
11+
'pachctl version'
12+
'pachctl list repo'
13+
'pachctl list repo --raw | jq -r ".repo.name" | while read r; do
14+
echo "---";
15+
pachctl inspect repo "${r}";
16+
echo "Commits:";
17+
pachctl list commit "${r}";
18+
echo;
19+
done'
20+
'pachctl list pipeline'
21+
'pachctl list job'
22+
'kubectl version'
23+
'kubectl get all --all-namespaces'
24+
'kubectl describe pod -l suite=pachyderm,app=pachd'
25+
'kubectl describe pod -l suite=pachyderm,app=etcd'
26+
# Set --tail b/c by default 'kubectl logs' only outputs 10 lines if -l is set
27+
'kubectl logs --tail=500 -l suite=pachyderm,app=pachd'
28+
'kubectl logs --tail=1000 -l suite=pachyderm,app=pachd --previous # if pachd restarted'
29+
'sudo dmesg | tail -n 40'
30+
'minikube logs | tail -n 100'
31+
'top -b -n 1 | head -n 40'
32+
'df -h'
33+
)
34+
for c in "${cmds[@]}"; do
35+
echo "======================================================================"
36+
echo "${c}"
37+
echo "----------------------------------------------------------------------"
38+
eval "${c}"
39+
done
40+
echo "======================================================================"

‎etc/testing/minikube_version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1.13.1

‎etc/testing/travis_cache.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
# Create all of our Travis cached dirs, or, if they exist, loosen their
4+
# permissions enough that our build process is guaranteed to be able to write
5+
# to them and Travis's caching process is guaranteed to be able to read them.
6+
# (Travis caching requires that all cached files be readable, and, I surmise,
7+
# that cached directories be executable, to be traversed).
8+
9+
set -ex
10+
11+
# If any directories are added or changed here, they must also be added to
12+
# .travis.yml
13+
dirs=(
14+
"${HOME}/.cache"
15+
"${HOME}/cached-deps"
16+
"${GOPATH}/pkg"
17+
"$(python3 -c 'import site; print(site.USER_BASE)')" # $HOME/.local
18+
)
19+
20+
for dir in "${dirs[@]}"; do
21+
if [[ -d "${dir}" ]]; then
22+
# change ownership on directory, in case it's owned by root
23+
sudo chown -R "${USER}:${USER}" "${dir}"
24+
else
25+
# create directory
26+
mkdir -p "${dir}"
27+
fi
28+
# Loosen permissions so build processes can write and Travis can cache
29+
chmod 777 -R "${dir}"
30+
done

‎etc/testing/travis_install.sh‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
# install latest version of docker
6+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
7+
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
8+
sudo apt-get update -y
9+
sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
10+
11+
# reconfigure & restart docker
12+
echo 'DOCKER_OPTS="-H unix:///var/run/docker.sock -s devicemapper"' | sudo tee /etc/default/docker > /dev/null
13+
echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
14+
sudo service docker restart
15+
16+
# Install deps
17+
sudo apt-get install -y -qq \
18+
jq \
19+
silversearcher-ag \
20+
python3 \
21+
python3-pip \
22+
python3-setuptools \
23+
pkg-config \
24+
fuse
25+
26+
27+
# Install kubectl
28+
# To get the latest kubectl version:
29+
# curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt
30+
if [ ! -f ~/cached-deps/kubectl ] ; then
31+
KUBECTL_VERSION=v1.19.2
32+
curl -L -o kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \
33+
chmod +x ./kubectl && \
34+
mv ./kubectl ~/cached-deps/kubectl
35+
fi
36+
37+
# Install minikube
38+
# To get the latest minikube version:
39+
# curl https://api.github.com/repos/kubernetes/minikube/releases | jq -r .[].tag_name | sort -V | tail -n1
40+
if [ ! -f ~/cached-deps/minikube ] ; then
41+
MINIKUBE_VERSION=v1.13.1 # If changed, also do etc/kube/start-minikube.sh
42+
curl -L -o minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64 && \
43+
chmod +x ./minikube && \
44+
mv ./minikube ~/cached-deps/minikube
45+
fi

‎proto/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
FROM python:3-onbuild
22
LABEL maintainer="[email protected]"
33

4-
ADD run /
5-
ENTRYPOINT ["/run"]
4+
ADD run /bin
5+
ENTRYPOINT ["/bin/run"]
66
WORKDIR /work

‎proto/pachyderm‎

Submodule pachyderm updated 95 files

0 commit comments

Comments
 (0)