Skip to content

Commit 767fdf5

Browse files
unidevelunidevel
committed
Add docker dev env
1 parent b516521 commit 767fdf5

File tree

4 files changed

+328
-47
lines changed

4 files changed

+328
-47
lines changed

‎.github/workflows/presto-dev-publish.yml‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
name: Presto Dev Images
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
prebuild_centos_dependency_image:
7+
description: 'Prebuilt CentOS dependency image (e.g., org/presto-centos-dependency:tag)'
8+
type: string
9+
required: false
10+
default: ''
11+
prebuild_ubuntu_dependency_image:
12+
description: 'Prebuilt Ubuntu dependency image (e.g., org/presto-ubuntu-dependency:tag)'
13+
type: string
14+
required: false
15+
default: ''
16+
publish_centos_image:
17+
description: 'Build and publish centos development image'
18+
type: boolean
19+
default: true
20+
required: false
21+
publish_ubuntu_image:
22+
description: 'Build and publish ubuntu development image'
23+
type: boolean
24+
default: true
25+
required: false
26+
tag_image_as_latest:
27+
description: 'Tag the published images as latest'
28+
type: boolean
29+
default: true
30+
required: false
31+
env:
32+
JAVA_VERSION: ${{ vars.JAVA_VERSION || '17' }}
33+
JAVA_DISTRIBUTION: ${{ vars.JAVA_DISTRIBUTION || 'temurin' }}
34+
MAVEN_OPTS: ${{ vars.MAVEN_OPTS }}
35+
DOCKER_REPO: ${{ github.repository }}
36+
ORG_NAME: ${{ github.repository_owner }}
37+
RELEASE_BRANCH: release-${{ inputs.RELEASE_VERSION }}
38+
RELEASE_TAG: ${{ inputs.RELEASE_VERSION }}
39+
RELEASE_NOTES_COMMIT: ${{ inputs.release-notes-commit }}
40+
41+
jobs:
42+
publish-centos-images:
43+
runs-on: ubuntu-latest
44+
if: ${{ github.event.inputs.publish_centos_image == 'true' }}
45+
environment: release
46+
timeout-minutes: 900
47+
steps:
48+
- name: Free disk space
49+
uses: jlumbroso/[email protected]
50+
with:
51+
tool-cache: false
52+
docker-images: false
53+
54+
- name: Set up JDK ${{ env.JAVA_DISTRIBUTION }}/${{ env.JAVA_VERSION }}
55+
uses: actions/setup-java@v4
56+
with:
57+
java-version: ${{ env.JAVA_VERSION }}
58+
distribution: ${{ env.JAVA_DISTRIBUTION }}
59+
60+
- name: Checkout
61+
uses: actions/checkout@v4
62+
63+
- name: Get presto release version
64+
run: |
65+
PRESTO_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \
66+
-Dexpression=project.version -q -ntp -DforceStdout | tail -n 1)
67+
echo "RELEASE_TAG=$PRESTO_VERSION" >> $GITHUB_ENV
68+
69+
- name: Initialize prestissimo submodules
70+
working-directory: presto-native-execution
71+
run: |
72+
df -h
73+
cd ${{ github.workspace }}/presto-native-execution && make submodules
74+
echo "COMMIT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
75+
76+
- name: Login to dockerHub
77+
uses: docker/login-action@v3
78+
with:
79+
username: ${{ secrets.DOCKERHUB_USERNAME }}
80+
password: ${{ secrets.DOCKERHUB_TOKEN }}
81+
82+
- name: Build centos dependency image
83+
working-directory: presto-native-execution
84+
run: |
85+
df -h
86+
if [[ -n "${{ github.event.inputs.prebuild_centos_dependency_image }}" ]]; then
87+
echo "Pulling prebuilt CentOS dependency image: ${{ github.event.inputs.prebuild_centos_dependency_image }}"
88+
docker pull ${{ github.event.inputs.prebuild_centos_dependency_image }}
89+
docker tag ${{ github.event.inputs.prebuild_centos_dependency_image }} presto/prestissimo-dependency:centos9
90+
else
91+
docker compose build centos-native-dependency
92+
docker tag presto/prestissimo-dependency:centos9 ${{ github.repository_owner }}/presto-centos-dependency:${{ env.RELEASE_TAG }}-${{ env.COMMIT_SHA }}
93+
docker push ${{ github.repository_owner }}/presto-centos-dependency:${{ env.RELEASE_TAG }}-${{ env.COMMIT_SHA }}
94+
if [[ "${{ github.event.inputs.tag_image_as_latest }}" == "true" ]]; then
95+
docker tag presto/prestissimo-dependency:centos9 ${{ github.repository_owner }}/presto-centos-dependency:latest
96+
docker push ${{ github.repository_owner }}/presto-centos-dependency:latest
97+
fi
98+
fi
99+
docker images
100+
101+
- name: Build centos dev images
102+
working-directory: docker
103+
run: |
104+
df -h
105+
docker compose build centos-dev
106+
docker images
107+
108+
- name: Publish centos dev images
109+
working-directory: docker
110+
run: |
111+
df -h
112+
docker tag presto/presto-dev:centos9 ${{ env.ORG_NAME }}/presto-centos-dev:${{env.RELEASE_TAG }}
113+
docker push ${{ env.ORG_NAME }}/presto-centos-dev:${{ env.RELEASE_TAG }}
114+
if [[ "${{ github.event.inputs.tag_image_as_latest }}" == "true" ]]; then
115+
docker tag presto/presto-dev:centos9 ${{ env.ORG_NAME }}/presto-centos-dev:latest
116+
docker push ${{ env.ORG_NAME }}/presto-centos-dev:latest
117+
fi
118+
docker images
119+
120+
publish-ubuntu-images:
121+
runs-on: ubuntu-latest
122+
if: ${{ github.event.inputs.publish_ubuntu_image == 'true' }}
123+
environment: release
124+
timeout-minutes: 900
125+
steps:
126+
- name: Free disk space
127+
uses: jlumbroso/[email protected]
128+
with:
129+
tool-cache: false
130+
docker-images: false
131+
132+
- name: Set up JDK ${{ env.JAVA_DISTRIBUTION }}/${{ env.JAVA_VERSION }}
133+
uses: actions/setup-java@v4
134+
with:
135+
java-version: ${{ env.JAVA_VERSION }}
136+
distribution: ${{ env.JAVA_DISTRIBUTION }}
137+
138+
- name: Checkout
139+
uses: actions/checkout@v4
140+
141+
- name: Get presto release version
142+
run: |
143+
PRESTO_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \
144+
-Dexpression=project.version -q -ntp -DforceStdout | tail -n 1)
145+
echo "RELEASE_TAG=$PRESTO_VERSION" >> $GITHUB_ENV
146+
147+
- name: Initialize prestissimo submodules
148+
working-directory: presto-native-execution
149+
run: |
150+
df -h
151+
cd ${{ github.workspace }}/presto-native-execution && make submodules
152+
echo "COMMIT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
153+
154+
- name: Login to dockerHub
155+
uses: docker/login-action@v3
156+
with:
157+
username: ${{ secrets.DOCKERHUB_USERNAME }}
158+
password: ${{ secrets.DOCKERHUB_TOKEN }}
159+
160+
- name: Build ubuntu dependency image
161+
working-directory: presto-native-execution
162+
run: |
163+
df -h
164+
if [[ -n "${{ github.event.inputs.prebuild_ubuntu_dependency_image }}" ]]; then
165+
echo "Pulling prebuilt Ubuntu dependency image: ${{ github.event.inputs.prebuild_ubuntu_dependency_image }}"
166+
docker pull ${{ github.event.inputs.prebuild_ubuntu_dependency_image }}
167+
docker tag ${{ github.event.inputs.prebuild_ubuntu_dependency_image }} presto/prestissimo-dependency:ubuntu-22.04
168+
else
169+
docker compose build ubuntu-native-dependency
170+
docker tag presto/prestissimo-dependency:ubuntu-22.04 ${{ github.repository_owner }}/presto-ubuntu-dependency:${{ env.RELEASE_TAG }}-${{ env.COMMIT_SHA }}
171+
docker push ${{ github.repository_owner }}/presto-ubuntu-dependency:${{ env.RELEASE_TAG }}-${{ env.COMMIT_SHA }}
172+
if [[ "${{ github.event.inputs.tag_image_as_latest }}" == "true" ]]; then
173+
docker tag presto/prestissimo-dependency:ubuntu-22.04 ${{ github.repository_owner }}/presto-ubuntu-dependency:latest
174+
docker push ${{ github.repository_owner }}/presto-ubuntu-dependency:latest
175+
fi
176+
fi
177+
docker images
178+
179+
- name: Build ubuntu dev Image
180+
if: ${{ github.event.inputs.publish_ubuntu_image }}
181+
working-directory: docker
182+
run: |
183+
df -h
184+
docker compose build ubuntu-dev
185+
docker images
186+
187+
- name: Publish ubuntu dev images
188+
if: ${{ github.event.inputs.publish_ubuntu_image }}
189+
working-directory: docker
190+
run: |
191+
df -h
192+
docker tag presto/presto-dev:ubuntu-22.04 ${{ env.ORG_NAME }}/presto-ubuntu-dev:${{env.RELEASE_TAG }}
193+
docker push ${{ env.ORG_NAME }}/presto-ubuntu-dev:${{ env.RELEASE_TAG }}
194+
if [[ "${{ github.event.inputs.tag_image_as_latest }}" == "true" ]]; then
195+
docker tag presto/presto-dev:ubuntu-22.04 ${{ env.ORG_NAME }}/presto-ubuntu-dev:latest
196+
docker push ${{ env.ORG_NAME }}/presto-ubuntu-dev:latest
197+
fi
198+
docker images

‎docker/Dockerfile-dev‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
# Licensed under the Apache License, Version 2.0 (the "License");
2+
# you may not use this file except in compliance with the License.
3+
# You may obtain a copy of the License at
4+
#
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
#
7+
# Unless required by applicable law or agreed to in writing, software
8+
# distributed under the License is distributed on an "AS IS" BASIS,
9+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
# See the License for the specific language governing permissions and
11+
# limitations under the License.
12+
13+
ARG DEPENDENCY_IMAGE=presto/prestissimo-dependency:centos9
14+
ARG BASE_IMAGE=quay.io/centos/centos:stream9
15+
FROM ${DEPENDENCY_IMAGE} as prestissimo-dev
16+
17+
ARG OSNAME=centos
18+
ARG EXTRA_CMAKE_FLAGS=''
19+
ARG NUM_THREADS=8
20+
21+
ENV PROMPT_ALWAYS_RESPOND=n
22+
ENV BUILD_BASE_DIR=_build
23+
ENV LANG=C.UTF-8
24+
ENV EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS}
25+
ENV BUILD_ROOT=/presto
26+
ENV PRESTISSIMO_HOME=/opt/presitissimo
27+
28+
RUN --mount=type=bind,source=.,target=${BUILD_ROOT},readonly=false \
29+
--mount=type=cache,target=/root/ccache-${OSNAME} \
30+
echo "[safe]" > /root/.gitconfig && \
31+
echo " directory = ${BUILD_ROOT}" >> /root/.gitconfig && \
32+
echo " directory = ${BUILD_ROOT}/presto-native-execution/velox" >> /root/.gitconfig && \
33+
cat /root/.gitconfig && \
34+
export CCACHE_DIR=/root/ccache-${OSNAME} && \
35+
cd ${BUILD_ROOT}/presto-native-execution && \
36+
make debug && \
37+
make release && \
38+
# EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS} NUM_THREADS=${NUM_THREADS} \
39+
# make --directory="${BUILD_ROOT}/presto-native-execution" cmake-and-build BUILD_TYPE=Release BUILD_DIR=release BUILD_BASE_DIR=${BUILD_BASE_DIR} && \
40+
# EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS} NUM_THREADS=${NUM_THREADS} \
41+
# make --directory="${BUILD_ROOT}/presto-native-execution" cmake-and-build BUILD_TYPE=Debug BUILD_DIR=debug BUILD_BASE_DIR=${BUILD_BASE_DIR} && \
42+
cd /root && cp -r ${CCACHE_DIR} .ccache && \
43+
ccache -sz -v && \
44+
mkdir -p ${PRESTISSIMO_HOME}/{bin,lib} && \
45+
cd ${BUILD_ROOT}/presto-native-execution && \
46+
cp ${BUILD_ROOT}/${BUILD_BASE_DIR}/release/presto_cpp/main/presto_server ${PRESTISSIMO_HOME}/bin && \
47+
chmod 0755 ${PRESTISSIMO_HOME}/bin/presto_server && \
48+
echo "${PRESTISSIMO_HOME}/lib" > /etc/ld.so.conf.d/prestissimo.conf && ldconfig && \
49+
make clean
50+
51+
COPY --chmod=0755 ./etc ${PRESTISSIMO_HOME}}/etc
52+
COPY --chmod=0775 ./entrypoint.sh ${PRESTISSIMO_HOME}/entrypoint.sh
53+
54+
FROM prestissimo-dev AS centos-dev
55+
56+
ENV PYTHON_VERSION=3.11.9 \
57+
JAVA_HOME=/usr/lib/jvm/jre-11-openjdk
58+
59+
RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked \
60+
--mount=type=cache,target=/usr/local/src,sharing=locked \
61+
dnf update -y && dnf install -y java-11-openjdk-headless less procps wget diffutils git vim && \
62+
cd /usr/local/src && \
63+
wget -c https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
64+
tar xvf Python-${PYTHON_VERSION}.tgz && \
65+
cd Python-${PYTHON_VERSION} && \
66+
./configure --enable-optimizations && \
67+
make -j$(nproc) && \
68+
make altinstall && \
69+
alternatives --install /usr/bin/python python /usr/local/bin/python3.11 1 && \
70+
alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.11 1 && \
71+
rm -rf Python-${PYTHON_VERSION}
72+
73+
RUN --mount=type=cache,target=/usr/local/src/.m2 \
74+
--mount=type=bind,source=..,target=${BUILD_ROOT},readonly=false \
75+
find /usr/local/src/.m2 || true && \
76+
cd /presto && \
77+
MAVEN_USER_HOME=/usr/local/src/.m2 ./mvnw clean install -Dmaven.repo.local=/usr/local/src/.m2/repository -DskipTests && \
78+
for dir in presto-*; do \
79+
rm -rf /usr/local/src/.m2/repository/com/facebook/presto/$dir; \
80+
done && \
81+
cp -a /usr/local/src/.m2 /root/
82+
83+
FROM prestissimo-dev AS ubuntu-dev
84+
85+
ENV PRESTO_HOME=/opt/presto \
86+
JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0
87+
88+
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
89+
--mount=type=cache,target=/usr/local/src,sharing=locked \
90+
apt-get update && \
91+
DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common wget less procps git vim diffutils curl xz-utils openjdk-11-jre-headless && \
92+
add-apt-repository ppa:deadsnakes/ppa && \
93+
apt-get update && \
94+
DEBIAN_FRONTEND=noninteractive apt-get install -y python3.11 python3.11-venv python3.11-distutils && \
95+
update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \
96+
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
97+
98+
RUN --mount=type=cache,target=/root/m2 \
99+
--mount=type=bind,source=..,target=${BUILD_ROOT},readonly=false \
100+
cd ${BUILD_ROOT} && \
101+
MAVEN_USER_HOME=/root/m2 ./mvnw clean install -Dmaven.repo.local=/root/.m2/repository -DskipTests && \
102+
for dir in presto-*; do \
103+
rm -rf /usr/local/src/.m2/repository/com/facebook/presto/$dir; \
104+
done && \
105+
cp -a /root/m2 /root/.m2 && \
106+
mkdir -p $PRESTO_HOME && \
107+
tar --strip-components=1 -C "$PRESTO_HOME" -zxf /$BUILD_ROOT/presto-server/target/presto-server-*.tar.gz \
108+
&& cp ${BUILD_ROOT}/presto-cli/target/presto-cli-*-executable.jar /opt/presto-cli \
109+
&& chmod 0755 /opt/presto-cli \
110+
&& ln -s /opt/presto-cli /usr/local/bin/ \
111+
# mkdir for config
112+
&& mkdir -p $PRESTO_HOME/etc/catalog \
113+
&& mkdir -p /var/lib/presto/data \
114+
&& mkdir -p /usr/lib/presto/utils \
115+
&& curl -o /usr/lib/presto/utils/jmx_prometheus_javaagent-$JMX_PROMETHEUS_JAVAAGENT_VERSION.jar https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/$JMX_PROMETHEUS_JAVAAGENT_VERSION/jmx_prometheus_javaagent-$JMX_PROMETHEUS_JAVAAGENT_VERSION.jar \
116+
&& ln -s /usr/lib/presto/utils/jmx_prometheus_javaagent-$JMX_PROMETHEUS_JAVAAGENT_VERSION.jar /usr/lib/presto/utils/jmx_prometheus_javaagent.jar
117+
118+
COPY etc/config.properties.example $PRESTO_HOME/etc/config.properties
119+
COPY etc/jvm.config.example $PRESTO_HOME/etc/jvm.config
120+
COPY etc/node.properties $PRESTO_HOME/etc/node.properties
121+
COPY etc/catalog $PRESTO_HOME/etc/catalog
122+
COPY entrypoint.sh $PRESTO_HOME/

‎presto-native-execution/docker-compose-dev.yml‎ renamed to ‎docker/docker-compose.yml‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ services:
1414
# Usage:
1515
# docker compose build ubuntu-dev
1616
# podman compose build ubuntu-dev
17-
image: presto/prestissimo-dev:ubuntu-22.04
17+
image: presto/presto-dev:ubuntu-22.04
1818
build:
1919
args:
2020
# A few files in Velox require significant memory to compile and link.
@@ -31,8 +31,9 @@ services:
3131
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS
3232
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER
3333
-DPRESTO_ENABLE_SPATIAL=ON
34-
context: .
35-
dockerfile: scripts/dockerfiles/prestissimo-dev.dockerfile
34+
context: ..
35+
dockerfile: docker/Dockerfile-dev
36+
target: ubuntu-dev
3637
stdin_open: true
3738
tty: true
3839
volumes:
@@ -44,7 +45,7 @@ services:
4445
# Usage:
4546
# docker compose build centos-dev
4647
# podman compose build centos-dev
47-
image: presto/prestissimo-dev:centos9
48+
image: presto/presto-dev:centos9
4849
build:
4950
args:
5051
# A few files in Velox require significant memory to compile and link.
@@ -59,8 +60,9 @@ services:
5960
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS
6061
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER
6162
-DPRESTO_ENABLE_SPATIAL=ON
62-
context: .
63-
dockerfile: scripts/dockerfiles/prestissimo-dev.dockerfile
63+
context: ..
64+
dockerfile: docker/Dockerfile-dev
65+
target: centos-dev
6466
stdin_open: true
6567
tty: true
6668
volumes:

0 commit comments

Comments
 (0)