Skip to content

Commit 3026e31

Browse files
wanglinsongwanglinsong
committed
Merge branch 'ci-fix-pr' into staging-0007f37c6-pr
Signed-off-by: Linsong Wang <[email protected]>
2 parents b77c762 + 65c22fc commit 3026e31

File tree

2 files changed

+229
-1
lines changed

2 files changed

+229
-1
lines changed

‎.github/workflows/gluten.yml‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Gluten Build
16+
17+
on:
18+
pull_request:
19+
paths:
20+
- .github/workflows/gluten.yml
21+
env:
22+
MVN_CMD: mvn -ntp
23+
24+
jobs:
25+
26+
gluten-cpp-build:
27+
name: gluten cpp build
28+
# prevent errors when forks ff their main branch
29+
if: ${{ github.repository == 'IBM/velox' }}
30+
runs-on: ubuntu-22.04
31+
env:
32+
CCACHE_DIR: "${{ github.workspace }}/.ccache"
33+
steps:
34+
- uses: actions/checkout@v4
35+
- name: Get Ccache
36+
uses: actions/cache/restore@v4
37+
with:
38+
path: '${{ env.CCACHE_DIR }}'
39+
key: ccache-ibm-velox-${{github.sha}}
40+
restore-keys: |
41+
ccache-ibm-velox
42+
43+
- name: Setup Gluten
44+
run: |
45+
git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
46+
sed -i 's/ibm/ibm-xxx/gI' ep/build-velox/src/get-velox.sh
47+
BRANCH=$(echo ${GITHUB_REF#refs/heads/})
48+
# sed -i 's/VELOX_BRANCH=2025.*/VELOX_BRANCH=${BRANCH}/g' ep/build-velox/src/get-velox.sh
49+
- name: Build Gluten native libraries
50+
run: |
51+
docker pull apache/gluten:vcpkg-centos-7
52+
docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c "
53+
git config --global --add safe.directory /work
54+
set -e
55+
df -a
56+
cd /work
57+
git log -n 3
58+
cd /work/gluten
59+
git log -n 3
60+
export CCACHE_DIR=/work/.ccache
61+
export CCACHE_SLOPPINESS=file_macro,locale,time_macros
62+
mkdir -p /work/.ccache
63+
ccache -M 1G
64+
ccache -sz
65+
source /opt/rh/devtoolset-11/enable
66+
source /opt/rh/rh-git227/enable
67+
export NUM_THREADS=4
68+
./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \
69+
--build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON --enable_enhanced_features=ON --velox_home=/work
70+
pushd /work
71+
git log -n 3
72+
popd
73+
ccache -s
74+
mkdir -p /work/.m2/repository/org/apache/arrow/
75+
cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
76+
"
77+
- name: "Save ccache"
78+
if: always()
79+
uses: actions/cache/save@v4
80+
id: ccache
81+
with:
82+
path: '${{ env.CCACHE_DIR }}'
83+
key: ccache-ibm-velox-${{github.sha}}
84+
85+
- uses: actions/upload-artifact@v4
86+
with:
87+
name: velox-native-lib-centos-7-${{github.sha}}
88+
path: ./gluten/cpp/build/releases/
89+
if-no-files-found: error
90+
- uses: actions/upload-artifact@v4
91+
with:
92+
name: arrow-jars-centos-7-${{github.sha}}
93+
path: .m2/repository/org/apache/arrow/
94+
if-no-files-found: error
95+
96+
spark-test-spark32:
97+
needs: gluten-cpp-build
98+
runs-on: ubuntu-22.04
99+
container: apache/gluten:centos-8-jdk8
100+
steps:
101+
- name: Setup Gluten
102+
run: |
103+
git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
104+
- name: Download All Artifacts
105+
uses: actions/download-artifact@v4
106+
with:
107+
name: velox-native-lib-centos-7-${{github.sha}}
108+
path: ./gluten/cpp/build/releases
109+
- name: Download Arrow Jars
110+
uses: actions/download-artifact@v4
111+
with:
112+
name: arrow-jars-centos-7-${{github.sha}}
113+
path: /root/.m2/repository/org/apache/arrow/
114+
- name: Build package for Spark 3.2
115+
run: |
116+
cd $GITHUB_WORKSPACE/gluten
117+
export SPARK_SCALA_VERSION=2.12
118+
yum install -y java-17-openjdk-devel
119+
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
120+
export PATH=$JAVA_HOME/bin:$PATH
121+
java -version
122+
$MVN_CMD clean package -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -Phudi -DskipTests
123+
124+
spark-test-spark34:
125+
needs: gluten-cpp-build
126+
runs-on: ubuntu-22.04
127+
container: apache/gluten:centos-8-jdk8
128+
steps:
129+
- name: Setup Gluten
130+
run: |
131+
git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
132+
- name: Download All Artifacts
133+
uses: actions/download-artifact@v4
134+
with:
135+
name: velox-native-lib-centos-7-${{github.sha}}
136+
path: ./gluten/cpp/build/releases
137+
- name: Download Arrow Jars
138+
uses: actions/download-artifact@v4
139+
with:
140+
name: arrow-jars-centos-7-${{github.sha}}
141+
path: /root/.m2/repository/org/apache/arrow/
142+
- name: Prepare spark.test.home for Spark 3.4.4 (other tests)
143+
run: |
144+
dnf module -y install python39 && \
145+
alternatives --set python3 /usr/bin/python3.9 && \
146+
pip3 install setuptools==77.0.3 && \
147+
pip3 install pyspark==3.4.4 cython && \
148+
pip3 install pandas==2.2.3 pyarrow==20.0.0
149+
- name: Build and Run unit test for Spark 3.4.4 (other tests)
150+
run: |
151+
cd $GITHUB_WORKSPACE/gluten
152+
export SPARK_SCALA_VERSION=2.12
153+
yum install -y java-17-openjdk-devel
154+
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
155+
export PATH=$JAVA_HOME/bin:$PATH
156+
java -version
157+
export SPARK_HOME=/opt/shims/spark34/spark_home/
158+
ls -l $SPARK_HOME
159+
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Pspark-ut \
160+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \
161+
-DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}"
162+
- name: Upload test report
163+
if: always()
164+
uses: actions/upload-artifact@v4
165+
with:
166+
name: ${{ github.job }}-report
167+
path: '**/surefire-reports/TEST-*.xml'
168+
- name: Upload unit tests log files
169+
if: ${{ !success() }}
170+
uses: actions/upload-artifact@v4
171+
with:
172+
name: ${{ github.job }}-test-log
173+
path: |
174+
**/target/*.log
175+
**/gluten-ut/**/hs_err_*.log
176+
**/gluten-ut/**/core.*
177+
- name: Upload golden files
178+
if: failure()
179+
uses: actions/upload-artifact@v4
180+
with:
181+
name: ${{ github.job }}-golden-files
182+
path: /tmp/tpch-approved-plan/**
183+
184+
spark-test-spark34-slow:
185+
needs: gluten-cpp-build
186+
runs-on: ubuntu-22.04
187+
container: apache/gluten:centos-8-jdk8
188+
steps:
189+
- name: Setup Gluten
190+
run: |
191+
git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
192+
- name: Download All Artifacts
193+
uses: actions/download-artifact@v4
194+
with:
195+
name: velox-native-lib-centos-7-${{github.sha}}
196+
path: ./gluten/cpp/build/releases
197+
- name: Download Arrow Jars
198+
uses: actions/download-artifact@v4
199+
with:
200+
name: arrow-jars-centos-7-${{github.sha}}
201+
path: /root/.m2/repository/org/apache/arrow/
202+
- name: Build and Run unit test for Spark 3.4.4 (slow tests)
203+
run: |
204+
cd $GITHUB_WORKSPACE/gluten
205+
yum install -y java-17-openjdk-devel
206+
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
207+
export PATH=$JAVA_HOME/bin:$PATH
208+
java -version
209+
export SPARK_HOME=/opt/shims/spark34/spark_home/
210+
ls -l $SPARK_HOME
211+
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut -Phudi \
212+
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \
213+
-DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}"
214+
- name: Upload test report
215+
if: always()
216+
uses: actions/upload-artifact@v4
217+
with:
218+
name: ${{ github.job }}-report
219+
path: '**/surefire-reports/TEST-*.xml'
220+
- name: Upload unit tests log files
221+
if: ${{ !success() }}
222+
uses: actions/upload-artifact@v4
223+
with:
224+
name: ${{ github.job }}-test-log
225+
path: |
226+
**/target/*.log
227+
**/gluten-ut/**/hs_err_*.log
228+
**/gluten-ut/**/core.*

‎velox/parse/IExpr.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ class IExpr {
9090
/// The last alias added will win if called multiple times. Throws in case the
9191
/// subclass does not implement it.
9292
virtual ExprPtr withAlias(const std::string& alias) const {
93-
VELOX_FAIL("Unable to add alias to expression '{}'", *this);
93+
VELOX_FAIL("Unable to add alias to expression '{}'", toString());
9494
}
9595

9696
/// Returns a copy of this expression with the alias removed.

0 commit comments

Comments
 (0)