Skip to content

Commit 5ed3b37

Browse files
YannByronYannByron
authored
[spark] Initiate fluss-spark and introduce spark catalog and table (#2219)
1 parent f847f63 commit 5ed3b37

File tree

25 files changed

+1978
-3
lines changed

25 files changed

+1978
-3
lines changed

‎.github/workflows/ci-template.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ jobs:
3636
strategy:
3737
fail-fast: false
3838
matrix:
39-
module: [ core, flink, lake ]
39+
module: [ core, flink, spark3, lake ]
4040
name: "${{ matrix.module }}"
4141
steps:
4242
- name: Checkout code

‎.github/workflows/stage.sh‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
STAGE_CORE="core"
2121
STAGE_FLINK="flink"
22+
STAGE_SPARK="spark3"
2223
STAGE_LAKE="lake"
2324

2425
MODULES_FLINK="\
@@ -28,6 +29,20 @@ fluss-flink/fluss-flink-2.2,\
2829
fluss-flink/fluss-flink-1.20,\
2930
"
3031

32+
MODULES_COMMON_SPARK="\
33+
fluss-spark,\
34+
fluss-spark/fluss-spark-common,\
35+
fluss-spark/fluss-spark-ut,\
36+
"
37+
38+
MODULES_SPARK3="\
39+
fluss-spark,\
40+
fluss-spark/fluss-spark-common,\
41+
fluss-spark/fluss-spark-ut,\
42+
fluss-spark/fluss-spark-3.5,\
43+
fluss-spark/fluss-spark-3.4,\
44+
"
45+
3146
# we move Flink legacy version tests to "lake" module for balancing testing time
3247
MODULES_LAKE="\
3348
fluss-flink/fluss-flink-1.19,\
@@ -42,10 +57,12 @@ function get_test_modules_for_stage() {
4257
local stage=$1
4358

4459
local modules_flink=$MODULES_FLINK
60+
local modules_spark3=$MODULES_SPARK3
4561
local modules_lake=$MODULES_LAKE
4662
local negated_flink=\!${MODULES_FLINK//,/,\!}
63+
local negated_spark=\!${MODULES_COMMON_SPARK//,/,\!}
4764
local negated_lake=\!${MODULES_LAKE//,/,\!}
48-
local modules_core="$negated_flink,$negated_lake"
65+
local modules_core="$negated_flink,$negated_spark,$negated_lake"
4966

5067
case ${stage} in
5168
(${STAGE_CORE})
@@ -54,6 +71,9 @@ function get_test_modules_for_stage() {
5471
(${STAGE_FLINK})
5572
echo "-pl fluss-test-coverage,$modules_flink"
5673
;;
74+
(${STAGE_SPARK})
75+
echo "-Pspark3 -pl fluss-test-coverage,$modules_spark3"
76+
;;
5777
(${STAGE_LAKE})
5878
echo "-pl fluss-test-coverage,$modules_lake"
5979
;;

‎.scalafmt.conf‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
runner.dialect = scala212
2+
3+
# Version is required to make sure IntelliJ picks the right version
4+
version = 3.10.2
5+
preset = default
6+
7+
# Max column
8+
maxColumn = 100
9+
10+
# This parameter simply says the .stripMargin method was not redefined by the user to assign
11+
# special meaning to indentation preceding the | character. Hence, that indentation can be modified.
12+
assumeStandardLibraryStripMargin = true
13+
align.stripMargin = true
14+
15+
# Align settings
16+
align.preset = none
17+
align.closeParenSite = false
18+
align.openParenCallSite = false
19+
danglingParentheses.defnSite = false
20+
danglingParentheses.callSite = false
21+
danglingParentheses.ctrlSite = true
22+
danglingParentheses.tupleSite = false
23+
align.openParenCallSite = false
24+
align.openParenDefnSite = false
25+
align.openParenTupleSite = false
26+
27+
# Newlines
28+
newlines.alwaysBeforeElseAfterCurlyIf = false
29+
newlines.beforeCurlyLambdaParams = multiline # Newline before lambda params
30+
newlines.afterCurlyLambdaParams = squash # No newline after lambda params
31+
newlines.inInterpolation = "avoid"
32+
newlines.avoidInResultType = true
33+
optIn.annotationNewlines = true
34+
35+
# Scaladoc
36+
docstrings.style = Asterisk # Javadoc style
37+
docstrings.removeEmpty = true
38+
docstrings.oneline = fold
39+
docstrings.forceBlankLineBefore = true
40+
41+
# Indentation
42+
indent.extendSite = 2 # This makes sure extend is not indented as the ctor parameters
43+
44+
# Rewrites
45+
rewrite.rules = [AvoidInfix, Imports, RedundantBraces, SortModifiers]
46+
47+
# Imports
48+
rewrite.imports.sort = scalastyle
49+
rewrite.imports.groups = [
50+
["org.apache.fluss\\..*"],
51+
["org.apache.fluss.shade\\..*"],
52+
[".*"],
53+
["javax\\..*"],
54+
["java\\..*"],
55+
["scala\\..*"]
56+
]
57+
rewrite.imports.contiguousGroups = no
58+
importSelectors = singleline # Imports in a single line, like IntelliJ
59+
60+
# Remove redundant braces in string interpolation.
61+
rewrite.redundantBraces.stringInterpolation = true
62+
rewrite.redundantBraces.defnBodies = false
63+
rewrite.redundantBraces.generalExpressions = false
64+
rewrite.redundantBraces.ifElseExpressions = false
65+
rewrite.redundantBraces.methodBodies = false
66+
rewrite.redundantBraces.includeUnitMethods = false
67+
rewrite.redundantBraces.maxBreaks = 1
68+
69+
# Remove trailing commas
70+
rewrite.trailingCommas.style = "never"

‎copyright.txt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+

‎fluss-common/src/main/java/org/apache/fluss/config/Configuration.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.fluss.config;
1919

2020
import org.apache.fluss.annotation.PublicStable;
21+
import org.apache.fluss.annotation.VisibleForTesting;
2122
import org.apache.fluss.utils.CollectionUtils;
2223

2324
import org.slf4j.Logger;
@@ -643,7 +644,8 @@ <T> void setValueInternal(String key, T value) {
643644
}
644645
}
645646

646-
private Optional<Object> getRawValue(String key) {
647+
@VisibleForTesting
648+
public Optional<Object> getRawValue(String key) {
647649
if (key == null) {
648650
throw new NullPointerException("Key must not be null.");
649651
}

‎fluss-spark/fluss-spark-3.4/pom.xml‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
-->
19+
20+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
22+
<modelVersion>4.0.0</modelVersion>
23+
<parent>
24+
<groupId>org.apache.fluss</groupId>
25+
<artifactId>fluss-spark</artifactId>
26+
<version>0.9-SNAPSHOT</version>
27+
</parent>
28+
29+
<artifactId>fluss-spark-3.4_${scala.binary.version}</artifactId>
30+
<name>Fluss : Engine Spark : 3.4</name>
31+
32+
<properties>
33+
<spark.version>3.4.3</spark.version>
34+
</properties>
35+
36+
<dependencies>
37+
<dependency>
38+
<groupId>org.apache.fluss</groupId>
39+
<artifactId>fluss-spark-common_${scala.binary.version}</artifactId>
40+
<version>${project.version}</version>
41+
</dependency>
42+
43+
<dependency>
44+
<groupId>org.apache.fluss</groupId>
45+
<artifactId>fluss-spark-ut_${scala.binary.version}</artifactId>
46+
<version>${project.version}</version>
47+
<classifier>tests</classifier>
48+
<scope>test</scope>
49+
</dependency>
50+
</dependencies>
51+
52+
<build>
53+
<plugins>
54+
<plugin>
55+
<groupId>org.apache.maven.plugins</groupId>
56+
<artifactId>maven-compiler-plugin</artifactId>
57+
<configuration>
58+
<!-- compilation of main sources -->
59+
<skipMain>${skip.on.java8}</skipMain>
60+
<!-- compilation of test sources -->
61+
<skip>${skip.on.java8}</skip>
62+
</configuration>
63+
</plugin>
64+
65+
<plugin>
66+
<groupId>org.apache.maven.plugins</groupId>
67+
<artifactId>maven-surefire-plugin</artifactId>
68+
<version>3.0.0-M5</version>
69+
<executions>
70+
<!-- Test end with ITCase is e2e test in this module -->
71+
<execution>
72+
<id>integration-tests</id>
73+
<phase>integration-test</phase>
74+
<inherited>false</inherited>
75+
<goals>
76+
<goal>test</goal>
77+
</goals>
78+
<configuration>
79+
<skip>${skip.on.java8}</skip>
80+
<includes>
81+
<include>**/*ITCase.*</include>
82+
</includes>
83+
<!-- e2e test with flink/zookeeper cluster, we set forkCount=1 -->
84+
<forkCount>1</forkCount>
85+
</configuration>
86+
</execution>
87+
<!-- others unit tests -->
88+
<execution>
89+
<id>default-test</id>
90+
<phase>test</phase>
91+
<inherited>false</inherited>
92+
<goals>
93+
<goal>test</goal>
94+
</goals>
95+
<configuration>
96+
<skip>${skip.on.java8}</skip>
97+
<excludes>
98+
<exclude>**/*ITCase.*</exclude>
99+
</excludes>
100+
</configuration>
101+
</execution>
102+
</executions>
103+
</plugin>
104+
105+
<plugin>
106+
<groupId>org.apache.maven.plugins</groupId>
107+
<artifactId>maven-shade-plugin</artifactId>
108+
<executions>
109+
<execution>
110+
<id>shade-fluss</id>
111+
<phase>package</phase>
112+
<goals>
113+
<goal>shade</goal>
114+
</goals>
115+
<configuration>
116+
<artifactSet>
117+
<includes combine.children="append">
118+
<include>org.apache.fluss:fluss-spark-common</include>
119+
<include>org.apache.fluss:fluss-client</include>
120+
</includes>
121+
</artifactSet>
122+
</configuration>
123+
</execution>
124+
</executions>
125+
</plugin>
126+
</plugins>
127+
</build>
128+
129+
</project>

0 commit comments

Comments
 (0)