Skip to content

Commit c38a040

Browse files
pvarypvary
authored
[FLINK-33559] Externalize Kafka Python connector code
1 parent 979791c commit c38a040

File tree

12 files changed

+2666
-1
lines changed

12 files changed

+2666
-1
lines changed

‎.github/workflows/push_pr.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,11 @@ jobs:
2929
uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
3030
with:
3131
flink_version: ${{ matrix.flink }}
32+
33+
python_test:
34+
strategy:
35+
matrix:
36+
flink: [ 1.17.1, 1.18.0 ]
37+
uses: apache/flink-connector-shared-utils/.github/workflows/python_ci.yml@ci_utils
38+
with:
39+
flink_version: ${{ matrix.flink }}

‎.gitignore

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,20 @@ out/
3535
tools/flink
3636
tools/flink-*
3737
tools/releasing/release
38-
tools/japicmp-output
38+
tools/japicmp-output
39+
40+
# Generated file, do not store in git
41+
flink-python/pyflink/datastream/connectors/kafka_connector_version.py
42+
flink-python/apache_flink_connectors_kafka.egg-info/
43+
flink-python/.tox/
44+
flink-python/build
45+
flink-python/dist
46+
flink-python/dev/download
47+
flink-python/dev/.conda/
48+
flink-python/dev/log/
49+
flink-python/dev/.stage.txt
50+
flink-python/dev/install_command.sh
51+
flink-python/dev/lint-python.sh
52+
flink-python/dev/build-wheels.sh
53+
flink-python/dev/glibc_version_fix.h
54+
flink-python/dev/dev-requirements.txt

‎flink-python/MANIFEST.in

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
graft pyflink
20+
global-exclude *.py[cod] __pycache__ .DS_Store
21+

‎flink-python/README.txt

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
This is official Apache Flink Kafka Python connector.
2+
3+
For the latest information about Flink connector, please visit our website at:
4+
5+
https://flink.apache.org
6+
7+
and our GitHub Account for Kafka connector
8+
9+
https://github.com/apache/flink-connector-kafka
10+
11+
If you have any questions, ask on our Mailing lists:
12+
13+
[email protected]
14+
[email protected]

‎flink-python/dev/integration_test.sh

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env bash
2+
################################################################################
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
################################################################################
19+
20+
function test_module() {
21+
module="$FLINK_PYTHON_DIR/pyflink/$1"
22+
echo "test module $module"
23+
pytest --durations=20 ${module} $2
24+
if [[ $? -ne 0 ]]; then
25+
echo "test module $module failed"
26+
exit 1
27+
fi
28+
}
29+
30+
function test_all_modules() {
31+
# test datastream module
32+
test_module "datastream"
33+
}
34+
35+
# CURRENT_DIR is "flink-connector-kafka/flink-python/dev/"
36+
CURRENT_DIR="$(cd "$( dirname "$0" )" && pwd)"
37+
38+
# FLINK_PYTHON_DIR is "flink-connector-kafka/flink-python"
39+
FLINK_PYTHON_DIR=$(dirname "$CURRENT_DIR")
40+
41+
# FLINK_SOURCE_DIR is "flink-connector-kafka"
42+
FLINK_SOURCE_DIR=$(dirname "$FLINK_PYTHON_DIR")
43+
44+
# set the FLINK_TEST_LIB_DIR to "flink-connector-kafka"
45+
export FLINK_TEST_LIBS="${FLINK_SOURCE_DIR}/flink-python/target/test-dependencies/*"
46+
47+
# Temporarily update the installed 'pyflink_gateway_server.py' files with the new one
48+
# Needed only until Flink 1.19 release
49+
echo "Checking ${FLINK_SOURCE_DIR} for 'pyflink_gateway_server.py'"
50+
find "${FLINK_SOURCE_DIR}/flink-python" -name pyflink_gateway_server.py
51+
find "${FLINK_SOURCE_DIR}/flink-python/.tox" -name pyflink_gateway_server.py -exec cp "${FLINK_SOURCE_DIR}/flink-python/pyflink/pyflink_gateway_server.py" {} \;
52+
53+
# python test
54+
test_all_modules

‎flink-python/pom.xml

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
24+
<modelVersion>4.0.0</modelVersion>
25+
26+
<parent>
27+
<groupId>org.apache.flink</groupId>
28+
<artifactId>flink-connector-kafka-parent</artifactId>
29+
<version>3.1-SNAPSHOT</version>
30+
</parent>
31+
32+
<artifactId>flink-connector-kafka-python</artifactId>
33+
<name>Flink : Connectors : SQL : Kafka : Python</name>
34+
35+
<packaging>pom</packaging>
36+
37+
<properties>
38+
</properties>
39+
40+
<dependencies>
41+
<dependency>
42+
<groupId>org.apache.flink</groupId>
43+
<artifactId>flink-sql-connector-kafka</artifactId>
44+
<version>${project.version}</version>
45+
</dependency>
46+
<dependency>
47+
<groupId>org.apache.flink</groupId>
48+
<artifactId>flink-runtime</artifactId>
49+
<version>${flink.version}</version>
50+
</dependency>
51+
<dependency>
52+
<groupId>org.apache.flink</groupId>
53+
<artifactId>flink-streaming-java</artifactId>
54+
<version>${flink.version}</version>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.apache.flink</groupId>
58+
<artifactId>flink-connector-test-utils</artifactId>
59+
<version>${flink.version}</version>
60+
</dependency>
61+
<dependency>
62+
<groupId>org.apache.flink</groupId>
63+
<artifactId>flink-sql-avro</artifactId>
64+
<version>${flink.version}</version>
65+
</dependency>
66+
</dependencies>
67+
68+
<build>
69+
<plugins>
70+
<plugin>
71+
<groupId>org.apache.maven.plugins</groupId>
72+
<artifactId>maven-antrun-plugin</artifactId>
73+
<executions>
74+
<execution>
75+
<id>clean</id>
76+
<phase>clean</phase>
77+
<goals>
78+
<goal>run</goal>
79+
</goals>
80+
<configuration>
81+
<target>
82+
<delete dir="${project.basedir}/.tox"/>
83+
<delete dir="${project.basedir}/apache_flink_connectors_kafka.egg-info"/>
84+
<delete dir="${project.basedir}/dev/.conda"/>
85+
<delete dir="${project.basedir}/dev/download"/>
86+
<delete dir="${project.basedir}/dev/log"/>
87+
<delete dir="${project.basedir}/build"/>
88+
<delete dir="${project.basedir}/dist"/>
89+
<delete dir="${project.basedir}/pyflink/lib"/>
90+
<delete file="${project.basedir}/dev/.stage.txt"/>
91+
<delete file="${project.basedir}/dev/install_command.sh"/>
92+
<delete file="${project.basedir}/dev/lint-python.sh"/>
93+
<delete file="${project.basedir}/dev/build-wheels.sh"/>
94+
<delete file="${project.basedir}/dev/glibc_version_fix.h"/>
95+
<delete file="${project.basedir}/dev/dev-requirements.txt"/>
96+
<delete file="${project.basedir}/pyflink/datastream/connectors/kafka_connector_version.py"/>
97+
</target>
98+
</configuration>
99+
</execution>
100+
</executions>
101+
</plugin>
102+
<plugin>
103+
<groupId>org.apache.maven.plugins</groupId>
104+
<artifactId>maven-dependency-plugin</artifactId>
105+
<executions>
106+
<execution>
107+
<phase>package</phase>
108+
<goals>
109+
<goal>copy</goal>
110+
</goals>
111+
<configuration>
112+
<artifactItems>
113+
<artifactItem>
114+
<groupId>org.apache.flink</groupId>
115+
<artifactId>flink-sql-connector-kafka</artifactId>
116+
</artifactItem>
117+
<artifactItem>
118+
<groupId>org.apache.flink</groupId>
119+
<artifactId>flink-runtime</artifactId>
120+
<!-- Don't use test-jar type because of a bug in the plugin (MDEP-587). -->
121+
<classifier>tests</classifier>
122+
</artifactItem>
123+
<artifactItem>
124+
<groupId>org.apache.flink</groupId>
125+
<artifactId>flink-streaming-java</artifactId>
126+
<!-- Don't use test-jar type because of a bug in the plugin (MDEP-587). -->
127+
<classifier>tests</classifier>
128+
</artifactItem>
129+
<artifactItem>
130+
<groupId>org.apache.flink</groupId>
131+
<artifactId>flink-connector-test-utils</artifactId>
132+
</artifactItem>
133+
<artifactItem>
134+
<groupId>org.apache.flink</groupId>
135+
<artifactId>flink-test-utils</artifactId>
136+
</artifactItem>
137+
<artifactItem>
138+
<groupId>org.apache.flink</groupId>
139+
<artifactId>flink-sql-avro</artifactId>
140+
</artifactItem>
141+
</artifactItems>
142+
<outputDirectory>${project.build.directory}/test-dependencies</outputDirectory>
143+
</configuration>
144+
</execution>
145+
</executions>
146+
</plugin>
147+
<plugin>
148+
<groupId>org.apache.maven.plugins</groupId>
149+
<artifactId>maven-dependency-plugin</artifactId>
150+
<executions>
151+
<execution>
152+
<id>copy-dependencies</id>
153+
<phase>package</phase>
154+
<goals>
155+
<goal>copy-dependencies</goal>
156+
</goals>
157+
<configuration>
158+
<includeGroupIds>junit</includeGroupIds>
159+
<outputDirectory>${project.build.directory}/test-dependencies</outputDirectory>
160+
</configuration>
161+
</execution>
162+
</executions>
163+
</plugin>
164+
<!-- Download the testing infra sources from the Flink main repository -->
165+
<plugin>
166+
<groupId>org.codehaus.mojo</groupId>
167+
<artifactId>wagon-maven-plugin</artifactId>
168+
<version>2.0.2</version>
169+
<executions>
170+
<execution>
171+
<id>download-install</id>
172+
<phase>validate</phase>
173+
<goals>
174+
<goal>download-single</goal>
175+
</goals>
176+
<configuration>
177+
<url>https://raw.githubusercontent.com/apache/flink-connector-shared-utils/ci_utils/python/install_command.sh</url>
178+
<toDir>${project.basedir}/dev</toDir>
179+
<skip>${python.infra.download.skip}</skip>
180+
</configuration>
181+
</execution>
182+
<execution>
183+
<id>download-lint</id>
184+
<phase>validate</phase>
185+
<goals>
186+
<goal>download-single</goal>
187+
</goals>
188+
<configuration>
189+
<url>https://raw.githubusercontent.com/apache/flink-connector-shared-utils/ci_utils/python/lint-python.sh</url>
190+
<toDir>${project.basedir}/dev</toDir>
191+
<skip>${python.infra.download.skip}</skip>
192+
</configuration>
193+
</execution>
194+
<execution>
195+
<id>download-build-wheels</id>
196+
<phase>validate</phase>
197+
<goals>
198+
<goal>download-single</goal>
199+
</goals>
200+
<configuration>
201+
<url>https://raw.githubusercontent.com/apache/flink-connector-shared-utils/ci_utils/python/build-wheels.sh</url>
202+
<toDir>${project.basedir}/dev</toDir>
203+
<skip>${python.infra.download.skip}</skip>
204+
</configuration>
205+
</execution>
206+
<execution>
207+
<id>download-build-version-header</id>
208+
<phase>validate</phase>
209+
<goals>
210+
<goal>download-single</goal>
211+
</goals>
212+
<configuration>
213+
<url>https://raw.githubusercontent.com/apache/flink-connector-shared-utils/ci_utils/python/glibc_version_fix.h</url>
214+
<toDir>${project.basedir}/dev</toDir>
215+
<skip>${python.infra.download.skip}</skip>
216+
</configuration>
217+
</execution>
218+
</executions>
219+
</plugin>
220+
</plugins>
221+
</build>
222+
</project>

0 commit comments

Comments
 (0)