Skip to content

Commit f648d98

Browse files
elahrvivazelahrvivaz
authored
Add geomesa_pyspark CI tests (#3506)
1 parent 0e2991f commit f648d98

File tree

6 files changed

+51
-49
lines changed

6 files changed

+51
-49
lines changed

‎.github/workflows/spark.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ jobs:
3636
# TODO we don't have any tests for hbase spark runtime
3737
- name: hbase
3838
list: geomesa-hbase/geomesa-hbase-spark
39+
- name: pyspark
40+
# note: we're sneaking in a profile to the project list arg here
41+
list: geomesa-spark/geomesa_pyspark -Ppython
3942
steps:
4043
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
4144
- uses: actions/setup-java@f2beeb24e141e01a676f977032f5a29d81c9e27e # v5.1.0
@@ -46,6 +49,15 @@ jobs:
4649
with:
4750
key: ${{ hashFiles('**/pom.xml') }}-spark-integration-tests-${{ matrix.scala-version }}-${{ matrix.projects.name }}
4851
path: ~/.m2/repository/
52+
- uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
53+
if: matrix.projects.name=='pyspark'
54+
with:
55+
python-version: '3.10'
56+
cache: 'pip'
57+
cache-dependency-path: geomesa-spark/geomesa_pyspark/src/test/python/requirements.txt
58+
- name: Install python dependencies
59+
if: matrix.projects.name=='pyspark'
60+
run: pip install -r geomesa-spark/geomesa_pyspark/src/test/python/requirements.txt
4961
- name: Set Scala version
5062
run: ./build/scripts/change-scala-version.sh ${{ matrix.scala-version }}
5163
- name: Compile

‎geomesa-spark/geomesa_pyspark/pom.xml‎

Lines changed: 35 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -50,19 +50,47 @@
5050
<artifactId>build-helper-maven-plugin</artifactId>
5151
<executions>
5252
<execution>
53-
<id>regex-property</id>
54-
<phase>validate</phase>
53+
<id>set-package-version-dev</id>
54+
<phase>initialize</phase>
5555
<goals>
5656
<goal>regex-property</goal>
5757
</goals>
5858
<configuration>
59-
<name>python.version</name>
59+
<name>python.version.dev</name>
6060
<value>${project.version}</value>
6161
<regex>-SNAPSHOT</regex>
6262
<replacement>.dev0</replacement>
6363
<failIfNoMatch>false</failIfNoMatch>
6464
</configuration>
6565
</execution>
66+
<execution>
67+
<id>set-package-version-rc</id>
68+
<phase>initialize</phase>
69+
<goals>
70+
<goal>regex-property</goal>
71+
</goals>
72+
<configuration>
73+
<name>python.version.rc</name>
74+
<value>${python.version.dev}</value>
75+
<regex>-rc\.?</regex>
76+
<replacement>rc</replacement>
77+
<failIfNoMatch>false</failIfNoMatch>
78+
</configuration>
79+
</execution>
80+
<execution>
81+
<id>set-package-version-alpha</id>
82+
<phase>initialize</phase>
83+
<goals>
84+
<goal>regex-property</goal>
85+
</goals>
86+
<configuration>
87+
<name>python.version</name>
88+
<value>${python.version.rc}</value>
89+
<regex>-m\.?</regex>
90+
<replacement>a</replacement>
91+
<failIfNoMatch>false</failIfNoMatch>
92+
</configuration>
93+
</execution>
6694
</executions>
6795
</plugin>
6896
</plugins>
@@ -115,15 +143,17 @@
115143
</goals>
116144
<configuration>
117145
<skip>${skipTests}</skip>
118-
<workingDirectory>${project.basedir}/src/test/python</workingDirectory>
146+
<workingDirectory>${project.basedir}</workingDirectory>
119147
<arguments>
120148
<argument>-m</argument>
121149
<argument>unittest</argument>
122150
<argument>discover</argument>
123151
<argument>-v</argument>
152+
<argument>-s</argument>
153+
<argument>${project.basedir}/src/test/python</argument>
124154
</arguments>
125155
<environmentVariables>
126-
<PYTHONPATH>${project.basedir}/src/main/python/</PYTHONPATH>
156+
<PYTHONPATH>${project.build.directory}/python</PYTHONPATH>
127157
</environmentVariables>
128158
</configuration>
129159
</execution>
@@ -133,48 +163,6 @@
133163
<groupId>org.codehaus.mojo</groupId>
134164
<artifactId>build-helper-maven-plugin</artifactId>
135165
<executions>
136-
<execution>
137-
<id>set-package-version-snapshot</id>
138-
<phase>validate</phase>
139-
<goals>
140-
<goal>regex-property</goal>
141-
</goals>
142-
<configuration>
143-
<name>python.version</name>
144-
<value>${project.version}</value>
145-
<regex>-SNAPSHOT</regex>
146-
<replacement>.dev0</replacement>
147-
<failIfNoMatch>false</failIfNoMatch>
148-
</configuration>
149-
</execution>
150-
<execution>
151-
<id>set-package-version-rc</id>
152-
<phase>validate</phase>
153-
<goals>
154-
<goal>regex-property</goal>
155-
</goals>
156-
<configuration>
157-
<name>python.version</name>
158-
<value>${project.version}</value>
159-
<regex>-rc\.?</regex>
160-
<replacement>rc</replacement>
161-
<failIfNoMatch>false</failIfNoMatch>
162-
</configuration>
163-
</execution>
164-
<execution>
165-
<id>set-package-version-milestone</id>
166-
<phase>validate</phase>
167-
<goals>
168-
<goal>regex-property</goal>
169-
</goals>
170-
<configuration>
171-
<name>python.version</name>
172-
<value>${project.version}</value>
173-
<regex>-m\.?</regex>
174-
<replacement>a</replacement>
175-
<failIfNoMatch>false</failIfNoMatch>
176-
</configuration>
177-
</execution>
178166
<execution>
179167
<id>attach-artifacts</id>
180168
<phase>package</phase>

‎geomesa-spark/geomesa_pyspark/src/main/python/geomesa_pyspark/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def configure(jars=[], packages=[], files=[], spark_home=None, spark_master='yar
3131

3232
py_files = pyspark_lib_zips + process_executor_packages(packages, tmp_path)
3333

34-
assert spark_master is 'yarn', 'only yarn master is supported with this release'
34+
assert spark_master == 'yarn', 'only yarn master is supported with this release'
3535

3636
import pyspark
3737
import geomesa_pyspark.types

‎geomesa-spark/geomesa_pyspark/src/main/python/setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
setup(
44
name='geomesa_pyspark',
55
version='${python.version}',
6-
url='http://www.geomesa.org',
6+
url='https://www.geomesa.org/',
77
packages=find_packages(),
88
install_requires=['pytz', 'shapely']
99
)

‎geomesa-spark/geomesa_pyspark/src/test/python/__init__.py‎

Whitespace-only changes.

‎geomesa-spark/geomesa_pyspark/src/test/python/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
pyspark
2+
shapely

0 commit comments

Comments
 (0)