Skip to content

Commit e00ad8f

Browse files
EnricoMiEnricoMi
authored
Run tests on the packaged jar (#325)
Running tests against the packaged jar helps identifying issues with dependencies and shaded classes. - Runs unit tests through Maven surefire plugin, Scala and Java tests together. - Runs integration tests through Maven failsafe plugin, Scala and Java tests together. Needed by #329, #330.
1 parent 217d703 commit e00ad8f

File tree

16 files changed

+190
-63
lines changed

16 files changed

+190
-63
lines changed

‎.github/actions/test-jvm/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ runs:
8686
run: |
8787
# Scala and Java Tests
8888
echo "::group::mvn test"
89-
mvn --batch-mode --update-snapshots -Dspotless.check.skip test
89+
mvn --batch-mode --update-snapshots -Dspotless.check.skip test integration-test
9090
echo "::endgroup::"
9191
shell: bash
9292

‎pom.xml‎

Lines changed: 62 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@
9696
<scope>test</scope>
9797
</dependency>
9898

99+
<dependency>
100+
<groupId>junit</groupId>
101+
<artifactId>junit</artifactId>
102+
<version>4.13.2</version>
103+
<scope>test</scope>
104+
</dependency>
105+
99106
<dependency>
100107
<groupId>org.scalatest</groupId>
101108
<artifactId>scalatest_${scala.compat.version}</artifactId>
@@ -104,9 +111,9 @@
104111
</dependency>
105112

106113
<dependency>
107-
<groupId>junit</groupId>
108-
<artifactId>junit</artifactId>
109-
<version>4.13.2</version>
114+
<groupId>org.scalatestplus</groupId>
115+
<artifactId>scalatestplus-junit_${scala.compat.version}</artifactId>
116+
<version>1.0.0-M2</version>
110117
<scope>test</scope>
111118
</dependency>
112119
</dependencies>
@@ -260,6 +267,7 @@
260267
<archive>
261268
<manifest>
262269
<addClasspath>true</addClasspath>
270+
<!-- add Diff app as main class -->
263271
<mainClass>uk.co.gresearch.spark.diff.App</mainClass>
264272
</manifest>
265273
</archive>
@@ -289,34 +297,75 @@
289297
</execution>
290298
</executions>
291299
</plugin>
292-
<!-- run java tests -->
300+
<!-- run tests -->
293301
<plugin>
294302
<groupId>org.apache.maven.plugins</groupId>
295303
<artifactId>maven-surefire-plugin</artifactId>
296304
<version>3.1.2</version>
297305
<configuration>
298306
<skipTests>false</skipTests>
307+
<includes>
308+
<include>**/*Tests.class</include>
309+
<include>**/*Suite.class</include>
310+
</includes>
299311
</configuration>
300312
</plugin>
301-
<!-- run scalatest -->
313+
<!-- packaging -->
314+
<plugin>
315+
<groupId>org.apache.maven.plugins</groupId>
316+
<artifactId>maven-source-plugin</artifactId>
317+
<version>3.3.0</version>
318+
<executions>
319+
<execution>
320+
<id>attach-sources</id>
321+
<goals>
322+
<goal>jar-no-fork</goal>
323+
</goals>
324+
</execution>
325+
</executions>
326+
</plugin>
327+
<plugin>
328+
<groupId>net.alchim31.maven</groupId>
329+
<artifactId>scala-maven-plugin</artifactId>
330+
<version>4.8.1</version>
331+
<executions>
332+
<execution>
333+
<id>attach-javadocs</id>
334+
<goals>
335+
<goal>doc-jar</goal>
336+
</goals>
337+
</execution>
338+
</executions>
339+
</plugin>
340+
<!-- run integration tests -->
302341
<plugin>
303-
<groupId>org.scalatest</groupId>
304-
<artifactId>scalatest-maven-plugin</artifactId>
305-
<version>2.2.0</version>
342+
<groupId>org.apache.maven.plugins</groupId>
343+
<artifactId>maven-failsafe-plugin</artifactId>
344+
<version>3.3.0</version>
306345
<configuration>
307-
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
308-
<junitxml>.</junitxml>
309-
<filereports>WDF TestSuite.txt</filereports>
346+
<additionalClasspathElements>
347+
<additionalClasspathElement>${project.build.directory}/${project.build.finalName}.jar</additionalClasspathElement>
348+
</additionalClasspathElements>
349+
<includes>
350+
<include>**/*Tests.class</include>
351+
<include>**/*Suite.class</include>
352+
</includes>
310353
</configuration>
311354
<executions>
312355
<execution>
313-
<id>test</id>
314356
<goals>
315-
<goal>test</goal>
357+
<goal>integration-test</goal>
358+
<goal>verify</goal>
316359
</goals>
360+
<configuration>
361+
<environmentVariables>
362+
<CI_INTEGRATION_TEST>true</CI_INTEGRATION_TEST>
363+
</environmentVariables>
364+
</configuration>
317365
</execution>
318366
</executions>
319367
</plugin>
368+
<!-- publishing -->
320369
<plugin>
321370
<groupId>org.sonatype.central</groupId>
322371
<artifactId>central-publishing-maven-plugin</artifactId>
@@ -342,32 +391,6 @@
342391
</execution>
343392
</executions>
344393
</plugin>
345-
<plugin>
346-
<groupId>org.apache.maven.plugins</groupId>
347-
<artifactId>maven-source-plugin</artifactId>
348-
<version>3.3.0</version>
349-
<executions>
350-
<execution>
351-
<id>attach-sources</id>
352-
<goals>
353-
<goal>jar-no-fork</goal>
354-
</goals>
355-
</execution>
356-
</executions>
357-
</plugin>
358-
<plugin>
359-
<groupId>net.alchim31.maven</groupId>
360-
<artifactId>scala-maven-plugin</artifactId>
361-
<version>4.8.1</version>
362-
<executions>
363-
<execution>
364-
<id>attach-javadocs</id>
365-
<goals>
366-
<goal>doc-jar</goal>
367-
</goals>
368-
</execution>
369-
</executions>
370-
</plugin>
371394
</plugins>
372395
</build>
373396

‎src/test/scala/uk/co/gresearch/spark/GroupBySuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
package uk.co.gresearch.spark
1818

1919
import org.apache.spark.sql.{DataFrame, Dataset, KeyValueGroupedDataset, Row}
20-
import org.scalatest.funspec.AnyFunSpec
2120
import uk.co.gresearch.spark.GroupBySortedSuite.{valueRowToTuple, valueToTuple}
2221
import uk.co.gresearch.spark.group.SortedGroupByDataset
22+
import uk.co.gresearch.test.Spec
2323

2424
import scala.language.implicitConversions
2525

@@ -33,7 +33,7 @@ case class State(init: Int) {
3333
}
3434
}
3535

36-
class GroupBySuite extends AnyFunSpec with SparkTestSession {
36+
class GroupBySuite extends Spec with SparkTestSession {
3737

3838
import spark.implicits._
3939

‎src/test/scala/uk/co/gresearch/spark/HistogramSuite.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@
1616

1717
package uk.co.gresearch.spark
1818

19-
import org.apache.spark.sql.{AnalysisException, Dataset, DataFrame}
19+
import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset}
2020
import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType}
21-
import org.scalatest.funsuite.AnyFunSuite
21+
import uk.co.gresearch.test.Suite
2222

2323
case class IntValue(id: Int, title: String, value: Int)
2424
case class DoubleValue(id: Int, title: String, value: Double)
2525

26-
class HistogramSuite extends AnyFunSuite with SparkTestSession {
26+
class HistogramSuite extends Suite with SparkTestSession {
2727

2828
import spark.implicits._
2929

‎src/test/scala/uk/co/gresearch/spark/SparkSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,15 @@ import org.apache.spark.sql.functions._
2323
import org.apache.spark.sql.types._
2424
import org.apache.spark.storage.StorageLevel
2525
import org.apache.spark.storage.StorageLevel.{DISK_ONLY, MEMORY_AND_DISK, MEMORY_ONLY, NONE, OFF_HEAP}
26-
import org.scalatest.funsuite.AnyFunSuite
2726
import uk.co.gresearch.ExtendedAny
2827
import uk.co.gresearch.spark.SparkSuite.{Value, collectJobDescription}
28+
import uk.co.gresearch.test.Suite
2929

3030
import java.nio.file.Paths
3131
import java.sql.Timestamp
3232
import java.time.Instant
3333

34-
class SparkSuite extends AnyFunSuite with SparkTestSession with SparkSuiteHelper {
34+
class SparkSuite extends Suite with SparkTestSession with SparkSuiteHelper {
3535

3636
import spark.implicits._
3737

‎src/test/scala/uk/co/gresearch/spark/WritePartitionedSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,15 @@ package uk.co.gresearch.spark
1919
import org.apache.spark.sql.Dataset
2020
import org.apache.spark.sql.functions.{col, reverse}
2121
import org.apache.spark.sql.internal.SQLConf
22-
import org.scalatest.funsuite.AnyFunSuite
2322
import uk.co.gresearch.spark.WritePartitionedSuite.Value
2423
import uk.co.gresearch.spark.UnpersistHandle.withUnpersist
24+
import uk.co.gresearch.test.Suite
2525

2626
import java.io.File
2727
import java.sql.Date
2828
import scala.io.Source
2929

30-
class WritePartitionedSuite extends AnyFunSuite with SparkTestSession {
30+
class WritePartitionedSuite extends Suite with SparkTestSession {
3131

3232
import spark.implicits._
3333

‎src/test/scala/uk/co/gresearch/spark/diff/AppSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@
1717
package uk.co.gresearch.spark.diff
1818

1919
import org.apache.spark.sql.SaveMode
20-
import org.scalatest.funsuite.AnyFunSuite
2120
import uk.co.gresearch.spark.SparkTestSession
21+
import uk.co.gresearch.test.Suite
2222

2323
import java.io.File
2424

25-
class AppSuite extends AnyFunSuite with SparkTestSession {
25+
class AppSuite extends Suite with SparkTestSession {
2626

2727
import spark.implicits._
2828

‎src/test/scala/uk/co/gresearch/spark/diff/DiffComparatorSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@ import org.apache.spark.sql.functions.{abs, lit, when}
2222
import org.apache.spark.sql.internal.SQLConf
2323
import org.apache.spark.sql.types._
2424
import org.apache.spark.unsafe.types.UTF8String
25-
import org.scalatest.funsuite.AnyFunSuite
2625
import uk.co.gresearch.spark.SparkTestSession
2726
import uk.co.gresearch.spark.diff.DiffComparatorSuite.{
2827
decimalEnc,
2928
optionsWithRelaxedComparators,
3029
optionsWithTightComparators
3130
}
3231
import uk.co.gresearch.spark.diff.comparator._
32+
import uk.co.gresearch.test.Suite
3333

3434
import java.sql.{Date, Timestamp}
3535
import java.time.Duration
@@ -49,7 +49,7 @@ case class Dates(id: Int, date: Date)
4949
case class Times(id: Int, time: Timestamp)
5050
case class Maps(id: Int, map: Map[Int, Long])
5151

52-
class DiffComparatorSuite extends AnyFunSuite with SparkTestSession {
52+
class DiffComparatorSuite extends Suite with SparkTestSession {
5353

5454
import spark.implicits._
5555

‎src/test/scala/uk/co/gresearch/spark/diff/DiffOptionsSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@ import org.apache.spark.sql.Column
2020
import org.apache.spark.sql.functions.lit
2121
import org.apache.spark.sql.internal.SQLConf
2222
import org.apache.spark.sql.types._
23-
import org.scalatest.funsuite.AnyFunSuite
2423
import uk.co.gresearch.spark.SparkTestSession
2524
import uk.co.gresearch.spark.diff.comparator.{DefaultDiffComparator, DiffComparator, EquivDiffComparator}
25+
import uk.co.gresearch.test.Suite
2626

27-
class DiffOptionsSuite extends AnyFunSuite with SparkTestSession {
27+
class DiffOptionsSuite extends Suite with SparkTestSession {
2828

2929
import spark.implicits._
3030

‎src/test/scala/uk/co/gresearch/spark/diff/DiffSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ import org.apache.spark.sql.functions.regexp_replace
2020
import org.apache.spark.sql.internal.SQLConf
2121
import org.apache.spark.sql.types._
2222
import org.apache.spark.sql.{Dataset, Encoders, Row, SparkSession}
23-
import org.scalatest.funsuite.AnyFunSuite
2423
import uk.co.gresearch.spark.{SparkTestSession, distinctPrefixFor}
24+
import uk.co.gresearch.test.Suite
2525

2626
case class Empty()
2727
case class Value(id: Int, value: Option[String])
@@ -112,7 +112,7 @@ object DiffSuite {
112112

113113
}
114114

115-
class DiffSuite extends AnyFunSuite with SparkTestSession {
115+
class DiffSuite extends Suite with SparkTestSession {
116116

117117
import spark.implicits._
118118

0 commit comments

Comments
 (0)