Skip to content

Commit 8d1146e

Browse files
groldanaaime
groldan
authored and
aaime
committed
Run Azure blob store integration tests with Azure and Azurite
Azurite is run as a testcontainer, and the github CI build runs all integration tests for Java 11, 17, and 21. In order to save on Azure resources, online integration tests are run only if the Azurite based tests succeeded, using github secrets to set up the account, key, and container; and for Java 11 only.
1 parent d04281e commit 8d1146e

23 files changed

+1172
-92
lines changed

‎.github/workflows/azure-integration.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: Azure BlobStore Integration
2+
3+
on:
4+
push:
5+
tags:
6+
- "**"
7+
pull_request:
8+
paths:
9+
- ".github/workflows/azure-integration.yml"
10+
- "pom.xml"
11+
- "geowebcache/pom.xml"
12+
- "geowebcache/core/**"
13+
- "geowebcache/azureblob/**"
14+
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
azurite:
21+
name: Azurite container
22+
runs-on: ubuntu-latest
23+
strategy:
24+
matrix:
25+
java-version: [ 11, 17, 21 ]
26+
steps:
27+
- uses: actions/checkout@v4
28+
- uses: actions/setup-java@v3
29+
with:
30+
distribution: 'temurin'
31+
java-version: ${{ matrix.java-version }}
32+
cache: 'maven'
33+
34+
- name: Tests against Azurite TestContainers
35+
#-PexcludeOnlineTests includes Azurite container tests and excludes Azure online tests
36+
run: |
37+
mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \
38+
-Ponline,excludeAzureOnlineTests \
39+
-DskipTests=true \
40+
-DskipITs=false -B -ntp
41+
42+
- name: Remove SNAPSHOT jars from repository
43+
run: |
44+
find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
45+
46+
azure:
47+
name: Azure online
48+
#if: github.repository == 'geowebcache/geowebcache'
49+
runs-on: ubuntu-latest
50+
needs: azurite
51+
if: |
52+
always() &&
53+
!contains(needs.*.result, 'cancelled') &&
54+
!contains(needs.*.result, 'failure')
55+
steps:
56+
- uses: actions/checkout@v4
57+
- uses: actions/setup-java@v3
58+
with:
59+
distribution: 'temurin'
60+
java-version: 11
61+
cache: 'maven'
62+
63+
- name: Tests against Azure
64+
env:
65+
azure_account: ${{ secrets.AZURE_ACCOUNT }}
66+
azure_account_key: ${{ secrets.AZURE_ACCOUNT_KEY }}
67+
azure_container: ${{ secrets.AZURE_CONTAINER }}
68+
if: ${{ env.azure_account != null }} && ${{ env.azure_account_key != null }}
69+
run: | #-PexcludeDockerTests includes Azure online tests and excludes Azurite container tests
70+
echo "accountName=$azure_account" > $HOME/.gwc_azure_tests.properties
71+
echo "accountKey=$azure_account_key" >> $HOME/.gwc_azure_tests.properties
72+
echo "container=$azure_container" >> $HOME/.gwc_azure_tests.properties
73+
echo 'maxConnections=8' >> $HOME/.gwc_azure_tests.properties
74+
echo 'useHTTPS=true' >> $HOME/.gwc_azure_tests.properties
75+
mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \
76+
-Ponline,excludeDockerTests \
77+
-DskipTests=true \
78+
-DskipITs=false -B -ntp
79+
80+
- name: Remove SNAPSHOT jars from repository
81+
run: |
82+
find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}

‎geowebcache/azureblob/REAME.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Azure BlobStore
2+
3+
GeoWebCache `BlobStore` implementation to store tiles on [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs)
4+
5+
6+
## Building
7+
8+
`mvn install|test|verify` will run only unit tests.
9+
10+
In order to run the integration tests, build with the `-Ponline` maven profile. For example:
11+
12+
```
13+
mvn verify -Ponline
14+
```
15+
16+
There are two sets of integration tests:
17+
18+
- `org.geowebcache.azure.tests.container.*IT.java`: run integration tests using [Testcontainers](https://testcontainers.com/), with a Microsoft [Azurite](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite) Docker image.
19+
- `org.geowebcache.azure.tests.online.*IT.java`: run integration tests against a real Azure account, only if there's a configuration file in `$HOME/.gwc_azure_tests.properties`, which must have the following contents:
20+
21+
```
22+
accountName=<Azure account name>
23+
accountKey=<Azure account key>
24+
container=<Azure blob storage container name>
25+
useHTTPS=true
26+
maxConnections=<optional, max number of concurrent connections>
27+
```
28+
29+
## Continuous integration
30+
31+
There's a Github Actions CI job defined at `<gwc git root>/.github/workflows/azure-integration.yml`
32+
that will run the Docker-based integration tests first, and then the online ones against
33+
a real Azure account, using Github repository secrets to define the values for the
34+
`$HOME/.gwc_azure_tests.properties` file's `accountName`, `accountSecret`, and `container` keys.

‎geowebcache/azureblob/pom.xml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,5 +51,76 @@
5151
<artifactId>javax.servlet-api</artifactId>
5252
<scope>provided</scope>
5353
</dependency>
54+
<dependency>
55+
<groupId>org.testcontainers</groupId>
56+
<artifactId>testcontainers</artifactId>
57+
<scope>test</scope>
58+
</dependency>
59+
<dependency>
60+
<groupId>org.awaitility</groupId>
61+
<artifactId>awaitility</artifactId>
62+
<scope>test</scope>
63+
</dependency>
5464
</dependencies>
65+
<profiles>
66+
<profile>
67+
<id>online</id>
68+
<activation>
69+
<activeByDefault>false</activeByDefault>
70+
</activation>
71+
<build>
72+
<plugins>
73+
<plugin>
74+
<artifactId>maven-failsafe-plugin</artifactId>
75+
<configuration>
76+
<forkCount>1</forkCount>
77+
<reuseForks>false</reuseForks>
78+
</configuration>
79+
</plugin>
80+
</plugins>
81+
</build>
82+
</profile>
83+
<profile>
84+
<!-- skips the integration tests against a real Azure account, used to split up the CI builds-->
85+
<id>excludeAzureOnlineTests</id>
86+
<activation>
87+
<activeByDefault>false</activeByDefault>
88+
</activation>
89+
<build>
90+
<plugins>
91+
<plugin>
92+
<artifactId>maven-failsafe-plugin</artifactId>
93+
<configuration>
94+
<forkCount>1</forkCount>
95+
<reuseForks>false</reuseForks>
96+
<excludes>
97+
<exclude>org.geowebcache.azure.tests.online.*IT</exclude>
98+
</excludes>
99+
</configuration>
100+
</plugin>
101+
</plugins>
102+
</build>
103+
</profile>
104+
<profile>
105+
<!-- skips the integration tests against the Azurite test container, used to split up the CI builds-->
106+
<id>excludeDockerTests</id>
107+
<activation>
108+
<activeByDefault>false</activeByDefault>
109+
</activation>
110+
<build>
111+
<plugins>
112+
<plugin>
113+
<artifactId>maven-failsafe-plugin</artifactId>
114+
<configuration>
115+
<forkCount>1</forkCount>
116+
<reuseForks>false</reuseForks>
117+
<excludes>
118+
<exclude>org.geowebcache.azure.tests.container.*IT</exclude>
119+
</excludes>
120+
</configuration>
121+
</plugin>
122+
</plugins>
123+
</build>
124+
</profile>
125+
</profiles>
55126
</project>

‎geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStoreData.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
* Azure Blobstore type-resolved data from a {@link AzureBlobStoreInfo} using enviroment variables
2323
* if enabled.
2424
*/
25-
class AzureBlobStoreData {
25+
public class AzureBlobStoreData {
2626

2727
private String container;
2828
private String prefix;
@@ -36,7 +36,7 @@ class AzureBlobStoreData {
3636
private String proxyPassword;
3737
private String serviceURL;
3838

39-
AzureBlobStoreData() {}
39+
public AzureBlobStoreData() {}
4040

4141
public AzureBlobStoreData(
4242
final AzureBlobStoreInfo storeInfo, final GeoWebCacheEnvironment environment) {

‎geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
import org.geowebcache.util.URLs;
5151
import org.springframework.http.HttpStatus;
5252

53-
class AzureClient implements Closeable {
53+
public class AzureClient implements Closeable {
5454

5555
private final NettyClient.Factory factory;
5656
private AzureBlobStoreData configuration;

‎geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreConformanceTest.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,25 +25,25 @@
2525
import java.util.stream.Stream;
2626
import org.easymock.EasyMock;
2727
import org.geowebcache.GeoWebCacheException;
28+
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreConformanceIT;
29+
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreConformanceIT;
2830
import org.geowebcache.layer.TileLayer;
2931
import org.geowebcache.layer.TileLayerDispatcher;
3032
import org.geowebcache.locks.LockProvider;
3133
import org.geowebcache.locks.NoOpLockProvider;
3234
import org.geowebcache.storage.AbstractBlobStoreTest;
33-
import org.junit.Assume;
34-
import org.junit.Rule;
3535

36-
public class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest<AzureBlobStore> {
37-
public PropertiesLoader testConfigLoader = new PropertiesLoader();
36+
/**
37+
* @see OnlineAzureBlobStoreConformanceIT
38+
* @see AzuriteAzureBlobStoreConformanceIT
39+
*/
40+
public abstract class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest<AzureBlobStore> {
3841

39-
@Rule
40-
public TemporaryAzureFolder tempFolder =
41-
new TemporaryAzureFolder(testConfigLoader.getProperties());
42+
protected abstract AzureBlobStoreData getConfiguration();
4243

4344
@Override
4445
public void createTestUnit() throws Exception {
45-
Assume.assumeTrue(tempFolder.isConfigured());
46-
AzureBlobStoreData config = tempFolder.getConfig();
46+
AzureBlobStoreData config = getConfiguration();
4747

4848
TileLayerDispatcher layers = createMock(TileLayerDispatcher.class);
4949
LockProvider lockProvider = new NoOpLockProvider();

geowebcache/azureblob/src/test/java/org/geowebcache/azure/AbstractAzureBlobStoreIntegrationTest.java renamed to geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreIntegrationTest.java

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
import java.util.Map;
4343
import java.util.logging.Logger;
4444
import org.geotools.util.logging.Logging;
45+
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreIntegrationIT;
46+
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreIntegrationIT;
4547
import org.geowebcache.config.DefaultGridsets;
4648
import org.geowebcache.grid.GridSet;
4749
import org.geowebcache.grid.GridSetBroker;
@@ -69,19 +71,20 @@
6971
* Integration tests for {@link AzureBlobStore}.
7072
*
7173
* <p>This is an abstract class for both online and offline integration tests.
74+
*
75+
* @see OnlineAzureBlobStoreIntegrationIT
76+
* @see AzuriteAzureBlobStoreIntegrationIT
7277
*/
73-
public abstract class AbstractAzureBlobStoreIntegrationTest {
78+
public abstract class AzureBlobStoreIntegrationTest {
7479

75-
private static Logger log = Logging.getLogger(PropertiesLoader.class.getName());
80+
private static Logger log = Logging.getLogger(AzureBlobStoreIntegrationTest.class.getName());
7681

7782
private static final String DEFAULT_FORMAT = "png";
7883

7984
private static final String DEFAULT_GRIDSET = "EPSG:4326";
8085

8186
private static final String DEFAULT_LAYER = "topp:world";
8287

83-
public PropertiesLoader testConfigLoader = new PropertiesLoader();
84-
8588
private AzureBlobStore blobStore;
8689

8790
protected abstract AzureBlobStoreData getConfiguration();

‎geowebcache/azureblob/src/test/java/org/geowebcache/azure/AzureBlobStoreSuitabilityTest.java‎

Lines changed: 13 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,12 @@
1717
import static org.hamcrest.Matchers.equalTo;
1818
import static org.hamcrest.Matchers.hasItemInArray;
1919
import static org.junit.Assert.assertTrue;
20-
import static org.junit.Assume.assumeFalse;
2120

2221
import io.reactivex.Flowable;
2322
import java.nio.ByteBuffer;
2423
import org.easymock.EasyMock;
24+
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreSuitabilityIT;
25+
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreSuitabilityIT;
2526
import org.geowebcache.layer.TileLayerDispatcher;
2627
import org.geowebcache.locks.LockProvider;
2728
import org.geowebcache.locks.NoOpLockProvider;
@@ -30,23 +31,14 @@
3031
import org.hamcrest.Matcher;
3132
import org.hamcrest.Matchers;
3233
import org.junit.Before;
33-
import org.junit.Rule;
3434
import org.junit.experimental.theories.DataPoints;
35-
import org.junit.experimental.theories.Theories;
36-
import org.junit.runner.RunWith;
37-
import org.junit.runners.model.FrameworkMethod;
38-
import org.junit.runners.model.InitializationError;
39-
import org.junit.runners.model.Statement;
4035
import org.springframework.http.HttpStatus;
4136

42-
@RunWith(AzureBlobStoreSuitabilityTest.MyTheories.class)
43-
public class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest {
44-
45-
public PropertiesLoader testConfigLoader = new PropertiesLoader();
46-
47-
@Rule
48-
public TemporaryAzureFolder tempFolder =
49-
new TemporaryAzureFolder(testConfigLoader.getProperties());
37+
/**
38+
* @see OnlineAzureBlobStoreSuitabilityIT
39+
* @see AzuriteAzureBlobStoreSuitabilityIT
40+
*/
41+
public abstract class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest {
5042

5143
@DataPoints
5244
public static String[][] persistenceLocations = {
@@ -67,6 +59,10 @@ public void setup() throws Exception {
6759
EasyMock.replay(tld);
6860
}
6961

62+
protected abstract AzureBlobStoreData getConfiguration();
63+
64+
protected abstract AzureClient getClient();
65+
7066
@SuppressWarnings("unchecked")
7167
@Override
7268
protected Matcher<Object> existing() {
@@ -81,13 +77,12 @@ protected Matcher<Object> empty() {
8177

8278
@Override
8379
public BlobStore create(Object dir) throws Exception {
84-
AzureBlobStoreData info = tempFolder.getConfig();
80+
AzureBlobStoreData info = getConfiguration();
8581
for (String path : (String[]) dir) {
8682
String fullPath = info.getPrefix() + "/" + path;
8783
ByteBuffer byteBuffer = ByteBuffer.wrap("testAbc".getBytes());
8884
int statusCode =
89-
tempFolder
90-
.getClient()
85+
getClient()
9186
.getBlockBlobURL(fullPath)
9287
.upload(Flowable.just(byteBuffer), byteBuffer.limit())
9388
.blockingGet()
@@ -96,27 +91,4 @@ public BlobStore create(Object dir) throws Exception {
9691
}
9792
return new AzureBlobStore(info, tld, locks);
9893
}
99-
100-
// Sorry, this bit of evil makes the Theories runner gracefully ignore the
101-
// tests if Azure is unavailable. There's probably a better way to do this.
102-
public static class MyTheories extends Theories {
103-
104-
public MyTheories(Class<?> klass) throws InitializationError {
105-
super(klass);
106-
}
107-
108-
@Override
109-
public Statement methodBlock(FrameworkMethod method) {
110-
if (new PropertiesLoader().getProperties().containsKey("container")) {
111-
return super.methodBlock(method);
112-
} else {
113-
return new Statement() {
114-
@Override
115-
public void evaluate() {
116-
assumeFalse("Azure unavailable", true);
117-
}
118-
};
119-
}
120-
}
121-
}
12294
}

0 commit comments

Comments
 (0)