Skip to content

Commit 4018479

Browse files
vga91vga91
authored
Fixes #4207: Integration Tests for Load Procedures with Cloud Object Storage (#4226)
* Fixes #4207: Integration Tests for Load Procedures with Cloud Object Storage * added google cloud tests * wip - adding other procs * cleanup * added jsonParams proc * fix tests * fix extended-it tests * cleanup * fix NoFileFound errors * fixed tests due to missing apoc config * added ignored export tests
1 parent 5748e1c commit 4018479

32 files changed

+1350
-454
lines changed

‎extended-it/build.gradle‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ dependencies {
2828
}
2929

3030
implementation project(":extended")
31+
32+
testImplementation group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.26.1'
3133
testImplementation project(':extended').sourceSets.main.allJava
3234

3335
testImplementation group: 'us.fatehi', name: 'schemacrawler-mysql', version: '16.20.8'
@@ -55,7 +57,9 @@ dependencies {
5557
testImplementation group: 'org.testcontainers', name: 'chromadb', version: '1.20.2'
5658
testImplementation group: 'org.testcontainers', name: 'weaviate', version: '1.20.2'
5759
testImplementation group: 'org.testcontainers', name: 'milvus', version: '1.20.2'
58-
60+
testImplementation group: 'org.apache.poi', name: 'poi', version: '5.1.0'
61+
testImplementation group: 'org.apache.poi', name: 'poi-ooxml', version: '5.1.0'
62+
testImplementation 'com.azure:azure-storage-blob:12.22.0'
5963
configurations.all {
6064
exclude group: 'org.slf4j', module: 'slf4j-nop'
6165
exclude group: 'ch.qos.logback', module: 'logback-classic'

‎extended-it/src/test/java/apoc/azure/ArrowAzureStorageTest.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package apoc.azure;
2+
3+
import apoc.export.arrow.ArrowTestUtil;
4+
import apoc.util.s3.S3BaseTest;
5+
import org.junit.Before;
6+
import org.junit.Ignore;
7+
import org.junit.Rule;
8+
import org.junit.Test;
9+
import org.neo4j.configuration.GraphDatabaseInternalSettings;
10+
import org.neo4j.test.rule.DbmsRule;
11+
import org.neo4j.test.rule.ImpermanentDbmsRule;
12+
13+
import java.util.Map;
14+
15+
import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED;
16+
import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED;
17+
import static apoc.ApocConfig.apocConfig;
18+
import static apoc.export.arrow.ArrowTestUtil.initDbCommon;
19+
import static apoc.export.arrow.ArrowTestUtil.testImportCommon;
20+
import static apoc.export.arrow.ArrowTestUtil.testLoadArrow;
21+
22+
@Ignore("This test won't work until the Azure Storage files will be correctly handled via FileUtils, placed in APOC Core")
23+
public class ArrowAzureStorageTest extends AzureStorageBaseTest {
24+
25+
@Rule
26+
public DbmsRule db = new ImpermanentDbmsRule()
27+
.withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true);
28+
29+
@Before
30+
public void beforeClass() {
31+
initDbCommon(db);
32+
apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true);
33+
apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
34+
}
35+
36+
@Test
37+
public void testFileRoundtripWithLoadArrow() {
38+
String url = putToAzureStorageAndGetUrl("test_all.arrow");
39+
40+
String file = db.executeTransactionally("CYPHER 25 CALL apoc.export.arrow.all($url) YIELD file",
41+
Map.of("url", url),
42+
ArrowTestUtil::extractFileName);
43+
44+
// check that the exported file is correct
45+
final String query = "CYPHER 25 CALL apoc.load.arrow($file, {})";
46+
testLoadArrow(db, query, Map.of("file", file));
47+
}
48+
49+
50+
@Test
51+
public void testFileRoundtripWithImportArrow() {
52+
db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})");
53+
54+
String url = putToAzureStorageAndGetUrl("test_all_import.arrow");
55+
String file = db.executeTransactionally("CYPHER 25 CALL apoc.export.arrow.all($url) YIELD file",
56+
Map.of("url", url),
57+
ArrowTestUtil::extractFileName);
58+
59+
// check that the exported file is correct
60+
testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL);
61+
}
62+
}

‎extended-it/src/test/java/apoc/azure/AzureStorageBaseTest.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package apoc.azure;
2+
3+
import com.azure.core.util.Context;
4+
import com.azure.storage.blob.BlobClient;
5+
import com.azure.storage.blob.BlobContainerClient;
6+
import com.azure.storage.blob.BlobContainerClientBuilder;
7+
import com.azure.storage.blob.sas.BlobSasPermission;
8+
import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
9+
import org.apache.commons.io.FileUtils;
10+
import org.junit.AfterClass;
11+
import org.junit.BeforeClass;
12+
import org.testcontainers.containers.GenericContainer;
13+
import org.testcontainers.utility.DockerImageName;
14+
15+
import java.io.ByteArrayInputStream;
16+
import java.io.File;
17+
import java.io.IOException;
18+
import java.time.OffsetDateTime;
19+
import java.util.UUID;
20+
21+
public class AzureStorageBaseTest {
22+
23+
public static GenericContainer<?> azuriteContainer;
24+
public static BlobContainerClient containerClient;
25+
26+
@BeforeClass
27+
public static void setUp() throws Exception {
28+
DockerImageName azuriteImg = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite");
29+
azuriteContainer = new GenericContainer<>(azuriteImg)
30+
.withExposedPorts(10000);
31+
32+
azuriteContainer.start();
33+
34+
var accountName = "devstoreaccount1";
35+
var accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
36+
var blobEndpoint = "http://%s:%d/%s".formatted(azuriteContainer.getHost(), azuriteContainer.getMappedPort(10000), accountName);
37+
var connectionString = "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=%s;"
38+
.formatted(accountName, accountKey, blobEndpoint);
39+
40+
containerClient = new BlobContainerClientBuilder()
41+
.connectionString(connectionString)
42+
.containerName("test-container")
43+
.buildClient();
44+
containerClient.create();
45+
}
46+
47+
@AfterClass
48+
public static void teardown() {
49+
azuriteContainer.close();
50+
}
51+
52+
public static String putToAzureStorageAndGetUrl(String url) {
53+
try {
54+
File file = new File(url);
55+
byte[] content = FileUtils.readFileToByteArray(file);
56+
57+
var blobClient = getBlobClient(content);
58+
BlobSasPermission permission = new BlobSasPermission().setReadPermission(true);
59+
OffsetDateTime expiryTime = OffsetDateTime.now().plusHours(1);
60+
String sasToken = blobClient.generateSas(new BlobServiceSasSignatureValues(expiryTime, permission), new Context("Azure-Storage-Log-String-To-Sign", "true"));
61+
return blobClient.getBlobUrl() + "?" + sasToken;
62+
} catch (IOException e) {
63+
throw new RuntimeException(e);
64+
}
65+
}
66+
67+
public static BlobClient getBlobClient(byte[] content) {
68+
var blobName = "blob-" + UUID.randomUUID();
69+
var blobClient = containerClient.getBlobClient(blobName);
70+
blobClient.upload(new ByteArrayInputStream(content));
71+
return blobClient;
72+
}
73+
74+
}

‎extended-it/src/test/java/apoc/azure/ImportAzureStorageTest.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package apoc.azure;
2+
3+
import apoc.load.Gexf;
4+
import apoc.util.TestUtil;
5+
import org.junit.Before;
6+
import org.junit.Rule;
7+
import org.junit.Test;
8+
import org.neo4j.test.rule.DbmsRule;
9+
import org.neo4j.test.rule.ImpermanentDbmsRule;
10+
11+
import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED;
12+
import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED;
13+
import static apoc.ApocConfig.apocConfig;
14+
import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL;
15+
import static apoc.export.arrow.ArrowTestUtil.initDbCommon;
16+
import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests;
17+
import static apoc.export.arrow.ArrowTestUtil.testImportCommon;
18+
import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH;
19+
import static apoc.util.GexfTestUtil.testImportGexfCommon;
20+
21+
public class ImportAzureStorageTest extends AzureStorageBaseTest {
22+
23+
@Rule
24+
public DbmsRule db = new ImpermanentDbmsRule();
25+
26+
@Before
27+
public void beforeClass() {
28+
apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true);
29+
apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
30+
}
31+
32+
@Test
33+
public void testImportArrow() {
34+
initDbCommon(db);
35+
createNodesForImportTests(db);
36+
37+
String fileWithPath = EXTENDED_RESOURCES_PATH + "test_all.arrow";
38+
String url = putToAzureStorageAndGetUrl(fileWithPath);
39+
40+
testImportCommon(db, url, MAPPING_ALL);
41+
}
42+
43+
@Test
44+
public void testImportGexf() {
45+
TestUtil.registerProcedure(db, Gexf.class);
46+
47+
String filename = EXTENDED_RESOURCES_PATH + "gexf/data.gexf";
48+
String url = putToAzureStorageAndGetUrl(filename);
49+
testImportGexfCommon(db, url);
50+
}
51+
}

‎extended-it/src/test/java/apoc/azure/LoadAzureStorageTest.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package apoc.azure;
2+
3+
import apoc.load.LoadCsv;
4+
import apoc.load.LoadDirectory;
5+
import apoc.load.LoadHtml;
6+
import apoc.load.LoadJsonExtended;
7+
import apoc.load.Xml;
8+
import apoc.load.xls.LoadXls;
9+
import apoc.util.TestUtil;
10+
import org.junit.BeforeClass;
11+
import org.junit.ClassRule;
12+
import org.junit.Test;
13+
import org.neo4j.configuration.GraphDatabaseInternalSettings;
14+
import org.neo4j.test.rule.DbmsRule;
15+
import org.neo4j.test.rule.ImpermanentDbmsRule;
16+
17+
import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED;
18+
import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG;
19+
import static apoc.ApocConfig.apocConfig;
20+
import static apoc.load.LoadCsvTest.commonTestLoadCsv;
21+
import static apoc.load.LoadHtmlTest.testLoadHtmlWithGetLinksCommon;
22+
import static apoc.load.xls.LoadXlsTest.testLoadXlsCommon;
23+
import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH;
24+
import static apoc.util.ExtendedITUtil.testLoadJsonCommon;
25+
import static apoc.util.ExtendedITUtil.testLoadXmlCommon;
26+
27+
28+
public class LoadAzureStorageTest extends AzureStorageBaseTest {
29+
30+
@ClassRule
31+
public static DbmsRule db = new ImpermanentDbmsRule()
32+
.withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true);
33+
34+
@BeforeClass
35+
public static void setUp() throws Exception {
36+
AzureStorageBaseTest.setUp();
37+
38+
TestUtil.registerProcedure(db, LoadCsv.class, LoadDirectory.class, LoadJsonExtended.class, LoadHtml.class, LoadXls.class, Xml.class);
39+
apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true);
40+
apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false);
41+
}
42+
43+
44+
@Test
45+
public void testLoadCsv() {
46+
String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "test.csv");
47+
commonTestLoadCsv(db, url);
48+
}
49+
50+
@Test
51+
public void testLoadJson() {
52+
String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "map.json");
53+
testLoadJsonCommon(db, url);
54+
}
55+
56+
@Test
57+
public void testLoadXml() {
58+
String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "xml/books.xml");
59+
testLoadXmlCommon(db, url);
60+
}
61+
62+
@Test
63+
public void testLoadXls() {
64+
String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "load_test.xlsx");
65+
testLoadXlsCommon(db, url);
66+
}
67+
68+
@Test
69+
public void testLoadHtml() {
70+
String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "wikipedia.html");
71+
testLoadHtmlWithGetLinksCommon(db, url);
72+
}
73+
74+
}

‎extended-it/src/test/java/apoc/azure/ParquetAzureStorageTest.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package apoc.azure;
2+
3+
import apoc.export.parquet.ParquetTestUtil;
4+
import org.junit.Before;
5+
import org.junit.BeforeClass;
6+
import org.junit.ClassRule;
7+
import org.junit.Ignore;
8+
import org.junit.Test;
9+
import org.neo4j.test.rule.DbmsRule;
10+
import org.neo4j.test.rule.ImpermanentDbmsRule;
11+
12+
import java.util.Map;
13+
14+
import static apoc.export.parquet.ParquetTest.MAPPING_ALL;
15+
import static apoc.export.parquet.ParquetTestUtil.beforeClassCommon;
16+
import static apoc.export.parquet.ParquetTestUtil.beforeCommon;
17+
import static apoc.export.parquet.ParquetTestUtil.testImportAllCommon;
18+
import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH;
19+
import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl;
20+
import static apoc.util.TestUtil.testResult;
21+
22+
public class ParquetAzureStorageTest extends AzureStorageBaseTest {
23+
24+
private final String EXPORT_FILENAME = "test_all.parquet";
25+
26+
@ClassRule
27+
public static DbmsRule db = new ImpermanentDbmsRule();
28+
29+
@BeforeClass
30+
public static void beforeClass() {
31+
beforeClassCommon(db);
32+
}
33+
34+
@Before
35+
public void before() {
36+
beforeCommon(db);
37+
}
38+
39+
@Test
40+
@Ignore("This test won't work until the Azure Storage files will be correctly handled via FileUtils, placed in APOC Core")
41+
public void testFileRoundtripParquetAll() {
42+
// given - when
43+
String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + EXPORT_FILENAME);
44+
String file = db.executeTransactionally("CALL apoc.export.parquet.all($url) YIELD file",
45+
Map.of("url", url),
46+
ParquetTestUtil::extractFileName);
47+
48+
// then
49+
final String query = "CALL apoc.load.parquet($file, $config) YIELD value " +
50+
"RETURN value";
51+
52+
testResult(db, query, Map.of("file", file, "config", MAPPING_ALL),
53+
ParquetTestUtil::roundtripLoadAllAssertions);
54+
}
55+
@Test
56+
public void testLoadParquet() {
57+
String query = "CALL apoc.load.parquet($url, $config) YIELD value " +
58+
"RETURN value";
59+
60+
String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + EXPORT_FILENAME);
61+
testResult(db, query, Map.of("url", url, "config", MAPPING_ALL),
62+
ParquetTestUtil::roundtripLoadAllAssertions);
63+
}
64+
65+
@Test
66+
public void testImportParquet() {
67+
String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + EXPORT_FILENAME);
68+
69+
Map<String, Object> params = Map.of("file", url, "config", MAPPING_ALL);
70+
testImportAllCommon(db, params);
71+
}
72+
}

0 commit comments

Comments
 (0)