Skip to content

Commit 1f4d160

Browse files
ferenc-csakyferenc-csaky
authored
Introduce AbstractFullUseCaseTest to bring FullUseCaseIT and ExternalUseCaseIT to a common parent layer (#1437)
1 parent d69778d commit 1f4d160

File tree

8 files changed

+242
-678
lines changed

8 files changed

+242
-678
lines changed

‎sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/AbstractFullUseCaseTest.java‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
/*
2+
* Copyright © 2021 DataSQRL ([email protected])
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.datasqrl;
17+
18+
import static com.datasqrl.config.SqrlConstants.BUILD_DIR_NAME;
19+
import static org.assertj.core.api.Assertions.fail;
20+
21+
import com.datasqrl.cli.AssertStatusHook;
22+
import com.datasqrl.cli.DatasqrlTest;
23+
import com.datasqrl.config.PackageJson;
24+
import com.datasqrl.config.SqrlConstants;
25+
import com.datasqrl.engines.TestContainersForTestGoal;
26+
import com.datasqrl.engines.TestContainersForTestGoal.TestContainerHook;
27+
import com.datasqrl.engines.TestEngine.EngineFactory;
28+
import com.datasqrl.error.ErrorCollector;
29+
import com.datasqrl.tests.TestExtension;
30+
import com.datasqrl.tests.UseCaseTestExtensions;
31+
import com.datasqrl.util.ConfigLoaderUtils;
32+
import com.datasqrl.util.SnapshotTest.Snapshot;
33+
import com.datasqrl.util.SqrlScriptExecutor;
34+
import java.nio.file.Path;
35+
import java.time.Duration;
36+
import java.util.HashMap;
37+
import java.util.Map;
38+
import lombok.SneakyThrows;
39+
import lombok.extern.slf4j.Slf4j;
40+
import org.apache.flink.configuration.CheckpointingOptions;
41+
import org.apache.flink.configuration.Configuration;
42+
import org.apache.flink.configuration.DeploymentOptions;
43+
import org.apache.flink.configuration.RestartStrategyOptions;
44+
import org.junit.jupiter.api.AfterAll;
45+
import org.junit.jupiter.api.AfterEach;
46+
import org.junit.jupiter.api.BeforeAll;
47+
48+
/** Abstract base class to run a full test on a given project in form of {@link UseCaseParam}. */
49+
@Slf4j
50+
abstract class AbstractFullUseCaseTest {
51+
52+
private static TestContainerHook containerHook;
53+
54+
UseCaseTestExtensions testExtensions = new UseCaseTestExtensions();
55+
56+
@BeforeAll
57+
static void beforeAll() {
58+
var engines = new EngineFactory().createAll();
59+
60+
containerHook = engines.accept(new TestContainersForTestGoal(), null);
61+
containerHook.start();
62+
}
63+
64+
@AfterAll
65+
static void afterAll() {
66+
if (containerHook != null) {
67+
containerHook.teardown();
68+
}
69+
}
70+
71+
@AfterEach
72+
void afterEach() {
73+
if (containerHook != null) {
74+
containerHook.clear();
75+
}
76+
}
77+
78+
void fullUseCaseTest(UseCaseParam param) {
79+
log.info("Testing {}", param.getPackageJsonName());
80+
81+
var snapshot =
82+
Snapshot.of(
83+
AbstractFullUseCaseTest.class,
84+
param.getUseCaseName(),
85+
param.getPackageJsonName().substring(0, param.getPackageJsonName().length() - 5));
86+
87+
TestExtension testExtension = testExtensions.create(param.getUseCaseName());
88+
try {
89+
testExtension.setup();
90+
91+
// Execute compile phase
92+
SqrlScriptExecutor executor = new SqrlScriptExecutor(param.packageJsonPath(), param.goal());
93+
AssertStatusHook hook = new AssertStatusHook();
94+
try {
95+
executor.execute(hook);
96+
} catch (Throwable e) {
97+
if (hook.failure() != null) {
98+
e.addSuppressed(hook.failure());
99+
}
100+
throw e;
101+
}
102+
103+
Path rootDir = param.packageJsonPath().getParent();
104+
105+
log.info(
106+
"""
107+
The test parameters
108+
Test name: {}
109+
Test path: {}
110+
Test package file: {}
111+
""",
112+
param.getUseCaseName(),
113+
rootDir,
114+
param.getPackageJsonName());
115+
116+
// Execute the test phase manually via DatasqrlTest
117+
PackageJson packageJson =
118+
ConfigLoaderUtils.loadResolvedConfig(
119+
ErrorCollector.root(), rootDir.resolve(BUILD_DIR_NAME));
120+
121+
var env = new HashMap<>(containerHook.getEnv());
122+
env.putAll(System.getenv());
123+
env.put("DATA_PATH", rootDir.resolve("build/deploy/flink/data").toAbsolutePath().toString());
124+
env.put("UDF_PATH", rootDir.resolve("build/deploy/flink/lib").toAbsolutePath().toString());
125+
126+
var planDir =
127+
rootDir
128+
.resolve(SqrlConstants.BUILD_DIR_NAME)
129+
.resolve(SqrlConstants.DEPLOY_DIR_NAME)
130+
.resolve(SqrlConstants.PLAN_DIR);
131+
var flinkConfig = loadInternalTestFlinkConfig(planDir, env);
132+
var test = new DatasqrlTest(rootDir, planDir, packageJson, flinkConfig, env);
133+
try {
134+
var run = test.run();
135+
if (run != 0) {
136+
fail(
137+
"Test runner returned error code while running test case '%s'. Check above for failed snapshot tests (in red) or exceptions"
138+
.formatted(param.getUseCaseName()));
139+
}
140+
} catch (Exception e) {
141+
fail(
142+
"Test runner threw exception while running test case '%s'"
143+
.formatted(param.getUseCaseName()),
144+
e);
145+
}
146+
147+
} finally {
148+
testExtension.teardown();
149+
containerHook.clear();
150+
}
151+
152+
if (snapshot.hasContent()) {
153+
snapshot.createOrValidate();
154+
}
155+
}
156+
157+
@SneakyThrows
158+
static Configuration loadInternalTestFlinkConfig(Path planDir, Map<String, String> env) {
159+
var flinkConfig = ConfigLoaderUtils.loadFlinkConfig(planDir);
160+
161+
flinkConfig.set(DeploymentOptions.TARGET, "local");
162+
if (env.get("FLINK_RESTART_STRATEGY") != null) {
163+
flinkConfig.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay");
164+
flinkConfig.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 0);
165+
flinkConfig.set(
166+
RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY, Duration.ofSeconds(5));
167+
}
168+
169+
flinkConfig.removeConfig(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
170+
flinkConfig.removeConfig(CheckpointingOptions.SAVEPOINT_DIRECTORY);
171+
172+
return flinkConfig;
173+
}
174+
}

‎sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/ExternalUseCaseIT.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright © 2021 DataSQRL ([email protected])
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.datasqrl;
17+
18+
import java.nio.file.Files;
19+
import java.nio.file.Path;
20+
import java.util.stream.Collectors;
21+
import java.util.stream.Stream;
22+
import lombok.extern.slf4j.Slf4j;
23+
import org.junit.jupiter.api.Disabled;
24+
import org.junit.jupiter.params.ParameterizedTest;
25+
import org.junit.jupiter.params.provider.MethodSource;
26+
27+
/** Tests external use cases manually, provided via {@code externalUseCaseProvider}. */
28+
@Slf4j
29+
@Disabled
30+
public class ExternalUseCaseIT extends AbstractFullUseCaseTest {
31+
32+
@ParameterizedTest
33+
@MethodSource("externalUseCaseProvider")
34+
void testCase(UseCaseParam param) {
35+
fullUseCaseTest(param);
36+
}
37+
38+
static Stream<UseCaseParam> externalUseCaseProvider() {
39+
// Provide absolute paths for the package.json file of the external project(s) to test
40+
Stream<Path> pathStream =
41+
Stream.of(
42+
// Path.of("<absolute-path-of-external-package-json>")
43+
);
44+
45+
var partitionedPaths = pathStream.collect(Collectors.partitioningBy(Files::exists));
46+
log.warn(
47+
"The following path point to a non-existing file, and will be ignored: {}",
48+
partitionedPaths.get(false));
49+
50+
var existingPaths = partitionedPaths.get(true);
51+
52+
return existingPaths.stream().map(UseCaseParam::new);
53+
}
54+
}

‎sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/ExternalUseCasesIT.java‎

Lines changed: 0 additions & 100 deletions
This file was deleted.

0 commit comments

Comments
 (0)