Skip to content

Commit 5f1bcc6

Browse files
dweissmocobeta
dweiss
authored and
mocobeta
committed
LUCENE-8930: script testing in the distribution (#550)
1 parent 73a3db9 commit 5f1bcc6

File tree

13 files changed

+402
-34
lines changed

13 files changed

+402
-34
lines changed

‎lucene/distribution.tests/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ configurations {
2929
dependencies {
3030
binaryDistribution project(path: ":lucene:distribution", configuration: "binaryDirForTests")
3131

32+
moduleTestImplementation "com.carrotsearch:procfork"
3233
moduleTestImplementation("com.carrotsearch.randomizedtesting:randomizedtesting-runner", {
3334
exclude group: "junit"
3435
})

‎lucene/distribution.tests/src/test/org/apache/lucene/distribution/AbstractLuceneDistributionTest.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.lucene.distribution;
18+
19+
import com.carrotsearch.randomizedtesting.RandomizedTest;
20+
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
21+
import java.nio.file.Files;
22+
import java.nio.file.Path;
23+
import java.nio.file.Paths;
24+
import java.util.Objects;
25+
import org.assertj.core.api.Assertions;
26+
import org.junit.BeforeClass;
27+
28+
/**
29+
* A parent scaffolding for tests that take a Lucene distribution as input. The location of the
30+
* distribution is pointed to by a system property {@link #DISTRIBUTION_PROPERTY}, which by default
31+
* is prepared and passed by the gradle build. It can be passed manually if you're testing from the
32+
* IDE, for example.
33+
*
34+
* <p>We do <em>not</em> want any distribution tests to depend on any Lucene classes (including the
35+
* test framework) so that there is no risk of accidental classpath space pollution. This also means
36+
* the default {@code LuceneTestCase} configuration setup is not used (you have to annotate test for
37+
* JUnit, for example).
38+
*/
39+
@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
40+
public abstract class AbstractLuceneDistributionTest extends RandomizedTest {
41+
/** A path to a directory with an expanded Lucene distribution. */
42+
public static final String DISTRIBUTION_PROPERTY = "lucene.distribution.dir";
43+
44+
/** The expected distribution version of Lucene modules. */
45+
public static final String VERSION_PROPERTY = "lucene.distribution.version";
46+
47+
/** Resolved and validated {@link #DISTRIBUTION_PROPERTY}. */
48+
private static Path distributionPath;
49+
50+
/** Ensure Lucene classes are not directly visible. */
51+
@BeforeClass
52+
public static void checkLuceneNotInClasspath() {
53+
Assertions.assertThatThrownBy(
54+
() -> {
55+
Class.forName("org.apache.lucene.index.IndexWriter");
56+
})
57+
.isInstanceOf(ClassNotFoundException.class);
58+
}
59+
60+
/** Verify the distribution property is provided and points at a valid location. */
61+
@BeforeClass
62+
public static void parseExternalProperties() {
63+
String distributionPropertyValue = System.getProperty(DISTRIBUTION_PROPERTY);
64+
if (distributionPropertyValue == null) {
65+
throw new AssertionError(DISTRIBUTION_PROPERTY + " property is required for this test.");
66+
}
67+
68+
distributionPath = Paths.get(distributionPropertyValue);
69+
70+
// Ensure the distribution path is sort of valid.
71+
Path topLevelReadme = distributionPath.resolve("README.md");
72+
if (!Files.isRegularFile(topLevelReadme)) {
73+
throw new AssertionError(
74+
DISTRIBUTION_PROPERTY
75+
+ " property does not seem to point to a top-level distribution directory"
76+
+ " where this file is present: "
77+
+ topLevelReadme.toAbsolutePath());
78+
}
79+
}
80+
81+
protected static Path getDistributionPath() {
82+
return Objects.requireNonNull(distributionPath, "Distribution path not set?");
83+
}
84+
}

‎lucene/distribution.tests/src/test/org/apache/lucene/distribution/Sync.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.lucene.distribution;
18+
19+
import java.io.IOException;
20+
import java.io.UncheckedIOException;
21+
import java.nio.file.*;
22+
import java.nio.file.attribute.BasicFileAttributes;
23+
import java.util.*;
24+
import java.util.stream.Collectors;
25+
26+
final class Sync {
27+
private static class Entry {
28+
String name;
29+
Path path;
30+
31+
public Entry(Path path) {
32+
this.path = path;
33+
this.name = path.getFileName().toString();
34+
}
35+
}
36+
37+
public void sync(Path source, Path target) throws IOException {
38+
List<Entry> sourceEntries = files(source);
39+
List<Entry> targetEntries = files(target);
40+
41+
for (Entry src : sourceEntries) {
42+
Path dst = target.resolve(src.name);
43+
if (Files.isDirectory(src.path)) {
44+
Files.createDirectories(dst);
45+
sync(src.path, dst);
46+
} else {
47+
if (!Files.exists(dst)
48+
|| Files.size(dst) != Files.size(src.path)
49+
|| Files.getLastModifiedTime(dst).compareTo(Files.getLastModifiedTime(src.path)) != 0) {
50+
Files.copy(
51+
src.path,
52+
dst,
53+
StandardCopyOption.COPY_ATTRIBUTES,
54+
StandardCopyOption.REPLACE_EXISTING);
55+
}
56+
}
57+
}
58+
59+
Set<String> atSource = sourceEntries.stream().map(e -> e.name).collect(Collectors.toSet());
60+
targetEntries.stream().filter(v -> !atSource.contains(v.name)).forEach(e -> remove(e.path));
61+
}
62+
63+
private List<Entry> files(Path source) throws IOException {
64+
ArrayList<Entry> entries = new ArrayList<>();
65+
try (DirectoryStream<Path> ds = Files.newDirectoryStream(source)) {
66+
ds.forEach(p -> entries.add(new Entry(p)));
67+
}
68+
return entries;
69+
}
70+
71+
private static void remove(Path p) {
72+
try {
73+
Files.walkFileTree(
74+
p,
75+
new SimpleFileVisitor<>() {
76+
@Override
77+
public FileVisitResult postVisitDirectory(Path dir, IOException exc)
78+
throws IOException {
79+
Files.delete(dir);
80+
return FileVisitResult.CONTINUE;
81+
}
82+
83+
@Override
84+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
85+
throws IOException {
86+
Files.delete(file);
87+
return FileVisitResult.CONTINUE;
88+
}
89+
});
90+
} catch (IOException e) {
91+
throw new UncheckedIOException(e);
92+
}
93+
}
94+
}

‎lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java‎

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -51,29 +51,13 @@
5151
* default {@code LuceneTestCase} configuration setup is not used (you have to annotate test for
5252
* JUnit, for example).
5353
*/
54-
public class TestModularLayer {
55-
/** A path to a directory with an expanded Lucene distribution. */
56-
private static final String DISTRIBUTION_PROPERTY = "lucene.distribution.dir";
57-
58-
/** The expected distribution version of Lucene modules. */
59-
private static final String VERSION_PROPERTY = "lucene.distribution.version";
60-
54+
public class TestModularLayer extends AbstractLuceneDistributionTest {
6155
/** Only core Lucene modules, no third party modules. */
6256
private static Set<ModuleReference> allCoreModules;
6357

6458
/** {@link ModuleFinder} resolving only the Lucene modules. */
6559
private static ModuleFinder coreModulesFinder;
6660

67-
/** Ensure Lucene classes are not directly visible. */
68-
@BeforeClass
69-
public static void checkLuceneNotInClasspath() {
70-
Assertions.assertThatThrownBy(
71-
() -> {
72-
Class.forName("org.apache.lucene.index.IndexWriter");
73-
})
74-
.isInstanceOf(ClassNotFoundException.class);
75-
}
76-
7761
/**
7862
* We accept external properties that point to the assembled set of distribution modules and to
7963
* their expected version. These properties are collected and passed by gradle but can be provided
@@ -86,7 +70,7 @@ public static void checkModulePathProvided() {
8670
throw new AssertionError(DISTRIBUTION_PROPERTY + " property is required for this test.");
8771
}
8872

89-
Path modulesPath = Paths.get(modulesPropertyValue).resolve("modules");
73+
Path modulesPath = getDistributionPath().resolve("modules");
9074
if (!Files.isDirectory(modulesPath)) {
9175
throw new AssertionError(
9276
DISTRIBUTION_PROPERTY

‎lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestScripts.java‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.lucene.distribution;
18+
19+
import com.carrotsearch.procfork.ForkedProcess;
20+
import com.carrotsearch.procfork.Launcher;
21+
import com.carrotsearch.procfork.ProcessBuilderLauncher;
22+
import com.carrotsearch.randomizedtesting.LifecycleScope;
23+
import com.carrotsearch.randomizedtesting.RandomizedTest;
24+
import java.nio.charset.Charset;
25+
import java.nio.charset.StandardCharsets;
26+
import java.nio.file.Files;
27+
import java.nio.file.Path;
28+
import java.util.ArrayList;
29+
import java.util.List;
30+
import java.util.Locale;
31+
import java.util.concurrent.TimeUnit;
32+
import java.util.function.Supplier;
33+
import org.assertj.core.api.Assertions;
34+
import org.assertj.core.api.ThrowingConsumer;
35+
import org.junit.Test;
36+
37+
/** Verify that scripts included in the distribution work. */
38+
public class TestScripts extends AbstractLuceneDistributionTest {
39+
@Test
40+
public void testLukeCanBeLaunched() throws Exception {
41+
Path distributionPath;
42+
if (randomBoolean()) {
43+
// Occasionally, be evil: put the distribution in a folder with a space inside. For Uwe.
44+
distributionPath = RandomizedTest.newTempDir(LifecycleScope.TEST).resolve("uh oh");
45+
Files.createDirectory(distributionPath);
46+
new Sync().sync(getDistributionPath(), distributionPath);
47+
} else {
48+
distributionPath = getDistributionPath();
49+
}
50+
51+
Path lukeScript = resolveScript(distributionPath.resolve("bin").resolve("luke"));
52+
53+
Launcher launcher =
54+
new ProcessBuilderLauncher()
55+
.executable(lukeScript)
56+
// tweak Windows launcher scripts so that they don't fork asynchronous java.
57+
.envvar("DISTRIBUTION_TESTING", "true")
58+
.viaShellLauncher()
59+
.cwd(distributionPath)
60+
.args("--sanity-check");
61+
62+
execute(
63+
launcher,
64+
0,
65+
5,
66+
(output) -> {
67+
Assertions.assertThat(output).contains("[Vader] Hello, Luke.");
68+
});
69+
}
70+
71+
/** The value of <code>System.getProperty("os.name")</code>. * */
72+
public static final String OS_NAME = System.getProperty("os.name");
73+
/** True iff running on Windows. */
74+
public static final boolean WINDOWS = OS_NAME.startsWith("Windows");
75+
76+
protected Path resolveScript(Path scriptPath) {
77+
List<Path> candidates = new ArrayList<>();
78+
candidates.add(scriptPath);
79+
80+
String fileName = scriptPath.getFileName().toString();
81+
if (WINDOWS) {
82+
candidates.add(scriptPath.resolveSibling(fileName + ".cmd"));
83+
candidates.add(scriptPath.resolveSibling(fileName + ".bat"));
84+
} else {
85+
candidates.add(scriptPath.resolveSibling(fileName + ".sh"));
86+
}
87+
88+
return candidates.stream()
89+
.sequential()
90+
.filter(Files::exists)
91+
.findFirst()
92+
.orElseThrow(() -> new AssertionError("No script found for the base path: " + scriptPath));
93+
}
94+
95+
private static Supplier<Charset> forkedProcessCharset =
96+
() -> {
97+
// The default charset for a forked java process could be computed for the current
98+
// platform but it adds more complexity. For now, assume it's just parseable ascii.
99+
return StandardCharsets.US_ASCII;
100+
};
101+
102+
protected String execute(
103+
Launcher launcher,
104+
int expectedExitCode,
105+
long timeoutInSeconds,
106+
ThrowingConsumer<String> consumer)
107+
throws Exception {
108+
109+
try (ForkedProcess forkedProcess = launcher.execute()) {
110+
String command = forkedProcess.getProcess().info().command().orElse("(unset command name)");
111+
112+
Charset charset = forkedProcessCharset.get();
113+
try {
114+
Process p = forkedProcess.getProcess();
115+
if (!p.waitFor(timeoutInSeconds, TimeUnit.SECONDS)) {
116+
throw new AssertionError("Forked process did not terminate in the expected time");
117+
}
118+
119+
int exitStatus = p.exitValue();
120+
121+
Assertions.assertThat(exitStatus)
122+
.as("forked process exit status")
123+
.isEqualTo(expectedExitCode);
124+
125+
String output = Files.readString(forkedProcess.getProcessOutputFile(), charset);
126+
consumer.accept(output);
127+
return output;
128+
} catch (Throwable t) {
129+
logSubprocessOutput(
130+
command, Files.readString(forkedProcess.getProcessOutputFile(), charset));
131+
throw t;
132+
}
133+
}
134+
}
135+
136+
protected void logSubprocessOutput(String command, String output) {
137+
System.out.printf(
138+
Locale.ROOT,
139+
"--- [forked subprocess output: %s] ---%n%s%n--- [end of subprocess output] ---%n",
140+
command,
141+
output);
142+
}
143+
}

‎lucene/distribution/src/binary-release/bin/luke.cmd‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,15 @@
1717

1818
SETLOCAL
1919
SET MODULES=%~dp0..
20-
start javaw --module-path "%MODULES%\modules;%MODULES%\modules-thirdparty" --module org.apache.lucene.luke
20+
21+
REM For distribution testing we want plain 'java' command, otherwise we can't block
22+
REM on luke invocation and can't intercept the return status.
23+
SET LAUNCH_CMD=start javaw
24+
IF NOT "%DISTRIBUTION_TESTING%"=="true" GOTO launch
25+
SET LAUNCH_CMD=java
26+
27+
:launch
28+
%LAUNCH_CMD% --module-path "%MODULES%\modules;%MODULES%\modules-thirdparty" --module org.apache.lucene.luke %*
29+
SET EXITVAL=%errorlevel%
30+
EXIT /b %EXITVAL%
2131
ENDLOCAL

‎lucene/distribution/src/binary-release/bin/luke.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,5 @@
1717

1818
MODULES=`dirname "$0"`/..
1919
MODULES=`cd "$MODULES" && pwd`
20-
java --module-path "$MODULES/modules:$MODULES/modules-thirdparty" --module org.apache.lucene.luke
20+
java --module-path "$MODULES/modules:$MODULES/modules-thirdparty" --module org.apache.lucene.luke "$@"
21+
exit $?

‎lucene/licenses/procfork-1.0.6.jar.sha1‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
7ac0dae744df9cc3aaa7a5fee72e289cad7790f9

0 commit comments

Comments
 (0)