Skip to content

Attempt to make crate-testing work under windows #92

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mackerl wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Attempt to make crate-testing work under windows #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion gradle/wrapper/gradle-wrapper.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-all.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
85 changes: 48 additions & 37 deletions src/main/java/io/crate/testing/CrateTestCluster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,9 @@
import io.crate.testing.download.FileDownloadSource;
import org.junit.rules.ExternalResource;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.io.*;
import java.net.*;
import java.nio.file.*;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeoutException;
Expand Down Expand Up @@ -201,7 +192,6 @@ public Builder numberOfNodes(int numberOfNodes) {
if (settings.isEmpty()) {
settings = new HashMap<>();
}
settings.put("node.max_local_storage_nodes", numberOfNodes);
return this;
}

Expand Down Expand Up @@ -269,14 +259,19 @@ private CrateTestServer[] buildServers() {

String[] unicastHosts = getUnicastHosts(hostAddress, transportPorts);
for (int i = 0; i < numberOfNodes; i++) {

Map<String,Object> nodeSettings = new HashMap<>(settings);
nodeSettings.put("node.name", "node-" + i);
nodeSettings.put("path.data", crateWorkingDir().resolve("node" + i));

servers[i] = new CrateTestServer(
clusterName,
httpPorts[i],
transportPorts[i],
psqlPorts[i],
crateWorkingDir(),
hostAddress,
settings,
nodeSettings,
commandLineArguments,
crateVersion,
unicastHosts
Expand Down Expand Up @@ -376,16 +371,22 @@ public void startCluster() throws Throwable {
}
}

public void prepareEnvironment() throws IOException {
public void prepareEnvironment() throws IOException, URISyntaxException {
createDirs();
Path downloadedCrateTarGz = downloadCrateTarGz();
Path downloadedCrateArchive = downloadCrateArchive();
Path crateWorkingDir = crateWorkingDir();

if (Files.notExists(crateWorkingDir)) {
Utils.uncompressTarGZ(
downloadedCrateTarGz.toFile(),
crateWorkingDir.toFile()
);
File archiveFile = downloadedCrateArchive.toFile();
String fileName = archiveFile.getName().toLowerCase();

if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
Utils.uncompressTarGZ(archiveFile, crateWorkingDir.toFile());
} else if (fileName.endsWith(".zip")) {
Utils.unzip(archiveFile, crateWorkingDir.toFile());
} else {
throw new IllegalArgumentException("Unsupported archive format: " + fileName);
}
}
}

Expand All @@ -398,32 +399,32 @@ private void createDirs() throws IOException {
}
}

private Path downloadCrateTarGz() throws IOException {
String tarGzFileName = fileNameFromDownloadSource(downloadSource);
private Path downloadCrateArchive() throws IOException, URISyntaxException {
String archiveFileName = fileNameFromDownloadSource(downloadSource);

Path tarGz;
Path archive;
if (downloadSource instanceof FileDownloadSource) {
tarGz = Paths.get(downloadSource.downloadUrl().getPath());
archive = Paths.get(downloadSource.downloadUrl().toURI());
} else {
tarGz = TMP_CACHE_DIR.resolve(tarGzFileName);
archive = TMP_CACHE_DIR.resolve(archiveFileName);
}

boolean isLatestDistribution = tarGzFileName.contains(LATEST_DISTRIBUTION_VERSION_IDENTIFIER);
if (!isLatestDistribution && Files.exists(tarGz)) {
Utils.log("No need to download crate. Already downloaded %s to: %s", downloadSource, tarGz);
boolean isLatestDistribution = archiveFileName.contains(LATEST_DISTRIBUTION_VERSION_IDENTIFIER);
if (!isLatestDistribution && Files.exists(archive)) {
Utils.log("No need to download crate. Already downloaded %s to: %s", downloadSource, archive);
} else {
Path tarGzPart = TMP_CACHE_DIR.resolve(String.format("%s.part-%s", tarGzFileName, clusterUUID));
Utils.log("Downloading Crate %s to: %s", downloadSource, tarGzPart);
Path archivePart = TMP_CACHE_DIR.resolve(String.format("%s.part-%s", archiveFileName, clusterUUID));
Utils.log("Downloading Crate %s to: %s", downloadSource, archivePart);
try (InputStream in = downloadSource.downloadUrl().openStream()) {
Files.copy(in, tarGzPart);
Files.copy(in, archivePart);
}
if(isLatestDistribution) {
Files.move(tarGzPart, tarGz, StandardCopyOption.REPLACE_EXISTING);
Files.move(archivePart, archive, StandardCopyOption.REPLACE_EXISTING);
} else {
Files.move(tarGzPart, tarGz);
Files.move(archivePart, archive);
}
}
return tarGz;
return archive;
}

private String fileNameFromDownloadSource(DownloadSource downloadSource) throws MalformedURLException {
Expand All @@ -445,16 +446,26 @@ public void after() {
}
try {
removeCrateDir();
} catch (IOException e) {
} catch (IOException | InterruptedException e) {
Utils.log("Error while deleting crate directory: %s error: %s", crateWorkingDir(), e);
}
servers = null;
}

private void removeCrateDir() throws IOException {
private void removeCrateDir() throws IOException, InterruptedException {
Path cratePath = crateWorkingDir();
if (Files.exists(cratePath) && !keepWorkingDir) {
Utils.deletePath(cratePath);

for (int i = 0; i < 10; i++) {
try {
Utils.deletePath(cratePath);
break;
} catch (AccessDeniedException e) {
Thread.sleep(1000); // wait and retry
Utils.log("Retrying to delete crate directory: %s error: %s", crateWorkingDir(), e);
}
}

assert Files.notExists(cratePath);
}
}
Expand Down
15 changes: 9 additions & 6 deletions src/main/java/io/crate/testing/CrateTestServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,14 @@ protected void after() {
Utils.log("Stopping crate server process...");
if (crateProcess != null) {
try {
crateProcess.destroy();
if(Utils.isWindows()) {
ProcessHandle handle = crateProcess.toHandle();
handle.descendants().forEach(ph -> {
Utils.log("Destroying child process: %d", ph.pid());
ph.destroy();
});
}
crateProcess.destroyForcibly();
crateProcess.waitFor();
} catch (Exception e) {
e.printStackTrace();
Expand All @@ -142,7 +149,7 @@ private long startCrateAsDaemon() throws IOException, InterruptedException {
int idx = 0;

String executable = Paths.get(workingDir.toString(), "bin", "crate").toString();
if (isWindows()) {
if (Utils.isWindows()) {
executable = executable.concat(".bat");
}
command[idx++] = executable;
Expand Down Expand Up @@ -187,10 +194,6 @@ public void run() {
return crateProcess.pid();
}

private static boolean isWindows() {
return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).contains("win");
}

Map<String, Object> prepareSettings() {
Map<String, Object> settings = new HashMap<>();
settings.put("network.host", crateHost);
Expand Down
80 changes: 72 additions & 8 deletions src/main/java/io/crate/testing/Utils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,16 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.ServerSocket;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.ThreadLocalRandom;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.ArrayList;

import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
Expand Down Expand Up @@ -78,20 +78,45 @@ static void log(String message, Object... params) {
}

public static void deletePath(Path path) throws IOException {
List<Path> lockedFiles = new ArrayList<>();
List<Path> lockedDirs = new ArrayList<>();

Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
try {
Files.delete(file);
} catch (AccessDeniedException e) {
Utils.log("Cannot delete locked file: %s (%s)", file, e.getMessage());
lockedFiles.add(file);
}
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
try {
Files.delete(dir);
} catch (AccessDeniedException e) {
Utils.log("Cannot delete locked directory: %s (%s)", dir, e.getMessage());
lockedDirs.add(dir);
}
return FileVisitResult.CONTINUE;
}

});

// If there are locked files or dirs, move the entire directory to a temp fallback
if (!lockedFiles.isEmpty() || !lockedDirs.isEmpty()) {
Path parent = path.getParent();
Path fallbackDir = parent.resolve(path.getFileName() + "_pending_delete_" + System.currentTimeMillis());
try {
Files.move(path, fallbackDir, StandardCopyOption.ATOMIC_MOVE);
Utils.log("Moved locked directory to fallback: %s", fallbackDir);
fallbackDir.toFile().deleteOnExit();
} catch (IOException e) {
Utils.log("Failed to move directory to fallback: %s (%s)", fallbackDir, e.getMessage());
}
}
}

public static String sha1(String input) {
Expand Down Expand Up @@ -170,4 +195,43 @@ static void uncompressTarGZ(File tarFile, File dest) throws IOException {
tarIn.close();
}

public static void unzip(File zipFile, File dest) throws IOException {
try (ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(new FileInputStream(zipFile)))) {
ZipEntry entry = zipIn.getNextEntry();

while (entry != null) {
Path entryPath = Paths.get(entry.getName());
if (entryPath.getNameCount() == 1) {
entry = zipIn.getNextEntry();
continue;
}
Path strippedPath = entryPath.subpath(1, entryPath.getNameCount());
File destFile = new File(dest, strippedPath.toString());

if (entry.isDirectory()) {
destFile.mkdirs();
} else {
Path destPath = destFile.toPath();
destPath.getParent().toFile().mkdirs();
destFile.createNewFile();
try (BufferedOutputStream bout = new BufferedOutputStream(new FileOutputStream(destFile))) {
zipIn.transferTo(bout);
}
// Optional: set executable for specific files
if (destFile.getPath().endsWith("bin/crate") || destFile.getPath().endsWith("/bin/java")) {
destFile.setExecutable(true);
}
}
zipIn.closeEntry();
entry = zipIn.getNextEntry();
}
}
}


public static boolean isWindows() {
String os = System.getProperty("os.name").toLowerCase();
return os.contains("win");
}

}
16 changes: 8 additions & 8 deletions src/main/java/io/crate/testing/download/FileDownloadSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,19 +36,19 @@ public class FileDownloadSource implements DownloadSource {

public static final String FOLDER_PREFIX = "crate-file-%s";

private final Path pathToTarGzDistribution;
private final Path pathToArchiveDistribution;
private final String folderName;

public FileDownloadSource(String pathToTarGzDistribution) {
Path path = Paths.get(pathToTarGzDistribution);
public FileDownloadSource(String pathToArchiveDistribution) {
Path path = Paths.get(pathToArchiveDistribution);
if (!Files.exists(path)) {
throw new IllegalArgumentException(String
.format(Locale.ENGLISH, "the for the given path [%s] does not exists", pathToTarGzDistribution));
.format(Locale.ENGLISH, "the for the given path [%s] does not exists", pathToArchiveDistribution));
}
this.pathToTarGzDistribution = path;
this.pathToArchiveDistribution = path;
this.folderName = String.format(Locale.ENGLISH,
FOLDER_PREFIX,
Utils.sha1(pathToTarGzDistribution));
Utils.sha1(pathToArchiveDistribution));
}

@Override
Expand All @@ -58,11 +58,11 @@ public File folder(File containingFolder) {

@Override
public URL downloadUrl() throws MalformedURLException {
return pathToTarGzDistribution.toAbsolutePath().toUri().toURL();
return pathToArchiveDistribution.toAbsolutePath().toUri().toURL();
}

@Override
public String toString() {
return String.format(Locale.ENGLISH, "FILE[%s]", pathToTarGzDistribution);
return String.format(Locale.ENGLISH, "FILE[%s]", pathToArchiveDistribution);
}
}
15 changes: 14 additions & 1 deletion src/main/java/io/crate/testing/download/VersionDownloadSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,27 @@

package io.crate.testing.download;

import io.crate.testing.Utils;

import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Locale;

class VersionDownloadSource implements DownloadSource {

public static final String VERSION_DOWNLOAD_URL = "https://cdn.crate.io/downloads/releases/crate-%s.tar.gz";
public static final String VERSION_DOWNLOAD_URL;

static {
VERSION_DOWNLOAD_URL=getVersionDownloadUrl();
}

public static String getVersionDownloadUrl() {
if(Utils.isWindows()) {
return "https://cdn2.crate.io/downloads/releases/cratedb/x64_windows/crate-%s.zip";
}
return "https://cdn.crate.io/downloads/releases/cratedb/crate-%s.tar.gz";
}

private final String version;
private final String folderName;
Expand Down
Loading
Loading