Skip to content

Commit 5bffaa7

Browse files
author
Lior Knaany
committed
begining of unit tests
1 parent 18524ed commit 5bffaa7

File tree

7 files changed

+310
-2
lines changed

7 files changed

+310
-2
lines changed

‎pom.xml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@
2828

2929
<tests.ifNoTests>warn</tests.ifNoTests>
3030
<elasticsearch.version>5.6.0</elasticsearch.version>
31+
<commons-io.version>2.4</commons-io.version>
32+
<httpcore.version>4.4.8</httpcore.version>
33+
<junit.version>4.12</junit.version>
34+
<jackson.version>2.7.4</jackson.version>
3135
</properties>
3236

3337

@@ -38,6 +42,90 @@
3842
<version>${elasticsearch.version}</version>
3943
<scope>provided</scope>
4044
</dependency>
45+
46+
<dependency>
47+
<groupId>junit</groupId>
48+
<artifactId>junit</artifactId>
49+
<version>${junit.version}</version>
50+
<scope>test</scope>
51+
</dependency>
52+
53+
<dependency>
54+
<groupId>org.codelibs.elasticsearch.module</groupId>
55+
<artifactId>reindex</artifactId>
56+
<version>${elasticsearch.version}</version>
57+
<scope>test</scope>
58+
<exclusions>
59+
<exclusion>
60+
<groupId>org.slf4j</groupId>
61+
<artifactId>slf4j-log4j12</artifactId>
62+
</exclusion>
63+
<exclusion>
64+
<artifactId>log4j</artifactId>
65+
<groupId>log4j</groupId>
66+
</exclusion>
67+
</exclusions>
68+
</dependency>
69+
<!--<dependency>-->
70+
<!--<groupId>org.elasticsearch.plugin</groupId>-->
71+
<!--<artifactId>transport-netty3-client</artifactId>-->
72+
<!--<version>${elasticsearch.version}</version>-->
73+
<!--<scope>test</scope>-->
74+
<!--</dependency>-->
75+
<dependency>
76+
<groupId>org.codelibs.elasticsearch.module</groupId>
77+
<artifactId>lang-painless</artifactId>
78+
<version>${elasticsearch.version}</version>
79+
<scope>test</scope>
80+
</dependency>
81+
<dependency>
82+
<groupId>commons-io</groupId>
83+
<artifactId>commons-io</artifactId>
84+
<version>${commons-io.version}</version>
85+
<scope>test</scope>
86+
</dependency>
87+
<dependency>
88+
<groupId>org.apache.httpcomponents</groupId>
89+
<artifactId>httpcore</artifactId>
90+
<version>${httpcore.version}</version>
91+
<scope>test</scope>
92+
</dependency>
93+
<dependency>
94+
<groupId>org.elasticsearch.client</groupId>
95+
<artifactId>elasticsearch-rest-client</artifactId>
96+
<version>${elasticsearch.version}</version>
97+
<!--<version>5.6.8</version>-->
98+
<exclusions>
99+
<exclusion>
100+
<groupId>org.apache.httpcomponents</groupId>
101+
<artifactId>httpclient</artifactId>
102+
</exclusion>
103+
<exclusion>
104+
<groupId>org.apache.httpcomponents</groupId>
105+
<artifactId>httpclient</artifactId>
106+
</exclusion>
107+
<exclusion>
108+
<groupId>org.apache.httpcomponents</groupId>
109+
<artifactId>httpcore</artifactId>
110+
</exclusion>
111+
<exclusion>
112+
<groupId>commons-codec</groupId>
113+
<artifactId>commons-codec</artifactId>
114+
</exclusion>
115+
<exclusion>
116+
<groupId>commons-logging</groupId>
117+
<artifactId>commons-logging</artifactId>
118+
</exclusion>
119+
</exclusions>
120+
</dependency>
121+
122+
<dependency>
123+
<groupId>com.fasterxml.jackson.core</groupId>
124+
<artifactId>jackson-databind</artifactId>
125+
<version>${jackson.version}</version>
126+
<scope>test</scope>
127+
</dependency>
128+
41129
<dependency>
42130
<groupId>com.google.collections</groupId>
43131
<artifactId>google-collections</artifactId>

‎src/main/java/com/liorkn/elasticsearch/plugin/VectorScoringPlugin.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,6 @@ public final class VectorScoringPlugin extends Plugin implements ScriptPlugin {
3030
public final ScriptEngineService getScriptEngineService(Settings settings) {
3131
return new VectorScoringScriptEngineService(settings);
3232
}
33+
34+
3335
}

‎src/main/java/com/liorkn/elasticsearch/script/VectorScoreScript.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.nio.ByteBuffer;
2626
import java.nio.DoubleBuffer;
2727
import java.util.ArrayList;
28+
import java.util.Base64;
2829
import java.util.Map;
2930

3031
/**
@@ -117,7 +118,6 @@ public VectorScoreScript(Map<String, Object> params) {
117118
this.field = field.toString();
118119

119120
// get query inputVector - convert to primitive
120-
121121
final Object vector = params.get("vector");
122122
if(vector != null) {
123123
final ArrayList<Double> tmp = (ArrayList<Double>) vector;
@@ -147,7 +147,6 @@ public VectorScoreScript(Map<String, Object> params) {
147147
}
148148

149149

150-
151150
/**
152151
* Called for each document
153152
* @return cosine similarity of the current document against the input inputVector

‎src/test/com/liorkn/elasticsearch/EmbeddedElasticsearchServer.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package com.liorkn.elasticsearch;
2+
3+
import com.google.common.annotations.VisibleForTesting;
4+
import com.liorkn.elasticsearch.plugin.VectorScoringPlugin;
5+
import org.apache.commons.io.FileUtils;
6+
import org.elasticsearch.client.Client;
7+
import org.elasticsearch.common.settings.Settings;
8+
import org.elasticsearch.http.BindHttpException;
9+
import org.elasticsearch.index.reindex.ReindexPlugin;
10+
import org.elasticsearch.node.Node;
11+
import org.elasticsearch.node.NodeValidationException;
12+
import org.elasticsearch.painless.PainlessPlugin;
13+
import org.elasticsearch.transport.Netty3Plugin;
14+
15+
import java.io.File;
16+
import java.io.IOException;
17+
import java.util.Arrays;
18+
import java.util.Random;
19+
20+
public class EmbeddedElasticsearchServer {
21+
private static final Random RANDOM = new Random();
22+
private static final String DEFAULT_DATA_DIRECTORY = "target/elasticsearch-data";
23+
private static final String DEFAULT_HOME_DIRECTORY = "target/elasticsearch-home";
24+
private static final int MAX_PORT_RETRIES = 20;
25+
26+
private Node node;
27+
private int port;
28+
private String dataDirectory;
29+
30+
public EmbeddedElasticsearchServer() throws NodeValidationException {
31+
this(DEFAULT_DATA_DIRECTORY);
32+
}
33+
34+
private EmbeddedElasticsearchServer(String dataDirectory) throws NodeValidationException {
35+
this(dataDirectory, randomPort());
36+
}
37+
38+
private EmbeddedElasticsearchServer(String defaultDataDirectory, int port) throws NodeValidationException {
39+
this.dataDirectory = defaultDataDirectory;
40+
this.port = port;
41+
42+
Settings.Builder settings = Settings.builder()
43+
.put("http.enabled", "true")
44+
.put("transport.type", "local")
45+
.put("http.type", "netty3")
46+
.put("path.data", dataDirectory)
47+
.put("path.home", DEFAULT_HOME_DIRECTORY)
48+
.put("script.inline", "on")
49+
.put("node.max_local_storage_nodes", 10000)
50+
.put("script.stored", "on");
51+
52+
startNodeInAvailablePort(settings);
53+
}
54+
55+
private void startNodeInAvailablePort(Settings.Builder settings) throws NodeValidationException {
56+
int findFreePortRetries = MAX_PORT_RETRIES;
57+
boolean success = false;
58+
59+
while(!success) {
60+
try {
61+
settings.put("http.port", String.valueOf(this.port));
62+
63+
// this a hack in order to load Groovy plug in since we want to enable the usage of scripts
64+
node = new NodeExt(settings.build() , Arrays.asList(Netty3Plugin.class, PainlessPlugin.class, ReindexPlugin.class, VectorScoringPlugin.class));
65+
node.start();
66+
success = true;
67+
System.out.println(EmbeddedElasticsearchServer.class.getName() + ": Using port: " + this.port);
68+
} catch (BindHttpException exception) {
69+
if(findFreePortRetries == 0) {
70+
System.out.println("Could not find any free port in range: [" + (this.port - MAX_PORT_RETRIES) + " - " + this.port+"]");
71+
throw exception;
72+
}
73+
findFreePortRetries--;
74+
System.out.println("Port already in use (" + this.port + "). Trying another port...");
75+
this.port = randomPort();
76+
}
77+
}
78+
}
79+
80+
public String getUrl() {
81+
return "http://localhost:" + port;
82+
}
83+
84+
public Client getClient() {
85+
return node.client();
86+
}
87+
88+
public void shutdown() {
89+
if ( node != null )
90+
try {
91+
node.close();
92+
} catch (IOException e) {
93+
e.printStackTrace();
94+
}
95+
deleteDataDirectory();
96+
}
97+
98+
private void deleteDataDirectory() {
99+
try {
100+
FileUtils.deleteDirectory(new File(dataDirectory));
101+
} catch (IOException e) {
102+
throw new RuntimeException("Could not delete data directory of embedded elasticsearch server", e);
103+
}
104+
}
105+
106+
private static int randomPort() {
107+
return RANDOM.nextInt(500) + 4200;
108+
}
109+
110+
@VisibleForTesting
111+
public int getPort() {
112+
return port;
113+
}
114+
}

‎src/test/com/liorkn/elasticsearch/NodeExt.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.liorkn.elasticsearch;
2+
3+
import org.elasticsearch.common.settings.Settings;
4+
import org.elasticsearch.node.InternalSettingsPreparer;
5+
import org.elasticsearch.node.Node;
6+
import org.elasticsearch.plugins.Plugin;
7+
8+
import java.util.Collection;
9+
10+
/**
11+
* a really god explanation for this is on: https://discuss.elastic.co/t/unsupported-http-type-netty3-when-trying-to-start-embedded-elasticsearch-node/69669/7
12+
*/
13+
public class NodeExt extends Node {
14+
15+
public NodeExt(Settings preparedSettings, Collection<Class<? extends Plugin>> classpathPlugins) {
16+
super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null),
17+
classpathPlugins);
18+
}
19+
20+
}

‎src/test/com/liorkn/elasticsearch/TestObject.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.liorkn.elasticsearch;
2+
3+
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
4+
import com.fasterxml.jackson.databind.annotation.JsonNaming;
5+
6+
/**
7+
* Created by Lior Knaany on 4/7/18.
8+
*/
9+
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
10+
public class TestObject {
11+
int jobId;
12+
String base64Vector;
13+
double[] vector;
14+
15+
public TestObject(int jobId, double[] vector) {
16+
this.jobId = jobId;
17+
this.vector = vector;
18+
this.base64Vector = Util.convertArrayToBase64(vector);
19+
}
20+
}

‎src/test/com/liorkn/elasticsearch/TestPlugin.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package com.liorkn.elasticsearch;
2+
3+
import com.fasterxml.jackson.core.JsonParser;
4+
import com.fasterxml.jackson.databind.ObjectMapper;
5+
import org.apache.http.HttpHost;
6+
import org.apache.http.entity.StringEntity;
7+
import org.elasticsearch.client.RestClient;
8+
import org.junit.AfterClass;
9+
import org.junit.BeforeClass;
10+
import org.junit.Test;
11+
12+
import java.io.IOException;
13+
import java.util.HashMap;
14+
import java.util.Map;
15+
16+
/**
17+
* Created by Lior Knaany on 4/7/18.
18+
*/
19+
public class TestPlugin {
20+
21+
private static EmbeddedElasticsearchServer esServer;
22+
private static RestClient esClient;
23+
24+
@BeforeClass
25+
public static void init() throws Exception {
26+
esServer = new EmbeddedElasticsearchServer();
27+
esClient = RestClient.builder(new HttpHost("localhost", esServer.getPort(), "http")).build();
28+
29+
// TODO: create test index
30+
}
31+
32+
public static final ObjectMapper mapper = new ObjectMapper();
33+
static {
34+
mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
35+
mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
36+
}
37+
38+
39+
//@Test TODO: fix this
40+
public void test() throws Exception {
41+
final Map<String, String> params = new HashMap<>();
42+
final ObjectMapper mapper = new ObjectMapper();
43+
final TestObject[] objs = {new TestObject(1, new double[] {0.0, 0.5, 1.0}),
44+
new TestObject(2, new double[] {0.2, 0.6, 0.99})};
45+
46+
for (int i = 0; i < objs.length; i++) {
47+
final TestObject t = objs[i];
48+
esClient.performRequest("PUT", "/test/" + t.jobId, params, new StringEntity(mapper.writeValueAsString(t)));
49+
}
50+
51+
// Test cosine score function
52+
53+
}
54+
55+
@AfterClass
56+
public static void shutdown() {
57+
try {
58+
esClient.close();
59+
esServer.shutdown();
60+
} catch (IOException e) {
61+
e.printStackTrace();
62+
}
63+
}
64+
65+
}

0 commit comments

Comments
 (0)