Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,11 @@ jobs:

steps:
- uses: actions/checkout@v2
- name: Set up JDK 1.8
uses: actions/setup-java@v1
- name: Set up JDK 11
uses: actions/setup-java@v5
with:
java-version: 1.8
distribution: temurin
java-version: 11
- name: Run tests
run: sbt "test-only tests.CITests"

17 changes: 12 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,19 @@ See the `.github/workflows/build.yml` file for details on the CI config used by

## Setup

Prerequisite: Java 8
Prerequisite: Java 11

`git clone https://github.com/hbz/nwbib.git ; cd nwbib`\
`wget http://downloads.typesafe.com/typesafe-activator/1.2.10/typesafe-activator-1.2.10-minimal.zip`\
`unzip typesafe-activator-1.2.10-minimal.zip`\
`./activator-1.2.10-minimal/activator test`
- `git clone https://github.com/hbz/nwbib.git ; cd nwbib`\
- `sbt clean`
- `sbt test`

Build the web application:

- `sbt clean`
- `sbt stage`
- `./target/universal/stage/bin/nwbib -no-version-check`

See the `.github/workflows/build.yml` file for details on the CI config used by Github Actions.

### Eclipse setup

Expand Down
51 changes: 35 additions & 16 deletions app/controllers/nwbib/Classification.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import java.text.Collator;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
Expand Down Expand Up @@ -39,14 +40,17 @@
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.node.InternalSettingsPreparer;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.transport.Netty4Plugin;

import com.fasterxml.jackson.databind.JsonNode;
import com.github.jsonldjava.core.JsonLdError;
Expand Down Expand Up @@ -200,6 +204,12 @@ private enum Label {
(JsonNode o1, JsonNode o2) -> Collator.getInstance(Locale.GERMAN)
.compare(labelText(o1), labelText(o2));

private static class EmbeddedNode extends Node {
public EmbeddedNode(Settings preparedSettings, Collection<Class<? extends Plugin>> classpathPlugins) {
super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), classpathPlugins);
}
}

private Classification() {
/* Use via static functions, no instantiation. */
}
Expand Down Expand Up @@ -238,7 +248,7 @@ public static List<String> toJsonLd(final URL turtleUrl) {
public static JsonNode ids(String q, String t) {
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
.must(QueryBuilders.idsQuery(Type.NWBIB.elasticsearchType,
Type.SPATIAL.elasticsearchType).ids(q));
Type.SPATIAL.elasticsearchType).addIds(q));
SearchRequestBuilder requestBuilder = client.prepareSearch(INDEX)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(queryBuilder);
if (t.isEmpty()) {
Expand Down Expand Up @@ -506,16 +516,21 @@ public static String shortId(String uri) {

/** Start up the embedded Elasticsearch classification index. */
public static void indexStartup() {
Settings clientSettings = ImmutableSettings.settingsBuilder()
.put("path.home", new File(".").getAbsolutePath())
.put("http.port",
play.Play.application().isTest() ? "8855"
: CONFIG.getString("index.es.port.http"))
.put("transport.tcp.port", play.Play.application().isTest() ? "8856"
: CONFIG.getString("index.es.port.tcp"))
.build();
node =
NodeBuilder.nodeBuilder().settings(clientSettings).local(true).node();
boolean isTest = play.Play.application().isTest();
String httpPort = isTest ? "8855" : CONFIG.getString("index.es.port.http");
String tcpPort = isTest ? "8856" : CONFIG.getString("index.es.port.tcp");
node = new EmbeddedNode(Settings.builder().put("transport.type", "netty4")//
.put("http.type", "netty4")//
.put("http.enabled", "true")//
.put("path.home", new File(".").getAbsolutePath())//
.put("http.port", httpPort)//
.put("transport.tcp.port", tcpPort)//
.build(), Arrays.asList(Netty4Plugin.class));
try {
node.start();
} catch (NodeValidationException e) {
e.printStackTrace();
}
client = node.client();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute()
.actionGet();
Expand All @@ -535,8 +550,8 @@ private static void indexData(String dataUrl, Type type) {
List<String> jsonLd = toJsonLd(new URL(dataUrl));
for (String concept : jsonLd) {
String id = Json.parse(concept).findValue("@id").textValue();
IndexRequestBuilder indexRequest = client
.prepareIndex(INDEX, type.elasticsearchType, id).setSource(concept);
IndexRequestBuilder indexRequest = client.prepareIndex(INDEX, type.elasticsearchType, id)
.setSource(concept.getBytes(), XContentType.JSON);
bulkRequest.add(indexRequest);
}
} catch (MalformedURLException e) {
Expand All @@ -550,7 +565,11 @@ private static void indexData(String dataUrl, Type type) {

/** Shut down the embedded Elasticsearch classification index. */
public static void indexShutdown() {
node.close();
try {
node.close();
} catch (IOException e) {
e.printStackTrace();
}
}

/**
Expand Down
2 changes: 1 addition & 1 deletion app/controllers/nwbib/Lobid.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
import java.util.stream.StreamSupport;

import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.common.collect.Lists;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.html.HtmlEscapers;

import controllers.nwbib.Classification.Type;
Expand Down
10 changes: 6 additions & 4 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,26 @@ name := "nwbib"

version := "0.1.0-SNAPSHOT"

scalaVersion := "2.11.11"
scalaVersion := "2.11.12"

libraryDependencies ++= Seq(
cache,
javaWs,
"com.typesafe.play" % "play-test_2.11" % "2.4.11",
"org.elasticsearch" % "elasticsearch" % "1.7.5" withSources(),
"org.elasticsearch" % "elasticsearch" % "5.6.16" withSources(),
"org.elasticsearch.plugin" % "transport-netty4-client" % "5.6.16" withSources(),
"org.mockito" % "mockito-core" % "1.9.5",
"org.apache.commons" % "commons-rdf-jena" % "0.5.0",
"org.apache.commons" % "commons-csv" % "1.6",
"org.apache.jena" % "jena-arq" % "3.17.0" exclude ("com.github.jsonld-java","jsonld-java"),
"org.apache.jena" % "jena-core" % "3.17.0",
"org.easytesting" % "fest-assert" % "1.4" % "test"
"org.easytesting" % "fest-assert" % "1.4" % "test",
"com.sun.xml.bind" % "jaxb-impl" % "2.3.3" withSources()
)

lazy val root = (project in file(".")).enablePlugins(PlayJava)

javacOptions ++= Seq("-source", "1.8", "-target", "1.8")
javacOptions ++= Seq("-source", "11", "-target", "11")

import com.typesafe.sbteclipse.core.EclipsePlugin.EclipseKeys

Expand Down
2 changes: 1 addition & 1 deletion conf/application.conf
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ play.http.router=nwbib.Routes
# ~~~~~
# The secret key is used to secure cryptographics functions.
# If you deploy your application to several instances be sure to use the same key!
application.secret="gNXZGXDQTEE=VfuVpyXXpTiMRuEYTnMcXx^28kThj3Jij42@j^>tGhh6pRmRAhRh"
play.crypto.secret="gNXZGXDQTEE=VfuVpyXXpTiMRuEYTnMcXx^28kThj3Jij42@j^>tGhh6pRmRAhRh"

# The application languages
# ~~~~~
Expand Down
5 changes: 4 additions & 1 deletion monit_restart.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ HOME="/home/sol"

# it is important to set the proper locale
. $HOME/.locale
export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
JAVA_OPTS=$(echo "$JAVA_OPTS" |sed 's#,#\ #g')

cd $HOME/git/$REPO
Expand All @@ -34,7 +35,9 @@ case $ACTION in
kill $(cat target/universal/stage/RUNNING_PID)
rm target/universal/stage/RUNNING_PID
fi
JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" $HOME/activator-dist-1.3.5/activator "start $PORT"
export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError -DpreferIPv4Stack"
sbt --java-home $JAVA_HOME stage >> ./target/universal/stage/logs/application.log 2>&1
./target/universal/stage/bin/nwbib -Dhttp.port=$PORT -no-version-check >> ./target/universal/stage/logs/application.log 2>&1
;;
stop)
kill $(cat target/universal/stage/RUNNING_PID)
Expand Down
2 changes: 1 addition & 1 deletion project/build.properties
Original file line number Diff line number Diff line change
@@ -1 +1 @@
sbt.version=0.13.16
sbt.version=0.13.18