MaRDI4NFDI · AndreG-P · Mar 11, 2020 · Apr 20, 2020 · Apr 20, 2020 · Apr 21, 2020
diff --git a/.gitignore b/.gitignore
@@ -84,3 +84,7 @@ dependency-reduced-pom.xml
 # Introducing the t-standard (copyrights reserved)
 t/
 /mathosphere-core/test/
+
+# ignore node folder (necessary to run mathoid service locally)
+node_modules/
+
diff --git a/.gitmodules b/.gitmodules
@@ -10,3 +10,7 @@
 	path = lib/GoUldI
 	url = https://github.com/ag-gipp/GoUldI.git
 	branch = master
+[submodule "lib/MathMLTools"]
+	path = lib/MathMLTools
+	url = https://github.com/ag-gipp/MathMLTools.git
+	branch = master
diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,7 @@
 language: java
 sudo: required
 jdk:
-- oraclejdk8
+- openjdk11
 
 before_install:
   - mvn clean -q

diff --git a/README.md b/README.md
@@ -68,3 +68,9 @@ If test fail due to encoding problems in windows, set the environment variable
 JAVA_TOOL_OPTIONS = -Dfile.encoding=UTF8
 ```
 as suggested on [stackoverflow.](http://stackoverflow.com/a/28470840)
+
+### Java 11
+This project is currently working for Java 11 (theoretically until Java 16 to be precise) but fails for Java 17 or newer.
+The reason is that the project uses flink with massive usages of reflections. Since Java 17, many of these reflections
+violate the visibility permissions. The errors are not obviously hinting towards visibility issues due to Java 17. 
+Hence, fixing the issues might be tricky. Simply make sure you run the project with Java 11 for now!
diff --git a/Test Results - com_formulasearchengine_in_mathosphere-core.html b/Test Results - com_formulasearchengine_in_mathosphere-core.html
diff --git a/basex/pom.xml b/basex/pom.xml
@@ -53,6 +53,12 @@
             <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
         </repository>
     </distributionManagement>
+    <repositories>
+        <repository>
+            <id>basex</id>
+            <url>https://files.basex.org/maven</url>
+        </repository>
+    </repositories>
     <dependencies>
         <dependency>
             <groupId>org.basex</groupId>
@@ -103,12 +109,12 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
-            <version>2.5.1</version>
+            <version>2.10.3</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.5.1</version>
+            <version>2.10.3</version>
         </dependency>
         <dependency>
             <groupId>com.intellij</groupId>
@@ -152,7 +158,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.9.1</version>
+                <version>3.2.0</version>
                 <executions>
                     <execution>
                         <id>attach-javadocs</id>

diff --git a/config.dev.yaml b/config.dev.yaml
@@ -9,7 +9,7 @@ worker_heap_limit_mb: 250
 
 # Logger info
 logging:
-  level: warn
+  level: info
 #  streams:
 #  # Use gelf-stream -> logstash
 #  - type: gelf

diff --git a/evaluation/pom.xml b/evaluation/pom.xml
@@ -29,14 +29,14 @@
     </developers>
     <build>
         <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
-                </configuration>
-            </plugin>
+<!--            <plugin>-->
+<!--                <groupId>org.apache.maven.plugins</groupId>-->
+<!--                <artifactId>maven-compiler-plugin</artifactId>-->
+<!--                <configuration>-->
+<!--                    <source>1.8</source>-->
+<!--                    <target>1.8</target>-->
+<!--                </configuration>-->
+<!--            </plugin>-->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -73,7 +73,7 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-csv</artifactId>
-            <version>RELEASE</version>
+            <version>1.8</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>

diff --git a/lib/GoUldI b/lib/GoUldI
diff --git a/lib/MathMLCan b/lib/MathMLCan
diff --git a/lib/MathMLTools b/lib/MathMLTools
diff --git a/lib/RTED b/lib/RTED
diff --git a/mathosphere-core/lacast.config.yaml b/mathosphere-core/lacast.config.yaml
@@ -0,0 +1,5 @@
+# This configuration defines the path the library and configuration folder for LaCASt.
+# You can either specify the location of this file with a flag: "-config=pathToThisConfig.yaml",
+# or you put this file into the local directory or your home directory in ".lacast" folder.
+lacast.libs.path: "/home/andreg-p/Projects/LaCASt/libs"
+lacast.config.path: "/home/andreg-p/Projects/LaCASt/config"
diff --git a/mathosphere-core/pom.xml b/mathosphere-core/pom.xml
@@ -24,7 +24,7 @@
     <repositories>
         <repository>
             <id>www2.ph.ed.ac.uk-releases</id>
-            <url>http://www2.ph.ed.ac.uk/maven2</url>
+            <url>https://www2.ph.ed.ac.uk/maven2</url>
         </repository>
         <repository>
             <id>apache.snapshots</id>
@@ -39,6 +39,11 @@
         </repository>
     </repositories>
     <dependencies>
+        <dependency>
+            <groupId>gov.nist.drmf.interpreter</groupId>
+            <artifactId>interpreter.pom</artifactId>
+            <version>2.1-SNAPSHOT</version>
+        </dependency>
         <!-- Logging via log4j2 -->
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
@@ -58,11 +63,22 @@
             <version>2.10.0</version>
         </dependency>
 
-        <!--dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.30</version>
+        </dependency>
+<!--        <dependency>-->
+<!--            <groupId>org.slf4j</groupId>-->
+<!--            <artifactId>slf4j-log4j12</artifactId>-->
+<!--            <version>1.7.30</version>-->
+<!--        </dependency>-->
+
+        <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-slf4j-impl</artifactId>
             <version>2.10.0</version>
-        </dependency-->
+        </dependency>
 
         <!-- Other things -->
         <dependency>
@@ -87,30 +103,44 @@
             <version>9.5.1-6</version>
         </dependency>
         <dependency>
-            <groupId>com.formulasearchengine</groupId>
-            <artifactId>mathmltools</artifactId>
-            <version>0.3.1-SNAPSHOT</version>
+            <groupId>com.formulasearchengine.mathmltools</groupId>
+            <artifactId>mathml-core</artifactId>
+            <version>2.0.5-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>cz.muni.fi.mir</groupId>
             <artifactId>mathml-canonicalizer</artifactId>
-            <version>1.2-Mathosphere-SNAPSHOT</version>
+            <version>1.3.1</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.3.2</version>
+            <version>3.10</version>
         </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.eclipse.mylyn.wikitext/wikitext.mediawiki -->
         <dependency>
-            <groupId>org.fusesource.wikitext</groupId>
-            <artifactId>wikitext-core</artifactId>
-            <version>1.4</version>
+            <groupId>org.eclipse.mylyn.wikitext</groupId>
+            <artifactId>wikitext.mediawiki</artifactId>
+            <version>0.9.4.I20090220-1600-e3x</version>
         </dependency>
+
         <dependency>
-            <groupId>org.fusesource.wikitext</groupId>
-            <artifactId>mediawiki-core</artifactId>
-            <version>1.4</version>
+            <groupId>org.netbeans.external</groupId>
+            <artifactId>org-eclipse-mylyn-wikitext-core</artifactId>
+            <version>RELEASE113</version>
         </dependency>
+
+        <!--        <dependency>-->
+<!--            <groupId>org.fusesource.wikitext</groupId>-->
+<!--            <artifactId>wikitext-core</artifactId>-->
+<!--            <version>1.4</version>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>org.fusesource.wikitext</groupId>-->
+<!--            <artifactId>mediawiki-core</artifactId>-->
+<!--            <version>1.4</version>-->
+<!--        </dependency>-->
         <dependency>
             <groupId>com.beust</groupId>
             <artifactId>jcommander</artifactId>
@@ -119,7 +149,7 @@
         <dependency>
             <groupId>org.sweble.wikitext</groupId>
             <artifactId>swc-engine</artifactId>
-            <version>3.1.5</version>
+            <version>3.1.9</version>
         </dependency>
         <!-- Flink -->
         <dependency>
@@ -129,14 +159,34 @@
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-java_2.10</artifactId>
+            <artifactId>flink-streaming-java_2.12</artifactId>
+            <version>${flink.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-clients_2.12</artifactId>
             <version>${flink.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-clients_2.10</artifactId>
+            <artifactId>flink-runtime-web_2.12</artifactId>
             <version>${flink.version}</version>
+            <scope>compile</scope>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.11.3</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core -->
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <version>2.11.3</version>
         </dependency>
+
+
         <dependency>
             <groupId>nz.ac.waikato.cms.weka</groupId>
             <artifactId>LibSVM</artifactId>
@@ -167,7 +217,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.jr</groupId>
             <artifactId>jackson-jr-objects</artifactId>
-            <version>2.5.0</version>
+            <version>${jackson.version}</version>
         </dependency>
 
         <dependency>
@@ -180,20 +230,20 @@
         <dependency>
             <groupId>edu.stanford.nlp</groupId>
             <artifactId>stanford-corenlp</artifactId>
-            <version>3.5.2</version>
+            <version>4.0.0</version>
         </dependency>
 
         <dependency>
             <groupId>edu.stanford.nlp</groupId>
             <artifactId>stanford-corenlp</artifactId>
-            <version>3.5.2</version>
-            <classifier>models</classifier>         <!-- English models -->
+            <version>4.0.0</version>
+            <classifier>models</classifier>         <!-- basic English models -->
         </dependency>
 
         <dependency>
             <groupId>edu.stanford.nlp</groupId>
             <artifactId>stanford-corenlp</artifactId>
-            <version>3.5.2</version>
+            <version>3.9.2</version>
             <classifier>models-german</classifier>  <!-- German models -->
         </dependency>
 
@@ -246,12 +296,13 @@
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-text</artifactId>
-            <version>1.0</version>
+            <version>1.8</version>
         </dependency>
+
         <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-text</artifactId>
-            <version>1.0</version>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+            <version>1.3.2</version>
         </dependency>
     </dependencies>
     <developers>
@@ -281,11 +332,14 @@
                 <configuration>
                     <includes>
                         <include>%regex[.*mathpd.*]</include>
+                        <include>%regex[.*mlp.*]</include>
+                        <include>%regex[.*wikitext.*]</include>
                     </includes>
                 </configuration>
             </plugin>
             <plugin>
                 <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.3.0</version>
                 <configuration>
                     <archive>
                         <manifest>

diff --git a/mathosphere-core/src/main/java/com/formulasearchengine/mathosphere/mathpd/Distances.java b/mathosphere-core/src/main/java/com/formulasearchengine/mathosphere/mathpd/Distances.java
@@ -1,8 +1,8 @@
 package com.formulasearchengine.mathosphere.mathpd;
 
+import com.formulasearchengine.mathmltools.helper.XMLHelper;
 import com.formulasearchengine.mathmltools.mml.CMMLInfo;
-import com.formulasearchengine.mathmltools.xmlhelper.NonWhitespaceNodeList;
-import com.formulasearchengine.mathmltools.xmlhelper.XMLHelper;
+import com.formulasearchengine.mathmltools.xml.NonWhitespaceNodeList;
 import com.formulasearchengine.mathosphere.mathpd.distances.earthmover.EarthMoverDistanceWrapper;
 import com.formulasearchengine.mathosphere.mathpd.distances.earthmover.JFastEMD;
 import com.formulasearchengine.mathosphere.mathpd.distances.earthmover.Signature;
@@ -21,6 +21,9 @@
 import java.text.DecimalFormat;
 import java.util.*;
 
+//import com.formulasearchengine.mathmltools.xmlhelper.NonWhitespaceNodeList;
+//import com.formulasearchengine.mathmltools.xmlhelper.XMLHelper;
+
 /**
  * Created by Felix Hamborg <[email protected]> on 05.12.16.
  */

diff --git a/mathosphere-core/src/main/java/com/formulasearchengine/mathosphere/mathpd/FlinkPd.java b/mathosphere-core/src/main/java/com/formulasearchengine/mathosphere/mathpd/FlinkPd.java
@@ -7,7 +7,7 @@
 import com.formulasearchengine.mathosphere.mathpd.pojos.ExtractedMathPDDocument;
 import com.formulasearchengine.mathosphere.mlp.contracts.CreateCandidatesMapper;
 import com.formulasearchengine.mathosphere.mlp.contracts.JsonSerializerMapper;
-import com.formulasearchengine.mathosphere.mlp.contracts.TextAnnotatorMapper;
+import com.formulasearchengine.mathosphere.mlp.contracts.WikiTextAnnotatorMapper;
 import com.formulasearchengine.mathosphere.mlp.pojos.ParsedWikiDocument;
 import com.formulasearchengine.mathosphere.mlp.pojos.WikiDocumentOutput;
 
@@ -459,7 +459,7 @@ public static DataSource<String> readPreprocessedFile(String pathname, Execution
     }
 
     public WikiDocumentOutput outDocFromText(FlinkPdCommandConfig config, String input) throws Exception {
-        final TextAnnotatorMapper textAnnotatorMapper = new TextAnnotatorMapper(config);
+        final WikiTextAnnotatorMapper textAnnotatorMapper = new WikiTextAnnotatorMapper(config);
         textAnnotatorMapper.open(null);
         final CreateCandidatesMapper candidatesMapper = new CreateCandidatesMapper(config);
 

diff --git a/...c/main/java/com/formulasearchengine/mathosphere/mathpd/contracts/TextExtractorMapper.java b/...c/main/java/com/formulasearchengine/mathosphere/mathpd/contracts/TextExtractorMapper.java
@@ -1,6 +1,6 @@
 package com.formulasearchengine.mathosphere.mathpd.contracts;
 
-import com.formulasearchengine.mathmltools.xmlhelper.NonWhitespaceNodeList;
+import com.formulasearchengine.mathmltools.xml.NonWhitespaceNodeList;
 import com.formulasearchengine.mathosphere.mathpd.Distances;
 import com.formulasearchengine.mathosphere.mathpd.pojos.ArxivDocument;
 import com.formulasearchengine.mathosphere.mathpd.pojos.ExtractedMathPDDocument;
+1 −1		.gitignore
+14 −10		pom.xml
+8 −5		src/main/java/cz/muni/fi/mir/mathmlcanonicalization/MathMLCanonicalizer.java