Fix gold

pomlp/pom.xml

@@ -110,7 +110,7 @@
             <version>1.0.0-SNAPSHOT</version>
         </dependency>
 
-        <!-- MathML Converter -->
+        <!-- MathML Tools -->
         <dependency>
             <groupId>com.formulasearchengine</groupId>
             <artifactId>mathmltools</artifactId>

pomlp/src/main/java/com/formulasearchengine/mathmltools/mml/elements/MathDocWrapper.java

@@ -0,0 +1,26 @@
+package com.formulasearchengine.mathmltools.mml.elements;
+
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.IOException;
+
+/**
+ * A wrapper for MathDoc from MathMLTools because of bad visibilities.
+ * @author Andre Greiner-Petter
+ */
+public class MathDocWrapper extends MathDoc {
+    public MathDocWrapper(String inputXMLString) throws ParserConfigurationException, SAXException, IOException {
+        super(inputXMLString);
+    }
+
+    @Override
+    public void changeTeXAnnotation(String newTeX){
+        super.changeTeXAnnotation(newTeX);
+    }
+
+    @Override
+    public void fixGoldCd(){
+        super.fixGoldCd();
+    }
+}

pomlp/src/main/java/com/formulasearchengine/mathosphere/pomlp/GoldStandardLoader.java

@@ -1,8 +1,17 @@
 package com.formulasearchengine.mathosphere.pomlp;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+
 import com.formulasearchengine.mathosphere.pomlp.gouldi.JsonGouldiBean;
 import com.formulasearchengine.mathosphere.pomlp.util.GoldUtils;
+import com.formulasearchengine.mathosphere.pomlp.util.GouldiRegenerator;
 import com.formulasearchengine.mathosphere.pomlp.util.config.ConfigLoader;
 import com.formulasearchengine.mathosphere.pomlp.util.rest.GitHubFileResponse;
 import com.formulasearchengine.mathosphere.pomlp.util.rest.RESTPathBuilder;
@@ -11,17 +20,8 @@
 import org.springframework.http.client.SimpleClientHttpRequestFactory;
 import org.springframework.web.client.RestTemplate;
 
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
 public class GoldStandardLoader {
-    private static final Logger LOG = LogManager.getLogger( GoldStandardLoader.class.getName() );
+    private static final Logger LOG = LogManager.getLogger(GoldStandardLoader.class.getName());
 
     // Time out is 5sec
     public static final int CONN_TIME_OUT = 5 * 1000;
@@ -46,63 +46,63 @@ public class GoldStandardLoader {
     private static JsonGouldiBean[] gouldi;
 
     /**
-     *
      * @throws RuntimeException if the configurations cannot be loaded from config.properties
      */
-    private GoldStandardLoader() {}
+    private GoldStandardLoader() {
+    }
 
     private static final GoldStandardLoader loader = new GoldStandardLoader();
 
-    public static GoldStandardLoader getInstance(){
+    public static GoldStandardLoader getInstance() {
         return loader;
     }
 
-    public void init(){
+    public void init() {
         props = ConfigLoader.CONFIG;
-        max = Integer.parseInt(props.getProperty( ConfigLoader.GOULDI_MAXIMUM_NUM ));
+        max = Integer.parseInt(props.getProperty(ConfigLoader.GOULDI_MAXIMUM_NUM));
 
-        String repo = props.getProperty( ConfigLoader.GITHUB_REPO_NAME );
-        String owner = props.getProperty( ConfigLoader.GITHUB_REPO_OWNER );
-        String path = props.getProperty( ConfigLoader.GITHUB_REPO_PATH );
-        String githubLink = props.getProperty( ConfigLoader.GITHUB_URL );
+        String repo = props.getProperty(ConfigLoader.GITHUB_REPO_NAME);
+        String owner = props.getProperty(ConfigLoader.GITHUB_REPO_OWNER);
+        String path = props.getProperty(ConfigLoader.GITHUB_REPO_PATH);
+        String githubLink = props.getProperty(ConfigLoader.GITHUB_URL);
 
-        if ( repo == null || owner == null || githubLink == null ){
+        if (repo == null || owner == null || githubLink == null) {
             LOG.info("Cannot find GitHub access -> switch to local initialization.");
             initLocally();
             return;
         }
 
         LOG.debug("Load all github properties.");
-        gitHubApiURL = new RESTPathBuilder( githubLink )
+        gitHubApiURL = new RESTPathBuilder(githubLink)
                 .setGithubContent(owner, repo)
                 .setInnerPath(path)
                 .getURL();
 
         SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
-        factory.setConnectTimeout( CONN_TIME_OUT );
-        factory.setReadTimeout( READ_TIME_OUT );
+        factory.setConnectTimeout(CONN_TIME_OUT);
+        factory.setReadTimeout(READ_TIME_OUT);
 
         rest = new RestTemplate(factory);
     }
 
     public int initLocally() {
         props = ConfigLoader.CONFIG;
-        max = Integer.parseInt(props.getProperty( ConfigLoader.GOULDI_MAXIMUM_NUM ));
+        max = Integer.parseInt(props.getProperty(ConfigLoader.GOULDI_MAXIMUM_NUM));
 
-        String goldPath = props.getProperty( ConfigLoader.GOULDI_LOCAL_PATH );
+        String goldPath = props.getProperty(ConfigLoader.GOULDI_LOCAL_PATH);
         Path path = Paths.get(goldPath);
         gouldi = new JsonGouldiBean[max];
 
-        ExecutorService executor = Executors.newFixedThreadPool( Runtime.getRuntime().availableProcessors()*2 );
-        for ( int i = 1; i <= max; i++ ){
-            executor.execute( new JSONReader( path, i ) );
+        ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
+        for (int i = 1; i <= max; i++) {
+            executor.execute(new JSONReader(path, i));
         }
 
         executor.shutdown();
 
         try {
-            executor.awaitTermination( PARALLEL_READING_TIMEOUT, PARALLEL_READING_TIMEOUT_UNIT );
-        } catch ( InterruptedException ie ){
+            executor.awaitTermination(PARALLEL_READING_TIMEOUT, PARALLEL_READING_TIMEOUT_UNIT);
+        } catch (InterruptedException ie) {
             LOG.warn("Executor service exceeds timeouts to read files. It maybe didn't properly load all gouldi-files.");
         }
 
@@ -111,50 +111,57 @@ public int initLocally() {
     }
 
     private class JSONReader implements Runnable {
-        private final Logger IN_LOG = LogManager.getLogger( JSONReader.class.getName() );
+        private final Logger IN_LOG = LogManager.getLogger(JSONReader.class.getName());
 
         private Path path;
         private int number;
 
-        public JSONReader( Path goldPath, int number ){
+        public JSONReader(Path goldPath, int number) {
             this.path = goldPath;
             this.number = number;
         }
 
         @Override
         public void run() {
             try {
-                Path p = path.resolve( number + ".json" );
-                gouldi[number-1] = GoldUtils.readGoldFile(p);
-            } catch ( Exception e ){
+                Path p = path.resolve(number + ".json");
+                gouldi[number - 1] = GoldUtils.readGoldFile(p);
+            } catch (Exception e) {
                 IN_LOG.error("Parallel process cannot parse " + path.toString() + number + ".json - " + e.getMessage(), e);
             }
         }
     }
 
-    public GitHubFileResponse getResponseFromGouldiRequest( int number ){
+    public GitHubFileResponse getResponseFromGouldiRequest(int number) {
         String file = number + ".json";
         return rest.getForObject(
                 gitHubApiURL + RESTPathBuilder.BIND + file,
                 GitHubFileResponse.class
         );
     }
 
-    public JsonGouldiBean getGouldiJson( int number )
-            throws IOException
-    {
-        if ( local ) {
+    public JsonGouldiBean getGouldiJson(int number)
+            throws IOException {
+        if (local) {
             LOG.trace("Local mode. Get Json: " + number);
-            return gouldi[number-1];
+            return gouldi[number - 1];
         }
 
-        GitHubFileResponse response = getResponseFromGouldiRequest( number );
+        GitHubFileResponse response = getResponseFromGouldiRequest(number);
         // TODO handle response codes here
         return response.getJsonBeanFromContent();
     }
 
-    public static void main(String[] args){
-        GoldStandardLoader gold = new GoldStandardLoader();
-        gold.initLocally();
+    public static void main(String[] args) {
+        // TODO: DO NOT USE THIS MAIN, IF YOU DON'T KNOW WHAT YOU ARE DOING!
+        // it's just a main for manually executions and changes constantly
+
+        // regenerate all gold files once!
+        //String goldPath = ConfigLoader.CONFIG.getProperty(ConfigLoader.GOULDI_LOCAL_PATH);
+        //Path outputPath = Paths.get( goldPath );
+        Path outputPath = Paths.get("outputTmp");
+        GouldiRegenerator regenerator = new GouldiRegenerator(outputPath);
+        regenerator.init();
+        regenerator.regenerateAllMML();
     }
 }

pomlp/src/main/java/com/formulasearchengine/mathosphere/pomlp/TreeFilesGenerator.java

@@ -83,46 +83,30 @@ public void generate( int number, Converters converter ) throws Exception{
      * Generates all third party MMLs at once
      */
     public void generateAllSubs(){
-//        ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(16);
-
-        // each number 1-300
         for ( int i = 1; i <= maxNumber; i++ ){
             try {
                 JsonGouldiBean bean = loader.getGouldiJson(i);
                 LOG.info("Start pre-processing of tex input.");
-                bean.setMathTex( Utility.latexPreProcessing(bean.getOriginalTex()) );
-                // each converter which is available
+                bean.setOriginalTex( Utility.latexPreProcessing(bean.getOriginalTex()) );
+                // use all available converters
                 for ( Converters c : Converters.values() ){
-                    final int index = i;
                     if (!c.skip()) {
-//                        executor.submit( () -> generate(index, bean, c) );
-                        generate(index, bean, c);
+                        generate(i, bean, c);
                     }
                 }
             } catch ( IOException e ){
                 LOG.error("SKIP: " + i, e);
             }
         }
-
-//        LOG.info("Shutdown thread pool. All threads in queue.");
-//        executor.shutdown();
-//        LOG.info("Wait for termination now.");
-//        try {
-//            executor.awaitTermination( 10, TimeUnit.MINUTES );
-//            LOG.info("Finished all threads in thread pool.");
-//        } catch ( InterruptedException ie ){
-//            LOG.error( "Waited 10 Minutes but still the thread pool is not terminated.", ie );
-//            LOG.error( "Shutdown now!" );
-//            executor.shutdownNow();
-//        }
     }
 
     public static void main(String[] args) throws Exception{
+        // TODO: WILL GENERATE AND OVERWRITE ALL MML FILES BY THIRD PARTY TOOLS
+        // TODO: DO NOT START IF YOU DON'T KNOW WHAT YOU ARE DOING HERE!
         preStartCommandCheck();
         TreeFilesGenerator gen = new TreeFilesGenerator();
         gen.init();
         gen.generateAllSubs();
-
     }
 
     /**

pomlp/src/main/java/com/formulasearchengine/mathosphere/pomlp/convertor/LatexmlGenerator.java

@@ -1,6 +1,6 @@
 package com.formulasearchengine.mathosphere.pomlp.convertor;
 
-import com.formulasearchengine.mathmlconverters.latexml.LaTeXMLConverter;
+import com.formulasearchengine.mathmltools.xmlhelper.XmlDocumentReader;
 import com.formulasearchengine.mathosphere.pomlp.xml.MathMLDocumentReader;
 import com.formulasearchengine.nativetools.CommandExecutor;
 import com.formulasearchengine.nativetools.NativeResponse;
@@ -11,50 +11,22 @@
 
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
+import static com.formulasearchengine.mathosphere.pomlp.util.config.LatexMLConfig.*;
+
 public class LatexmlGenerator implements Parser, Canonicalizable{
 
     private static Logger LOG = LogManager.getLogger(LatexmlGenerator.class.getName());
 
     private static final String NAME = "LaTeXML";
-    private static final String COMMAND = "latexmlc";
-
-    private static final String[] DEFAULT_CMD = new String[]{
-            COMMAND,
-            "--includestyles",
-            "--format=xhtml",
-            "--whatsin=math",
-            "--whatsout=math",
-            "--pmml",
-            "--cmml",
-            "--nodefaultresources",
-            "--linelength=90",
-            "--preload", "LaTeX.pool",
-            "--preload", "article.cls",
-            "--preload", "amsmath.sty",
-            "--preload", "amsthm.sty",
-            "--preload", "amstext.sty",
-            "--preload", "amssymb.sty",
-            "--preload", "eucal.sty",
-            "--preload", "[dvipsnames]xcolor.sty",
-            "--preload", "url.sty",
-            "--preload", "hyperref.sty",
-            "--preload", "[ids]latexml.sty",
-            "--preload", "texvc"
-    };
 
-    private LaTeXMLConverter converter;
     private NativeResponse response;
 
     public LatexmlGenerator(){}
 
     @Override
-    public void init() {
-        // we only need latexmlc -> no configuration needed
-        converter = new LaTeXMLConverter(null);
-    }
+    public void init() {}
 
     @Override
     public Document parse(String latex) {
@@ -63,7 +35,7 @@ public Document parse(String latex) {
         response = executor.exec( CommandExecutor.DEFAULT_TIMEOUT, Level.TRACE );
         if ( handleResponseCode(response, NAME, LOG) != 0 ) return null;
         LOG.info(NAME + " conversion successful.");
-        return MathMLDocumentReader.getDocumentFromXMLString( response.getResult() );
+        return XmlDocumentReader.getDocumentFromXMLString( response.getResult() );
     }
 
     @Override
@@ -78,17 +50,17 @@ public void parseToFile(String latex, Path outputFile) {
 
     @Override
     public String getNativeCommand(){
-        return COMMAND;
+        return NATIVE_COMMAND;
     }
 
     public ArrayList<String> buildArguments(String latex ){
-        ArrayList<String> args = new ArrayList<>(Arrays.asList(DEFAULT_CMD));
+        ArrayList<String> args = asList( GENERIC_CONFIG );
         args.add( "literal:" + latex );
         return args;
     }
 
     public List<String> buildArguments(String latex, Path outputFile ){
-        ArrayList<String> args = new ArrayList<>(Arrays.asList(DEFAULT_CMD));
+        ArrayList<String> args = asList( GENERIC_CONFIG );
         args.add( "--dest=" + outputFile.toAbsolutePath().toString() );
         args.add( "literal:" + latex );
         return args;

pomlp/src/main/java/com/formulasearchengine/mathosphere/pomlp/convertor/MathToWebConverter.java

@@ -1,8 +1,7 @@
 package com.formulasearchengine.mathosphere.pomlp.convertor;
 
+import com.formulasearchengine.mathmltools.xmlhelper.XmlDocumentReader;
 import com.formulasearchengine.mathosphere.pomlp.util.Utility;
-import com.formulasearchengine.mathosphere.pomlp.xml.MathMLDocumentReader;
-import com.formulasearchengine.mathosphere.pomlp.xml.XmlDocumentReader;
 import mathtoweb.engine.MathToWeb;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;

pomlp/src/main/java/com/formulasearchengine/mathosphere/pomlp/convertor/NativeConverter.java

@@ -1,7 +1,7 @@
 package com.formulasearchengine.mathosphere.pomlp.convertor;
 
+import com.formulasearchengine.mathmltools.xmlhelper.XmlDocumentReader;
 import com.formulasearchengine.mathosphere.pomlp.util.Utility;
-import com.formulasearchengine.mathosphere.pomlp.xml.XmlDocumentReader;
 import com.formulasearchengine.nativetools.CommandExecutor;
 import com.formulasearchengine.nativetools.NativeResponse;
 import org.apache.logging.log4j.LogManager;