Ability to extend supported extension - file parser mapping #3690

jmix-search/search/search.gradle

@@ -69,8 +69,9 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-engine'
     testImplementation 'org.junit.jupiter:junit-jupiter-params'
     testImplementation 'org.junit.vintage:junit-vintage-engine'
+    testImplementation 'org.spockframework:spock-core'
     testImplementation 'org.mockito:mockito-core'
-    testImplementation "org.spockframework:spock-core"
+    testImplementation 'ch.qos.logback:logback-classic'
     testRuntimeOnly 'org.slf4j:slf4j-simple'
     testRuntimeOnly 'org.hsqldb:hsqldb'
     testRuntimeOnly 'org.junit.platform:junit-platform-launcher'

jmix-search/search/src/main/java/io/jmix/search/exception/UnsupportedFileFormatException.java

@@ -16,13 +16,29 @@
 
 package io.jmix.search.exception;
 
-import org.apache.commons.io.FilenameUtils;
+import java.util.List;
 
+/**
+ * An exception that is thrown when a user added some file of the type that is not supported
+ * and there are no any known parser for.
+ */
 public class UnsupportedFileFormatException extends Exception {
 
-    public static final String MESSAGE = "The file %s with the '%s' extension is not supported.";
+    private static final String MESSAGE = "The file %s can't be parsed. " +
+            "Only the following file parsing criteria are supported:\n  -%s";
+
+    /**
+     * @param fileName            the name of the file which type is not supported
+     * @param supportedExtensions the list of the criteria that are supported in the application
+     */
+    public UnsupportedFileFormatException(String fileName, List<String> supportedExtensions) {
+        super(String.format(
+                MESSAGE,
+                fileName,
+                getSupportedExtensionsString(supportedExtensions)));
+    }
 
-    public UnsupportedFileFormatException(String fileName) {
-        super(String.format(MESSAGE, fileName, FilenameUtils.getExtension(fileName)));
+    protected static String getSupportedExtensionsString(List<String> supportedExtensions) {
+        return String.join("\n  -", supportedExtensions);
     }
 }

jmix-search/search/src/main/java/io/jmix/search/index/fileparsing/AbstractExtensionBasedFileParserResolver.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2024 Haulmont.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.jmix.search.index.fileparsing;
+
+import com.google.common.base.Strings;
+import io.jmix.core.FileRef;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.BodyContentHandler;
+import org.xml.sax.ContentHandler;
+
+import java.io.StringWriter;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * Implements the common logic for all extension based file parser resolvers.
+ */
+public abstract class AbstractExtensionBasedFileParserResolver implements FileParserResolver {
+
+    /**
+     * Returns a collection of supported extensions of the supported file type.
+     * Note that the extension checking mechanism is case-sensitive. So in order to support
+     * the both uppercase one and lowercase option of the extension they should be defined explicitly.
+     * E.g. ["xlsx", "XLSX", "docx", "DOCX"].
+     *
+     * @return collection of supported extensions
+     */
+    public abstract Set<String> getSupportedExtensions();
+
+    @Override
+    public String getCriteriaDescription() {
+        return String.format(
+                "File parser resolver: %s. Supported extensions: %s.",
+                this.getClass().getSimpleName(),
+                getSupportedExtensionsString(getSupportedExtensions()));
+    }
+
+    @Override
+    public boolean supports(FileRef fileRef) {
+        String fileName = fileRef.getFileName();
+        String fileExtension = FilenameUtils.getExtension(fileName);
+        if (Strings.isNullOrEmpty(fileExtension)) {
+            return false;
+        }
+
+        return getSupportedExtensions().contains(fileExtension);
+    }
+
+    protected String getSupportedExtensionsString(Set<String> supportedExtensions) {
+        return String.join(", ", supportedExtensions);
+    }
+
+    @Override
+    public FileParserKit getParserKit() {
+        return new FileParserKit(
+                getParser(),
+                getContentHandlerGenerator(),
+                getMetadata(),
+                getParseContext());
+    }
+
+    /**
+     * Returns a parser for the supported file type.
+     */
+    protected abstract Parser getParser();
+
+    /**
+     * Returns a function for the ContentHandler generating that is necessary for the given file parsing.
+     */
+    protected Function<StringWriter, ContentHandler> getContentHandlerGenerator() {
+        return stringWriter -> new BodyContentHandler(stringWriter);
+    }
+
+    /**
+     * Returns a Metadata object for the given file parsing.
+     */
+    protected Metadata getMetadata() {
+        return new Metadata();
+    }
+
+    /**
+     * Returns a ParseContext object for the given file parsing.
+     */
+    protected ParseContext getParseContext() {
+        return new ParseContext();
+    }
+}

jmix-search/search/src/main/java/io/jmix/search/index/fileparsing/FileParserKit.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2024 Haulmont.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.jmix.search.index.fileparsing;
+
+import jakarta.validation.constraints.NotNull;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.xml.sax.ContentHandler;
+
+import java.io.StringWriter;
+import java.util.function.Function;
+
+public record FileParserKit(
+        @NotNull Parser parser,
+        @NotNull Function<StringWriter, ContentHandler> contentHandlerGenerator,
+        @NotNull Metadata metadata,
+        @NotNull ParseContext parseContext) {}

jmix-search/search/src/main/java/io/jmix/search/index/fileparsing/FileParserResolver.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2024 Haulmont.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.jmix.search.index.fileparsing;
+
+import io.jmix.core.FileRef;
+
+/**
+ * Interface to be implemented for adding a custom file parser resolver
+ * or modifying the behavior of the existing file parser resolvers. It gives an ability to define the exact parser
+ * for the exact file types with a custom implementation of the file checking logic. These parsers are used to extract
+ * file content for sending it to the search server and indexing.
+ */
+public interface FileParserResolver {
+
+    /**
+     * Returns the description of the criteria for the files that are supported with this resolver.
+     * This text is used for generating the log message that is written into the log
+     * while no one of the resolvers supports the processing of the given file.
+     *
+     * @return criteria description
+     */
+    String getCriteriaDescription();
+
+    /**
+     * Returns a complex object that contains all necessary objects for the supported file type parsing.
+     *
+     * @return an instance of a file parser kit
+     */
+    FileParserKit getParserKit();
+
+    /**
+     * Returns the result of the checking if the file with the given fileRef is supported by the resolver or not.
+     *
+     * @param fileRef object with the file information
+     * @return the given FileRef's checking result
+     */
+    boolean supports(FileRef fileRef);
+}

jmix-search/search/src/main/java/io/jmix/search/index/fileparsing/package-info.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2020 Haulmont.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@NonNullApi
+package io.jmix.search.index.fileparsing;
+
+import org.springframework.lang.NonNullApi;

jmix-search/search/src/main/java/io/jmix/search/index/fileparsing/resolvers/LegacyMSOfficeDocumentsParserResolver.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2024 Haulmont.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.jmix.search.index.fileparsing.resolvers;
+
+import io.jmix.search.index.fileparsing.AbstractExtensionBasedFileParserResolver;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.microsoft.OfficeParser;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+
+import java.util.Set;
+
+@Component("search_LegacyMSOfficeDocumentsParserResolver")
+@Order(100)
+public class LegacyMSOfficeDocumentsParserResolver extends AbstractExtensionBasedFileParserResolver {
+
+    @Override
+    public Set<String> getSupportedExtensions() {
+        return Set.of("doc", "xls", "DOC", "XLS");
+    }
+
+    @Override
+    public Parser getParser() {
+        return new OfficeParser();
+    }
+}

jmix-search/search/src/main/java/io/jmix/search/index/fileparsing/resolvers/MSOfficeDocumentsParserResolver.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024 Haulmont.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.jmix.search.index.fileparsing.resolvers;
+
+import io.jmix.search.index.fileparsing.AbstractExtensionBasedFileParserResolver;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.microsoft.OfficeParserConfig;
+import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+
+import java.util.Set;
+
+@Component("search_OfficeDocumentsParserResolver")
+@Order(100)
+public class MSOfficeDocumentsParserResolver extends AbstractExtensionBasedFileParserResolver {
+
+    @Override
+    public Set<String> getSupportedExtensions() {
+        return Set.of("docx", "xlsx", "DOCX", "XLSX");
+    }
+
+    @Override
+    public Parser getParser() {
+        return new OOXMLParser();
+    }
+
+    @Override
+    protected ParseContext getParseContext() {
+        ParseContext parseContext = super.getParseContext();
+
+        OfficeParserConfig officeParserConfig = new OfficeParserConfig();
+        officeParserConfig.setIncludeHeadersAndFooters(false);
+        parseContext.set(OfficeParserConfig.class, officeParserConfig);
+
+        return parseContext;
+    }
+}

jmix-search/search/src/main/java/io/jmix/search/index/fileparsing/resolvers/OpenOfficeDocumentsParserResolver.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2024 Haulmont.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.jmix.search.index.fileparsing.resolvers;
+
+import io.jmix.search.index.fileparsing.AbstractExtensionBasedFileParserResolver;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.odf.OpenDocumentParser;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+
+import java.util.Set;
+
+@Component("search_OpenOfficeDocumentsParserResolver")
+@Order(100)
+public class OpenOfficeDocumentsParserResolver extends AbstractExtensionBasedFileParserResolver {
+
+    @Override
+    public Set<String> getSupportedExtensions() {
+        return Set.of("odt", "ods", "ODT", "ODS");
+    }
+
+    @Override
+    public Parser getParser() {
+        return new OpenDocumentParser();
+    }
+}