Add ability to define custom analysers

Author: Guillaume Vaudaux-Ruth

elasticsearch-annotations/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.alien4cloud</groupId>
         <artifactId>elasticsearch-mapping-parent</artifactId>
-        <version>1.7.4</version>
+        <version>1.7.5</version>
     </parent>
 
     <artifactId>elasticsearch-annotations</artifactId>

elasticsearch-annotations/src/main/java/org/elasticsearch/annotation/ESObject.java

@@ -22,4 +22,6 @@
 	boolean source() default true;
 
 	boolean all() default true;
+
+	IndexAnalyserDefinition[] analyzerDefinitions() default {};
 }

elasticsearch-annotations/src/main/java/org/elasticsearch/annotation/IndexAnalyserDefinition.java

@@ -0,0 +1,22 @@
+package org.elasticsearch.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE })
+public @interface IndexAnalyserDefinition {
+    String name();
+
+    String tokenizer() default "";
+
+    String[] filters() default {};
+
+    String type() default "";
+
+    String[] stopwords() default {};
+
+    String[] char_filter() default {};
+}

elasticsearch-mapping/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.alien4cloud</groupId>
         <artifactId>elasticsearch-mapping-parent</artifactId>
-        <version>1.7.4</version>
+        <version>1.7.5</version>
     </parent>
 
     <artifactId>elasticsearch-mapping</artifactId>

elasticsearch-mapping/src/main/java/org/elasticsearch/mapping/MappingBuilder.java

@@ -7,19 +7,24 @@
 
 import org.elasticsearch.annotation.ESAll;
 import org.elasticsearch.annotation.ESObject;
+import org.elasticsearch.annotation.IndexAnalyserDefinition;
 import org.elasticsearch.annotation.TypeName;
+import org.elasticsearch.common.collect.Maps;
+import org.elasticsearch.common.lang3.ArrayUtils;
 import org.elasticsearch.util.AnnotationScanner;
 import org.elasticsearch.util.MapUtil;
 import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 /**
  * Helps to parse the ES annotations.
- * 
+ *
  * @author luc boutier
  */
 @Component
@@ -28,6 +33,7 @@ public class MappingBuilder {
     private FieldsMappingBuilder fieldsMappingBuilder = new FieldsMappingBuilder();
 
     private Map<String, String> classesMappings = new HashMap<String, String>();
+    private Map<String, String> settingsByClassName = new HashMap<String, String>();
     private Map<String, String> typeByClassName = new HashMap<String, String>();
 
     private Map<String, List<IFilterBuilderHelper>> filtersByClassName = new HashMap<String, List<IFilterBuilderHelper>>();
@@ -37,7 +43,7 @@ public class MappingBuilder {
     /**
      * Helper to return a valid index type from a class. Currently uses clazz.getSimpleName().toLowerCase();
      * Using the helper make sure this is done in a consistent way in different locations of the code.
-     * 
+     *
      * @param clazz The class for which to get an index type.
      * @return The index type.
      */
@@ -47,7 +53,7 @@ public static String indexTypeFromClass(Class<?> clazz) {
 
     /**
      * Build mapping for the given packages.
-     * 
+     *
      * @param packages list of packages in which to look for {@link ESObject} annotated classes.
      * @throws IntrospectionException Java reflection usage related exception.
      * @throws IOException In case of an IO error while creating Json.
@@ -64,7 +70,7 @@ public void initialize(String... packages) throws IntrospectionException, JsonGe
 
     /**
      * Get the mapping json string for a given class.
-     * 
+     *
      * @param clazz The class for which to get the mapping.
      * @return The json mapping for this class.
      * @throws JsonGenerationException In case the mapping failed to be serialized to json.
@@ -75,15 +81,24 @@ public void initialize(String... packages) throws IntrospectionException, JsonGe
     public String getMapping(Class<?> clazz) throws JsonGenerationException, JsonMappingException, IntrospectionException, IOException {
         String classMapping = classesMappings.get(clazz.getName());
         if (classMapping == null) {
-            parseClassMapping(clazz, "");
+            parseClassAnnotations(clazz, "");
         }
         classMapping = classesMappings.get(clazz.getName());
         return classMapping;
     }
 
+    public String getIndexSettings(Class<?> clazz) throws JsonGenerationException, JsonMappingException, IntrospectionException, IOException {
+        String settings = settingsByClassName.get(clazz.getName());
+        if (settings == null) {
+            parseClassAnnotations(clazz, "");
+        }
+        settings = settingsByClassName.get(clazz.getName());
+        return settings;
+    }
+
     /**
      * Get the name of the type in elastic search for the given class.
-     * 
+     *
      * @param clazz The class for which to get the type.
      * @return The type name in elastic search.
      */
@@ -93,7 +108,7 @@ public String getTypeName(Class<?> clazz) {
 
     /**
      * Get the list of es fields that should be filtered in a filter search for the given class.
-     * 
+     *
      * @param clazz The class for which to get the facets.
      * @return The list of filters builders for this class.
      */
@@ -103,7 +118,7 @@ public List<IFilterBuilderHelper> getFilters(Class<?> clazz) {
 
     /**
      * Get the list of es fields that should be filtered in a filter search for the given class.
-     * 
+     *
      * @param className The name for which to get the facets.
      * @return The list of filters builders for this class.
      */
@@ -113,7 +128,7 @@ public List<IFilterBuilderHelper> getFilters(String className) {
 
     /**
      * Get the list of es fields that should be faceted in a facet search for the given class.
-     * 
+     *
      * @param clazz The class for which to get the facets.
      * @return The list of facet builders for this class.
      */
@@ -123,7 +138,7 @@ public List<IFacetBuilderHelper> getFacets(Class<?> clazz) {
 
     /**
      * Get the list of es fields that should be faceted in a facet search for the given class.
-     * 
+     *
      * @param className The name for which to get the facets.
      * @return The list of facet builders for this class.
      */
@@ -133,7 +148,7 @@ public List<IFacetBuilderHelper> getFacets(String className) {
 
     /**
      * Get the {@link SourceFetchContext} for a given fetch context.
-     * 
+     *
      * @param className The class for which to get the fetch context.
      * @param fetchContext The fetch context for which to get the field list.
      * @return The requested {@link SourceFetchContext} or null if no context match the given class and fetch context key.
@@ -149,11 +164,12 @@ public SourceFetchContext getFetchSource(String className, String fetchContext)
     private void initialize(String packageName) throws IntrospectionException, JsonGenerationException, JsonMappingException, IOException {
         Set<Class<?>> classSet = org.elasticsearch.util.AnnotationScanner.scan(packageName, ESObject.class);
         for (Class<?> clazz : classSet) {
-            parseClassMapping(clazz, "");
+            parseClassAnnotations(clazz, "");
         }
     }
 
-    public void parseClassMapping(Class<?> clazz, String pathPrefix) throws IntrospectionException, JsonGenerationException, JsonMappingException, IOException {
+    public void parseClassAnnotations(Class<?> clazz, String pathPrefix)
+            throws IntrospectionException, JsonGenerationException, JsonMappingException, IOException {
         ESObject esObject = AnnotationScanner.getAnnotation(ESObject.class, clazz);
         ESAll esAll = AnnotationScanner.getAnnotation(ESAll.class, clazz);
 
@@ -196,5 +212,35 @@ public void parseClassMapping(Class<?> clazz, String pathPrefix) throws Introspe
         this.facetByClassName.put(clazz.getName(), facetFields);
         this.filtersByClassName.put(clazz.getName(), filteredFields);
         this.fetchSourceContextByClass.put(clazz.getName(), fetchContexts);
+
+        this.settingsByClassName.put(clazz.getName(), buildSettings(mapper, esObject.analyzerDefinitions()));
+    }
+
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    private static class AnalyserFields {
+        public String tokenizer;
+        public String type;
+        public String[] char_filter;
+        public String[] filter;
+        public String[] stopwords;
+    }
+
+    private String buildSettings(ObjectMapper mapper, IndexAnalyserDefinition[] customAnalyserDefinitions) throws JsonProcessingException {
+        if (ArrayUtils.isEmpty(customAnalyserDefinitions)) {
+            return null;
+        }
+
+        Map<Object, Object> analysers = Maps.newHashMap();
+        for (IndexAnalyserDefinition analyserDefinition : customAnalyserDefinitions) {
+            AnalyserFields analyserFields = new AnalyserFields();
+            analyserFields.char_filter = analyserDefinition.char_filter();
+            analyserFields.filter = analyserDefinition.filters();
+            analyserFields.stopwords = analyserDefinition.stopwords();
+            analyserFields.tokenizer = analyserDefinition.tokenizer();
+            analyserFields.type = analyserDefinition.type();
+            analysers.put(analyserDefinition.name(), analyserFields);
+        }
+
+        return "{\"analysis\":{\"analyzer\":" + mapper.writeValueAsString(analysers) + "}}";
     }
 }

elasticsearch-mapping/src/test/java/org/elasticsearch/mapping/MappingBuilderTest.java

@@ -6,25 +6,46 @@
 import java.io.IOException;
 import java.nio.file.Paths;
 
+import org.elasticsearch.mapping.model.City;
 import org.elasticsearch.mapping.model.Person;
+import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
-import junit.framework.Assert;
-
 /**
  * Test the mappings.
  * 
  * @author luc boutier
  */
 public class MappingBuilderTest {
 
+    private MappingBuilder mappingBuilder;
+
+    @Before
+    public void setUp() throws IntrospectionException, IOException {
+        mappingBuilder = new MappingBuilder();
+        mappingBuilder.initialize("org.elasticsearch.mapping.model");
+    }
+
     @Test
     public void testSimpleClassMapping() throws IntrospectionException, IOException {
-        MappingBuilder mappingBuilder = new MappingBuilder();
-        mappingBuilder.initialize("org.elasticsearch.mapping.model");
         String personMapping = mappingBuilder.getMapping(Person.class);
-        BufferedReader brTest = new BufferedReader(new FileReader(Paths.get("src/test/resources/mapping.json").toFile()));
-        String expected = brTest.readLine();
-        Assert.assertEquals(expected, personMapping);
+        assertSameContent(personMapping, "src/test/resources/person-mapping.json");
     }
+
+    @Test
+    public void testSettingsAndMapping() throws IntrospectionException, IOException {
+        String citySettings = mappingBuilder.getIndexSettings(City.class);
+        assertSameContent(citySettings, "src/test/resources/city-settings.json");
+
+        String cityMapping = mappingBuilder.getMapping(City.class);
+        assertSameContent(cityMapping, "src/test/resources/city-mapping.json");
+    }
+
+    private void assertSameContent(String content, String expectedContentFromFile) throws IOException {
+        BufferedReader brMappingTest = new BufferedReader(new FileReader(Paths.get(expectedContentFromFile).toFile()));
+        String expectedMapping = brMappingTest.readLine();
+        Assert.assertEquals(expectedMapping, content);
+    }
+
 }

elasticsearch-mapping/src/test/java/org/elasticsearch/mapping/model/City.java

@@ -0,0 +1,24 @@
+package org.elasticsearch.mapping.model;
+
+import org.elasticsearch.annotation.ESObject;
+import org.elasticsearch.annotation.IndexAnalyserDefinition;
+import org.elasticsearch.annotation.StringField;
+import org.elasticsearch.annotation.StringFieldMulti;
+import org.elasticsearch.mapping.IndexType;
+
+@ESObject(analyzerDefinitions = @IndexAnalyserDefinition(name = "lowerCaseAnalyser", filters = "lowercase", tokenizer = "keyword"))
+public class City {
+
+    @StringFieldMulti(main = @StringField(indexType = IndexType.not_analyzed),
+            multiNames = "lower_case",
+            multi = @StringField(indexType = IndexType.analyzed, analyser = "lowerCaseAnalyser", includeInAll = false))
+    private String city;
+
+    public String getCity() {
+        return city;
+    }
+
+    public void setCity(String city) {
+        this.city = city;
+    }
+}

elasticsearch-mapping/src/test/resources/city-mapping.json

@@ -0,0 +1 @@
+{"city":{"_type":{"index":"not_analyzed","store":false},"_source":{"enabled":true},"_all":{"enabled":true},"properties":{"city":{"include_in_all":true,"term_vector":"no","index":"not_analyzed","boost":1.0,"store":false,"type":"string","fields":{"lower_case":{"include_in_all":false,"analyzer":"lowerCaseAnalyser","term_vector":"no","index":"analyzed","boost":1.0,"store":false,"type":"string"}}}}}}

elasticsearch-mapping/src/test/resources/city-settings.json

@@ -0,0 +1 @@
+{"analysis":{"analyzer":{"lowerCaseAnalyser":{"tokenizer":"keyword","filters":["lowercase"]}}}}