feat(analysis): Add package-based Hunspell dictionary support with ref_path parameter and cache invalidation API

mise.toml

@@ -0,0 +1,2 @@
+[tools]
+java = "corretto-21"

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -117,6 +117,7 @@
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -1701,6 +1702,7 @@ private static void validateErrors(String indexName, List<String> validationErro
 
     List<String> getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings, String indexName) {
         List<String> validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings, Optional.of(indexName));
+        validationErrors.addAll(validateRefPath(settings, env.configDir()));
         return validationErrors;
     }
 
@@ -1732,6 +1734,38 @@ List<String> getIndexSettingsValidationErrors(
         }
         return validationErrors;
     }
+     /**
+     * Validates the ref_path setting if present.
+     * Checks that the path format is valid and the directory exists.
+     *
+     * @param settings the index settings
+     * @param configDir the config directory path
+     * @return a list containing validation errors or an empty list if valid
+     */
+    private List<String> validateRefPath(Settings settings, Path configDir) {
+        List<String> validationErrors = new ArrayList<>();
+        String refPath = settings.get(IndexSettings.INDEX_REF_PATH_SETTING.getKey());
+
+        if (refPath != null && !refPath.isEmpty()) {
+            try {
+                // Validate format: should be in packages/<package_id> format
+                if (!refPath.startsWith("packages/")) {
+                    validationErrors.add("ref_path [" + refPath + "] must start with 'packages/'");
+                    return validationErrors;
+                }
+
+                // Resolve and check if path exists
+                Path resolvedPath = configDir.resolve(refPath).normalize();
+                if (!Files.isDirectory(resolvedPath)) {
+                    validationErrors.add("ref_path [" + refPath + "] does not exist or is not a directory");
+                }
+            } catch (Exception e) {
+                validationErrors.add("invalid ref_path [" + refPath + "]: " + e.getMessage());
+            }
+        }
+
+        return validationErrors;
+    }
 
     private static List<String> validatePrivateSettingsNotExplicitlySet(Settings settings, IndexScopedSettings indexScopedSettings) {
         List<String> validationErrors = new ArrayList<>();

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

@@ -146,6 +146,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 IndexSortConfig.INDEX_SORT_ORDER_SETTING,
                 IndexSortConfig.INDEX_SORT_MISSING_SETTING,
                 IndexSortConfig.INDEX_SORT_MODE_SETTING,
+                IndexSettings.INDEX_REF_PATH_SETTING,
                 IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING,
                 IndexSettings.INDEX_TRANSLOG_READ_FORWARD_SETTING,
                 IndexSettings.INDEX_WARMER_ENABLED_SETTING,

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -917,6 +917,13 @@ public static IndexMergePolicy fromString(String text) {
         Property.Dynamic
     );
 
+    public static final Setting<String> INDEX_REF_PATH_SETTING = Setting.simpleString(
+        "index.ref_path",
+        "",
+        Property.IndexScope,
+        Property.Dynamic
+    );
+
     private final Index index;
     private final Version version;
     private final Logger logger;
@@ -974,6 +981,7 @@ public static IndexMergePolicy fromString(String text) {
     private volatile boolean allowDerivedField;
     private final boolean derivedSourceEnabled;
     private volatile boolean derivedSourceEnabledForTranslog;
+    private volatile String refPath;
 
     /**
      * The maximum age of a retention lease before it is considered expired.
@@ -1168,6 +1176,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         this.defaultAllowUnmappedFields = scopedSettings.get(ALLOW_UNMAPPED);
         this.allowDerivedField = scopedSettings.get(ALLOW_DERIVED_FIELDS);
         this.durability = scopedSettings.get(INDEX_TRANSLOG_DURABILITY_SETTING);
+        this.refPath = scopedSettings.get(INDEX_REF_PATH_SETTING);
         this.translogReadForward = INDEX_TRANSLOG_READ_FORWARD_SETTING.get(settings);
         defaultFields = scopedSettings.get(DEFAULT_FIELD_SETTING);
         syncInterval = INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.get(settings);
@@ -1381,6 +1390,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
             this::setRemoteStoreTranslogRepository
         );
         scopedSettings.addSettingsUpdateConsumer(StarTreeIndexSettings.STAR_TREE_SEARCH_ENABLED_SETTING, this::setStarTreeIndexEnabled);
+        scopedSettings.addSettingsUpdateConsumer(INDEX_REF_PATH_SETTING, this::setRefPath);
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
@@ -2002,6 +2012,14 @@ public boolean getStarTreeIndexEnabled() {
         return isStarTreeIndexEnabled;
     }
 
+    private void setRefPath(String refPath){
+        this.refPath = refPath;
+    }
+
+    public String getRefPath(){
+        return refPath;
+    }
+
     /**
      * Returns the merge policy that should be used for this index.
      *

server/src/main/java/org/opensearch/index/analysis/HunspellTokenFilterFactory.java

@@ -35,13 +35,38 @@
 import org.apache.lucene.analysis.hunspell.Dictionary;
 import org.apache.lucene.analysis.hunspell.HunspellStemFilter;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.env.Environment;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.indices.analysis.HunspellService;
 
 import java.util.Locale;
 
 /**
  * The token filter factory for the hunspell analyzer
+ * * 
+ * Supports hot-reload when used with {@code updateable: true} setting.
+ * The dictionary is loaded from either:
+ * <ul>
+ *   <li>A ref_path (package ID, e.g., "pkg-1234") combined with locale for package-based dictionaries</li>
+ *   <li>A locale (e.g., "en_US") for traditional hunspell dictionaries from config/hunspell/</li>
+ * </ul>
+ * 
+ * <h2>Usage Examples:</h2>
+ * <pre>
+ * // Traditional locale-based (loads from config/hunspell/en_US/)
+ * {
+ *   "type": "hunspell",
+ *   "locale": "en_US"
+ * }
+ * 
+ * // Package-based (loads from config/packages/pkg-1234/hunspell/en_US/)
+ * {
+ *   "type": "hunspell",
+ *   "ref_path": "pkg-1234",
+ *   "locale": "en_US"
+ * }
+ * </pre>
+ *
  *
  * @opensearch.internal
  */
@@ -50,18 +75,58 @@ public class HunspellTokenFilterFactory extends AbstractTokenFilterFactory {
     private final Dictionary dictionary;
     private final boolean dedup;
     private final boolean longestOnly;
+    private final AnalysisMode analysisMode;
 
-    public HunspellTokenFilterFactory(IndexSettings indexSettings, String name, Settings settings, HunspellService hunspellService) {
+    public HunspellTokenFilterFactory(IndexSettings indexSettings, String name, Settings settings, HunspellService hunspellService, Environment env) {
         super(indexSettings, name, settings);
+        // Check for updateable flag - enables hot-reload support (same pattern as SynonymTokenFilterFactory)
+        boolean updateable = settings.getAsBoolean("updateable", false);
+        this.analysisMode = updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL;
 
+        // Get both ref_path and locale parameters
+        String refPath = settings.get("ref_path");  // Package ID only (optional)
         String locale = settings.get("locale", settings.get("language", settings.get("lang", null)));
-        if (locale == null) {
-            throw new IllegalArgumentException("missing [locale | language | lang] configuration for hunspell token filter");
-        }
-
-        dictionary = hunspellService.getDictionary(locale);
-        if (dictionary == null) {
-            throw new IllegalArgumentException(String.format(Locale.ROOT, "Unknown hunspell dictionary for locale [%s]", locale));
+
+        if (refPath != null) {
+            // Package-based loading: ref_path (package ID) + locale (required)
+            if (locale == null) {
+                throw new IllegalArgumentException(
+                    "When using ref_path, the 'locale' parameter is required for hunspell token filter"
+                );
+            }
+
+            // Validate ref_path is just package ID (no slashes allowed)
+            if (refPath.contains("/")) {
+                throw new IllegalArgumentException(
+                    String.format(Locale.ROOT, 
+                        "ref_path should contain only the package ID, not a full path. Got: [%s]. " +
+                        "Use ref_path for package ID and locale for the dictionary locale.", 
+                        refPath)
+                );
+            }
+
+            // Load from package directory: config/packages/{ref_path}/hunspell/{locale}/
+            dictionary = hunspellService.getDictionaryFromPackage(refPath, locale, env);
+            if (dictionary == null) {
+                throw new IllegalArgumentException(
+                    String.format(Locale.ROOT, 
+                        "Could not find hunspell dictionary for locale [%s] in package [%s]", 
+                        locale, refPath)
+                );
+            }
+        } else if (locale != null) {
+            // Traditional locale-based loading (backward compatible)
+            // Loads from config/hunspell/{locale}/
+            dictionary = hunspellService.getDictionary(locale);
+            if (dictionary == null) {
+                throw new IllegalArgumentException(
+                    String.format(Locale.ROOT, "Unknown hunspell dictionary for locale [%s]", locale)
+                );
+            }
+        } else {
+            throw new IllegalArgumentException(
+                "missing [locale | language | lang] configuration for hunspell token filter"
+            );
         }
 
         dedup = settings.getAsBoolean("dedup", true);
@@ -73,6 +138,16 @@ public TokenStream create(TokenStream tokenStream) {
         return new HunspellStemFilter(tokenStream, dictionary, dedup, longestOnly);
     }
 
+    /**
+     * Returns the analysis mode for this filter.
+     * When {@code updateable: true} is set, returns {@code SEARCH_TIME} which enables hot-reload
+     * via the _reload_search_analyzers API.
+     */
+    @Override
+    public AnalysisMode getAnalysisMode() {
+        return this.analysisMode;
+    }
+
     public boolean dedup() {
         return dedup;
     }

server/src/main/java/org/opensearch/indices/analysis/AnalysisModule.java

@@ -119,7 +119,7 @@ public AnalysisModule(Environment environment, List<AnalysisPlugin> plugins) thr
         );
     }
 
-    HunspellService getHunspellService() {
+    public HunspellService getHunspellService() {
         return hunspellService;
     }
 
@@ -161,7 +161,7 @@ public boolean requiresAnalysisSettings() {
         tokenFilters.register(
             "hunspell",
             requiresAnalysisSettings(
-                (indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService)
+                (indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService, env)
             )
         );