Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public ReaderWithOrigin getRulesReader(SynonymTokenFilterFactory factory, IndexC
);
} else {
reader = new ReaderWithOrigin(
Analysis.getReaderFromIndex(synonymsSet, factory.synonymsManagementAPIService),
Analysis.getReaderFromIndex(synonymsSet, factory.synonymsManagementAPIService, factory.lenient),
"[" + synonymsSet + "] synonyms_set in .synonyms index",
synonymsSet
);
Expand Down
3 changes: 0 additions & 3 deletions qa/mixed-cluster/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,6 @@ excludeList.add('aggregations/percentiles_hdr_metric/Negative values test')
// sync_id is removed in 9.0
excludeList.add("cat.shards/10_basic/Help")

// Can't work until auto-expand replicas is 0-1 for synonyms index
excludeList.add("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unrelated to this change, but a good opportunity to enable this test again


def clusterPath = getPath()

buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,3 +313,158 @@ setup:
indices.stats: { index: test_index }

- length: { indices: 0 }

---
"Load index with non existent synonyms set":
- requires:
test_runner_features: [ capabilities ]
capabilities:
- method: PUT
path: /{index}
capabilities: [ synonyms_set_lenient_on_non_existing ]
reason: "requires synonyms_set_lenient_on_non_existing bug fix"
- do:
indices.create:
index: test_index
body:
settings:
index:
number_of_shards: 1
number_of_replicas: 0
analysis:
filter:
my_synonym_filter:
type: synonym
synonyms_set: set1
updateable: true
analyzer:
my_analyzer:
type: custom
tokenizer: whitespace
filter: [ lowercase, my_synonym_filter ]
mappings:
properties:
my_field:
type: text
search_analyzer: my_analyzer

- match: { acknowledged: true }
- match: { shards_acknowledged: true }

- do:
indices.stats: { index: test_index }

- match: { indices.test_index.health: "green" }

# Synonyms are not applied
- do:
indices.analyze:
index: test_index
body:
analyzer: my_analyzer
text: foo

- length: { tokens: 1 }
- match: { tokens.0.token: foo }


# Create synonyms set and check synonyms are applied
- do:
synonyms.put_synonym:
id: set1
body:
synonyms_set:
synonyms: "foo => bar, baz"

# This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
- do:
cluster.health:
index: .synonyms
wait_for_status: green


- do:
indices.stats: { index: test_index }

- match: { indices.test_index.health: "green" }

# Synonyms are applied
- do:
indices.analyze:
index: test_index
body:
analyzer: my_analyzer
text: foo

- length: { tokens: 2 }

---
"Load index with non existent synonyms set and lenient set to false":

- do:
indices.create:
index: test_index
body:
settings:
index:
number_of_shards: 1
number_of_replicas: 0
analysis:
filter:
my_synonym_filter:
type: synonym
synonyms_set: set1
updateable: true
lenient: false
analyzer:
my_analyzer:
type: custom
tokenizer: whitespace
filter: [ lowercase, my_synonym_filter ]
mappings:
properties:
my_field:
type: text
search_analyzer: my_analyzer

- match: { acknowledged: true }
- match: { shards_acknowledged: false }

- do:
indices.stats: { index: test_index }

- length: { indices: 0 }

# Create synonyms set and check synonyms are applied
- do:
synonyms.put_synonym:
id: set1
body:
synonyms_set:
synonyms: "foo => bar, baz"

# This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
- do:
cluster.health:
index: .synonyms
wait_for_status: green

- do:
cluster.reroute:
retry_failed: true

- do:
cluster.health:
index: test_index
wait_for_status: green

# Synonyms are applied
- do:
indices.analyze:
index: test_index
body:
analyzer: my_analyzer
text: foo

- length: { tokens: 2 }

Original file line number Diff line number Diff line change
Expand Up @@ -165,34 +165,6 @@ setup:
query: hola
- match: { hits.total.value: 1 }

---
"Fail loading synonyms from index if synonyms_set doesn't exist":
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I moved this test to 100_synonyms_invalid

- do:
indices.create:
index: another_index
body:
settings:
index:
number_of_shards: 1
analysis:
filter:
my_synonym_filter:
type: synonym
synonyms_set: set_missing
updateable: true
analyzer:
my_analyzer:
type: custom
tokenizer: standard
filter: [ lowercase, my_synonym_filter ]
mappings:
properties:
my_field:
type: text
search_analyzer: my_analyzer
- match: { acknowledged: true }
- match: { shards_acknowledged: false }

---
"Load empty synonyms set from index for an analyzer":
- do:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.apache.lucene.analysis.th.ThaiAnalyzer;
import org.apache.lucene.analysis.tr.TurkishAnalyzer;
import org.apache.lucene.analysis.util.CSVUtil;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
Expand Down Expand Up @@ -353,10 +354,27 @@ public static Reader getReaderFromFile(Environment env, String filePath, String
}
}

public static Reader getReaderFromIndex(String synonymsSet, SynonymsManagementAPIService synonymsManagementAPIService) {
public static Reader getReaderFromIndex(
String synonymsSet,
SynonymsManagementAPIService synonymsManagementAPIService,
boolean ignoreMissing
) {
final PlainActionFuture<PagedResult<SynonymRule>> synonymsLoadingFuture = new PlainActionFuture<>();
synonymsManagementAPIService.getSynonymSetRules(synonymsSet, synonymsLoadingFuture);
PagedResult<SynonymRule> results = synonymsLoadingFuture.actionGet();

PagedResult<SynonymRule> results;
try {
results = synonymsLoadingFuture.actionGet();
} catch (ResourceNotFoundException e) {
if (ignoreMissing == false) {
throw e;
}
logger.warn(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be at the error level?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also wonder if we should handle all exceptions rather than just the resource not found one?
Maybe we should even retry if that's appropriate (based on the exception)?
I am fine if we do that in a follow up but a comment here would help to understand why we do that.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be at the error level?

I was discussing this with @mayya-sharipova - I think this is not something that prevents ES from working, it's more of a user error. I think WARN is the appropriate level.

I also wonder if we should handle all exceptions rather than just the resource not found one?
That's a good point - Adding that as a log with ERROR level in ecb2f26

Maybe we should even retry if that's appropriate (based on the exception)?

I'm not sure, what errors would benefit from a retry?

Shard allocation already is tried a number of times, so I think we should be covered here. In case it can be fixed after some additional action, reroute can be used to recover the shard.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shard allocation already is tried a number of times, so I think we should be covered here. In case it can be fixed after some additional action, reroute can be used to recover the shard.

Yep that's fine, let's monitor the exception we catch here to be proactive.

Regarding the catching of exceptions, I am not sure that only ElasticsearchException are thrown by the call to actionGet, should we catch all Exception instead?

Copy link
Member Author

@carlosdelest carlosdelest Mar 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure that only ElasticsearchException are thrown by the call to actionGet, should we catch all Exception instead?

That makes sense, done in b88fdc4

"Synonyms set {} not found - synonyms will not be applied to search results on indices that use this synonym set",
synonymsSet
);
results = new PagedResult<>(0, new SynonymRule[0]);
}

SynonymRule[] synonymRules = results.pageResults();
StringBuilder sb = new StringBuilder();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,14 @@ public class CreateIndexCapabilities {

private static final String HUNSPELL_DICT_400 = "hunspell_dict_400";

private static final String SYNONYMS_SET_LENIENT_ON_NON_EXISTING = "synonyms_set_lenient_on_non_existing";

public static final Set<String> CAPABILITIES = Set.of(
LOGSDB_INDEX_MODE_CAPABILITY,
LOOKUP_INDEX_MODE_CAPABILITY,
NESTED_DENSE_VECTOR_SYNTHETIC_TEST,
POORLY_FORMATTED_BAD_REQUEST,
HUNSPELL_DICT_400
HUNSPELL_DICT_400,
SYNONYMS_SET_LENIENT_ON_NON_EXISTING
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's just for tests no? Can we separate it and only add it when testing?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe create a test feature?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah thanks, I'm too used to caps right now - changed on 636bea9

);
}