Skip to content

Commit 48fe7cc

Browse files
prdoyleprdoyle
authored
Merge branch 'main' into bootstrap-entitlements-for-testing
2 parents 4b36d55 + 4ca96c1 commit 48fe7cc

File tree

33 files changed

+446
-141
lines changed

33 files changed

+446
-141
lines changed

‎libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -237,20 +237,21 @@ public static void subtract(float[] v1, float[] v2, float[] result) {
237237
}
238238

239239
/**
240-
* calculates the spill-over score for a vector and a centroid, given its residual with
241-
* its actually nearest centroid
240+
* calculates the soar distance for a vector and a centroid
242241
* @param v1 the vector
243242
* @param centroid the centroid
244243
* @param originalResidual the residual with the actually nearest centroid
245-
* @return the spill-over score (soar)
244+
* @param soarLambda the lambda parameter
245+
* @param rnorm distance to the nearest centroid
246+
* @return the soar distance
246247
*/
247-
public static float soarResidual(float[] v1, float[] centroid, float[] originalResidual) {
248+
public static float soarDistance(float[] v1, float[] centroid, float[] originalResidual, float soarLambda, float rnorm) {
248249
if (v1.length != centroid.length) {
249250
throw new IllegalArgumentException("vector dimensions differ: " + v1.length + "!=" + centroid.length);
250251
}
251252
if (originalResidual.length != v1.length) {
252253
throw new IllegalArgumentException("vector dimensions differ: " + originalResidual.length + "!=" + v1.length);
253254
}
254-
return IMPL.soarResidual(v1, centroid, originalResidual);
255+
return IMPL.soarDistance(v1, centroid, originalResidual, soarLambda, rnorm);
255256
}
256257
}

‎libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.apache.lucene.util.BitUtil;
1313
import org.apache.lucene.util.Constants;
14+
import org.apache.lucene.util.VectorUtil;
1415

1516
final class DefaultESVectorUtilSupport implements ESVectorUtilSupport {
1617

@@ -139,15 +140,16 @@ public void centerAndCalculateOSQStatsDp(float[] target, float[] centroid, float
139140
}
140141

141142
@Override
142-
public float soarResidual(float[] v1, float[] centroid, float[] originalResidual) {
143+
public float soarDistance(float[] v1, float[] centroid, float[] originalResidual, float soarLambda, float rnorm) {
143144
assert v1.length == centroid.length;
144145
assert v1.length == originalResidual.length;
146+
float dsq = VectorUtil.squareDistance(v1, centroid);
145147
float proj = 0;
146148
for (int i = 0; i < v1.length; i++) {
147149
float djk = v1[i] - centroid[i];
148150
proj = fma(djk, originalResidual[i], proj);
149151
}
150-
return proj;
152+
return dsq + soarLambda * proj * proj / rnorm;
151153
}
152154

153155
public static int ipByteBitImpl(byte[] q, byte[] d) {

‎libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,6 @@ public interface ESVectorUtilSupport {
3737

3838
void centerAndCalculateOSQStatsDp(float[] target, float[] centroid, float[] centered, float[] stats);
3939

40-
float soarResidual(float[] v1, float[] centroid, float[] originalResidual);
40+
float soarDistance(float[] v1, float[] centroid, float[] originalResidual, float soarLambda, float rnorm);
4141

4242
}

‎libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,14 +368,17 @@ public float calculateOSQLoss(float[] target, float[] interval, float step, floa
368368
}
369369

370370
@Override
371-
public float soarResidual(float[] v1, float[] centroid, float[] originalResidual) {
371+
public float soarDistance(float[] v1, float[] centroid, float[] originalResidual, float soarLambda, float rnorm) {
372372
assert v1.length == centroid.length;
373373
assert v1.length == originalResidual.length;
374374
float proj = 0;
375+
float dsq = 0;
375376
int i = 0;
376377
if (v1.length > 2 * FLOAT_SPECIES.length()) {
377378
FloatVector projVec1 = FloatVector.zero(FLOAT_SPECIES);
378379
FloatVector projVec2 = FloatVector.zero(FLOAT_SPECIES);
380+
FloatVector acc1 = FloatVector.zero(FLOAT_SPECIES);
381+
FloatVector acc2 = FloatVector.zero(FLOAT_SPECIES);
379382
int unrolledLimit = FLOAT_SPECIES.loopBound(v1.length) - FLOAT_SPECIES.length();
380383
for (; i < unrolledLimit; i += 2 * FLOAT_SPECIES.length()) {
381384
// one
@@ -384,13 +387,15 @@ public float soarResidual(float[] v1, float[] centroid, float[] originalResidual
384387
FloatVector originalResidualVec0 = FloatVector.fromArray(FLOAT_SPECIES, originalResidual, i);
385388
FloatVector djkVec0 = v1Vec0.sub(centroidVec0);
386389
projVec1 = fma(djkVec0, originalResidualVec0, projVec1);
390+
acc1 = fma(djkVec0, djkVec0, acc1);
387391

388392
// two
389393
FloatVector v1Vec1 = FloatVector.fromArray(FLOAT_SPECIES, v1, i + FLOAT_SPECIES.length());
390394
FloatVector centroidVec1 = FloatVector.fromArray(FLOAT_SPECIES, centroid, i + FLOAT_SPECIES.length());
391395
FloatVector originalResidualVec1 = FloatVector.fromArray(FLOAT_SPECIES, originalResidual, i + FLOAT_SPECIES.length());
392396
FloatVector djkVec1 = v1Vec1.sub(centroidVec1);
393397
projVec2 = fma(djkVec1, originalResidualVec1, projVec2);
398+
acc2 = fma(djkVec1, djkVec1, acc2);
394399
}
395400
// vector tail
396401
for (; i < FLOAT_SPECIES.loopBound(v1.length); i += FLOAT_SPECIES.length()) {
@@ -399,15 +404,18 @@ public float soarResidual(float[] v1, float[] centroid, float[] originalResidual
399404
FloatVector originalResidualVec = FloatVector.fromArray(FLOAT_SPECIES, originalResidual, i);
400405
FloatVector djkVec = v1Vec.sub(centroidVec);
401406
projVec1 = fma(djkVec, originalResidualVec, projVec1);
407+
acc1 = fma(djkVec, djkVec, acc1);
402408
}
403409
proj += projVec1.add(projVec2).reduceLanes(ADD);
410+
dsq += acc1.add(acc2).reduceLanes(ADD);
404411
}
405412
// tail
406413
for (; i < v1.length; i++) {
407414
float djk = v1[i] - centroid[i];
408415
proj = fma(djk, originalResidual[i], proj);
416+
dsq = fma(djk, djk, dsq);
409417
}
410-
return proj;
418+
return dsq + soarLambda * proj * proj / rnorm;
411419
}
412420

413421
private static final VectorSpecies<Byte> BYTE_SPECIES_128 = ByteVector.SPECIES_128;

‎libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ public void testOsqGridPoints() {
268268
}
269269
}
270270

271-
public void testSoarOverspillScore() {
271+
public void testSoarDistance() {
272272
int size = random().nextInt(128, 512);
273273
float deltaEps = 1e-5f * size;
274274
var vector = new float[size];
@@ -279,8 +279,10 @@ public void testSoarOverspillScore() {
279279
centroid[i] = random().nextFloat();
280280
preResidual[i] = random().nextFloat();
281281
}
282-
var expected = defaultedProvider.getVectorUtilSupport().soarResidual(vector, centroid, preResidual);
283-
var result = defOrPanamaProvider.getVectorUtilSupport().soarResidual(vector, centroid, preResidual);
282+
float soarLambda = random().nextFloat();
283+
float rnorm = random().nextFloat();
284+
var expected = defaultedProvider.getVectorUtilSupport().soarDistance(vector, centroid, preResidual, soarLambda, rnorm);
285+
var result = defOrPanamaProvider.getVectorUtilSupport().soarDistance(vector, centroid, preResidual, soarLambda, rnorm);
284286
assertEquals(expected, result, deltaEps);
285287
}
286288

‎modules/streams/src/yamlRestTest/java/org/elasticsearch/streams/StreamsYamlTestSuiteIT.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
1414

1515
import org.elasticsearch.test.cluster.ElasticsearchCluster;
16+
import org.elasticsearch.test.cluster.FeatureFlag;
1617
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
1718
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
1819
import org.junit.ClassRule;
@@ -28,10 +29,11 @@ public static Iterable<Object[]> parameters() throws Exception {
2829
}
2930

3031
@ClassRule
31-
public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("streams").build();
32+
public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("streams").feature(FeatureFlag.LOGS_STREAM).build();
3233

3334
@Override
3435
protected String getTestRestCluster() {
3536
return cluster.getHttpAddresses();
3637
}
38+
3739
}

‎muted-tests.yml‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -448,9 +448,6 @@ tests:
448448
- class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT
449449
method: testCCSClusterDetailsWhereAllShardsSkippedInCanMatch
450450
issue: https://github.com/elastic/elasticsearch/issues/128418
451-
- class: org.elasticsearch.xpack.esql.action.ForkIT
452-
method: testProfile
453-
issue: https://github.com/elastic/elasticsearch/issues/128377
454451
- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithFiltersIT
455452
method: testTimestampFilterFromQuery
456453
issue: https://github.com/elastic/elasticsearch/issues/127332
@@ -568,15 +565,12 @@ tests:
568565
- class: org.elasticsearch.search.query.RescoreKnnVectorQueryIT
569566
method: testKnnSearchRescore
570567
issue: https://github.com/elastic/elasticsearch/issues/129713
571-
- class: org.elasticsearch.streams.StreamsYamlTestSuiteIT
572-
method: test {yaml=streams/logs/10_basic/Basic toggle of logs state enable to disable and back}
573-
issue: https://github.com/elastic/elasticsearch/issues/129733
574-
- class: org.elasticsearch.streams.StreamsYamlTestSuiteIT
575-
method: test {yaml=streams/logs/10_basic/Check for repeated toggle to same state}
576-
issue: https://github.com/elastic/elasticsearch/issues/129735
577-
- class: org.elasticsearch.snapshots.GetSnapshotsIT
578-
method: testFilterByState
579-
issue: https://github.com/elastic/elasticsearch/issues/129740
568+
- class: org.elasticsearch.snapshots.SnapshotShutdownIT
569+
method: testSnapshotShutdownProgressTracker
570+
issue: https://github.com/elastic/elasticsearch/issues/129752
571+
- class: org.elasticsearch.xpack.security.SecurityRolesMultiProjectIT
572+
method: testUpdatingFileBasedRoleAffectsAllProjects
573+
issue: https://github.com/elastic/elasticsearch/issues/129775
580574

581575
# Examples:
582576
#

‎qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,8 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
9090
// geohex_grid requires gold license
9191
.setting("xpack.license.self_generated.type", "trial")
9292
.feature(FeatureFlag.TIME_SERIES_MODE)
93-
.feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED);
93+
.feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
94+
.feature(FeatureFlag.IVF_FORMAT);
9495

9596
private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local()
9697
.name(REMOTE_CLUSTER_NAME)

‎qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
9292
.setting("xpack.security.remote_cluster_client.ssl.enabled", "false")
9393
.feature(FeatureFlag.TIME_SERIES_MODE)
9494
.feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
95+
.feature(FeatureFlag.IVF_FORMAT)
9596
.user("test_admin", "x-pack-test-password");
9697

9798
private static ElasticsearchCluster fulfillingCluster = ElasticsearchCluster.local()

‎server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java‎

Lines changed: 36 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryDataBlobName;
7272
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
7373
import static org.hamcrest.Matchers.empty;
74+
import static org.hamcrest.Matchers.hasItem;
7475
import static org.hamcrest.Matchers.hasSize;
7576
import static org.hamcrest.Matchers.in;
7677
import static org.hamcrest.Matchers.is;
@@ -655,40 +656,47 @@ public void testFilterByState() throws Exception {
655656
assertThat(snapshots, hasSize(1));
656657
assertThat(snapshots.getFirst().state(), is(SnapshotState.SUCCESS));
657658

659+
// Add some more state (so the next snapshot has some work to do)
660+
indexRandomDocs(randomIdentifier(), 100);
661+
658662
// Create a snapshot in progress
659663
blockAllDataNodes(repoName);
660-
startFullSnapshot(repoName, "snapshot-in-progress");
661-
awaitNumberOfSnapshotsInProgress(1);
662-
663-
// Fetch snapshots with state=IN_PROGRESS
664-
snapshots = getSnapshotsForStates.apply(EnumSet.of(SnapshotState.IN_PROGRESS));
665-
assertThat(snapshots, hasSize(1));
666-
assertThat(snapshots.getFirst().state(), is(SnapshotState.IN_PROGRESS));
667-
668-
// Fetch snapshots with multiple states (SUCCESS, IN_PROGRESS)
669-
snapshots = getSnapshotsForStates.apply(EnumSet.of(SnapshotState.SUCCESS, SnapshotState.IN_PROGRESS));
670-
assertThat(snapshots, hasSize(2));
671-
var states = snapshots.stream().map(SnapshotInfo::state).collect(Collectors.toSet());
672-
assertTrue(states.contains(SnapshotState.SUCCESS));
673-
assertTrue(states.contains(SnapshotState.IN_PROGRESS));
674-
675-
// Fetch all snapshots (without state)
676-
snapshots = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoName).get().getSnapshots();
677-
assertThat(snapshots, hasSize(2));
678-
679-
// Fetch snapshots with an invalid state
680-
IllegalArgumentException e = expectThrows(
681-
IllegalArgumentException.class,
682-
() -> getSnapshotsForStates.apply(EnumSet.of(SnapshotState.valueOf("FOO")))
683-
);
684-
assertThat(e.getMessage(), is("No enum constant org.elasticsearch.snapshots.SnapshotState.FOO"));
664+
try {
665+
startFullSnapshot(repoName, "snapshot-in-progress");
666+
awaitNumberOfSnapshotsInProgress(1);
667+
668+
// Fetch snapshots with state=IN_PROGRESS
669+
snapshots = getSnapshotsForStates.apply(EnumSet.of(SnapshotState.IN_PROGRESS));
670+
assertThat(snapshots, hasSize(1));
671+
assertThat(snapshots.getFirst().state(), is(SnapshotState.IN_PROGRESS));
672+
673+
// Fetch snapshots with multiple states (SUCCESS, IN_PROGRESS)
674+
snapshots = getSnapshotsForStates.apply(EnumSet.of(SnapshotState.SUCCESS, SnapshotState.IN_PROGRESS));
675+
assertThat(snapshots, hasSize(2));
676+
var states = snapshots.stream().map(SnapshotInfo::state).collect(Collectors.toSet());
677+
assertThat(states, hasItem(SnapshotState.SUCCESS));
678+
assertThat(states, hasItem(SnapshotState.IN_PROGRESS));
679+
680+
// Fetch all snapshots (without state)
681+
snapshots = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoName).get().getSnapshots();
682+
assertThat(snapshots, hasSize(2));
683+
684+
// Fetch snapshots with an invalid state
685+
IllegalArgumentException e = expectThrows(
686+
IllegalArgumentException.class,
687+
() -> getSnapshotsForStates.apply(EnumSet.of(SnapshotState.valueOf("FOO")))
688+
);
689+
assertThat(e.getMessage(), is("No enum constant org.elasticsearch.snapshots.SnapshotState.FOO"));
690+
} finally {
691+
// Allow the IN_PROGRESS snapshot to finish, then verify GET using SUCCESS has results and IN_PROGRESS does not.
692+
// Do this in a finally, so the block doesn't interfere with teardown in the event of a failure
693+
unblockAllDataNodes(repoName);
694+
}
685695

686-
// Allow the IN_PROGRESS snapshot to finish, then verify GET using SUCCESS has results and IN_PROGRESS does not.
687-
unblockAllDataNodes(repoName);
688696
awaitNumberOfSnapshotsInProgress(0);
689697
snapshots = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoName).get().getSnapshots();
690698
assertThat(snapshots, hasSize(2));
691-
states = snapshots.stream().map(SnapshotInfo::state).collect(Collectors.toSet());
699+
var states = snapshots.stream().map(SnapshotInfo::state).collect(Collectors.toSet());
692700
assertThat(states, hasSize(1));
693701
assertTrue(states.contains(SnapshotState.SUCCESS));
694702
snapshots = getSnapshotsForStates.apply(EnumSet.of(SnapshotState.IN_PROGRESS));

0 commit comments

Comments
 (0)