Skip to content

Commit 79bb2a6

Browse files
jimczijimczi
authored
Merge branch 'main' into fleet_kb_thread_pool
2 parents d9974a0 + a79ea5d commit 79bb2a6

File tree

172 files changed

+3422
-641
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

172 files changed

+3422
-641
lines changed

‎.buildkite/pipelines/pull-request-performance-benchmark.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
steps:
2+
- label: ":pipeline: Generate steps"
3+
command: bash .buildkite/scripts/generate-pr-performance-benchmark.sh | buildkite-agent pipeline upload

‎.buildkite/pipelines/pull-request/performance-benchmark.yml‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎.buildkite/pull-requests.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
"set_commit_status": false,
2828
"build_on_commit": false,
2929
"build_on_comment": true,
30-
"trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
30+
"target_branch": "main",
31+
"trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\S+)( please)?$"
3132
}
3233
]
3334
}

‎.buildkite/scripts/generate-pr-performance-benchmark.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
env_id_baseline=$(python3 -c 'import uuid; print(uuid.uuid4())')
6+
env_id_contender=$(python3 -c 'import uuid; print(uuid.uuid4())')
7+
merge_base=$(git merge-base "${GITHUB_PR_TARGET_BRANCH}" HEAD)
8+
9+
buildkite-agent meta-data set pr_comment:custom-body:body \
10+
"This build attempted two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
11+
buildkite-agent meta-data set pr_comment:custom-baseline:head \
12+
"* Baseline: ${merge_base} (env ID ${env_id_baseline})"
13+
buildkite-agent meta-data set pr_comment:custom-contender:head \
14+
"* Contender: ${GITHUB_PR_TRIGGERED_SHA} (env ID ${env_id_contender})"
15+
16+
cat << _EOF_
17+
steps:
18+
- label: Trigger baseline benchmark
19+
trigger: elasticsearch-performance-esbench-pr
20+
build:
21+
message: Baseline benchmark for PR${GITHUB_PR_NUMBER}
22+
branch: master
23+
env:
24+
CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
25+
ENV_ID: ${env_id_baseline}
26+
REVISION: ${merge_base}
27+
- label: Trigger contender benchmark
28+
trigger: elasticsearch-performance-esbench-pr
29+
build:
30+
message: Contender benchmark for PR${GITHUB_PR_NUMBER}
31+
branch: master
32+
env:
33+
CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
34+
ENV_ID: ${env_id_contender}
35+
ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
36+
REVISION: ${GITHUB_PR_TRIGGERED_SHA}
37+
- wait: ~
38+
- label: Modify PR comment
39+
command: buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
40+
_EOF_

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11

2+
# claude
3+
.claude
4+
25
# intellij files
36
.idea/
47
*.iml

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,11 +142,26 @@ static void selfTest() {
142142
private static List<ValuesSourceReaderOperator.FieldInfo> fields(String name) {
143143
return switch (name) {
144144
case "3_stored_keywords" -> List.of(
145-
new ValuesSourceReaderOperator.FieldInfo("keyword_1", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_1")),
146-
new ValuesSourceReaderOperator.FieldInfo("keyword_2", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_2")),
147-
new ValuesSourceReaderOperator.FieldInfo("keyword_3", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_3"))
145+
new ValuesSourceReaderOperator.FieldInfo(
146+
"keyword_1",
147+
ElementType.BYTES_REF,
148+
false,
149+
shardIdx -> blockLoader("stored_keyword_1")
150+
),
151+
new ValuesSourceReaderOperator.FieldInfo(
152+
"keyword_2",
153+
ElementType.BYTES_REF,
154+
false,
155+
shardIdx -> blockLoader("stored_keyword_2")
156+
),
157+
new ValuesSourceReaderOperator.FieldInfo(
158+
"keyword_3",
159+
ElementType.BYTES_REF,
160+
false,
161+
shardIdx -> blockLoader("stored_keyword_3")
162+
)
148163
);
149-
default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, elementType(name), shardIdx -> blockLoader(name)));
164+
default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, elementType(name), false, shardIdx -> blockLoader(name)));
150165
};
151166
}
152167

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,11 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
167167
}
168168
}
169169
}
170+
171+
tasks.named('assemble').configure {
172+
dependsOn buildTar
173+
}
174+
170175
artifacts {
171176
it.add("default", buildTar)
172177
it.add("extracted", buildExpanded)

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -204,18 +204,18 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
204204
unreferencedTransportVersion("initial_10.0.0", "10000000")
205205
latestTransportVersion("10.0", "initial_10.0.0", "10000000")
206206
when:
207-
def result = gradleRunner(":myserver:validateTransportVersionDefinitions").build()
207+
def result = gradleRunner(":myserver:validateTransportVersionResources").build()
208208
then:
209-
result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS
209+
result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
210210
}
211211

212212
def "named and unreferenced definitions cannot have the same name"() {
213213
given:
214214
unreferencedTransportVersion("existing_92", "10000000")
215215
when:
216-
def result = validateDefinitionsFails()
216+
def result = validateResourcesFails()
217217
then:
218-
assertDefinitionsFailure(result, "Transport version definition file " +
218+
assertValidateResourcesFailure(result, "Transport version definition file " +
219219
"[myserver/src/main/resources/transport/definitions/named/existing_92.csv] " +
220220
"has same name as unreferenced definition " +
221221
"[myserver/src/main/resources/transport/definitions/unreferenced/existing_92.csv]")
@@ -225,8 +225,8 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
225225
given:
226226
unreferencedTransportVersion("initial_10.0.1", "10000001")
227227
when:
228-
def result = gradleRunner(":myserver:validateTransportVersionDefinitions").build()
228+
def result = gradleRunner(":myserver:validateTransportVersionResources").build()
229229
then:
230-
result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS
230+
result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
231231
}
232232
}

‎build-tools-internal/src/main/groovy/elasticsearch.fips.gradle‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@ if (buildParams.inFipsJvm) {
2323
File fipsSecurity = new File(fipsResourcesDir, javaSecurityFilename)
2424
File fipsPolicy = new File(fipsResourcesDir, 'fips_java.policy')
2525
File fipsTrustStore = new File(fipsResourcesDir, 'cacerts.bcfks')
26-
def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.5')
26+
def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.6')
2727
def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19')
2828
def manualDebug = false; //change this to manually debug bouncy castle in an IDE
2929
if(manualDebug) {
30-
bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.5')
30+
bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.6')
3131
bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19'){
3232
exclude group: 'org.bouncycastle', module: 'bc-fips' // to avoid jar hell
3333
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ private void registerAndConfigureDistributionArchivesExtension(Project project)
7474
project.project(subProjectName, sub -> {
7575
sub.getPlugins().apply(BasePlugin.class);
7676
sub.getArtifacts().add(DEFAULT_CONFIGURATION_NAME, distributionArchive.getArchiveTask());
77+
sub.getTasks().named("assemble").configure(task -> task.dependsOn(distributionArchive.getArchiveTask()));
7778
var extractedConfiguration = sub.getConfigurations().create(EXTRACTED_CONFIGURATION_NAME);
7879
extractedConfiguration.setCanBeResolved(false);
7980
extractedConfiguration.setCanBeConsumed(true);

0 commit comments

Comments
 (0)