Skip to content

Commit af2b45a

Browse files
strtgbbstrtgbb
authored
Merge pull request #1002 from Altinity/fixing_24.8
24.8 Update runner labels & other fixes
2 parents 83d2eb0 + 5178f8c commit af2b45a

File tree

10 files changed

+228
-54
lines changed

10 files changed

+228
-54
lines changed

‎.github/workflows/docker_test_images.yml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ name: Build docker images
1010
description: set latest tag for resulting multiarch manifest
1111
required: false
1212
type: boolean
13-
default: false
13+
default: false
1414
secrets:
1515
secret_envs:
1616
description: if given, it's passed to the environments
@@ -26,10 +26,14 @@ env:
2626
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
2727
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
2828
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
29+
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
30+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
31+
ROBOT_TOKEN: ${{ secrets.GITHUB_TOKEN }}
32+
2933

3034
jobs:
3135
DockerBuildAarch64:
32-
runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none]
36+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
3337
if: |
3438
!failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_aarch64) != '[]'
3539
steps:
@@ -44,7 +48,7 @@ jobs:
4448
--image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
4549
--missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_aarch64) }}'
4650
DockerBuildAmd64:
47-
runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
51+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
4852
if: |
4953
!failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_amd64) != '[]'
5054
steps:
@@ -60,7 +64,7 @@ jobs:
6064
--missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_amd64) }}'
6165
DockerMultiArchManifest:
6266
needs: [DockerBuildAmd64, DockerBuildAarch64]
63-
runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
67+
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
6468
if: |
6569
!failure() && !cancelled() && (toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]' || inputs.set_latest)
6670
steps:

‎.github/workflows/grype_scan.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ jobs:
6161
TAG_SUFFIX: ${{ inputs.tag-suffix }}
6262
SPECIFIED_VERSION: ${{ inputs.version }}
6363
run: |
64-
python3 ./tests/ci/version_helper.py | tee /tmp/version_info
64+
python3 ./tests/ci/version_helper.py | grep = | tee /tmp/version_info
6565
source /tmp/version_info
6666
if [ -z "$SPECIFIED_VERSION" ]; then
6767
VERSION=$CLICKHOUSE_VERSION_STRING

‎.github/workflows/regression.yml‎

Lines changed: 170 additions & 23 deletions
Large diffs are not rendered by default.

‎.github/workflows/release_branches.yml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,7 @@ jobs:
492492
secrets: inherit
493493
with:
494494
test_name: Integration tests (aarch64)
495-
runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none
495+
runner_type: altinity-on-demand, altinity-func-tester-aarch64
496496
data: ${{ needs.RunConfig.outputs.data }}
497497
#############################################################################################
498498
####################################### AST FUZZERS #########################################
@@ -504,7 +504,7 @@ jobs:
504504
secrets: inherit
505505
with:
506506
test_name: AST fuzzer (asan)
507-
runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none
507+
runner_type: altinity-on-demand, altinity-func-tester
508508
data: ${{ needs.RunConfig.outputs.data }}
509509
ASTFuzzerTsan:
510510
needs: [RunConfig, BuilderDebTsan]
@@ -513,7 +513,7 @@ jobs:
513513
secrets: inherit
514514
with:
515515
test_name: AST fuzzer (tsan)
516-
runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none
516+
runner_type: altinity-on-demand, altinity-func-tester
517517
data: ${{ needs.RunConfig.outputs.data }}
518518
ASTFuzzerMsan:
519519
needs: [RunConfig, BuilderDebMsan]
@@ -522,7 +522,7 @@ jobs:
522522
secrets: inherit
523523
with:
524524
test_name: AST fuzzer (msan)
525-
runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none
525+
runner_type: altinity-on-demand, altinity-func-tester
526526
data: ${{ needs.RunConfig.outputs.data }}
527527
ASTFuzzerUBsan:
528528
needs: [RunConfig, BuilderDebUBsan]
@@ -531,7 +531,7 @@ jobs:
531531
secrets: inherit
532532
with:
533533
test_name: AST fuzzer (ubsan)
534-
runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none
534+
runner_type: altinity-on-demand, altinity-func-tester
535535
data: ${{ needs.RunConfig.outputs.data }}
536536
ASTFuzzerDebug:
537537
needs: [RunConfig, BuilderDebDebug]
@@ -540,7 +540,7 @@ jobs:
540540
secrets: inherit
541541
with:
542542
test_name: AST fuzzer (debug)
543-
runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none
543+
runner_type: altinity-on-demand, altinity-func-tester
544544
data: ${{ needs.RunConfig.outputs.data }}
545545
#############################################################################################
546546
##################################### REGRESSION TESTS ######################################
@@ -552,7 +552,7 @@ jobs:
552552
secrets: inherit
553553
with:
554554
runner_type: altinity-on-demand, altinity-regression-tester
555-
commit: e3c00be97a045aa04e9d1a6ec50cc64f4c387b70
555+
commit: fd33d9e73ebd14392d601d87902ddf0e7c90709c
556556
arch: release
557557
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
558558
timeout_minutes: 300
@@ -563,7 +563,7 @@ jobs:
563563
secrets: inherit
564564
with:
565565
runner_type: altinity-on-demand, altinity-regression-tester-aarch64
566-
commit: e3c00be97a045aa04e9d1a6ec50cc64f4c387b70
566+
commit: fd33d9e73ebd14392d601d87902ddf0e7c90709c
567567
arch: aarch64
568568
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
569569
timeout_minutes: 300

‎docker/test/fuzzer/run-fuzzer.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ BASE_REF=${BASE_REF:="master"}
2424
function git_clone_with_retry
2525
{
2626
for _ in 1 2 3 4; do
27-
if git clone --depth 1 https://github.com/Altinity/ClickHouse.git -b "${BASE_REF}" -- "$1" 2>&1 | ts '%Y-%m-%d %H:%M:%S';then
27+
# Strip refs/tags/ prefix if present, as --branch expects just the tag/branch name
28+
local ref_name="${BASE_REF#refs/tags/}"
29+
30+
if git clone --depth 1 https://github.com/Altinity/ClickHouse.git --branch "${ref_name}" -- "$1" 2>&1 | ts '%Y-%m-%d %H:%M:%S';then
2831
return 0
2932
else
3033
sleep 0.5

‎docker/test/stress/run.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
6060

6161
config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml
6262

63+
# NOTE(strtgbb): Trying to avoid errors that may be related to running out of resources
64+
export CLICKHOUSE_MAX_THREADS=8
65+
export CLICKHOUSE_MAX_CONCURRENT_QUERIES=4
66+
6367
start_server
6468

6569
setup_logs_replication

‎tests/broken_tests.json‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,29 @@
11
{
22
"02700_s3_part_INT_MAX": {
3-
"reason": "Fails on asan"
3+
"reason": "INVESTIGATE: Fails on asan"
44
},
55
"02815_no_throw_in_simple_queries": {
6-
"reason": "Fails on asan,msan,tsan,debug,Aarch64"
6+
"reason": "INVESTIGATE: Fails on asan,msan,tsan,debug,Aarch64"
77
},
88
"03206_no_exceptions_clickhouse_local": {
9-
"reason": "Fails on asan,msan,tsan,debug,Aarch64"
9+
"reason": "INVESTIGATE: Fails on asan,msan,tsan,debug,Aarch64"
10+
},
11+
"02479_race_condition_between_insert_and_droppin_mv": {
12+
"reason": "INVESTIGATE: Unstable on ubsan"
13+
},
14+
"01825_new_type_json_ghdata": {
15+
"reason": "INVESTIGATE: Sometime out of memory on msan"
16+
},
17+
"00159_parallel_formatting_http": {
18+
"reason": "INVESTIGATE: Unstable on tsan"
19+
},
20+
"02941_variant_type_3": {
21+
"reason": "INVESTIGATE: Memory limit exceeded on asan"
22+
},
23+
"03008_local_plain_rewritable": {
24+
"reason": "INVESTIGATE: Memory limit exceeded on asan"
25+
},
26+
"02490_benchmark_max_consecutive_errors": {
27+
"reason": "INVESTIGATE: Unstable on msan"
1028
}
1129
}

‎tests/ci/ci_config.py‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,8 @@ class CI:
275275
required_builds=[BuildNames.PACKAGE_ASAN]
276276
),
277277
JobNames.STATEFUL_TEST_TSAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
278-
required_builds=[BuildNames.PACKAGE_TSAN]
278+
required_builds=[BuildNames.PACKAGE_TSAN],
279+
timeout=2 * 3600,
279280
),
280281
JobNames.STATEFUL_TEST_MSAN: CommonJobConfigs.STATEFUL_TEST.with_properties(
281282
required_builds=[BuildNames.PACKAGE_MSAN]
@@ -397,18 +398,19 @@ class CI:
397398
required_builds=[BuildNames.PACKAGE_DEBUG], pr_only=True
398399
),
399400
JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
400-
required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4,
401-
timeout=9000, # the job timed out with default value (7200)
401+
required_builds=[BuildNames.PACKAGE_ASAN],
402+
num_batches=4,
403+
timeout=3 * 3600, # the job timed out with default value (7200)
402404
),
403405
JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties(
404406
required_builds=[BuildNames.PACKAGE_ASAN],
405407
num_batches=6,
406-
timeout=12000, # the job timed out with default value (7200)
408+
timeout=4 * 3600, # the job timed out with default value (7200)
407409
),
408410
JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties(
409411
required_builds=[BuildNames.PACKAGE_TSAN],
410412
num_batches=6,
411-
timeout=12000, # the job timed out with default value (7200)
413+
timeout=4 * 3600, # the job timed out with default value (7200)
412414
),
413415
JobNames.INTEGRATION_TEST_ARM: CommonJobConfigs.INTEGRATION_TEST.with_properties(
414416
required_builds=[BuildNames.PACKAGE_AARCH64],
@@ -419,7 +421,7 @@ class CI:
419421
JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties(
420422
required_builds=[BuildNames.PACKAGE_RELEASE],
421423
num_batches=4,
422-
#release_only=True,
424+
# release_only=True,
423425
timeout=12000, # the job timed out with default value (7200)
424426
),
425427
JobNames.INTEGRATION_TEST_FLAKY: CommonJobConfigs.INTEGRATION_TEST.with_properties(
@@ -558,11 +560,11 @@ class CI:
558560
),
559561
JobNames.SIGN_RELEASE: JobConfig(
560562
required_builds=[BuildNames.PACKAGE_RELEASE],
561-
runner_type=Runners.STYLE_CHECKER
563+
runner_type=Runners.STYLE_CHECKER,
562564
),
563565
JobNames.SIGN_AARCH64: JobConfig(
564566
required_builds=[BuildNames.PACKAGE_AARCH64],
565-
runner_type=Runners.STYLE_CHECKER_ARM
567+
runner_type=Runners.STYLE_CHECKER_ARM,
566568
),
567569
}
568570

‎tests/ci/s3_helper.py‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@
2020
S3_URL,
2121
)
2222

23-
sensitive_var_pattern = re.compile(
24-
r"\b[A-Z_]*(?<!WRONG_)(SECRET|PASSWORD|ACCESS_KEY|TOKEN)[A-Z_]*\b(?!%)(?!=clickhouse$)(?!=minio)(?!: \*{3}$)(?! '\[HIDDEN\]')"
25-
)
23+
sensitive_var_pattern = re.compile(r"[A-Z_]*(SECRET|PASSWORD|KEY|TOKEN|AZURE)[A-Z_]*")
2624
sensitive_strings = {
2725
var: value for var, value in os.environ.items() if sensitive_var_pattern.match(var)
2826
}
@@ -41,8 +39,6 @@ def clean_line(line):
4139

4240
matches = []
4341
for line_number, line in enumerate(file_content.splitlines(), start=1):
44-
for match in sensitive_var_pattern.finditer(line):
45-
matches.append((file_name, line_number, clean_line(line)))
4642
for name, value in sensitive_strings.items():
4743
if value in line:
4844
matches.append((file_name, line_number, clean_line(line)))

‎tests/integration/compose/docker_compose_azurite.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: '2.3'
22

33
services:
44
azurite1:
5-
image: mcr.microsoft.com/azure-storage/azurite
5+
image: altinityinfra/azurite:3.35.0 # NOTE(strtgbb): Use mirror to avoid access denied error from mcr.microsoft.com
66
ports:
77
- "${AZURITE_PORT}:${AZURITE_PORT}"
88
volumes:

0 commit comments

Comments
 (0)