Skip to content

Commit b76a7d7

Browse files
jkartsevajkartseva
committed
Merge remote-tracking branch 'upstream/master' into plain-rw-imitate-hardlink
2 parents e6c17bf + 5345879 commit b76a7d7

File tree

118 files changed

+2707
-830
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

118 files changed

+2707
-830
lines changed

‎.github/workflows/master.yml‎

Lines changed: 185 additions & 109 deletions
Large diffs are not rendered by default.

‎.github/workflows/nightly.yml‎

Lines changed: 0 additions & 91 deletions
This file was deleted.

‎.github/workflows/nightly_fuzzers.yml‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
# generated by praktika
2+
3+
name: NightlyFuzzers
4+
on:
5+
schedule:
6+
- cron: 13 3 * * *
7+
workflow_dispatch:
8+
9+
concurrency:
10+
group: ${{ github.workflow }}
11+
12+
env:
13+
PYTHONUNBUFFERED: 1
14+
CHECKOUT_REF: ""
15+
16+
jobs:
17+
18+
config_workflow:
19+
runs-on: [self-hosted, style-checker-aarch64]
20+
needs: []
21+
name: "Config Workflow"
22+
outputs:
23+
data: ${{ steps.run.outputs.DATA }}
24+
steps:
25+
- name: Checkout code
26+
uses: actions/checkout@v4
27+
with:
28+
ref: ${{ env.CHECKOUT_REF }}
29+
30+
- name: Prepare env script
31+
run: |
32+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
33+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
34+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
35+
export PYTHONPATH=./ci:.:
36+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
37+
${{ needs.config_workflow.outputs.data }}
38+
EOF
39+
cat > ./ci/tmp/workflow_status.json << 'EOF'
40+
${{ toJson(needs) }}
41+
EOF
42+
ENV_SETUP_SCRIPT_EOF
43+
44+
- name: Run
45+
id: run
46+
run: |
47+
. ./ci/tmp/praktika_setup_env.sh
48+
set -o pipefail
49+
if command -v ts &> /dev/null; then
50+
python3 -m praktika run 'Config Workflow' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
51+
else
52+
python3 -m praktika run 'Config Workflow' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
53+
fi
54+
55+
dockers_build_arm:
56+
runs-on: [self-hosted, style-checker-aarch64]
57+
needs: [config_workflow]
58+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYXJtKQ==') }}
59+
name: "Dockers Build (arm)"
60+
outputs:
61+
data: ${{ steps.run.outputs.DATA }}
62+
steps:
63+
- name: Checkout code
64+
uses: actions/checkout@v4
65+
with:
66+
ref: ${{ env.CHECKOUT_REF }}
67+
68+
- name: Prepare env script
69+
run: |
70+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
71+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
72+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
73+
export PYTHONPATH=./ci:.:
74+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
75+
${{ needs.config_workflow.outputs.data }}
76+
EOF
77+
cat > ./ci/tmp/workflow_status.json << 'EOF'
78+
${{ toJson(needs) }}
79+
EOF
80+
ENV_SETUP_SCRIPT_EOF
81+
82+
- name: Run
83+
id: run
84+
run: |
85+
. ./ci/tmp/praktika_setup_env.sh
86+
set -o pipefail
87+
if command -v ts &> /dev/null; then
88+
python3 -m praktika run 'Dockers Build (arm)' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
89+
else
90+
python3 -m praktika run 'Dockers Build (arm)' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
91+
fi
92+
93+
dockers_build_amd_and_merge:
94+
runs-on: [self-hosted, style-checker]
95+
needs: [config_workflow, dockers_build_arm]
96+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VycyBCdWlsZCAoYW1kKSBhbmQgTWVyZ2U=') }}
97+
name: "Dockers Build (amd) and Merge"
98+
outputs:
99+
data: ${{ steps.run.outputs.DATA }}
100+
steps:
101+
- name: Checkout code
102+
uses: actions/checkout@v4
103+
with:
104+
ref: ${{ env.CHECKOUT_REF }}
105+
106+
- name: Prepare env script
107+
run: |
108+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
109+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
110+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
111+
export PYTHONPATH=./ci:.:
112+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
113+
${{ needs.config_workflow.outputs.data }}
114+
EOF
115+
cat > ./ci/tmp/workflow_status.json << 'EOF'
116+
${{ toJson(needs) }}
117+
EOF
118+
ENV_SETUP_SCRIPT_EOF
119+
120+
- name: Run
121+
id: run
122+
run: |
123+
. ./ci/tmp/praktika_setup_env.sh
124+
set -o pipefail
125+
if command -v ts &> /dev/null; then
126+
python3 -m praktika run 'Dockers Build (amd) and Merge' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
127+
else
128+
python3 -m praktika run 'Dockers Build (amd) and Merge' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
129+
fi
130+
131+
build_fuzzers:
132+
runs-on: [self-hosted, builder-aarch64]
133+
needs: [config_workflow, dockers_build_amd_and_merge]
134+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGZ1enplcnMp') }}
135+
name: "Build (fuzzers)"
136+
outputs:
137+
data: ${{ steps.run.outputs.DATA }}
138+
steps:
139+
- name: Checkout code
140+
uses: actions/checkout@v4
141+
with:
142+
ref: ${{ env.CHECKOUT_REF }}
143+
144+
- name: Prepare env script
145+
run: |
146+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
147+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
148+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
149+
export PYTHONPATH=./ci:.:
150+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
151+
${{ needs.config_workflow.outputs.data }}
152+
EOF
153+
cat > ./ci/tmp/workflow_status.json << 'EOF'
154+
${{ toJson(needs) }}
155+
EOF
156+
ENV_SETUP_SCRIPT_EOF
157+
158+
- name: Run
159+
id: run
160+
run: |
161+
. ./ci/tmp/praktika_setup_env.sh
162+
set -o pipefail
163+
if command -v ts &> /dev/null; then
164+
python3 -m praktika run 'Build (fuzzers)' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
165+
else
166+
python3 -m praktika run 'Build (fuzzers)' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
167+
fi
168+
169+
libfuzzer_tests:
170+
runs-on: [self-hosted, func-tester]
171+
needs: [config_workflow, dockers_build_amd_and_merge, build_fuzzers]
172+
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'bGliRnV6emVyIHRlc3Rz') }}
173+
name: "libFuzzer tests"
174+
outputs:
175+
data: ${{ steps.run.outputs.DATA }}
176+
steps:
177+
- name: Checkout code
178+
uses: actions/checkout@v4
179+
with:
180+
ref: ${{ env.CHECKOUT_REF }}
181+
182+
- name: Prepare env script
183+
run: |
184+
rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
185+
mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
186+
cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
187+
export PYTHONPATH=./ci:.:
188+
cat > ./ci/tmp/workflow_config_nightlyfuzzers.json << 'EOF'
189+
${{ needs.config_workflow.outputs.data }}
190+
EOF
191+
cat > ./ci/tmp/workflow_status.json << 'EOF'
192+
${{ toJson(needs) }}
193+
EOF
194+
ENV_SETUP_SCRIPT_EOF
195+
196+
- name: Run
197+
id: run
198+
run: |
199+
. ./ci/tmp/praktika_setup_env.sh
200+
set -o pipefail
201+
if command -v ts &> /dev/null; then
202+
python3 -m praktika run 'libFuzzer tests' --workflow "NightlyFuzzers" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
203+
else
204+
python3 -m praktika run 'libFuzzer tests' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
205+
fi

‎base/poco/Crypto/include/Poco/Crypto/OpenSSLInitializer.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020

2121
#include <openssl/crypto.h>
22+
#include <openssl/provider.h>
2223
#include "Poco/AtomicCounter.h"
2324
#include "Poco/Crypto/Crypto.h"
2425
#include "Poco/Mutex.h"
@@ -83,6 +84,8 @@ namespace Crypto
8384
private:
8485
static Poco::FastMutex * _mutexes;
8586
static Poco::AtomicCounter _rc;
87+
88+
static OSSL_PROVIDER * legacy_provider;
8689
};
8790

8891

‎base/poco/Crypto/src/OpenSSLInitializer.cpp‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include <openssl/rand.h>
2020
#include <openssl/crypto.h>
2121
#include <openssl/err.h>
22+
#include <openssl/provider.h>
2223
#if OPENSSL_VERSION_NUMBER >= 0x0907000L
2324
#include <openssl/conf.h>
2425
#endif
@@ -36,6 +37,7 @@ namespace Crypto {
3637

3738
Poco::FastMutex* OpenSSLInitializer::_mutexes(0);
3839
Poco::AtomicCounter OpenSSLInitializer::_rc;
40+
OSSL_PROVIDER * OpenSSLInitializer::legacy_provider;
3941

4042

4143
OpenSSLInitializer::OpenSSLInitializer()
@@ -67,21 +69,25 @@ void OpenSSLInitializer::initialize()
6769
SSL_library_init();
6870
SSL_load_error_strings();
6971
OpenSSL_add_all_algorithms();
70-
72+
7173
char seed[SEEDSIZE];
7274
RandomInputStream rnd;
7375
rnd.read(seed, sizeof(seed));
7476
RAND_seed(seed, SEEDSIZE);
75-
77+
78+
legacy_provider = OSSL_PROVIDER_load(NULL, "legacy");
79+
if (!legacy_provider)
80+
throw std::runtime_error("Failed to load OpenSSL legacy provider");
81+
7682
int nMutexes = CRYPTO_num_locks();
7783
_mutexes = new Poco::FastMutex[nMutexes];
7884
CRYPTO_set_locking_callback(&OpenSSLInitializer::lock);
7985
// Not needed on Windows (see SF #110: random unhandled exceptions when linking with ssl).
8086
// https://sourceforge.net/p/poco/bugs/110/
8187
//
8288
// From http://www.openssl.org/docs/crypto/threads.html :
83-
// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(),
84-
// then a default implementation is used - on Windows and BeOS this uses the system's
89+
// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(),
90+
// then a default implementation is used - on Windows and BeOS this uses the system's
8591
// default thread identifying APIs"
8692
CRYPTO_set_id_callback(&OpenSSLInitializer::id);
8793
CRYPTO_set_dynlock_create_callback(&OpenSSLInitializer::dynlockCreate);
@@ -100,7 +106,8 @@ void OpenSSLInitializer::uninitialize()
100106
CRYPTO_set_locking_callback(0);
101107
CRYPTO_set_id_callback(0);
102108
delete [] _mutexes;
103-
109+
110+
OSSL_PROVIDER_unload(legacy_provider);
104111
CONF_modules_free();
105112
}
106113
}

‎ci/defs/defs.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ class CIFiles:
251251
Docker.Config(
252252
name="clickhouse/docs-builder",
253253
path="./docker/docs/builder",
254-
platforms=[Docker.Platforms.AMD],
254+
platforms=Docker.Platforms.arm_amd,
255255
depends_on=[],
256256
),
257257
Docker.Config(
@@ -473,6 +473,7 @@ class ArtifactConfigs:
473473
name="...",
474474
type=Artifact.Type.S3,
475475
path=f"{TEMP_DIR}/build/src/unit_tests_dbms",
476+
compress_zst=True,
476477
).parametrize(
477478
names=[
478479
ArtifactNames.UNITTEST_AMD_ASAN,

0 commit comments

Comments
 (0)