Skip to content

Commit 2cd9430

Browse files
MyroTkMyroTk
authored
Merge pull request #1055 from Altinity/25.6_update_s3_paths
25.6 update s3 paths
2 parents f5d064f + 6123b5c commit 2cd9430

File tree

13 files changed

+101
-29
lines changed

13 files changed

+101
-29
lines changed

‎.github/actions/create_workflow_report/ci_run_report.html.jinja‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
66
<meta charset="UTF-8">
77
<meta name="viewport" content="width=device-width, initial-scale=1.0">
8+
{%- if is_preview %}
9+
<meta http-equiv="Cache-Control" content="max-age=300">
10+
{%- endif %}
811
<style>
912
/* Base colors for Altinity */
1013
:root {

‎.github/actions/create_workflow_report/create_workflow_report.py‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,7 @@ def get_cached_job(job_name: str) -> dict:
386386
return workflow_config["cache_jobs"].get(job_name, {})
387387

388388

389-
def get_cves(pr_number, commit_sha):
389+
def get_cves(pr_number, commit_sha, branch):
390390
"""
391391
Fetch Grype results from S3.
392392
@@ -395,19 +395,33 @@ def get_cves(pr_number, commit_sha):
395395
s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL"))
396396
prefixes_to_check = set()
397397

398+
def format_prefix(pr_number, commit_sha, branch):
399+
if pr_number == 0:
400+
return f"REFs/{branch}/{commit_sha}/grype/"
401+
else:
402+
return f"PRs/{pr_number}/{commit_sha}/grype/"
403+
398404
cached_server_job = get_cached_job("Docker server image")
399405
if cached_server_job:
400406
prefixes_to_check.add(
401-
f"{cached_server_job['pr_number']}/{cached_server_job['sha']}/grype/"
407+
format_prefix(
408+
cached_server_job["pr_number"],
409+
cached_server_job["sha"],
410+
cached_server_job["branch"],
411+
)
402412
)
403413
cached_keeper_job = get_cached_job("Docker keeper image")
404414
if cached_keeper_job:
405415
prefixes_to_check.add(
406-
f"{cached_keeper_job['pr_number']}/{cached_keeper_job['sha']}/grype/"
416+
format_prefix(
417+
cached_keeper_job["pr_number"],
418+
cached_keeper_job["sha"],
419+
cached_keeper_job["branch"],
420+
)
407421
)
408422

409423
if not prefixes_to_check:
410-
prefixes_to_check = {f"{pr_number}/{commit_sha}/grype/"}
424+
prefixes_to_check = {format_prefix(pr_number, commit_sha, branch)}
411425

412426
grype_result_dirs = []
413427
for s3_prefix in prefixes_to_check:
@@ -690,7 +704,7 @@ def create_workflow_report(
690704
"checks_errors": get_checks_errors(db_client, commit_sha, branch_name),
691705
"regression_fails": get_regression_fails(db_client, actions_run_url),
692706
"docker_images_cves": (
693-
[] if not check_cves else get_cves(pr_number, commit_sha)
707+
[] if not check_cves else get_cves(pr_number, commit_sha, branch_name)
694708
),
695709
}
696710

@@ -809,7 +823,10 @@ def create_workflow_report(
809823
print(f"Report saved to {report_path}")
810824
exit(0)
811825

812-
report_destination_key = f"{pr_number}/{commit_sha}/{report_name}"
826+
if pr_number == 0:
827+
report_destination_key = f"REFs/{branch_name}/{commit_sha}/{report_name}"
828+
else:
829+
report_destination_key = f"PRs/{pr_number}/{commit_sha}/{report_name}"
813830

814831
# Upload the report to S3
815832
s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL"))

‎.github/grype/transform_and_upload_results_s3.sh‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,18 @@
11
DOCKER_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g')
22

3-
S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE"
4-
HTTPS_S3_PATH="https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE"
5-
echo "https_s3_path=$HTTPS_S3_PATH" >> $GITHUB_OUTPUT
3+
if [ "$PR_NUMBER" -eq 0 ]; then
4+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA/grype/$DOCKER_IMAGE"
5+
else
6+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE"
7+
fi
8+
9+
S3_PATH="s3://$S3_BUCKET/$PREFIX"
10+
HTTPS_RESULTS_PATH="https://$S3_BUCKET.s3.amazonaws.com/index.html#$PREFIX/"
11+
HTTPS_REPORT_PATH="https://s3.amazonaws.com/$S3_BUCKET/$PREFIX/results.html"
12+
echo "https_report_path=$HTTPS_REPORT_PATH" >> $GITHUB_OUTPUT
613

714
tfs --no-colors transform nice raw.log nice.log.txt
8-
tfs --no-colors report results -a $HTTPS_S3_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html
15+
tfs --no-colors report results -a $HTTPS_RESULTS_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html
916

1017
aws s3 cp --no-progress nice.log.txt $S3_PATH/nice.log.txt --content-type "text/plain; charset=utf-8" || echo "nice log file not found".
1118
aws s3 cp --no-progress results.html $S3_PATH/results.html || echo "results file not found".

‎.github/workflows/backport_branches.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,12 @@ jobs:
6363
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
6464
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
6565
run: |
66-
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PR_NUMBER/$COMMIT_SHA/ci_run_report.html
66+
if [ "$PR_NUMBER" -eq 0 ]; then
67+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
68+
else
69+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
70+
fi
71+
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/ci_run_report.html
6772
echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
6873
6974
- name: Prepare env script
@@ -1524,7 +1529,7 @@ jobs:
15241529
secrets: inherit
15251530
with:
15261531
runner_type: altinity-on-demand, altinity-regression-tester
1527-
commit: eadf0647501a547d57c49b71ca256e10c6c304f6
1532+
commit: bf856e2a535d5bb7a010cc2c2992c75d14df3c0e
15281533
arch: release
15291534
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
15301535
timeout_minutes: 300
@@ -1536,7 +1541,7 @@ jobs:
15361541
secrets: inherit
15371542
with:
15381543
runner_type: altinity-on-demand, altinity-regression-tester-aarch64
1539-
commit: eadf0647501a547d57c49b71ca256e10c6c304f6
1544+
commit: bf856e2a535d5bb7a010cc2c2992c75d14df3c0e
15401545
arch: aarch64
15411546
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
15421547
timeout_minutes: 300

‎.github/workflows/grype_scan.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ jobs:
139139
repo: context.repo.repo,
140140
sha: '${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}',
141141
state: hasError ? 'error' : hasVulnerabilities ? 'failure' : 'success',
142-
target_url: '${{ steps.upload_results.outputs.https_s3_path }}/results.html',
142+
target_url: '${{ steps.upload_results.outputs.https_report_path }}',
143143
description: hasError ? 'An error occurred' : `Grype Scan Completed with ${totalHighCritical} high/critical vulnerabilities`,
144144
context: 'Grype Scan ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}'
145145
});

‎.github/workflows/master.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,12 @@ jobs:
6262
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
6363
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
6464
run: |
65-
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PR_NUMBER/$COMMIT_SHA/ci_run_report.html
65+
if [ "$PR_NUMBER" -eq 0 ]; then
66+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
67+
else
68+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
69+
fi
70+
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/ci_run_report.html
6671
echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
6772
6873
- name: Prepare env script
@@ -3998,7 +4003,7 @@ jobs:
39984003
secrets: inherit
39994004
with:
40004005
runner_type: altinity-on-demand, altinity-regression-tester
4001-
commit: eadf0647501a547d57c49b71ca256e10c6c304f6
4006+
commit: bf856e2a535d5bb7a010cc2c2992c75d14df3c0e
40024007
arch: release
40034008
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
40044009
timeout_minutes: 300
@@ -4010,7 +4015,7 @@ jobs:
40104015
secrets: inherit
40114016
with:
40124017
runner_type: altinity-on-demand, altinity-regression-tester-aarch64
4013-
commit: eadf0647501a547d57c49b71ca256e10c6c304f6
4018+
commit: bf856e2a535d5bb7a010cc2c2992c75d14df3c0e
40144019
arch: aarch64
40154020
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
40164021
timeout_minutes: 300

‎.github/workflows/merge_queue.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,12 @@ jobs:
5151
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
5252
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
5353
run: |
54-
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PR_NUMBER/$COMMIT_SHA/ci_run_report.html
54+
if [ "$PR_NUMBER" -eq 0 ]; then
55+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
56+
else
57+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
58+
fi
59+
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/ci_run_report.html
5560
echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
5661
5762
- name: Prepare env script

‎.github/workflows/nightly_fuzzers.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,12 @@ jobs:
3939
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
4040
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
4141
run: |
42-
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PR_NUMBER/$COMMIT_SHA/ci_run_report.html
42+
if [ "$PR_NUMBER" -eq 0 ]; then
43+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
44+
else
45+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
46+
fi
47+
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/ci_run_report.html
4348
echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
4449
4550
- name: Prepare env script

‎.github/workflows/nightly_jepsen.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,12 @@ jobs:
3939
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
4040
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
4141
run: |
42-
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PR_NUMBER/$COMMIT_SHA/ci_run_report.html
42+
if [ "$PR_NUMBER" -eq 0 ]; then
43+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
44+
else
45+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
46+
fi
47+
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/ci_run_report.html
4348
echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
4449
4550
- name: Prepare env script

‎.github/workflows/nightly_statistics.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,12 @@ jobs:
3939
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
4040
COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
4141
run: |
42-
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PR_NUMBER/$COMMIT_SHA/ci_run_report.html
42+
if [ "$PR_NUMBER" -eq 0 ]; then
43+
PREFIX="REFs/$GITHUB_REF_NAME/$COMMIT_SHA"
44+
else
45+
PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
46+
fi
47+
REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/ci_run_report.html
4348
echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
4449
4550
- name: Prepare env script

0 commit comments

Comments
 (0)