Skip to content

Commit 658f89d

Browse files
lucia-sblucia-sb
committed
measure disk usage
1 parent 132d3d3 commit 658f89d

File tree

3 files changed

+191
-25
lines changed

3 files changed

+191
-25
lines changed

‎.github/workflows/measure-disk-usage.yml‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
name: Measure Disk Usage
22

33
on:
4-
workflow_run:
5-
workflows: ['Resolve Dependencies and Build Wheels']
6-
types:
7-
- completed
8-
4+
workflow_call:
5+
# workflow_run:
6+
# workflows: ['Resolve Dependencies and Build Wheels']
7+
# types:
8+
# - completed
9+
pull_request:
10+
branches:
11+
- master
912
env:
1013
PYTHON_VERSION: "3.12"
1114

@@ -43,24 +46,18 @@ jobs:
4346
4447
- name: Measure disk usage
4548
run: |
46-
cmd="ddev size status \
47-
--platform ${{ matrix.platform }} \
48-
--format json"
49-
49+
cmd="python .github/workflows/scripts/measure-disk-usage.py \
50+
--current-commit ${{ github.sha }} \
51+
--base-commit ${{ github.event.pull_request.base.sha }} \
52+
--platform ${{ matrix.platform }} "
53+
5054
# Send metrics to Datadog only on push to default branch
5155
if [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref_name }}" = "${{ github.event.repository.default_branch }}" ]; then
5256
cmd="$cmd --to-dd-key ${{ secrets.DD_API_KEY }}"
5357
fi
5458
55-
if [ "${{ github.event_name }}" = "workflow_run" ]; then
56-
cmd="$cmd --dependency-sizes dependency_sizes_${{ matrix.platform }}.json"
57-
fi
58-
# Uncompressed metrics
5959
$cmd
6060
61-
# Compressed metrics
62-
$cmd --compressed
63-
6461
- name: Upload JSON uncompressed sizes artifact
6562
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
6663
with:

‎.github/workflows/resolve-build-deps.yaml‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ jobs:
5656
echo "No dependency files changed."
5757
fi
5858
- name: Cancel workflow if no dependency changes
59-
if: true # steps.dependency-check.outputs.changed == 'false'
59+
if: steps.dependency-check.outputs.changed == 'false'
6060
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
6161
with:
6262
script: |
@@ -275,14 +275,12 @@ jobs:
275275
name: target-macos-${{ matrix.job.arch }}
276276
path: output
277277

278-
# measure-disk-usage:
279-
# needs:
280-
# - build
281-
# - build-macos
282-
# uses: ./.github/workflows/measure-disk-usage.yml
283-
# with:
284-
# platforms: linux-aarch64 linux-x86_64 windows-x86_64 macos-x86_64 macos-aarch64
285-
# resolved-deps: true
278+
measure-disk-usage:
279+
needs:
280+
# - build
281+
- build-macos
282+
uses: ./.github/workflows/measure-disk-usage.yml
283+
286284

287285

288286

‎.github/workflows/scripts/measure-disk-usage.py‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import argparse
2+
import os
3+
import shutil
4+
import subprocess
5+
import sys
6+
import tempfile
7+
import json
8+
9+
10+
def get_run_id(commit, workflow):
11+
try:
12+
result = subprocess.run(
13+
[
14+
'gh',
15+
'run',
16+
'list',
17+
'--workflow',
18+
workflow,
19+
'-c',
20+
commit,
21+
'--status',
22+
'completed',
23+
'--json',
24+
'databaseId',
25+
'--jq',
26+
'.[-1].databaseId',
27+
],
28+
)
29+
except subprocess.CalledProcessError as e:
30+
stderr = (e.stderr or '').strip()
31+
if stderr:
32+
print(stderr)
33+
return None
34+
return result.stdout.strip()
35+
36+
def get_dep_sizes_json(current_commit, platform):
37+
run_id = get_run_id(current_commit, '.github/workflows/resolve-build-deps.yaml')
38+
if run_id and check_artifact_exists(run_id, f'target-{platform}'):
39+
dep_sizes_json = get_current_sizes_json(run_id, platform)
40+
return dep_sizes_json
41+
else:
42+
return None
43+
44+
45+
def check_artifact_exists(run_id, artifact_name):
46+
result = subprocess.run(
47+
[
48+
'gh',
49+
'api',
50+
f'repos/Datadog/integrations-core/actions/runs/{run_id}/artifacts',
51+
'--jq',
52+
'.artifacts[].name',
53+
],
54+
check=True,
55+
capture_output=True,
56+
text=True,
57+
)
58+
59+
artifact_names = {n.strip() for n in (result.stdout or '').splitlines() if n.strip()}
60+
if artifact_name not in artifact_names:
61+
print(f"Artifact '{artifact_name}' not found in run {run_id}")
62+
return False
63+
64+
print(f"Found artifact: {artifact_name}")
65+
return True
66+
67+
def get_current_sizes_json(run_id, platform):
68+
with tempfile.TemporaryDirectory() as tmpdir:
69+
print(f"Downloading artifacts to {tmpdir}")
70+
_ = subprocess.run(
71+
['gh', 'run', 'download', run_id, '--name', f'target-{platform}', '--dir', tmpdir],
72+
check=True,
73+
capture_output=True,
74+
text=True,
75+
)
76+
print(f"Downloaded artifacts to {tmpdir}")
77+
# Look for the sizes.json file in the downloaded artifacts
78+
sizes_file = os.path.join(tmpdir, platform, 'py3', 'sizes.json')
79+
80+
if os.path.exists(sizes_file):
81+
print(f"Found sizes.json at {sizes_file}")
82+
shutil.move(sizes_file, os.path.join(os.getcwd(), f'{platform}.json'))
83+
return os.getcwd() / f'{platform}.json'
84+
else:
85+
print(f"sizes.json not found at {sizes_file}")
86+
return None
87+
88+
def get_artifact(run_id, artifact_name):
89+
_ = subprocess.run(
90+
['gh', 'run', 'download', run_id, '--name', artifact_name],
91+
check=True,
92+
capture_output=True,
93+
text=True,
94+
)
95+
return os.path.join(os.getcwd(), artifact_name)
96+
97+
def get_previous_dep_sizes_json(base_commit, platform):
98+
# Get the previous commit in master branch
99+
result = subprocess.run(
100+
['git', 'rev-parse', f'{base_commit}~1'],
101+
check=True,
102+
capture_output=True,
103+
text=True,
104+
)
105+
prev_commit = result.stdout.strip()
106+
run_id = get_run_id(prev_commit, '.github/workflows/measure-disk-usage.yml')
107+
if run_id and check_artifact_exists(run_id, f'status_uncompressed_{platform}.json'):
108+
uncompressed_json = get_artifact(run_id, f'status_uncompressed_{platform}.json')
109+
if run_id and check_artifact_exists(run_id, f'status_compressed_{platform}.json'):
110+
compressed_json = get_artifact(run_id, f'status_compressed_{platform}.json')
111+
112+
sizes_json = parse_sizes_json(compressed_json, uncompressed_json)
113+
with open(f'{platform}.json', 'w') as f:
114+
json.dump(sizes_json, f, indent=2)
115+
return f'{platform}.json'
116+
117+
118+
def parse_sizes_json(compressed_json_path, uncompressed_json_path):
119+
with open(compressed_json_path, 'r') as f:
120+
compressed_list = json.load(f)
121+
with open(uncompressed_json_path, 'r') as f:
122+
uncompressed_list = json.load(f)
123+
124+
sizes_json = {
125+
dep["Name"]: {
126+
"compressed": dep["Size_Bytes"],
127+
"version": dep["Version"]
128+
}
129+
for dep in compressed_list
130+
}
131+
132+
for dep in uncompressed_list:
133+
sizes_json[dep["Name"]]["uncompressed"] = dep["Size_Bytes"]
134+
135+
return sizes_json
136+
137+
138+
def main():
139+
parser = argparse.ArgumentParser(description='Calculate the current repo size')
140+
parser.add_argument('--current-commit', required=True, help='Current commit hash')
141+
parser.add_argument('--base-commit', required=True, help='Base commit hash')
142+
parser.add_argument('--platform', required=True, help='Platform to compare')
143+
parser.add_argument('--to-dd-key', required=False, help='Send to Datadog')
144+
args = parser.parse_args()
145+
146+
dep_sizes_json = get_dep_sizes_json(args.current_commit, args.platform)
147+
if not dep_sizes_json:
148+
dep_sizes_json = get_previous_dep_sizes_json(args.base_commit, args.platform)
149+
150+
command = (
151+
f"ddev size status "
152+
f"--platform {args.platform} "
153+
f"--dependency-sizes {dep_sizes_json} "
154+
f"--format json"
155+
)
156+
if args.send_to_dd:
157+
command += f" --to-dd-key {args.send_to_dd}"
158+
159+
subprocess.run(command, check=True)
160+
161+
command += "--compressed"
162+
163+
subprocess.run(command, check=True)
164+
165+
return 0
166+
167+
168+
169+
170+
if __name__ == '__main__':
171+
sys.exit(main())

0 commit comments

Comments
 (0)