Skip to content

Commit 28fac9a

Browse files
lucia-sblucia-sb
committed
ddev
1 parent 8756ce2 commit 28fac9a

File tree

3 files changed

+205
-16
lines changed

3 files changed

+205
-16
lines changed

‎.github/workflows/measure-disk-usage.yml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -49,23 +49,23 @@ jobs:
4949
- name: Determine commit SHAs
5050
id: commits
5151
run: |
52-
if [ "${{ github.event_name }}" = "pull_request" ]; then
52+
if [ "${{ github.event_name }}" = "pull_request" ]; then # CHANGE TO github.event.workflow_run.event
5353
echo "current_sha=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"
5454
echo "base_sha=${{ github.event.pull_request.base.sha }}" >> "$GITHUB_OUTPUT"
55-
elif [ "${{ github.event_name }}" = "push" ]; then
55+
elif [ "${{ github.event_name }}" = "push" ]; then # CHANGE TO github.event.workflow_run.event
5656
echo "current_sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
5757
echo "base_sha=${{ github.event.before }}" >> "$GITHUB_OUTPUT"
5858
fi
5959
6060
- name: Measure disk usage
6161
run: |
62-
cmd="python .github/workflows/scripts/measure-disk-usage.py \
63-
--current-commit ${{ steps.commits.outputs.current_sha }} \
64-
--base-commit ${{ steps.commits.outputs.base_sha }} \
65-
--platform ${{ matrix.platform }} "
66-
--run-id ${{ github.run_id }}
62+
cmd="ddev size status \
63+
--platform ${{ matrix.platform }} \
64+
--commit ${{ steps.commits.outputs.current_sha }} \
65+
--format json"
66+
6767
# Send metrics to Datadog only on push to default branch
68-
if [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref_name }}" = "${{ github.event.repository.default_branch }}" ]; then
68+
if [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref_name }}" = "${{ github.event.repository.default_branch }}" ]; then # CHANGE TO github.event.workflow_run.event
6969
cmd="$cmd --to-dd-key ${{ secrets.DD_API_KEY }}"
7070
fi
7171

‎ddev/src/ddev/cli/size/status.py‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
import os
66
from pathlib import Path
7-
from typing import Optional
87

98
import click
109
from rich.console import Console
@@ -16,7 +15,9 @@
1615
export_format,
1716
format_modules,
1817
get_dependencies,
18+
get_dependencies_from_json,
1919
get_files,
20+
get_last_dependency_sizes_artifact,
2021
get_valid_platforms,
2122
get_valid_versions,
2223
plot_treemap,
@@ -32,17 +33,21 @@
3233
@click.option("--to-dd-org", type=str, help="Send metrics to Datadog using the specified organization name.")
3334
@click.option("--to-dd-key", type=str, help="Send metrics to datadoghq.com using the specified API key.")
3435
@click.option("--python", "version", help="Python version (e.g 3.12). If not specified, all versions will be analyzed")
36+
@click.option("--dependency-sizes", help="Path to the dependency sizes file.")
37+
@click.option("--commit", help="Commit hash to check the status of.")
3538
@common_params # platform, compressed, format, show_gui
3639
@click.pass_obj
3740
def status(
3841
app: Application,
39-
platform: Optional[str],
40-
version: Optional[str],
42+
platform: str | None,
43+
version: str | None,
4144
compressed: bool,
4245
format: list[str],
4346
show_gui: bool,
4447
to_dd_org: str,
4548
to_dd_key: str,
49+
dependency_sizes: str | None,
50+
commit: str | None,
4651
) -> None:
4752
"""
4853
Show the current size of all integrations and dependencies.
@@ -65,7 +70,10 @@ def status(
6570
platforms = valid_platforms if platform is None else [platform]
6671
versions = valid_versions if version is None else [version]
6772
combinations = [(p, v) for p in platforms for v in versions]
73+
6874
for plat, ver in combinations:
75+
if commit and not dependency_sizes:
76+
dependency_sizes = get_last_dependency_sizes_artifact(commit, plat)
6977
parameters: CLIParameters = {
7078
"app": app,
7179
"platform": plat,
@@ -78,6 +86,7 @@ def status(
7886
status_mode(
7987
repo_path,
8088
parameters,
89+
dependency_sizes,
8190
)
8291
)
8392

@@ -92,11 +101,17 @@ def status(
92101
def status_mode(
93102
repo_path: Path,
94103
params: CLIParameters,
104+
dependency_sizes: str | None,
95105
) -> list[FileDataEntryPlatformVersion]:
96106
with console.status("[cyan]Calculating sizes...", spinner="dots"):
97-
modules = get_files(repo_path, params["compressed"], params["version"]) + get_dependencies(
98-
repo_path, params["platform"], params["version"], params["compressed"]
99-
)
107+
if dependency_sizes:
108+
modules = get_files(repo_path, params["compressed"], params["version"]) + get_dependencies_from_json(
109+
dependency_sizes, params["platform"], params["version"], params["compressed"]
110+
)
111+
else:
112+
modules = get_files(repo_path, params["compressed"], params["version"]) + get_dependencies(
113+
repo_path, params["platform"], params["version"], params["compressed"]
114+
)
100115

101116
formatted_modules = format_modules(modules, params["platform"], params["version"])
102117
formatted_modules.sort(key=lambda x: x["Size_Bytes"], reverse=True)

‎ddev/src/ddev/cli/size/utils/common_funcs.py‎

Lines changed: 176 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
from ddev.utils.toml import load_toml_file
2525

2626
METRIC_VERSION = 2
27-
27+
METRIC_NAME = "datadog.agent_integrations"
2828
if TYPE_CHECKING:
2929
from matplotlib.axes import Axes
3030
from matplotlib.patches import Patch
@@ -227,6 +227,10 @@ def get_files(repo_path: str | Path, compressed: bool, py_version: str) -> list[
227227
relative_path = os.path.relpath(file_path, repo_path)
228228
if not is_valid_integration_file(relative_path, str(repo_path)):
229229
continue
230+
path = Path(relative_path)
231+
parts = path.parts
232+
233+
integration_name = parts[0]
230234

231235
size = compress(file_path) if compressed else os.path.getsize(file_path)
232236
integration_sizes[integration_name] = integration_sizes.get(integration_name, 0) + size
@@ -837,7 +841,7 @@ def send_metrics_to_dd(
837841
key: str,
838842
compressed: bool,
839843
) -> None:
840-
metric_name = "datadog.agent_integrations"
844+
metric_name = METRIC_NAME
841845
size_type = "compressed" if compressed else "uncompressed"
842846

843847
config_file_info = get_org(app, org) if org else {"api_key": key, "site": "datadoghq.com"}
@@ -1090,3 +1094,173 @@ def __exit__(
10901094
) -> None:
10911095
if self.repo_dir and os.path.exists(self.repo_dir):
10921096
shutil.rmtree(self.repo_dir)
1097+
1098+
1099+
def get_last_dependency_sizes_artifact(commit: str, platform: str) -> str:
1100+
dep_sizes_json = get_dep_sizes_json(commit, platform)
1101+
if not dep_sizes_json:
1102+
dep_sizes_json = get_previous_dep_sizes_json(commit, platform)
1103+
return dep_sizes_json
1104+
1105+
1106+
def get_run_id(commit, workflow):
1107+
print(f"Getting run id for commit: {commit}, workflow: {workflow}")
1108+
try:
1109+
result = subprocess.run(
1110+
[
1111+
'gh',
1112+
'run',
1113+
'list',
1114+
'--workflow',
1115+
workflow,
1116+
'-c',
1117+
commit,
1118+
'--json',
1119+
'databaseId',
1120+
'--jq',
1121+
'.[-1].databaseId',
1122+
],
1123+
capture_output=True,
1124+
text=True,
1125+
)
1126+
except subprocess.CalledProcessError as e:
1127+
stderr = (e.stderr or '').strip()
1128+
if stderr:
1129+
print(stderr)
1130+
print("Failed to get run id")
1131+
return None
1132+
run_id = result.stdout.strip() if result.stdout else None
1133+
print(f"Run id: {run_id}")
1134+
return run_id
1135+
1136+
1137+
def get_dep_sizes_json(current_commit, platform):
1138+
print(f"Getting dependency sizes json for commit: {current_commit}, platform: {platform}")
1139+
run_id = get_run_id(current_commit, '.github/workflows/resolve-build-deps.yaml')
1140+
if run_id and check_artifact_exists(run_id, f'target-{platform}'):
1141+
dep_sizes_json = get_current_sizes_json(run_id, platform)
1142+
print(f"Dependency sizes json path: {dep_sizes_json}")
1143+
return dep_sizes_json
1144+
else:
1145+
print("Dependency sizes json not found for current commit.")
1146+
return None
1147+
1148+
1149+
def check_artifact_exists(run_id, artifact_name):
1150+
print(f"Checking if artifact exists: run_id={run_id}, artifact_name={artifact_name}")
1151+
result = subprocess.run(
1152+
[
1153+
'gh',
1154+
'api',
1155+
f'repos/Datadog/integrations-core/actions/runs/{run_id}/artifacts',
1156+
'--jq',
1157+
'.artifacts[].name',
1158+
],
1159+
check=True,
1160+
capture_output=True,
1161+
text=True,
1162+
)
1163+
1164+
artifact_names = {n.strip() for n in (result.stdout or '').splitlines() if n.strip()}
1165+
print(f"Available artifacts: {artifact_names}")
1166+
if artifact_name not in artifact_names:
1167+
print(f"Artifact '{artifact_name}' not found in run {run_id}")
1168+
return False
1169+
1170+
print(f"Found artifact: {artifact_name}")
1171+
return True
1172+
1173+
1174+
def get_current_sizes_json(run_id, platform):
1175+
print(f"Getting current sizes json for run_id={run_id}, platform={platform}")
1176+
with tempfile.TemporaryDirectory() as tmpdir:
1177+
print(f"Downloading artifacts to {tmpdir}")
1178+
_ = subprocess.run(
1179+
[
1180+
'gh',
1181+
'run',
1182+
'download',
1183+
run_id,
1184+
'--name',
1185+
f'target-{platform}',
1186+
'--dir',
1187+
tmpdir,
1188+
],
1189+
check=True,
1190+
capture_output=True,
1191+
text=True,
1192+
)
1193+
print(f"Downloaded artifacts to {tmpdir}")
1194+
# Look for the sizes.json file in the downloaded artifacts
1195+
sizes_file = os.path.join(tmpdir, platform, 'py3', 'sizes.json')
1196+
1197+
if os.path.exists(sizes_file):
1198+
print(f"Found sizes.json at {sizes_file}")
1199+
dest_path = os.path.join(os.getcwd(), f'{platform}.json')
1200+
shutil.move(sizes_file, dest_path)
1201+
return dest_path
1202+
else:
1203+
print(f"sizes.json not found at {sizes_file}")
1204+
return None
1205+
1206+
1207+
def get_artifact(run_id, artifact_name):
1208+
print(f"Downloading artifact: {artifact_name} from run_id={run_id}")
1209+
_ = subprocess.run(
1210+
[
1211+
'gh',
1212+
'run',
1213+
'download',
1214+
run_id,
1215+
'--name',
1216+
artifact_name,
1217+
],
1218+
check=True,
1219+
capture_output=True,
1220+
text=True,
1221+
)
1222+
artifact_path = os.path.join(os.getcwd(), artifact_name)
1223+
print(f"Artifact downloaded to: {artifact_path}")
1224+
return artifact_path
1225+
1226+
1227+
def get_previous_dep_sizes_json(base_commit, platform):
1228+
print(f"Getting previous dependency sizes json for base_commit={base_commit}, platform={platform}")
1229+
run_id = get_run_id(base_commit, '.github/workflows/measure-disk-usage.yml')
1230+
print(f"Previous run_id: {run_id}")
1231+
compressed_json = None
1232+
uncompressed_json = None
1233+
if run_id and check_artifact_exists(run_id, f'status_compressed_{platform}.json'):
1234+
compressed_json = get_artifact(run_id, f'status_compressed_{platform}.json')
1235+
if run_id and check_artifact_exists(run_id, f'status_uncompressed_{platform}.json'):
1236+
uncompressed_json = get_artifact(run_id, f'status_uncompressed_{platform}.json')
1237+
print(f"Compressed json: {compressed_json}")
1238+
print(f"Uncompressed json: {uncompressed_json}")
1239+
sizes_json = parse_sizes_json(compressed_json, uncompressed_json)
1240+
output_path = f'{platform}.json'
1241+
with open(output_path, 'w') as f:
1242+
json.dump(sizes_json, f, indent=2)
1243+
print(f"Wrote merged sizes json to {output_path}")
1244+
return output_path
1245+
1246+
1247+
def parse_sizes_json(compressed_json_path, uncompressed_json_path):
1248+
with open(compressed_json_path, 'r') as f:
1249+
compressed_list = list(json.load(f))
1250+
with open(uncompressed_json_path, 'r') as f:
1251+
uncompressed_list = list(json.load(f))
1252+
1253+
sizes_json = {
1254+
dep["Name"]: {
1255+
"compressed": int(dep["Size_Bytes"]),
1256+
"version": dep.get("Version"),
1257+
}
1258+
for dep in compressed_list
1259+
}
1260+
1261+
for dep in uncompressed_list:
1262+
name = dep["Name"]
1263+
entry = sizes_json.setdefault(name, {"version": dep.get("Version")})
1264+
entry["uncompressed"] = int(dep["Size_Bytes"])
1265+
1266+
return sizes_json

0 commit comments

Comments
 (0)