Skip to content

Commit 4f37a2a

Browse files
lucia-sblucia-sb
committed
calculate differences
1 parent e613bd6 commit 4f37a2a

File tree

3 files changed

+312
-1
lines changed

3 files changed

+312
-1
lines changed

‎.github/workflows/measure-disk-usage.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,14 +61,24 @@ jobs:
6161
name: ${{ inputs.platform }}_uncompressed_status.json
6262
path: ${{ inputs.platform }}_uncompressed_status.json
6363
if-no-files-found: error
64-
64+
6565
- name: Upload JSON compressed sizes artifact
6666
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
6767
with:
6868
name: ${{ inputs.platform }}_compressed_status.json
6969
path: ${{ inputs.platform }}_compressed_status.json
7070
if-no-files-found: error
7171

72+
calculate-diffs:
73+
needs: measure-disk-usage
74+
runs-on: ubuntu-22.04
75+
uses: DataDog/integrations-core/.github/workflows/quality-gates.yml@lucia-sb/calculate-diffs
76+
with:
77+
platform: ${{ inputs.platform }}
78+
run-id: ${{ github.run_id }}
79+
80+
81+
7282

7383
# run: |
7484
# ddev size status --to-dd-key ${{secrets.DD_API_KEY}} > size-uncompressed.txt

‎.github/workflows/quality-gates.yml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
name: Quality Gates
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
platform:
7+
required: true
8+
type: string
9+
run-id:
10+
required: true
11+
type: string
12+
13+
env:
14+
PYTHON_VERSION: "3.12"
15+
16+
jobs:
17+
get-merge-base:
18+
needs: check-file-changes
19+
if: needs.check-file-changes.outputs.builders_changed == 'false'
20+
runs-on: ubuntu-22.04
21+
outputs:
22+
run_id: ${{ steps.find_run.outputs.run_id }}
23+
steps:
24+
- name: Fetch master and PR branch
25+
run: |
26+
git fetch origin master:refs/remotes/origin/master
27+
git fetch origin ${{ github.head_ref }}:refs/remotes/origin/temp-${{ github.head_ref }}
28+
29+
- name: Get merge base commit
30+
id: merge_base
31+
run: |
32+
base=$(git merge-base origin/temp-${{ github.head_ref }} origin/master)
33+
34+
- name: Find workflow run at exact merge-base SHA
35+
id: find_run
36+
env:
37+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
38+
run: |
39+
MERGE_BASE=${{ steps.merge_base.outputs.sha }}
40+
41+
count=$(git rev-list --count ${{ steps.merge_base.outputs.sha }}..origin/master)
42+
count_plus_one=$((count + 1))
43+
run_id=$(
44+
(
45+
gh run list --workflow "Measure Disk Usage" --limit $count_plus_one --branch "master" --json databaseId,headSha,event
46+
gh run list --workflow "Fetch Previous Sizes JSON" --limit $count_plus_one --branch "master" --json databaseId,headSha,event
47+
) | jq -s '[.[][]] | .[] | select(.headSha == "'"$MERGE_BASE"'") | .databaseId' | head -n 1
48+
)
49+
if [ -z "$run_id" ]; then
50+
echo "No workflow run found for that SHA" >> $GITHUB_STEP_SUMMARY
51+
exit 1
52+
fi
53+
54+
download-artifacts:
55+
needs: get-merge-base
56+
runs-on: ubuntu-22.04
57+
strategy:
58+
matrix:
59+
platform: [macos-x86_64, macos-aarch64, linux-x86_64, linux-aarch64, windows-x86_64]
60+
steps:
61+
- name: Download artifact from merge-base run
62+
run: gh run download ${{ needs.get-merge-base.outputs.run_id }} --name ${{ matrix.platform }}_compressed_status.json
63+
env:
64+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
65+
- name: Rename previous sizes artifact
66+
run: |
67+
mv ${{ matrix.platform }}_compressed_status.json ${{ matrix.platform }}_prev_compressed_status.json
68+
- name: Download current sizes artifact
69+
run: gh run download ${{ inputs.run-id }} --name ${{ inputs.platform }}_compressed_status.json
70+
env:
71+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
72+
- name: Rename current sizes artifact
73+
run: |
74+
mv ${{ matrix.platform }}_compressed_status.json ${{ matrix.platform }}_curr_compressed_status.json
75+
76+
calculate-diffs:
77+
runs-on: ubuntu-22.04
78+
steps:
79+
- name: Set up Python ${{ env.PYTHON_VERSION }}
80+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
81+
with:
82+
python-version: ${{ env.PYTHON_VERSION }}
83+
- name: Calculate diffs
84+
run: |
85+
python -m ddev.cli.size.utils.gha_diff --prev-sizes ${{ matrix.platform }}_prev_compressed_status.json --curr-sizes ${{ matrix.platform }}_curr_compressed_status.json > $GITHUB_STEP_SUMMARY
86+
87+
88+
89+
90+
91+
92+
93+
94+
95+
96+
97+
98+
99+
100+
101+
102+

‎ddev/src/ddev/cli/size/utils/gha_diff.py‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
import argparse
2+
import json
3+
4+
from ddev.cli.size.utils.common_funcs import convert_to_human_readable_size
5+
6+
7+
def calculate_diffs(prev_sizes, curr_sizes):
8+
def key(entry):
9+
return (
10+
entry.get("Name"),
11+
entry.get("Platform"),
12+
entry.get("Python_Version"),
13+
entry.get("Type"),
14+
)
15+
16+
prev_map = {key(e): e for e in prev_sizes}
17+
curr_map = {key(e): e for e in curr_sizes}
18+
19+
added = []
20+
removed = []
21+
changed = []
22+
23+
total_diff = 0
24+
# Find added and changed
25+
for curr_key, curr_entry in curr_map.items():
26+
if curr_key not in prev_map:
27+
added.append(curr_entry)
28+
total_diff += curr_entry.get("Size_Bytes", 0)
29+
else:
30+
prev_entry = prev_map[curr_key]
31+
prev_size = prev_entry.get("Size_Bytes", 0)
32+
curr_size = curr_entry.get("Size_Bytes", 0)
33+
if prev_size != curr_size:
34+
changed.append(
35+
{
36+
"Name": curr_entry.get("Name"),
37+
"Version": curr_entry.get("Version"),
38+
"Prev Version": prev_entry.get("Version"),
39+
"Platform": curr_entry.get("Platform"),
40+
"Python_Version": curr_entry.get("Python_Version"),
41+
"Type": curr_entry.get("Type"),
42+
"prev_Size_Bytes": prev_size,
43+
"curr_Size_Bytes": curr_size,
44+
"diff": curr_size - prev_size,
45+
}
46+
)
47+
total_diff += curr_size - prev_size
48+
# Find removed
49+
for prev_key, prev_entry in prev_map.items():
50+
if prev_key not in curr_map:
51+
removed.append(prev_entry)
52+
total_diff -= prev_entry.get("Size_Bytes", 0)
53+
54+
return {
55+
"added": added,
56+
"removed": removed,
57+
"changed": changed,
58+
"total_diff": total_diff,
59+
}
60+
61+
62+
def display_diffs(diffs):
63+
# Print a well-formatted summary of the diffs
64+
sign = "+" if diffs['total_diff'] > 0 else "-"
65+
print("=" * 60)
66+
print(
67+
f"Dependency Size Differences for {diffs['added'][0]['Platform']} and"
68+
f" Python {diffs['added'][0]['Python_Version']}"
69+
)
70+
print("=" * 60)
71+
print(f"Total size difference: {sign}{convert_to_human_readable_size(diffs['total_diff'])}")
72+
print()
73+
74+
if diffs["added"]:
75+
print("Added:")
76+
for entry in diffs["added"]:
77+
name = entry.get("Name", "")
78+
version = entry.get("Version", "")
79+
size = entry.get("Size", 0)
80+
typ = entry.get("Type", "")
81+
print(f" + [{typ}] {name} {version}: + {size}")
82+
print()
83+
else:
84+
print("Added: None\n")
85+
86+
if diffs["removed"]:
87+
print("Removed:")
88+
for entry in diffs["removed"]:
89+
name = entry.get("Name", "")
90+
version = entry.get("Version", "")
91+
size = entry.get("Size", 0)
92+
typ = entry.get("Type", "")
93+
print(f" - [{typ}] {name} {version}: - {size}")
94+
print()
95+
else:
96+
print("Removed: None\n")
97+
98+
if diffs["changed"]:
99+
print("Changed:")
100+
for entry in diffs["changed"]:
101+
name = entry.get("Name", "")
102+
version = entry.get("Version", "")
103+
typ = entry.get("Type", "")
104+
prev_size = entry.get("prev_Size_Bytes", 0)
105+
# curr_size = entry.get("curr_Size_Bytes", 0)
106+
diff = entry.get("diff", 0)
107+
percentage = (diff / prev_size) * 100 if prev_size != 0 else 0
108+
sign = "+" if diff > 0 else "-"
109+
version_diff = (
110+
f"{entry.get('Prev Version', version)} -> {entry.get('Version', version)} "
111+
if entry.get('Prev Version', version) != entry.get('Version', version)
112+
else "version unchanged"
113+
)
114+
print(
115+
f" * [{typ}] {name} {version_diff}: "
116+
f"{sign} {convert_to_human_readable_size(diff)} ({sign}{percentage:.2f}%)"
117+
)
118+
print()
119+
else:
120+
print("Changed: None\n")
121+
print("=" * 60)
122+
123+
124+
def main():
125+
parser = argparse.ArgumentParser(prog='gha_diff', allow_abbrev=False)
126+
parser.add_argument('--prev-sizes', required=True)
127+
parser.add_argument('--curr-sizes', required=True)
128+
args = parser.parse_args()
129+
130+
with open(args.prev_sizes, "r") as f:
131+
prev_sizes = json.load(f)
132+
with open(args.curr_sizes, "r") as f:
133+
curr_sizes = json.load(f)
134+
135+
# prev_sizes = [
136+
# {
137+
# "Name": "cryptography",
138+
# "Version": "45.0.5",
139+
# "Size_Bytes": 21933835,
140+
# "Size": "20.92 MB",
141+
# "Type": "Dependency",
142+
# "Platform": "macos-x86_64",
143+
# "Python_Version": "3.12",
144+
# },
145+
# {
146+
# "Name": "PyYAML",
147+
# "Version": "6.0.1",
148+
# "Size_Bytes": 1234567,
149+
# "Size": "1.18 MB",
150+
# "Type": "Dependency",
151+
# "Platform": "macos-x86_64",
152+
# "Python_Version": "3.12",
153+
# },
154+
# {
155+
# "Name": "hola",
156+
# "Version": "3.9.10",
157+
# "Size_Bytes": 2345678,
158+
# "Size": "2.24 MB",
159+
# "Type": "Dependency",
160+
# },
161+
# ]
162+
163+
# curr_sizes = [
164+
# {
165+
# "Name": "cryptography",
166+
# "Version": "45.0.6",
167+
# "Size_Bytes": 22933835, # Increased size
168+
# "Size": "21.88 MB",
169+
# "Type": "Dependency",
170+
# "Platform": "macos-x86_64",
171+
# "Python_Version": "3.12",
172+
# },
173+
# {
174+
# "Name": "PyYAML",
175+
# "Version": "6.0.1",
176+
# "Size_Bytes": 1234567,
177+
# "Size": "1.18 MB",
178+
# "Type": "Dependency",
179+
# "Platform": "macos-x86_64",
180+
# "Python_Version": "3.12",
181+
# },
182+
# {
183+
# "Name": "orjson",
184+
# "Version": "3.9.10",
185+
# "Size_Bytes": 2345678,
186+
# "Size": "2.24 MB",
187+
# "Type": "Dependency",
188+
# "Platform": "macos-x86_64",
189+
# "Python_Version": "3.12",
190+
# },
191+
# ]
192+
193+
diffs = calculate_diffs(prev_sizes, curr_sizes)
194+
195+
display_diffs(diffs)
196+
197+
198+
if __name__ == "__main__":
199+
main()

0 commit comments

Comments
 (0)