Skip to content

Commit f744b57

Browse files
syuilosyuilo
authored
enhance(dev): improve mem report (#17117)
* wip * Update report-backend-memory.yml
1 parent 2b3d72b commit f744b57

File tree

2 files changed

+69
-80
lines changed

2 files changed

+69
-80
lines changed

‎.github/workflows/report-backend-memory.yml‎

Lines changed: 31 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -54,55 +54,50 @@ jobs:
5454
BASE_MEMORY=$(cat ./artifacts/memory-base.json)
5555
HEAD_MEMORY=$(cat ./artifacts/memory-head.json)
5656
57-
BASE_RSS=$(echo "$BASE_MEMORY" | jq -r '.memory.rss // 0')
58-
HEAD_RSS=$(echo "$HEAD_MEMORY" | jq -r '.memory.rss // 0')
57+
calc() {
58+
BASE=$(echo "$BASE_MEMORY" | jq -r '.memory.'"$1"' // 0')
59+
HEAD=$(echo "$HEAD_MEMORY" | jq -r '.memory.'"$1"' // 0')
5960
60-
# Calculate difference
61-
if [ "$BASE_RSS" -gt 0 ] && [ "$HEAD_RSS" -gt 0 ]; then
62-
DIFF=$((HEAD_RSS - BASE_RSS))
63-
DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE_RSS" | bc)
61+
DIFF=$((HEAD - BASE))
62+
if [ "$BASE" -gt 0 ]; then
63+
DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc)
64+
else
65+
DIFF_PERCENT=0
66+
fi
6467
65-
# Convert to MB for readability
66-
BASE_MB=$(echo "scale=2; $BASE_RSS / 1048576" | bc)
67-
HEAD_MB=$(echo "scale=2; $HEAD_RSS / 1048576" | bc)
68-
DIFF_MB=$(echo "scale=2; $DIFF / 1048576" | bc)
68+
# Convert KB to MB for readability
69+
BASE_MB=$(echo "scale=2; $BASE / 1024" | bc)
70+
HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc)
71+
DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc)
6972
70-
echo "base_mb=$BASE_MB" >> "$GITHUB_OUTPUT"
71-
echo "head_mb=$HEAD_MB" >> "$GITHUB_OUTPUT"
72-
echo "diff_mb=$DIFF_MB" >> "$GITHUB_OUTPUT"
73-
echo "diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT"
74-
echo "has_data=true" >> "$GITHUB_OUTPUT"
73+
echo "$1-base=$BASE_MB" >> "$GITHUB_OUTPUT"
74+
echo "$1-head=$HEAD_MB" >> "$GITHUB_OUTPUT"
75+
echo "$1-diff=$DIFF_MB" >> "$GITHUB_OUTPUT"
76+
echo "$1-diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT"
77+
}
7578
76-
# Determine if this is a significant change (more than 5% increase)
77-
if [ "$(echo "$DIFF_PERCENT > 5" | bc)" -eq 1 ]; then
78-
echo "significant_increase=true" >> "$GITHUB_OUTPUT"
79-
else
80-
echo "significant_increase=false" >> "$GITHUB_OUTPUT"
81-
fi
82-
else
83-
echo "has_data=false" >> "$GITHUB_OUTPUT"
84-
fi
79+
calc VmRSS
80+
calc VmHWM
81+
calc VmSize
8582
- id: build-comment
8683
name: Build memory comment
8784
run: |
88-
HEADER="## Backend Memory Usage Comparison"
85+
HEADER="## Backend memory usage comparison"
8986
FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
9087
9188
echo "$HEADER" > ./output.md
9289
echo >> ./output.md
9390
94-
if [ "${{ steps.compare.outputs.has_data }}" == "true" ]; then
95-
echo "| Metric | base | head | Diff |" >> ./output.md
96-
echo "|--------|------|------|------|" >> ./output.md
97-
echo "| RSS | ${{ steps.compare.outputs.base_mb }} MB | ${{ steps.compare.outputs.head_mb }} MB | ${{ steps.compare.outputs.diff_mb }} MB (${{ steps.compare.outputs.diff_percent }}%) |" >> ./output.md
98-
echo >> ./output.md
91+
echo "| Metric | base | head | Diff |" >> ./output.md
92+
echo "|--------|------|------|------|" >> ./output.md
93+
echo "| RSS | ${{ steps.compare.outputs.VmRSS-base }} MB | ${{ steps.compare.outputs.VmRSS-head }} MB | ${{ steps.compare.outputs.VmRSS-diff }} MB (${{ steps.compare.outputs.VmRSS-diff_percent }}%) |" >> ./output.md
94+
echo "| HWM | ${{ steps.compare.outputs.VmHWM-base }} MB | ${{ steps.compare.outputs.VmHWM-head }} MB | ${{ steps.compare.outputs.VmHWM-diff }} MB (${{ steps.compare.outputs.VmHWM-diff_percent }}%) |" >> ./output.md
95+
echo "| VMS | ${{ steps.compare.outputs.VmSize-base }} MB | ${{ steps.compare.outputs.VmSize-head }} MB | ${{ steps.compare.outputs.VmSize-diff }} MB (${{ steps.compare.outputs.VmSize-diff_percent }}%) |" >> ./output.md
96+
echo >> ./output.md
9997
100-
if [ "${{ steps.compare.outputs.significant_increase }}" == "true" ]; then
101-
echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
102-
echo >> ./output.md
103-
fi
104-
else
105-
echo "Could not retrieve memory usage data." >> ./output.md
98+
# Determine if this is a significant change (more than 5% increase)
99+
if [ "$(echo "${{ steps.compare.outputs.VmRSS-diff_percent }} > 5" | bc)" -eq 1 ]; then
100+
echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
106101
echo >> ./output.md
107102
fi
108103

‎packages/backend/scripts/measure-memory.mjs‎

Lines changed: 38 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import { fork } from 'node:child_process';
1414
import { setTimeout } from 'node:timers/promises';
1515
import { fileURLToPath } from 'node:url';
1616
import { dirname, join } from 'node:path';
17+
import * as fs from 'node:fs/promises';
1718

1819
const __filename = fileURLToPath(import.meta.url);
1920
const __dirname = dirname(__filename);
@@ -22,6 +23,35 @@ const SAMPLE_COUNT = 3; // Number of samples to measure
2223
const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup
2324
const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle
2425

26+
const keys = {
27+
VmPeak: 0,
28+
VmSize: 0,
29+
VmHWM: 0,
30+
VmRSS: 0,
31+
VmData: 0,
32+
VmStk: 0,
33+
VmExe: 0,
34+
VmLib: 0,
35+
VmPTE: 0,
36+
VmSwap: 0,
37+
};
38+
39+
async function getMemoryUsage(pid) {
40+
const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
41+
42+
const result = {};
43+
for (const key of Object.keys(keys)) {
44+
const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
45+
if (match) {
46+
result[key] = parseInt(match[1], 10);
47+
} else {
48+
throw new Error(`Failed to parse ${key} from /proc/${pid}/status`);
49+
}
50+
}
51+
52+
return result;
53+
}
54+
2555
async function measureMemory() {
2656
// Start the Misskey backend server using fork to enable IPC
2757
const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], {
@@ -76,39 +106,7 @@ async function measureMemory() {
76106

77107
// Get memory usage from the server process via /proc
78108
const pid = serverProcess.pid;
79-
let memoryInfo;
80-
81-
try {
82-
const fs = await import('node:fs/promises');
83-
84-
// Read /proc/[pid]/status for detailed memory info
85-
const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
86-
const vmRssMatch = status.match(/VmRSS:\s+(\d+)\s+kB/);
87-
const vmDataMatch = status.match(/VmData:\s+(\d+)\s+kB/);
88-
const vmSizeMatch = status.match(/VmSize:\s+(\d+)\s+kB/);
89-
90-
memoryInfo = {
91-
rss: vmRssMatch ? parseInt(vmRssMatch[1], 10) * 1024 : null,
92-
heapUsed: vmDataMatch ? parseInt(vmDataMatch[1], 10) * 1024 : null,
93-
vmSize: vmSizeMatch ? parseInt(vmSizeMatch[1], 10) * 1024 : null,
94-
};
95-
} catch (err) {
96-
// Fallback: use ps command
97-
process.stderr.write(`Warning: Could not read /proc/${pid}/status: ${err}\n`);
98-
99-
const { execSync } = await import('node:child_process');
100-
try {
101-
const ps = execSync(`ps -o rss= -p ${pid}`, { encoding: 'utf-8' });
102-
const rssKb = parseInt(ps.trim(), 10);
103-
memoryInfo = {
104-
rss: rssKb * 1024,
105-
heapUsed: null,
106-
vmSize: null,
107-
};
108-
} catch {
109-
throw new Error('Failed to get memory usage via ps command');
110-
}
111-
}
109+
const memoryInfo = await getMemoryUsage(pid);
112110

113111
// Stop the server
114112
serverProcess.kill('SIGTERM');
@@ -146,19 +144,15 @@ async function main() {
146144
}
147145

148146
// Calculate averages
149-
const avgMemory = {
150-
rss: 0,
151-
heapUsed: 0,
152-
vmSize: 0,
153-
};
147+
const avgMemory = structuredClone(keys);
154148
for (const res of results) {
155-
avgMemory.rss += res.memory.rss ?? 0;
156-
avgMemory.heapUsed += res.memory.heapUsed ?? 0;
157-
avgMemory.vmSize += res.memory.vmSize ?? 0;
149+
for (const key of Object.keys(avgMemory)) {
150+
avgMemory[key] += res.memory[key];
151+
}
152+
}
153+
for (const key of Object.keys(avgMemory)) {
154+
avgMemory[key] = Math.round(avgMemory[key] / SAMPLE_COUNT);
158155
}
159-
avgMemory.rss = Math.round(avgMemory.rss / SAMPLE_COUNT);
160-
avgMemory.heapUsed = Math.round(avgMemory.heapUsed / SAMPLE_COUNT);
161-
avgMemory.vmSize = Math.round(avgMemory.vmSize / SAMPLE_COUNT);
162156

163157
const result = {
164158
timestamp: new Date().toISOString(),

0 commit comments

Comments
 (0)