Skip to content

Commit daa5e5f

Browse files
alexandraBaraalexandraBara
committed
added numactl -H call + datamodel update for lsmem output parsing
1 parent 95d6ac9 commit daa5e5f

File tree

3 files changed

+451
-59
lines changed

3 files changed

+451
-59
lines changed

‎nodescraper/plugins/inband/memory/memory_collector.py‎

Lines changed: 190 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,15 @@
3030
from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily
3131
from nodescraper.models import TaskResult
3232

33-
from .memorydata import MemoryDataModel
33+
from .memorydata import (
34+
LsmemData,
35+
MemoryBlock,
36+
MemoryDataModel,
37+
MemorySummary,
38+
NumaDistance,
39+
NumaNode,
40+
NumaTopology,
41+
)
3442

3543

3644
class MemoryCollector(InBandDataCollector[MemoryDataModel, None]):
@@ -42,7 +50,8 @@ class MemoryCollector(InBandDataCollector[MemoryDataModel, None]):
4250
"wmic OS get FreePhysicalMemory /Value; wmic ComputerSystem get TotalPhysicalMemory /Value"
4351
)
4452
CMD = "free -b"
45-
CMD_LSMEM = "/usr/bin/lsmem"
53+
CMD_LSMEM = "lsmem"
54+
CMD_NUMACTL = "numactl -H"
4655

4756
def collect_data(self, args=None) -> tuple[TaskResult, Optional[MemoryDataModel]]:
4857
"""
@@ -84,12 +93,23 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[MemoryDataModel]
8493
lsmem_cmd = self._run_sut_cmd(self.CMD_LSMEM)
8594
if lsmem_cmd.exit_code == 0:
8695
lsmem_data = self._parse_lsmem_output(lsmem_cmd.stdout)
87-
self._log_event(
88-
category=EventCategory.OS,
89-
description="lsmem output collected",
90-
data=lsmem_data,
91-
priority=EventPriority.INFO,
92-
)
96+
if lsmem_data:
97+
self._log_event(
98+
category=EventCategory.OS,
99+
description="lsmem output collected",
100+
data={
101+
"memory_blocks": len(lsmem_data.memory_blocks),
102+
"total_online_memory": lsmem_data.summary.total_online_memory,
103+
},
104+
priority=EventPriority.INFO,
105+
)
106+
else:
107+
self._log_event(
108+
category=EventCategory.OS,
109+
description="Failed to parse lsmem output",
110+
priority=EventPriority.WARNING,
111+
console_log=False,
112+
)
93113
else:
94114
self._log_event(
95115
category=EventCategory.OS,
@@ -103,9 +123,48 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[MemoryDataModel]
103123
console_log=False,
104124
)
105125

126+
# Collect NUMA topology information
127+
numa_topology = None
128+
if self.system_info.os_family != OSFamily.WINDOWS:
129+
numactl_cmd = self._run_sut_cmd(self.CMD_NUMACTL)
130+
if numactl_cmd.exit_code == 0:
131+
numa_topology = self._parse_numactl_hardware(numactl_cmd.stdout)
132+
if numa_topology:
133+
self._log_event(
134+
category=EventCategory.MEMORY,
135+
description="NUMA topology collected",
136+
data={
137+
"available_nodes": numa_topology.available_nodes,
138+
"node_count": len(numa_topology.nodes),
139+
},
140+
priority=EventPriority.INFO,
141+
)
142+
else:
143+
self._log_event(
144+
category=EventCategory.MEMORY,
145+
description="Failed to parse numactl output",
146+
priority=EventPriority.WARNING,
147+
console_log=False,
148+
)
149+
else:
150+
self._log_event(
151+
category=EventCategory.MEMORY,
152+
description="Error running numactl command",
153+
data={
154+
"command": numactl_cmd.command,
155+
"exit_code": numactl_cmd.exit_code,
156+
"stderr": numactl_cmd.stderr,
157+
},
158+
priority=EventPriority.WARNING,
159+
console_log=False,
160+
)
161+
106162
if mem_free and mem_total:
107163
mem_data = MemoryDataModel(
108-
mem_free=mem_free, mem_total=mem_total, lsmem_output=lsmem_data
164+
mem_free=mem_free,
165+
mem_total=mem_total,
166+
lsmem_data=lsmem_data,
167+
numa_topology=numa_topology,
109168
)
110169
self._log_event(
111170
category=EventCategory.OS,
@@ -122,19 +181,19 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[MemoryDataModel]
122181

123182
return self.result, mem_data
124183

125-
def _parse_lsmem_output(self, output: str) -> dict:
184+
def _parse_lsmem_output(self, output: str):
126185
"""
127-
Parse lsmem command output into a structured dictionary.
186+
Parse lsmem command output into a structured LsmemData object.
128187
129188
Args:
130189
output: Raw stdout from lsmem command
131190
132191
Returns:
133-
dict: Parsed lsmem data with memory blocks and summary information
192+
LsmemData: Parsed lsmem data with memory blocks and summary information
134193
"""
135194
lines = output.strip().split("\n")
136195
memory_blocks = []
137-
summary = {}
196+
summary_dict = {}
138197

139198
for line in lines:
140199
line = line.strip()
@@ -146,21 +205,126 @@ def _parse_lsmem_output(self, output: str) -> dict:
146205
parts = line.split()
147206
if len(parts) >= 4:
148207
memory_blocks.append(
149-
{
150-
"range": parts[0],
151-
"size": parts[1],
152-
"state": parts[2],
153-
"removable": parts[3] if len(parts) > 3 else None,
154-
"block": parts[4] if len(parts) > 4 else None,
155-
}
208+
MemoryBlock(
209+
range=parts[0],
210+
size=parts[1],
211+
state=parts[2],
212+
removable=parts[3] if len(parts) > 3 else None,
213+
block=parts[4] if len(parts) > 4 else None,
214+
)
156215
)
157216
# Parse summary lines
158217
elif ":" in line:
159218
key, value = line.split(":", 1)
160-
summary[key.strip().lower().replace(" ", "_")] = value.strip()
219+
summary_dict[key.strip().lower().replace(" ", "_")] = value.strip()
220+
221+
summary = MemorySummary(
222+
memory_block_size=summary_dict.get("memory_block_size"),
223+
total_online_memory=summary_dict.get("total_online_memory"),
224+
total_offline_memory=summary_dict.get("total_offline_memory"),
225+
)
226+
227+
if not memory_blocks:
228+
return None
229+
230+
return LsmemData(memory_blocks=memory_blocks, summary=summary)
231+
232+
def _parse_numactl_hardware(self, output: str):
233+
"""
234+
Parse 'numactl -H' output into NumaTopology structure.
235+
236+
Args:
237+
output: Raw stdout from numactl -H command
238+
239+
Returns:
240+
NumaTopology object or None if parsing fails
241+
"""
242+
lines = output.strip().split("\n")
243+
available_nodes = []
244+
nodes = []
245+
distances = []
246+
distance_matrix = {}
247+
248+
current_section = None
249+
250+
for line in lines:
251+
line = line.strip()
252+
if not line:
253+
continue
254+
255+
# Parse available nodes line
256+
if line.startswith("available:"):
257+
match = re.search(r"available:\s*(\d+)\s+nodes?\s*\(([^)]+)\)", line)
258+
if match:
259+
node_range = match.group(2)
260+
if "-" in node_range:
261+
start, end = node_range.split("-")
262+
available_nodes = list(range(int(start), int(end) + 1))
263+
else:
264+
available_nodes = [int(x.strip()) for x in node_range.split()]
265+
266+
# Parse node CPU line
267+
elif line.startswith("node") and "cpus:" in line:
268+
match = re.search(r"node\s+(\d+)\s+cpus:\s*(.+)", line)
269+
if match:
270+
node_id = int(match.group(1))
271+
cpu_list_str = match.group(2).strip()
272+
if cpu_list_str:
273+
cpus = [int(x) for x in cpu_list_str.split()]
274+
else:
275+
cpus = []
276+
nodes.append(NumaNode(node_id=node_id, cpus=cpus))
277+
278+
# Parse node memory size
279+
elif line.startswith("node") and "size:" in line:
280+
match = re.search(r"node\s+(\d+)\s+size:\s*(\d+)\s*MB", line)
281+
if match:
282+
node_id = int(match.group(1))
283+
size_mb = int(match.group(2))
284+
# Find existing node and update
285+
for node in nodes:
286+
if node.node_id == node_id:
287+
node.memory_size_mb = size_mb
288+
break
289+
290+
# Parse node free memory
291+
elif line.startswith("node") and "free:" in line:
292+
match = re.search(r"node\s+(\d+)\s+free:\s*(\d+)\s*MB", line)
293+
if match:
294+
node_id = int(match.group(1))
295+
free_mb = int(match.group(2))
296+
# Find existing node and update
297+
for node in nodes:
298+
if node.node_id == node_id:
299+
node.memory_free_mb = free_mb
300+
break
301+
302+
# Parse distance matrix
303+
elif line.startswith("node distances:"):
304+
current_section = "distances"
305+
306+
elif current_section == "distances":
307+
if line.startswith("node") and ":" not in line:
308+
continue
309+
elif ":" in line:
310+
parts = line.split(":")
311+
if len(parts) == 2:
312+
from_node = int(parts[0].strip())
313+
dist_values = [int(x) for x in parts[1].split()]
314+
315+
distance_matrix[from_node] = {}
316+
for to_node, dist in enumerate(dist_values):
317+
distance_matrix[from_node][to_node] = dist
318+
distances.append(
319+
NumaDistance(from_node=from_node, to_node=to_node, distance=dist)
320+
)
321+
322+
if not nodes:
323+
return None
161324

162-
return {
163-
"raw_output": output,
164-
"memory_blocks": memory_blocks,
165-
"summary": summary,
166-
}
325+
return NumaTopology(
326+
available_nodes=available_nodes if available_nodes else [n.node_id for n in nodes],
327+
nodes=nodes,
328+
distances=distances,
329+
distance_matrix=distance_matrix if distance_matrix else None,
330+
)

‎nodescraper/plugins/inband/memory/memorydata.py‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,66 @@
2525
###############################################################################
2626
from typing import Optional
2727

28+
from pydantic import BaseModel
29+
2830
from nodescraper.models import DataModel
2931

3032

33+
class MemoryBlock(BaseModel):
34+
"""Memory block information from lsmem"""
35+
36+
range: str
37+
size: str
38+
state: str
39+
removable: Optional[str] = None
40+
block: Optional[str] = None
41+
42+
43+
class MemorySummary(BaseModel):
44+
"""Summary information from lsmem"""
45+
46+
memory_block_size: Optional[str] = None
47+
total_online_memory: Optional[str] = None
48+
total_offline_memory: Optional[str] = None
49+
50+
51+
class LsmemData(BaseModel):
52+
"""Complete lsmem output data"""
53+
54+
memory_blocks: list[MemoryBlock]
55+
summary: MemorySummary
56+
57+
58+
class NumaNode(BaseModel):
59+
"""NUMA node information"""
60+
61+
node_id: int
62+
cpus: list[int]
63+
memory_size_mb: Optional[int] = None
64+
memory_free_mb: Optional[int] = None
65+
66+
67+
class NumaDistance(BaseModel):
68+
"""Distance between two NUMA nodes"""
69+
70+
from_node: int
71+
to_node: int
72+
distance: int
73+
74+
75+
class NumaTopology(BaseModel):
76+
"""Complete NUMA topology from 'numactl --hardware'"""
77+
78+
available_nodes: list[int]
79+
nodes: list[NumaNode]
80+
distances: list[NumaDistance]
81+
distance_matrix: Optional[dict[int, dict[int, int]]] = None
82+
83+
3184
class MemoryDataModel(DataModel):
85+
"""Memory data model"""
86+
3287
mem_free: str
3388
mem_total: str
34-
lsmem_output: Optional[dict] = None
89+
lsmem_data: Optional[LsmemData] = None
90+
numa_topology: Optional[NumaTopology] = None

0 commit comments

Comments
 (0)