Skip to content

Commit d91af35

Browse files
GabrielCWTGabrielCWT
authored
Add CPU and memory information to web console (#18613)
* Display CPU and memory information on web console * Add equality checks * Update to use new DiscoveryDruidNode constructor * Add documentation for getTotalMemory method * Update DiscoveryDruidNode tests * Update services view snapshot * Run autofix * Use JvmUtils instead * Update doc wording * Update .spelling * Update server field size in test * Fix based on comments * Update unknown value to be static * Update integration tests * Use Runtime to get available processors * Update unknown value to int
1 parent da8a96c commit d91af35

File tree

12 files changed

+227
-41
lines changed

12 files changed

+227
-41
lines changed

‎docs/querying/sql-metadata-tables.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,8 @@ Servers table lists all discovered servers in the cluster.
238238
|start_time|STRING|Timestamp in ISO8601 format when the server was announced in the cluster|
239239
|version|VARCHAR|Druid version running on the server|
240240
|labels|VARCHAR|Labels for the server configured using the property [`druid.labels`](../configuration/index.md)|
241+
|available_processors|BIGINT|Total number of CPU processors available to the server|
242+
|total_memory|BIGINT|Total memory in bytes available to the server|
241243

242244
To retrieve information about all servers, use the query:
243245

‎embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AbstractAuthConfigurationTest.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1018,9 +1018,8 @@ protected String getBrokerUrl()
10181018

10191019
/**
10201020
* curr_size on historicals changes because cluster state is not isolated across
1021-
* different
1022-
* integration tests, zero it out for consistent test results
1023-
* version and start_time are not configurable therefore we zero them as well
1021+
* different integration tests, zero it out for consistent test results.
1022+
* version, start_time, available_processors, total_memory are not configurable therefore we zero them as well
10241023
*/
10251024
protected static List<Map<String, Object>> getServersWithoutNonConfigurableFields(List<Map<String, Object>> servers)
10261025
{
@@ -1031,6 +1030,8 @@ protected static List<Map<String, Object>> getServersWithoutNonConfigurableField
10311030
newServer.put("curr_size", 0);
10321031
newServer.put("start_time", "0");
10331032
newServer.put("version", "0.0.0");
1033+
newServer.put("available_processors", 0);
1034+
newServer.put("total_memory", 0);
10341035
return newServer;
10351036
}
10361037
);

‎integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
"is_leader": null,
1212
"start_time": "0",
1313
"version": "0.0.0",
14-
"labels": null
14+
"labels": null,
15+
"available_processors": 0,
16+
"total_memory": 0
1517
},
1618
{
1719
"server": "%%BROKER%%:8282",
@@ -25,6 +27,8 @@
2527
"is_leader": null,
2628
"start_time": "0",
2729
"version": "0.0.0",
28-
"labels": null
30+
"labels": null,
31+
"available_processors": 0,
32+
"total_memory": 0
2933
}
3034
]

‎processing/src/main/java/org/apache/druid/utils/JvmUtils.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import com.google.common.primitives.Ints;
2323
import com.google.inject.Inject;
24+
import com.sun.management.OperatingSystemMXBean;
2425

2526
import java.io.File;
2627
import java.lang.management.ManagementFactory;
@@ -44,6 +45,7 @@ public class JvmUtils
4445
private static RuntimeInfo RUNTIME_INFO = new RuntimeInfo();
4546

4647
private static final ThreadMXBean THREAD_MX_BEAN = ManagementFactory.getThreadMXBean();
48+
private static final OperatingSystemMXBean OPERATING_SYSTEM_MX_BEAN = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
4749

4850
private static int computeMajorVersion()
4951
{
@@ -139,4 +141,15 @@ public static List<URL> systemClassPath()
139141
).collect(Collectors.toList());
140142
return jobURLs;
141143
}
144+
145+
/**
146+
* Get the total memory of the machine it is running on. This function is container aware.
147+
* If the machine is running in a container, the function will return the total memory of the container.
148+
* If the machine is not running in a container, the function will return the total memory of the machine.
149+
* @return the total memory of the machine it is running on in bytes.
150+
*/
151+
public static long getTotalMemory()
152+
{
153+
return OPERATING_SYSTEM_MX_BEAN.getTotalPhysicalMemorySize();
154+
}
142155
}

‎server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.fasterxml.jackson.annotation.JsonIgnore;
2525
import com.fasterxml.jackson.annotation.JsonProperty;
2626
import com.fasterxml.jackson.databind.ObjectMapper;
27+
import com.google.common.annotations.VisibleForTesting;
2728
import com.google.common.collect.Maps;
2829
import org.apache.druid.client.DruidServer;
2930
import org.apache.druid.jackson.StringObjectPairList;
@@ -32,6 +33,7 @@
3233
import org.apache.druid.java.util.common.NonnullPair;
3334
import org.apache.druid.java.util.common.logger.Logger;
3435
import org.apache.druid.server.DruidNode;
36+
import org.apache.druid.utils.JvmUtils;
3537
import org.joda.time.DateTime;
3638

3739
import javax.annotation.Nullable;
@@ -50,10 +52,13 @@
5052
public class DiscoveryDruidNode
5153
{
5254
private static final Logger LOG = new Logger(DiscoveryDruidNode.class);
55+
private static final int UNKNOWN_VALUE = -1;
5356

5457
private final DruidNode druidNode;
5558
private final NodeRole nodeRole;
5659
private final DateTime startTime;
60+
private final Integer availableProcessors;
61+
private final Long totalMemory;
5762

5863
/**
5964
* Map of service name -> DruidServices.
@@ -65,20 +70,36 @@ public class DiscoveryDruidNode
6570
*/
6671
private final Map<String, DruidService> services = new HashMap<>();
6772

73+
/**
74+
* Constructor for tests. In production, the @Inject constructor is used instead.
75+
*/
76+
@VisibleForTesting
77+
public DiscoveryDruidNode(
78+
DruidNode druidNode,
79+
NodeRole nodeRole,
80+
Map<String, DruidService> services,
81+
DateTime startTime
82+
)
83+
{
84+
this(druidNode, nodeRole, services, startTime, Runtime.getRuntime().availableProcessors(), JvmUtils.getTotalMemory());
85+
}
86+
6887
public DiscoveryDruidNode(
6988
DruidNode druidNode,
7089
NodeRole nodeRole,
7190
Map<String, DruidService> services
7291
)
7392
{
74-
this(druidNode, nodeRole, services, DateTimes.nowUtc());
93+
this(druidNode, nodeRole, services, DateTimes.nowUtc(), Runtime.getRuntime().availableProcessors(), JvmUtils.getTotalMemory());
7594
}
7695

7796
public DiscoveryDruidNode(
7897
DruidNode druidNode,
7998
NodeRole nodeRole,
8099
Map<String, DruidService> services,
81-
DateTime startTime
100+
DateTime startTime,
101+
Integer availableProcessors,
102+
Long totalMemory
82103
)
83104
{
84105
this.druidNode = druidNode;
@@ -88,6 +109,10 @@ public DiscoveryDruidNode(
88109
this.services.putAll(services);
89110
}
90111
this.startTime = startTime;
112+
113+
// Happens if service is running older version of Druid
114+
this.availableProcessors = availableProcessors != null ? availableProcessors : UNKNOWN_VALUE;
115+
this.totalMemory = totalMemory != null ? totalMemory : UNKNOWN_VALUE;
91116
}
92117

93118
@JsonCreator
@@ -96,6 +121,8 @@ private static DiscoveryDruidNode fromJson(
96121
@JsonProperty("nodeType") NodeRole nodeRole,
97122
@JsonProperty("services") Map<String, StringObjectPairList> rawServices,
98123
@JsonProperty("startTime") DateTime startTime,
124+
@JsonProperty("availableProcessors") Integer availableProcessors,
125+
@JsonProperty("totalMemory") Long totalMemory,
99126
@JacksonInject ObjectMapper jsonMapper
100127
)
101128
{
@@ -111,7 +138,7 @@ private static DiscoveryDruidNode fromJson(
111138
}
112139
}
113140
}
114-
return new DiscoveryDruidNode(druidNode, nodeRole, services, startTime);
141+
return new DiscoveryDruidNode(druidNode, nodeRole, services, startTime, availableProcessors, totalMemory);
115142
}
116143

117144
/**
@@ -188,6 +215,18 @@ public DateTime getStartTime()
188215
return startTime;
189216
}
190217

218+
@JsonProperty
219+
public Integer getAvailableProcessors()
220+
{
221+
return availableProcessors;
222+
}
223+
224+
@JsonProperty
225+
public Long getTotalMemory()
226+
{
227+
return totalMemory;
228+
}
229+
191230
@Nullable
192231
@JsonIgnore
193232
public <T extends DruidService> T getService(String key, Class<T> clazz)
@@ -235,13 +274,15 @@ public boolean equals(Object o)
235274
DiscoveryDruidNode that = (DiscoveryDruidNode) o;
236275
return Objects.equals(druidNode, that.druidNode) &&
237276
Objects.equals(nodeRole, that.nodeRole) &&
238-
Objects.equals(services, that.services);
277+
Objects.equals(services, that.services) &&
278+
Objects.equals(availableProcessors, that.availableProcessors) &&
279+
Objects.equals(totalMemory, that.totalMemory);
239280
}
240281

241282
@Override
242283
public int hashCode()
243284
{
244-
return Objects.hash(druidNode, nodeRole, services);
285+
return Objects.hash(druidNode, nodeRole, services, availableProcessors, totalMemory);
245286
}
246287

247288
@Override
@@ -252,6 +293,8 @@ public String toString()
252293
", nodeRole='" + nodeRole + '\'' +
253294
", services=" + services + '\'' +
254295
", startTime=" + startTime +
296+
", availableProcessors=" + availableProcessors +
297+
", totalMemory=" + totalMemory +
255298
'}';
256299
}
257300
}

‎server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public DiscoveryDruidNodeTest()
5858
public void testEquals()
5959
{
6060
EqualsVerifier.forClass(DiscoveryDruidNode.class)
61-
.withNonnullFields("druidNode", "nodeRole", "services")
61+
.withNonnullFields("druidNode", "nodeRole", "services", "availableProcessors", "totalMemory")
6262
.withIgnoredFields("startTime")
6363
.usingGetClass()
6464
.verify();
@@ -156,7 +156,9 @@ public void testDeserializeWithDataNodeServiceWithAWrongPropertyOrder() throws J
156156
+ " \"serverType\" : \"broker\",\n"
157157
+ " \"priority\" : 0\n"
158158
+ " }\n"
159-
+ " }\n"
159+
+ " },\n"
160+
+ " \"availableProcessors\" : 3,\n"
161+
+ " \"totalMemory\" : 1234\n"
160162
+ "}";
161163
Assert.assertEquals(
162164
new DiscoveryDruidNode(
@@ -175,7 +177,10 @@ public void testDeserializeWithDataNodeServiceWithAWrongPropertyOrder() throws J
175177
ImmutableMap.of(
176178
"dataNodeService",
177179
new DataNodeService("_default_tier", 1000000000, ServerType.BROKER, 0)
178-
)
180+
),
181+
null,
182+
3,
183+
1234L
179184
),
180185
mapper.readValue(json, DiscoveryDruidNode.class)
181186
);
@@ -206,7 +211,9 @@ public void testDeserialize_duplicateProperties_shouldSucceedToDeserialize() thr
206211
+ " \"serverType\" : \"broker\",\n"
207212
+ " \"priority\" : 0\n"
208213
+ " }\n"
209-
+ " }\n"
214+
+ " },\n"
215+
+ " \"availableProcessors\" : 6,\n"
216+
+ " \"totalMemory\" : 5432\n"
210217
+ "}";
211218
Assert.assertEquals(
212219
new DiscoveryDruidNode(
@@ -225,7 +232,10 @@ public void testDeserialize_duplicateProperties_shouldSucceedToDeserialize() thr
225232
ImmutableMap.of(
226233
"dataNodeService",
227234
new DataNodeService("_default_tier", 1000000000, ServerType.BROKER, 0)
228-
)
235+
),
236+
null,
237+
6,
238+
5432L
229239
),
230240
mapper.readValue(json, DiscoveryDruidNode.class)
231241
);
@@ -257,7 +267,9 @@ public void testDeserialize_duplicateKeysWithDifferentValus_shouldIgnoreDataNode
257267
+ " \"serverType\" : \"broker\",\n"
258268
+ " \"priority\" : 0\n"
259269
+ " }\n"
260-
+ " }\n"
270+
+ " },\n"
271+
+ " \"availableProcessors\" : 4,\n"
272+
+ " \"totalMemory\" : 246810\n"
261273
+ "}";
262274
Assert.assertEquals(
263275
new DiscoveryDruidNode(
@@ -273,7 +285,10 @@ public void testDeserialize_duplicateKeysWithDifferentValus_shouldIgnoreDataNode
273285
null
274286
),
275287
NodeRole.BROKER,
276-
ImmutableMap.of()
288+
ImmutableMap.of(),
289+
null,
290+
4,
291+
246810L
277292
),
278293
mapper.readValue(json, DiscoveryDruidNode.class)
279294
);

‎sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,8 @@ public class SystemSchema extends AbstractSchema
185185
.add("start_time", ColumnType.STRING)
186186
.add("version", ColumnType.STRING)
187187
.add("labels", ColumnType.STRING)
188+
.add("available_processors", ColumnType.LONG)
189+
.add("total_memory", ColumnType.LONG)
188190
.build();
189191

190192
static final RowSignature SERVER_SEGMENTS_SIGNATURE = RowSignature
@@ -653,7 +655,9 @@ private Object[] buildRowForNonDataServer(DiscoveryDruidNode discoveryDruidNode)
653655
null,
654656
toStringOrNull(discoveryDruidNode.getStartTime()),
655657
node.getVersion(),
656-
node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels())
658+
node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels()),
659+
(long) discoveryDruidNode.getAvailableProcessors(),
660+
discoveryDruidNode.getTotalMemory()
657661
};
658662
}
659663

@@ -678,7 +682,9 @@ private Object[] buildRowForNonDataServerWithLeadership(
678682
isLeader ? 1L : 0L,
679683
toStringOrNull(discoveryDruidNode.getStartTime()),
680684
node.getVersion(),
681-
node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels())
685+
node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels()),
686+
(long) discoveryDruidNode.getAvailableProcessors(),
687+
discoveryDruidNode.getTotalMemory()
682688
};
683689
}
684690

@@ -715,7 +721,9 @@ private Object[] buildRowForDiscoverableDataServer(
715721
null,
716722
toStringOrNull(discoveryDruidNode.getStartTime()),
717723
node.getVersion(),
718-
node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels())
724+
node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels()),
725+
(long) discoveryDruidNode.getAvailableProcessors(),
726+
discoveryDruidNode.getTotalMemory()
719727
};
720728
}
721729

0 commit comments

Comments
 (0)