IGNITE-27179 Fixed flaky the CommunicationConnectionPoolMetricsTest#testMetricsBasics test (#12538)

Thank you for submitting the pull request to the Apache Ignite.

In order to streamline the review of the contribution 
we ask you to ensure the following steps have been taken:

### The Contribution Checklist
- [ ] There is a single JIRA ticket related to the pull request. 
- [ ] The web-link to the pull request is attached to the JIRA ticket.
- [ ] The JIRA ticket has the _Patch Available_ state.
- [ ] The pull request body describes changes that have been made. 
The description explains _WHAT_ and _WHY_ was made instead of _HOW_.
- [ ] The pull request title is treated as the final commit message. 
The following pattern must be used: `IGNITE-XXXX Change summary` where
`XXXX` - number of JIRA issue.
- [ ] A reviewer has been mentioned through the JIRA comments 
(see [the Maintainers
list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers))
- [ ] The pull request has been checked by the Teamcity Bot and 
the `green visa` attached to the JIRA ticket (see [TC.Bot: Check
PR](https://mtcga.gridgain.com/prs.html))

### Notes
- [How to
Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute)
- [Coding abbreviation
rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules)
- [Coding
Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines)
- [Apache Ignite Teamcity
Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot)

If you need any help, please email [email protected] or ask anу
advice on http://asf.slack.com _#ignite_ channel.

Author: Vladsz83

docs/_docs/monitoring-metrics/custom-metrics.adoc

@@ -16,9 +16,9 @@
 
 WARNING: This feature is experimental and may change in future releases.
 
-Ignite provides various internal link:monitoring-metrics/new-metrics.adoc[metrics]. However, these metrics might
+Ignite provides various internal link:monitoring-metrics/new-metrics[metrics]. However, these metrics might
 not be enough. Users can design and publish their own custom metrics. Custom Metrics are based on
-link:monitoring-metrics/new-metrics-system.adoc[Metric System].
+link:monitoring-metrics/new-metrics-system[Metric System].
 
 [NOTE]
 ====

modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionClientPool.java

@@ -631,8 +631,7 @@ public void addNodeClient(ClusterNode node, int connIdx, GridCommunicationClient
                 curClients = clients.compute(node.id(), (nodeId0, clients0) -> {
                     if (clients0 == null) {
                         // Syncs metrics creation on this map.
-                        if (metricsMgr != null)
-                            createNodeMetrics(node);
+                        createNodeMetrics(node);
 
                         return newClients;
                     }
@@ -737,7 +736,7 @@ private NodeMetrics updatedNodeMetrics(UUID nodeId) {
                     return clients;
                 });
             }
-            else {
+            else if (res != null) {
                 removeNodeMetrics(nodeId);
 
                 res = null;

modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/CommunicationConnectionPoolMetricsTest.java

@@ -241,8 +241,8 @@ public void testMetricsBasics() throws Exception {
 
         Ignite ldr = clientLdr ? cli : srvr;
 
-        GridMetricManager metricsMgr = ((IgniteEx)ldr).context().metric();
-        MetricRegistryImpl mreg0 = metricsMgr.registry(ConnectionClientPool.SHARED_METRICS_REGISTRY_NAME);
+        GridMetricManager ldrMetricsMgr = ((IgniteEx)ldr).context().metric();
+        MetricRegistryImpl mreg0 = ldrMetricsMgr.registry(ConnectionClientPool.SHARED_METRICS_REGISTRY_NAME);
 
         assertEquals(connsPerNode, mreg0.<IntMetric>findMetric(ConnectionClientPool.METRIC_NAME_POOL_SIZE).value());
         assertEquals(pairedConns, mreg0.<BooleanGauge>findMetric(ConnectionClientPool.METRIC_NAME_PAIRED_CONNS).value());
@@ -271,7 +271,7 @@ public void testMetricsBasics() throws Exception {
 
             UUID nodeId = node.cluster().localNode().id();
 
-            MetricRegistryImpl mreg = metricsMgr.registry(nodeMetricsRegName(nodeId));
+            MetricRegistryImpl mreg = ldrMetricsMgr.registry(nodeMetricsRegName(nodeId));
 
             // We assume that entire pool was used at least once.
             assertTrue(waitForCondition(() -> connsPerNode == mreg.<IntMetric>findMetric(METRIC_NAME_CUR_CNT).value(),
@@ -295,6 +295,10 @@ public void testMetricsBasics() throws Exception {
 
         dumpMetrics(ldr);
 
+        // Stop the loading, free the messages queue to prevent the connection recreation attempts.
+        runFlag.set(false);
+        loadFut.get(getTestTimeout());
+
         // Check node metrics are cleared if a node stops.
         for (Ignite node : G.allGrids()) {
             if (node == ldr)
@@ -306,18 +310,24 @@ public void testMetricsBasics() throws Exception {
 
             UUID nodeId = node.cluster().localNode().id();
 
-            assertTrue(G.stop(node.name(), true));
+            // Wait until there is no messages to this node.
+            assertTrue(waitForCondition(() -> {
+                MetricRegistryImpl mreg = ldrMetricsMgr.registry(nodeMetricsRegName(nodeId));
+
+                assert mreg != null;
+
+                return mreg.<IntMetric>findMetric(METRIC_NAME_MSG_QUEUE_SIZE).value() == 0;
+            }, getTestTimeout(), 100));
+
+            assertTrue(G.stop(node.name(), false));
 
             assertTrue(waitForCondition(() -> {
-                MetricRegistryImpl mreg = metricsMgr.registry(nodeMetricsRegName(nodeId));
+                MetricRegistryImpl mreg = ldrMetricsMgr.registry(nodeMetricsRegName(nodeId));
 
                 return mreg == null || !mreg.iterator().hasNext();
             }, getTestTimeout()));
         }
 
-        runFlag.set(false);
-        loadFut.get(getTestTimeout());
-
         // Ensure that all the possible nodes are stopped.
         assertTrue(waitForCondition(() -> ldr.cluster().nodes().size() == (clientLdr ? 2 : 1), getTestTimeout()));
     }