Skip to content

Commit 6835427

Browse files
kostorrkostorr
committed
[roc-pkt-mon] Add monitoring
1 parent 8dacae8 commit 6835427

File tree

5 files changed

+222
-25
lines changed

5 files changed

+222
-25
lines changed

‎README.md‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,50 @@ To directly send metrics to the Alice O2 Monitoring library, the argument `--mon
403403
| `tags::Key::ID` | ID of the card |
404404
| `tags::Key::Type` | Type of the card |
405405
406+
### roc-pkt-monitor
407+
Monitors packet statistics per link and per CRU wrapper. Output may be in an ASCII table (default) or in JSON (`json-out` option)
408+
format. Example outputs can be found [here](doc/examples/roc-pkt-monitor/).
409+
410+
Parameter information can be extracted from the monitoring table below.
411+
412+
#### Monitoring metrics
413+
414+
To directly send metrics to the Alice O2 Monitoring library, the argument `--monitoring` is necessary.
415+
416+
###### Metric: `"link"`
417+
418+
| Value name | Value | Type |
419+
| ---------------- | ------ | ------ |
420+
| `"pciAddress"` | - | string |
421+
| `"serial"` | - | int |
422+
| `"endpoint"` | - | int |
423+
| `"accepted"` | - | int |
424+
| `"rejected"` | - | int |
425+
| `"forced"` | - | int |
426+
427+
| Tag key | Value |
428+
| ------------------ | -------------------- |
429+
| `tags::Key::CRU` | ID of the CRU |
430+
| `tags::Key::ID` | ID of the link |
431+
| `tags::Key::Type` | `tags::Value::CRU` |
432+
433+
##### Metric: `"wrapper"`
434+
435+
| Value name | Value | Type |
436+
| -------------------------- | ------ | ------ |
437+
| `"pciAddress"` | - | string |
438+
| `"serial"` | - | int |
439+
| `"endpoint"` | - | int |
440+
| `"dropped"` | - | int |
441+
| `"totalPacketsPerSec"` | - | int |
442+
| `"forced"` | - | int |
443+
444+
| Tag key | Value |
445+
| ------------------ | -------------------- |
446+
| `tags::Key::CRU` | ID of the CRU |
447+
| `tags::Key::ID` | ID of the wrapper |
448+
| `tags::Key::Type` | `tags::Value::CRU` |
449+
406450
### roc-reg-[read, read-range, write]
407451
Writes and reads registers to/from a card's BAR.
408452
By convention, registers are 32-bit unsigned integers.

‎doc/examples/roc-pkt-monitor/roc-pkt-monitor-cru.json‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
{
2+
"gbtLinks": {
3+
"0": {
4+
"linkId": "0",
5+
"accepted": "1230564670",
6+
"rejected": "107289",
7+
"forced": "0"
8+
},
9+
"1": {
10+
"linkId": "1",
11+
"accepted": "1230564673",
12+
"rejected": "107289",
13+
"forced": "0"
14+
},
15+
"2": {
16+
"linkId": "2",
17+
"accepted": "1230564673",
18+
"rejected": "107289",
19+
"forced": "0"
20+
},
21+
"3": {
22+
"linkId": "3",
23+
"accepted": "0",
24+
"rejected": "0",
25+
"forced": "0"
26+
},
27+
"4": {
28+
"linkId": "4",
29+
"accepted": "0",
30+
"rejected": "0",
31+
"forced": "0"
32+
},
33+
"5": {
34+
"linkId": "5",
35+
"accepted": "0",
36+
"rejected": "0",
37+
"forced": "0"
38+
},
39+
"6": {
40+
"linkId": "6",
41+
"accepted": "0",
42+
"rejected": "0",
43+
"forced": "0"
44+
},
45+
"7": {
46+
"linkId": "7",
47+
"accepted": "0",
48+
"rejected": "0",
49+
"forced": "0"
50+
},
51+
"8": {
52+
"linkId": "8",
53+
"accepted": "0",
54+
"rejected": "0",
55+
"forced": "0"
56+
},
57+
"9": {
58+
"linkId": "9",
59+
"accepted": "0",
60+
"rejected": "0",
61+
"forced": "0"
62+
},
63+
"10": {
64+
"linkId": "10",
65+
"accepted": "0",
66+
"rejected": "0",
67+
"forced": "0"
68+
},
69+
"11": {
70+
"linkId": "11",
71+
"accepted": "0",
72+
"rejected": "0",
73+
"forced": "0"
74+
},
75+
"13": {
76+
"linkId": "13",
77+
"accepted": "0",
78+
"rejected": "0",
79+
"forced": "0"
80+
},
81+
"15": {
82+
"linkId": "15",
83+
"accepted": "0",
84+
"rejected": "0",
85+
"forced": "0"
86+
}
87+
},
88+
"wrapper": {
89+
"wrapperId": "0",
90+
"dropped": "3604563062",
91+
"totalPacketsPerSec": "304347"
92+
}
93+
}

‎doc/examples/roc-pkt-monitor/roc-pkt-monitor-cru.mon‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
2020-08-13 15:55:41 InfluxDB backend initialized
2+
link,hostname=flp-kostas.cern.ch,CRU=1,id=0,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=1231196187i,rejected=107289i,forced=0i 1597326941681864570
3+
link,hostname=flp-kostas.cern.ch,CRU=1,id=1,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=1231196188i,rejected=107289i,forced=0i 1597326941681962027
4+
link,hostname=flp-kostas.cern.ch,CRU=1,id=2,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=1231196188i,rejected=107289i,forced=0i 1597326941681973819
5+
link,hostname=flp-kostas.cern.ch,CRU=1,id=3,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941681982570
6+
link,hostname=flp-kostas.cern.ch,CRU=1,id=4,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941681990492
7+
link,hostname=flp-kostas.cern.ch,CRU=1,id=5,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941681998179
8+
link,hostname=flp-kostas.cern.ch,CRU=1,id=6,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941682005737
9+
link,hostname=flp-kostas.cern.ch,CRU=1,id=7,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941682013295
10+
link,hostname=flp-kostas.cern.ch,CRU=1,id=8,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941682020724
11+
link,hostname=flp-kostas.cern.ch,CRU=1,id=9,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941682028192
12+
link,hostname=flp-kostas.cern.ch,CRU=1,id=10,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941682035597
13+
link,hostname=flp-kostas.cern.ch,CRU=1,id=11,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941682043187
14+
link,hostname=flp-kostas.cern.ch,CRU=1,id=13,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941682050683
15+
link,hostname=flp-kostas.cern.ch,CRU=1,id=15,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,accepted=0i,rejected=0i,forced=0i 1597326941682058139
16+
wrapper,hostname=flp-kostas.cern.ch,CRU=1,id=0,type=CRU pciAddress="af:00.0",serial=243i,endpoint=0i,dropped=-688509689i,totalPacketsPerSec=304347i 1597326941682073479

‎doc/examples/roc-pkt-monitor/roc-pkt-monitor-cru.out‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
=======================================================
2+
Link ID Accepted Rejected Forced
3+
-------------------------------------------------------
4+
0 1229826148 107289 0
5+
1 1229826151 107289 0
6+
2 1229826151 107289 0
7+
3 0 0 0
8+
4 0 0 0
9+
5 0 0 0
10+
6 0 0 0
11+
7 0 0 0
12+
8 0 0 0
13+
9 0 0 0
14+
10 0 0 0
15+
11 0 0 0
16+
13 0 0 0
17+
15 0 0 0
18+
=======================================================
19+
Wrapper Dropped Total Packets per second
20+
-------------------------------------------------------
21+
0 3602347496 304350
22+
=======================================================

‎src/CommandLineUtilities/ProgramPacketMonitor.cxx‎

Lines changed: 47 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
#include <boost/property_tree/ptree.hpp>
2525
#include <boost/property_tree/json_parser.hpp>
2626

27+
#include "Monitoring/MonitoringFactory.h"
28+
using namespace o2::monitoring;
29+
2730
using namespace AliceO2::roc::CommandLineUtilities;
2831
using namespace AliceO2::roc;
2932
using namespace AliceO2::InfoLogger;
@@ -36,7 +39,9 @@ class ProgramPacketMonitor : public Program
3639
virtual Description getDescription()
3740
{
3841
return { "Packet Monitor", "Return RoC packet monitoring information",
39-
"roc-pkt-monitor --id 42:00.0\n" };
42+
"roc-pkt-monitor --id 42:00.0\n"
43+
"roc-pkt-monitor --id 42:00.0 --json\n"
44+
"roc-pkt-monitor --id 42:00.0 --monitoring\n" };
4045
}
4146

4247
virtual void addOptions(boost::program_options::options_description& options)
@@ -45,9 +50,9 @@ class ProgramPacketMonitor : public Program
4550
options.add_options()("json-out",
4651
po::bool_switch(&mOptions.jsonOut),
4752
"Toggle json-formatted output");
48-
options.add_options()("csv-out",
49-
po::bool_switch(&mOptions.csvOut),
50-
"Toggle csv-formatted output");
53+
options.add_options()("monitoring",
54+
po::bool_switch(&mOptions.monitoring),
55+
"Toggle monitoring metrics sending");
5156
}
5257

5358
virtual void run(const boost::program_options::variables_map& map)
@@ -56,8 +61,9 @@ class ProgramPacketMonitor : public Program
5661
auto cardId = Options::getOptionCardId(map);
5762
auto params = Parameters::makeParameters(cardId, 2); //status available on BAR2
5863
auto bar2 = ChannelFactory().getBar(params);
64+
auto card = RocPciDevice(cardId).getCardDescriptor();
65+
auto cardType = card.cardType;
5966

60-
CardType::type cardType = bar2->getCardType();
6167
if (cardType == CardType::type::Crorc) {
6268
std::cout << "CRORC packet monitoring not yet supported" << std::endl;
6369
return;
@@ -68,6 +74,12 @@ class ProgramPacketMonitor : public Program
6874

6975
auto cruBar2 = std::dynamic_pointer_cast<CruBar>(bar2);
7076

77+
// Monitoring instance to send metrics
78+
std::unique_ptr<Monitoring> monitoring;
79+
if (mOptions.monitoring) {
80+
monitoring = MonitoringFactory::Get(getMonitoringUri());
81+
}
82+
7183
Cru::PacketMonitoringInfo packetMonitoringInfo = cruBar2->monitorPackets();
7284

7385
/* HEADER */
@@ -78,10 +90,7 @@ class ProgramPacketMonitor : public Program
7890
auto lineFat = std::string(header.length(), '=') + '\n';
7991
auto lineThin = std::string(header.length(), '-') + '\n';
8092

81-
if (mOptions.csvOut) {
82-
auto csvHeader = "Link ID,Accepted,Rejected,Forced\n";
83-
std::cout << csvHeader;
84-
} else if (!mOptions.jsonOut) {
93+
if (!mOptions.jsonOut) {
8594
table << lineFat << header << lineThin;
8695
}
8796

@@ -97,7 +106,19 @@ class ProgramPacketMonitor : public Program
97106
uint32_t rejected = linkMonitoringInfoMap.rejected;
98107
uint32_t forced = linkMonitoringInfoMap.forced;
99108

100-
if (mOptions.jsonOut) {
109+
if (mOptions.monitoring) {
110+
monitoring->send(Metric{ "link" }
111+
.addValue(card.pciAddress.toString(), "pciAddress")
112+
.addValue(card.serialId.getSerial(), "serial")
113+
.addValue(card.serialId.getEndpoint(), "endpoint")
114+
.addValue((int)accepted, "accepted")
115+
.addValue((int)rejected, "rejected")
116+
.addValue((int)forced, "forced")
117+
.addTag(tags::Key::CRU, card.sequenceId)
118+
.addTag(tags::Key::ID, globalId)
119+
.addTag(tags::Key::Type, tags::Value::CRU));
120+
121+
} else if (mOptions.jsonOut) {
101122
pt::ptree linkNode;
102123

103124
// add kv pairs for this link
@@ -107,9 +128,6 @@ class ProgramPacketMonitor : public Program
107128
linkNode.put("forced", std::to_string(forced));
108129

109130
gbtLinks.add_child(std::to_string(globalId), linkNode);
110-
} else if (mOptions.csvOut) {
111-
auto csvLine = std::to_string(globalId) + "," + std::to_string(accepted) + "," + std::to_string(rejected) + "," + std::to_string(forced) + "\n";
112-
std::cout << csvLine;
113131
} else {
114132
auto format = boost::format(formatRow) % globalId % accepted % rejected % forced;
115133
table << format;
@@ -120,7 +138,7 @@ class ProgramPacketMonitor : public Program
120138
root.add_child("gbtLinks", gbtLinks);
121139

122140
/* PRINT */
123-
if (!mOptions.csvOut) {
141+
if (!mOptions.jsonOut && !mOptions.monitoring) {
124142
std::cout << table.str();
125143
}
126144

@@ -132,10 +150,7 @@ class ProgramPacketMonitor : public Program
132150
lineFat = std::string(header.length(), '=') + '\n';
133151
lineThin = std::string(header.length(), '-') + '\n';
134152

135-
if (mOptions.csvOut) {
136-
auto csvHeader = "Wrapper,Dropped,Total Packets per second\n";
137-
std::cout << csvHeader;
138-
} else {
153+
if (!mOptions.jsonOut && !mOptions.monitoring) {
139154
otherTable << lineFat << header << lineThin;
140155
}
141156

@@ -148,7 +163,17 @@ class ProgramPacketMonitor : public Program
148163
uint32_t dropped = wrapperMonitoringInfoMap.dropped;
149164
uint32_t totalPacketsPerSec = wrapperMonitoringInfoMap.totalPacketsPerSec;
150165

151-
if (mOptions.jsonOut) {
166+
if (mOptions.monitoring) {
167+
monitoring->send(Metric{ "wrapper" }
168+
.addValue(card.pciAddress.toString(), "pciAddress")
169+
.addValue(card.serialId.getSerial(), "serial")
170+
.addValue(card.serialId.getEndpoint(), "endpoint")
171+
.addValue((int)dropped, "dropped")
172+
.addValue((int)totalPacketsPerSec, "totalPacketsPerSec")
173+
.addTag(tags::Key::CRU, card.sequenceId)
174+
.addTag(tags::Key::ID, wrapper)
175+
.addTag(tags::Key::Type, tags::Value::CRU));
176+
} else if (mOptions.jsonOut) {
152177
pt::ptree wrapperNode;
153178

154179
// add kv pairs for this wrapper
@@ -159,10 +184,7 @@ class ProgramPacketMonitor : public Program
159184
// add the wrapper node to the tree
160185
root.add_child("wrapper", wrapperNode);
161186
}
162-
if (mOptions.csvOut) {
163-
auto csvLine = std::to_string(wrapper) + "," + std::to_string(dropped) + "," + std::to_string(totalPacketsPerSec) + "\n";
164-
std::cout << csvLine;
165-
} else {
187+
if (!mOptions.jsonOut && !mOptions.monitoring) {
166188
auto format = boost::format(formatRow) % wrapper % dropped % totalPacketsPerSec;
167189
otherTable << format;
168190
}
@@ -171,7 +193,7 @@ class ProgramPacketMonitor : public Program
171193
/* BREAK + PRINT */
172194
if (mOptions.jsonOut) {
173195
pt::write_json(std::cout, root);
174-
} else if (!mOptions.csvOut) {
196+
} else if (!mOptions.monitoring) {
175197
auto lineFat = std::string(header.length(), '=') + '\n';
176198
otherTable << lineFat;
177199
std::cout << otherTable.str();
@@ -181,7 +203,7 @@ class ProgramPacketMonitor : public Program
181203
private:
182204
struct OptionsStruct {
183205
bool jsonOut = false;
184-
bool csvOut = false;
206+
bool monitoring = false;
185207
} mOptions;
186208
};
187209

0 commit comments

Comments
 (0)