Skip to content

Commit 99c49da

Browse files
pippohubpippohub
authored
Merge pull request #7 from AliceO2Group/ORC-69/stress_bar
Add SWT support and a tool to stress CRU bar accesses
2 parents fbf6482 + 9d5817f commit 99c49da

File tree

5 files changed

+392
-1
lines changed

5 files changed

+392
-1
lines changed

‎CMakeLists.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ if(ALICEO2_READOUTCARD_PDA_ENABLED)
119119
src/Pda/PdaDevice.cxx
120120
src/Pda/PdaDmaBuffer.cxx
121121
src/RocPciDevice.cxx
122+
src/Swt/Swt.cxx
122123
)
123124
if(PYTHONLIBS_FOUND)
124125
list(APPEND SRCS
@@ -147,6 +148,7 @@ build_util_exec(roc-reset CommandLineUtilities/ProgramReset.cxx)
147148
build_util_exec(roc-reg-read CommandLineUtilities/ProgramRegisterRead.cxx)
148149
build_util_exec(roc-reg-read-range CommandLineUtilities/ProgramRegisterReadRange)
149150
build_util_exec(roc-reg-write CommandLineUtilities/ProgramRegisterWrite.cxx)
151+
build_util_exec(roc-bar-stress CommandLineUtilities/ProgramBarStress.cxx)
150152

151153
if(ALICEO2_READOUTCARD_PDA_ENABLED)
152154
build_util_exec(roc-channel-cleanup CommandLineUtilities/ProgramCleanup.cxx)
@@ -224,4 +226,4 @@ install(
224226
FILES src/roc-setup-hugetlbfs.sh
225227
DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
226228
PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
227-
RENAME roc-setup-hugetlbfs)
229+
RENAME roc-setup-hugetlbfs)

‎src/CommandLineUtilities/ProgramBarStress.cxx‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
/// \file ProgramBarStress.cxx
2+
/// \brief Tool that stress the Bar Accessor
3+
///
4+
/// \author Kostas Alexopoulos ([email protected])
5+
6+
#include <iostream>
7+
#include <iomanip>
8+
#include <cstdint>
9+
#include <chrono>
10+
#include <thread>
11+
#include <cstddef>
12+
#include "Cru/Constants.h"
13+
#include "ReadoutCard/ChannelFactory.h"
14+
#include "Swt/Swt.h"
15+
#include "CommandLineUtilities/Options.h"
16+
#include "CommandLineUtilities/Program.h"
17+
18+
using namespace AliceO2::roc::CommandLineUtilities;
19+
using namespace AliceO2::roc;
20+
using namespace AliceO2::InfoLogger;
21+
namespace po = boost::program_options;
22+
23+
class ProgramBarStress: public Program
24+
{
25+
public:
26+
27+
virtual Description getDescription()
28+
{
29+
return {"Bar Stress", "Stress the Bar Accessor",
30+
"roc-bar-stress --pci-address 42:00.0 --channel 0 --cycles 100000 --print-freq 10000 --errorcheck"};
31+
}
32+
33+
virtual void addOptions(boost::program_options::options_description& options)
34+
{
35+
options.add_options()
36+
("pci-address",
37+
po::value<std::string>(&mOptions.pciAddress)->default_value("-1"),
38+
"Card's PCI Address")
39+
("gbt-link",
40+
po::value<uint32_t>(&mOptions.gbtLink)->default_value(0),
41+
"GBT link over which the bar writes will be performed. CRU is 0-17")
42+
("cycles",
43+
po::value<long long>(&mOptions.cycles)->default_value(100),
44+
"Cycles of SWT writes(/reads) to perform")
45+
("print-freq",
46+
po::value<long long>(&mOptions.printFrequency)->default_value(10),
47+
"Print every #print-freq cycles")
48+
("errorcheck",
49+
po::bool_switch(&mOptions.errorCheck),
50+
"Perform data validation");
51+
}
52+
53+
int stress(Swt *swt, long long cycles, long long printFrequency, bool errorCheck)
54+
{
55+
56+
SwtWord swtWordWr = SwtWord(0x0, 0x0, 0x0);
57+
SwtWord swtWordRd = SwtWord(0x0, 0x0, 0x0);
58+
59+
/* swtWordCheck for testing */
60+
// SwtWord swtWordCheck = SwtWord(0x42, 0x42, 0x42);
61+
62+
std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
63+
std::chrono::high_resolution_clock::time_point finish = std::chrono::high_resolution_clock::now();
64+
65+
for (long long i=0;; i++){
66+
swtWordWr.setLow((0x1 + i)%0xffffffff);
67+
swtWordWr.setMed((0x2 + i)%0xffffffff);
68+
swtWordWr.setHigh((0x3 + i)%0xffff);
69+
70+
uint32_t mon = swt->write(swtWordWr);
71+
if (isVerbose())
72+
getLogger() << "WR MON: 0x" << std::setfill('0') << std::hex << mon << InfoLogger::endm;
73+
74+
if (errorCheck){
75+
uint32_t mon = swt->read(swtWordRd);
76+
if (swtWordRd != swtWordWr){
77+
getLogger() << "SWT validation failed" << InfoLogger::endm;
78+
getLogger() << "Read: " << swtWordRd << " | Expected: " << swtWordWr << InfoLogger::endm;
79+
return -1;
80+
}
81+
82+
if (isVerbose()){
83+
getLogger() << "RD MON: 0x" << std::setfill('0') << std::hex << mon << InfoLogger::endm;
84+
getLogger() << "Read swtWord: " << swtWordRd << InfoLogger::endm;
85+
}
86+
}
87+
88+
if (i && (i%printFrequency == 0)){
89+
finish = std::chrono::high_resolution_clock::now();
90+
getLogger() << "loops [" << i-printFrequency+1 << " - " << i << "]: " <<
91+
std::chrono::duration_cast<std::chrono::nanoseconds> (finish - start).count() << "ns" << InfoLogger::endm;
92+
93+
if (i == cycles || isSigInt()){ //sigInt only stops at cycles = printFrequency * X
94+
double throughput = (barOps /(std::chrono::duration_cast<std::chrono::nanoseconds> (finish - start).count() * 1e-9)); //throughput (ops/time) [ops/sec]
95+
getLogger() << "Throughput :" << throughput << " ops/sec" << InfoLogger::endm;
96+
97+
double latency = ((std::chrono::duration_cast<std::chrono::nanoseconds> (finish - start).count() * 1e-9) / barOps); // latency (time/ops) [sec]
98+
getLogger() << "Operation latency: " << latency << " sec " << InfoLogger::endm;
99+
return i;
100+
}
101+
102+
start = std::chrono::high_resolution_clock::now();
103+
}
104+
}
105+
}
106+
107+
virtual void run(const boost::program_options::variables_map& map)
108+
{
109+
110+
getLogger() << "PCI Address: " << mOptions.pciAddress << InfoLogger::endm;
111+
getLogger() << "GBT Link: " << mOptions.gbtLink << InfoLogger::endm;
112+
getLogger() << "Cycles of SWT write(/read) operations: " << mOptions.cycles << InfoLogger::endm;
113+
getLogger() << "Print frequency: " << mOptions.printFrequency << InfoLogger::endm;
114+
getLogger() << "Error Check enabled: " << mOptions.errorCheck << InfoLogger::endm;
115+
116+
/* bar(Writes | Reads | Ops) every #printFrequency cycles */
117+
barWrites = (SWT_WRITE_BAR_WRITES + SWT_READ_BAR_WRITES*mOptions.errorCheck)*mOptions.printFrequency;
118+
barReads = (SWT_WRITE_BAR_READS + SWT_READ_BAR_READS*mOptions.errorCheck)*mOptions.printFrequency;
119+
barOps = barWrites + barReads;
120+
121+
getLogger() << "Logging time every " << barOps << " bar operations, of which:" << InfoLogger::endm;
122+
getLogger() << "barWrites: " << barWrites << " | barReads: " << barReads << InfoLogger::endm;
123+
124+
125+
std::shared_ptr<BarInterface>
126+
bar0 = ChannelFactory().getBar(mOptions.pciAddress, 0);
127+
std::shared_ptr<BarInterface>
128+
bar2 = ChannelFactory().getBar(mOptions.pciAddress, 2);
129+
130+
if(isVerbose())
131+
getLogger() << "Resetting card..." << InfoLogger::endm;
132+
bar0->writeRegister(Cru::Registers::RESET_CONTROL.index, 0x1);
133+
134+
if(isVerbose())
135+
getLogger() << "Initializing SWT..." << InfoLogger::endm;
136+
auto swt = Swt(*bar2, mOptions.gbtLink);
137+
138+
if(isVerbose())
139+
getLogger() << "Running operations..." << InfoLogger::endm;
140+
141+
auto start = std::chrono::high_resolution_clock::now();
142+
long long cycles_run = stress(&swt, mOptions.cycles, mOptions.printFrequency, mOptions.errorCheck);
143+
auto finish = std::chrono::high_resolution_clock::now();
144+
145+
if (!cycles_run)
146+
getLogger() << "Execution terminated because of error..." << InfoLogger::endm;
147+
148+
getLogger() << "Total duration: " << std::chrono::duration_cast<std::chrono::seconds> (finish-start).count() << "s" << InfoLogger::endm;
149+
getLogger() << "Total bar operations: " << barOps * (cycles_run/mOptions.printFrequency) << InfoLogger::endm;
150+
getLogger() << "Total bar writes " << barWrites * (cycles_run/mOptions.printFrequency) << InfoLogger::endm;
151+
getLogger() << "Total bar reads: " << barReads * (cycles_run/mOptions.printFrequency) << InfoLogger::endm;
152+
}
153+
154+
struct OptionsStruct
155+
{
156+
std::string pciAddress = "-1";
157+
uint32_t gbtLink = 0;
158+
long long cycles = 100;
159+
long long printFrequency = 10;
160+
bool errorCheck = true;
161+
}mOptions;
162+
163+
long long barOps = 0;
164+
long long barWrites = 0;
165+
long long barReads = 0;
166+
167+
private:
168+
};
169+
170+
int main(int argc, char** argv)
171+
{
172+
return ProgramBarStress().execute(argc, argv);
173+
}

‎src/Swt/Swt.cxx‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/// \file Swt.cxx
2+
/// \brief Definition of SWT operations
3+
///
4+
/// \author Kostas Alexopoulos ([email protected])
5+
6+
#include "Swt/Swt.h"
7+
#include <iostream>
8+
#include <chrono>
9+
#include <thread>
10+
11+
namespace AliceO2 {
12+
namespace roc {
13+
14+
namespace Registers {
15+
constexpr uint32_t SWT_BASE_INDEX = 0x0f00000 / 4;
16+
17+
constexpr uint32_t SWT_WR_WORD_L = 0x40 / 4;
18+
constexpr uint32_t SWT_WR_WORD_M = 0x44 / 4;
19+
constexpr uint32_t SWT_WR_WORD_H = 0x48 / 4;
20+
constexpr uint32_t SWT_WR_CMD = 0x4c / 4;
21+
22+
constexpr uint32_t SWT_RD_WORD_L = 0x50 / 4;
23+
constexpr uint32_t SWT_RD_WORD_M = 0x54 / 4;
24+
constexpr uint32_t SWT_RD_WORD_H = 0x58 / 4;
25+
constexpr uint32_t SWT_RD_CMD = 0x4c / 4;
26+
27+
constexpr uint32_t SWT_RD_WORD_MON = 0x5c / 4;
28+
29+
constexpr uint32_t SWT_SET_CHANNEL = 0x60 / 4;
30+
constexpr uint32_t SWT_RESET_CORE = 0x64 / 4;
31+
32+
} //namespace Registers
33+
34+
Swt::Swt(RegisterReadWriteInterface &bar2, int gbtChannel) : mBar2(bar2)
35+
{
36+
reset();
37+
setChannel(gbtChannel);
38+
}
39+
40+
void Swt::setChannel(int gbtChannel)
41+
{
42+
barWrite(Registers::SWT_SET_CHANNEL, gbtChannel);
43+
}
44+
45+
void Swt::reset()
46+
{
47+
barWrite(Registers::SWT_RESET_CORE, 0x1);
48+
barWrite(Registers::SWT_RESET_CORE, 0x0); //void cmd to sync clocks
49+
}
50+
51+
uint32_t Swt::write(SwtWord& swtWord)
52+
{
53+
// prep the swt word
54+
barWrite(Registers::SWT_WR_WORD_L, swtWord.getLow());
55+
barWrite(Registers::SWT_WR_WORD_M, swtWord.getMed());
56+
barWrite(Registers::SWT_WR_WORD_H, swtWord.getHigh());
57+
58+
// perform write
59+
barWrite(Registers::SWT_WR_CMD, 0x1);
60+
barWrite(Registers::SWT_WR_CMD, 0x0); //void cmd to sync clocks
61+
62+
return barRead(Registers::SWT_RD_WORD_MON);
63+
}
64+
65+
uint32_t Swt::read(SwtWord& swtWord)
66+
{
67+
barWrite(Registers::SWT_RD_CMD, 0x2);
68+
barWrite(Registers::SWT_RD_CMD, 0x0); // void cmd to sync clocks
69+
70+
swtWord.setLow(barRead(Registers::SWT_RD_WORD_L));
71+
swtWord.setMed(barRead(Registers::SWT_RD_WORD_M));
72+
swtWord.setHigh(barRead(Registers::SWT_RD_WORD_H));
73+
74+
//return 0x0;
75+
return barRead(Registers::SWT_RD_WORD_MON);
76+
}
77+
78+
void Swt::barWrite(uint32_t offset, uint32_t data)
79+
{
80+
mBar2.writeRegister(Registers::SWT_BASE_INDEX + offset, data);
81+
}
82+
83+
uint32_t Swt::barRead(uint32_t offset)
84+
{
85+
uint32_t read = mBar2.readRegister(Registers::SWT_BASE_INDEX + offset);
86+
return read;
87+
}
88+
89+
} //namespace roc
90+
} //namespace AliceO2

‎src/Swt/Swt.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/// \file Swt.h
2+
/// \brief Definition of SWT operations
3+
///
4+
/// \author Kostas Alexopoulos ([email protected])
5+
6+
#ifndef ALICEO2_READOUTCARD_UTILITIES_SWT_H
7+
#define ALICEO2_READOUTCARD_UTILITIES_SWT_H
8+
9+
#include <string>
10+
#include "ReadoutCard/RegisterReadWriteInterface.h"
11+
#include "Swt/SwtWord.h"
12+
13+
namespace AliceO2 {
14+
namespace roc {
15+
16+
#define SWT_WRITE_BAR_WRITES 5
17+
#define SWT_WRITE_BAR_READS 1
18+
#define SWT_READ_BAR_WRITES 2
19+
#define SWT_READ_BAR_READS 4
20+
21+
/// Class for Single Word Transactions with the CRU
22+
class Swt
23+
{
24+
public:
25+
Swt(RegisterReadWriteInterface& bar2, int gbtChannel);
26+
27+
void reset();
28+
uint32_t write(SwtWord& swtWord);
29+
uint32_t read(SwtWord& swtWord);
30+
31+
private:
32+
void setChannel(int gbtChannel);
33+
void barWrite(uint32_t offset, uint32_t data);
34+
uint32_t barRead(uint32_t index);
35+
36+
RegisterReadWriteInterface& mBar2;
37+
};
38+
39+
} // namespace roc
40+
} // namespace AliceO2
41+
42+
#endif // ALICEO2_READOUTCARD_UTILITIES_SWT_H

0 commit comments

Comments
 (0)