Skip to content

Commit 59fe3ca

Browse files
PascalBoeschotenPascalBoeschoten
committed
RoC: Fix hugepage check for C-RORC ReadyFIFO DMA buffer
1 parent 70e5253 commit 59fe3ca

File tree

5 files changed

+136
-31
lines changed

5 files changed

+136
-31
lines changed

‎src/Crorc/CrorcDmaChannel.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ CrorcDmaChannel::CrorcDmaChannel(const Parameters& parameters)
5353
constexpr auto FIFO_SIZE = sizeof(ReadyFifo);
5454
Utilities::resetSmartPtr(mBufferFifoFile, getPaths().fifo(), FIFO_SIZE);
5555
Utilities::resetSmartPtr(mPdaDmaBufferFifo, getRocPciDevice().getPciDevice(), mBufferFifoFile->getAddress(),
56-
FIFO_SIZE, getPdaDmaBufferIndexFifo(getChannelNumber()));
56+
FIFO_SIZE, getPdaDmaBufferIndexFifo(getChannelNumber()), false);// note the 'false' at the end specifies non-hugepage memory
5757

5858
const auto& entry = mPdaDmaBufferFifo->getScatterGatherList().at(0);
5959
if (entry.size < FIFO_SIZE) {

‎src/Cru/DataFormat.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
//
2+
// Created by pboescho on 8/4/17.
3+
//
4+
5+
#ifndef READOUTCARD_DATAFORMAT_H
6+
#define READOUTCARD_DATAFORMAT_H
7+
8+
#endif //READOUTCARD_DATAFORMAT_H

‎src/Pda/PdaDmaBuffer.cxx‎

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ auto MUTEX_NAME = "AliceO2_roc_Pda_PdaDmaBuffer_Mutex";
1919
} // Anonymous namespace
2020

2121
PdaDmaBuffer::PdaDmaBuffer(PdaDevice::PdaPciDevice pciDevice, void* userBufferAddress, size_t userBufferSize,
22-
int dmaBufferId) : mPciDevice(pciDevice)
22+
int dmaBufferId, bool requireHugepage) : mPciDevice(pciDevice)
2323
{
2424
// Safeguard against PDA kernel module deadlocks
2525
Mutex mutex {boost::interprocess::open_or_create, MUTEX_NAME};
@@ -61,37 +61,42 @@ PdaDmaBuffer::PdaDmaBuffer(PdaDevice::PdaPciDevice pciDevice, void* userBufferAd
6161
" help, but ensure no channels are open before reinsertion (modprobe -r uio_pci_dma; modprobe uio_pci_dma"});
6262
}
6363

64-
DMABuffer_SGNode* sgList;
65-
if (DMABuffer_getSGList(mDmaBuffer, &sgList) != PDA_SUCCESS) {
66-
BOOST_THROW_EXCEPTION(PdaException() << ErrorInfo::Message("Failed to get scatter-gather list"));
67-
}
64+
try {
65+
DMABuffer_SGNode *sgList;
66+
if (DMABuffer_getSGList(mDmaBuffer, &sgList) != PDA_SUCCESS) {
67+
BOOST_THROW_EXCEPTION(PdaException() << ErrorInfo::Message("Failed to get scatter-gather list"));
68+
}
6869

69-
auto node = sgList;
70-
71-
while (node != nullptr) {
72-
// Check if scatter-gather list is not suspicious
73-
{
74-
auto hugePageMinSize = 1024 * 1024 * 2; // 2 MiB, the smallest hugepage size
75-
if (node->length < hugePageMinSize) {
76-
BOOST_THROW_EXCEPTION(
77-
Exception() << ErrorInfo::Message("Scatter-gather node smaller than 2 MiB (minimum hugepage"
78-
" size. This means the IOMMU is off and the buffer is not backed by hugepages - an unsupported buffer "
79-
"configuration."));
70+
auto node = sgList;
71+
while (node != nullptr) {
72+
if (requireHugepage) {
73+
size_t hugePageMinSize = 1024 * 1024 * 2; // 2 MiB, the smallest hugepage size
74+
printf("node->length=%lu\n", node->length);
75+
if (node->length < hugePageMinSize) {
76+
BOOST_THROW_EXCEPTION(
77+
PdaException() << ErrorInfo::Message("Scatter-gather node smaller than 2 MiB (minimum hugepage"
78+
" size. This means the IOMMU is off and the buffer is not backed by hugepages - an unsupported buffer "
79+
"configuration."));
80+
}
8081
}
81-
}
8282

83-
ScatterGatherEntry e;
84-
e.size = node->length;
85-
e.addressUser = reinterpret_cast<uintptr_t>(node->u_pointer);
86-
e.addressBus = reinterpret_cast<uintptr_t>(node->d_pointer);
87-
e.addressKernel = reinterpret_cast<uintptr_t>(node->k_pointer);
88-
mScatterGatherVector.push_back(e);
89-
node = node->next;
90-
}
83+
ScatterGatherEntry e;
84+
e.size = node->length;
85+
e.addressUser = reinterpret_cast<uintptr_t>(node->u_pointer);
86+
e.addressBus = reinterpret_cast<uintptr_t>(node->d_pointer);
87+
e.addressKernel = reinterpret_cast<uintptr_t>(node->k_pointer);
88+
mScatterGatherVector.push_back(e);
89+
node = node->next;
90+
}
9191

92-
if (mScatterGatherVector.empty()) {
93-
BOOST_THROW_EXCEPTION(PdaException() << ErrorInfo::Message(
92+
if (mScatterGatherVector.empty()) {
93+
BOOST_THROW_EXCEPTION(PdaException() << ErrorInfo::Message(
9494
"Failed to initialize scatter-gather list, was empty"));
95+
}
96+
}
97+
catch (const PdaException& ) {
98+
PciDevice_deleteDMABuffer(mPciDevice.get(), mDmaBuffer);
99+
throw;
95100
}
96101
}
97102

@@ -108,8 +113,6 @@ uintptr_t PdaDmaBuffer::getBusOffsetAddress(size_t offset) const
108113
{
109114
const auto& list = mScatterGatherVector;
110115

111-
// TODO shortcut for SGL size 1 (happens with small buffers, or when IOMMU is enabled)
112-
113116
auto userBase = list.at(0).addressUser;
114117
auto userWithOffset = userBase + offset;
115118

‎src/Pda/PdaDmaBuffer.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@ class PdaDmaBuffer
2323
/// \param userBufferAddress Address of the user-allocated buffer
2424
/// \param userBufferSize Size of the user-allocated buffer
2525
/// \param dmaBufferId Unique ID to use for registering the buffer (uniqueness must be card-wide)
26+
/// \param requireHugepage Require the buffer to have hugepage-sized scatter-gather list nodes
2627
PdaDmaBuffer(PdaDevice::PdaPciDevice pciDevice, void* userBufferAddress, size_t userBufferSize,
27-
int dmaBufferId);
28+
int dmaBufferId, bool requireHugepage = true);
2829

2930
~PdaDmaBuffer();
3031

‎test/TestCruDataFormat.cxx‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/// \file TestBarAcessor.cxx
2+
/// \brief Tests for BarAccessor
3+
///
4+
/// \author Pascal Boeschoten ([email protected])
5+
6+
#define BOOST_TEST_MODULE RORC_TestBarAccessor
7+
#define BOOST_TEST_MAIN
8+
#define BOOST_TEST_DYN_LINK
9+
#include <boost/test/unit_test.hpp>
10+
#include <assert.h>
11+
#include "ReadoutCard/BarInterface.h"
12+
#include "Cru/BarAccessor.h"
13+
14+
using namespace AliceO2::roc;
15+
16+
BOOST_AUTO_TEST_CASE(TestFirmwareFeatures)
17+
{
18+
{
19+
// Integrated firmware should have everything
20+
FirmwareFeatures f = Cru::BarAccessor::convertToFirmwareFeatures(0x40000000);
21+
BOOST_CHECK(!f.standalone);
22+
BOOST_CHECK(f.serial);
23+
BOOST_CHECK(f.dataSelection);
24+
BOOST_CHECK(f.temperature);
25+
BOOST_CHECK(f.firmwareInfo);
26+
}
27+
{
28+
// Standalone with everything enabled
29+
FirmwareFeatures f = Cru::BarAccessor::convertToFirmwareFeatures(0x40005AFE);
30+
BOOST_CHECK(f.standalone);
31+
BOOST_CHECK(f.serial);
32+
BOOST_CHECK(f.dataSelection);
33+
BOOST_CHECK(f.temperature);
34+
BOOST_CHECK(f.firmwareInfo);
35+
}
36+
{
37+
// Standalone with everything disabled
38+
FirmwareFeatures f = Cru::BarAccessor::convertToFirmwareFeatures(0x40005AFE + (0b1111 << 16));
39+
BOOST_CHECK(f.standalone);
40+
BOOST_CHECK(!f.serial);
41+
BOOST_CHECK(!f.dataSelection);
42+
BOOST_CHECK(!f.temperature);
43+
BOOST_CHECK(!f.firmwareInfo);
44+
}
45+
{
46+
// Standalone individual features disabled
47+
BOOST_CHECK(!Cru::BarAccessor::convertToFirmwareFeatures(0x40005AFE + (1 << 16)).dataSelection);
48+
BOOST_CHECK(!Cru::BarAccessor::convertToFirmwareFeatures(0x40005AFE + (1 << 17)).temperature);
49+
BOOST_CHECK(!Cru::BarAccessor::convertToFirmwareFeatures(0x40005AFE + (1 << 18)).serial);
50+
BOOST_CHECK(!Cru::BarAccessor::convertToFirmwareFeatures(0x40005AFE + (1 << 19)).firmwareInfo);
51+
}
52+
}
53+
54+
BOOST_AUTO_TEST_CASE(TestDataGeneratorConfiguration)
55+
{
56+
{
57+
uint32_t bits = 0;
58+
Cru::BarAccessor::setDataGeneratorEnableBits(bits, true);
59+
BOOST_CHECK(bits == 0x1);
60+
Cru::BarAccessor::setDataGeneratorEnableBits(bits, false);
61+
BOOST_CHECK(bits == 0x0);
62+
}
63+
{
64+
uint32_t bits = 0;
65+
Cru::BarAccessor::setDataGeneratorRandomSizeBits(bits, true);
66+
BOOST_CHECK(bits == (1 << 16));
67+
Cru::BarAccessor::setDataGeneratorRandomSizeBits(bits, false);
68+
BOOST_CHECK(bits != (1 << 16));
69+
}
70+
{
71+
uint32_t bits = 0;
72+
Cru::BarAccessor::setDataGeneratorEnableBits(bits, true);
73+
BOOST_CHECK(bits == 0x1);
74+
Cru::BarAccessor::setDataGeneratorPatternBits(bits, GeneratorPattern::Incremental);
75+
BOOST_CHECK(bits == 0x3);
76+
Cru::BarAccessor::setDataGeneratorSizeBits(bits, 8*1024);
77+
BOOST_CHECK(bits == 0xff03);
78+
}
79+
{
80+
uint32_t bits = 0;
81+
Cru::BarAccessor::setDataGeneratorEnableBits(bits, true);
82+
Cru::BarAccessor::setDataGeneratorPatternBits(bits, GeneratorPattern::Incremental);
83+
Cru::BarAccessor::setDataGeneratorSizeBits(bits, 32);
84+
BOOST_CHECK(bits == 0x0003);
85+
}
86+
{
87+
uint32_t bits = 0;
88+
// Too high value
89+
BOOST_CHECK_THROW(Cru::BarAccessor::setDataGeneratorSizeBits(bits, 8*1024 + 1), std::exception);
90+
// Not a multiple of 256 bits
91+
BOOST_CHECK_THROW(Cru::BarAccessor::setDataGeneratorSizeBits(bits, 257), std::exception);
92+
}
93+
}

0 commit comments

Comments
 (0)