[cru] Limit DMA links from 0 to 11 per endpoint

Author: kostorr

src/Cru/Constants.h

@@ -27,7 +27,7 @@ namespace Cru
 {
 
 /// Maximum amount of available links
-static constexpr int MAX_LINKS = 32;
+static constexpr int MAX_LINKS = 12;
 
 /// Amount of available superpage descriptors per link
 static constexpr int MAX_SUPERPAGE_DESCRIPTORS = 128;

src/Cru/CruBar.cxx

@@ -67,6 +67,8 @@ CruBar::CruBar(const Parameters& parameters, std::unique_ptr<RocPciDevice> rocPc
   } else {
     mAllowRejection = 0x0;
   }
+
+  mEndpoint = mRocPciDevice->getSerialId().getEndpoint();
 }
 
 CruBar::CruBar(std::shared_ptr<Pda::PdaBar> bar)
@@ -141,7 +143,6 @@ uint32_t CruBar::getSuperpageSize(uint32_t link)
   uint32_t superpageSizeIndex = Utilities::getBits(superpageSizeFifo, 24, 31); // [24-31] -> superpage index (0-255)
 
   while (superpageSizeIndex != mSuperpageSizeIndexCounter[link]) { // In case the PCIe bus wasn't fast enough
-    //std::cout << "[SP INDEX] link " << link << " : " << superpageSizeIndex << " instead of " << mSuperpageSizeIndexCounter[link] << std::endl;
     superpageSizeFifo = readRegister(Cru::Registers::LINK_SUPERPAGE_SIZE.get(link).index);
     superpageSize = Utilities::getBits(superpageSizeFifo, 0, 23);
     superpageSizeIndex = Utilities::getBits(superpageSizeFifo, 24, 31);
@@ -606,7 +607,10 @@ std::map<int, Link> CruBar::initializeLinkMap()
     uint32_t address = Cru::getWrapperBaseAddress(wrapper) + Cru::Registers::GBT_WRAPPER_CONF0.address;
     uint32_t wrapperConfig = readRegister(address / 4);
 
-    for (int bank = 0; bank < 6; bank++) {
+    for (int bank = mEndpoint * 2; bank < (mEndpoint * 2 + 2); bank++) {
+      //for (int bank = 0; bank < 4; bank++) { //for 0-23 range
+      // endpoint 0 -> banks {0,1}
+      // endpoint 1 -> banks {2,3}
       int dwrapper = (bank < 2) ? 0 : 1;
       int lpbLSB = (4 * bank) + 4;
       int lpbMSB = lpbLSB + 4 - 1;
@@ -638,7 +642,8 @@ std::map<int, Link> CruBar::initializeLinkMap()
   std::map<int, Link> newLinkMap;
   for (std::size_t i = 0; i < links.size(); i++) {
     Link link = links.at(i);
-    int newPos = (i - link.bank * 6) * 2 + 12 * (int)(link.bank / 2) + (link.bank % 2);
+    int newPos = (i - (link.bank - mEndpoint * 2) * 6) * 2 + (link.bank % 2);
+    //int newPos = (i - link.bank * 6) * 2 + + 12 * (int)(link.bank/2) + (link.bank % 2); //for 0-23 range
     link.dwrapperId = newPos % 12;
     newLinkMap.insert({ newPos, link });
   }

src/Cru/CruBar.h

@@ -136,6 +136,8 @@ class CruBar final : public BarInterfaceBase
   bool mDynamicOffset;
   uint32_t mTriggerWindowSize;
 
+  int mEndpoint;
+
   /// Per-link counter to verify superpage sizes received are valid
   uint32_t mSuperpageSizeIndexCounter[Cru::MAX_LINKS] = { 0 };
 

src/Cru/CruDmaChannel.cxx

@@ -77,7 +77,7 @@ CruDmaChannel::CruDmaChannel(const Parameters& parameters)
     mLinks.reserve(linkMask.size());
     for (uint32_t id : linkMask) {
       if (id >= Cru::MAX_LINKS) {
-        BOOST_THROW_EXCEPTION(InvalidLinkId() << ErrorInfo::Message("CRU does not support given link ID")
+        BOOST_THROW_EXCEPTION(InvalidLinkId() << ErrorInfo::Message("Each endpoint supports up to 12 links.")
                                               << ErrorInfo::LinkId(id));
       }
       stream << id << " ";

src/Parameters.cxx

@@ -180,6 +180,14 @@ auto Parameters::linkMaskFromString(const std::string& string) -> LinkMaskType
 {
   std::set<uint32_t> links;
 
+  auto checkLink = [](int link) {
+    if (!(link >= 0 && link < 12)) {
+      BOOST_THROW_EXCEPTION(
+        ParseException() << ErrorInfo::Message(std::string("Link: " + std::to_string(link) + " out of [0-11] range")));
+    }
+    return;
+  };
+
   try {
     // Separate by comma
     std::vector<std::string> commaSeparateds;
@@ -195,9 +203,12 @@ auto Parameters::linkMaskFromString(const std::string& string) -> LinkMaskType
         auto start = boost::lexical_cast<uint32_t>(dashSeparateds[0]);
         auto end = boost::lexical_cast<uint32_t>(dashSeparateds[1]);
         for (uint32_t i = start; i <= end; ++i) {
+          checkLink(i);
           links.insert(i);
         }
       } else {
+        uint32_t link = boost::lexical_cast<uint32_t>(commaSeparated);
+        checkLink(link);
         links.insert(boost::lexical_cast<uint32_t>(commaSeparated));
       }
     }

src/RocPciDevice.h

@@ -24,6 +24,7 @@
 #include "ReadoutCard/CardDescriptor.h"
 #include "ReadoutCard/Parameters.h"
 #include "ReadoutCard/ParameterTypes/PciAddress.h"
+#include "ReadoutCard/ParameterTypes/SerialId.h"
 #include "ReadoutCard/PciId.h"
 
 namespace AliceO2
@@ -64,9 +65,9 @@ class RocPciDevice
     return mPciDevice;
   }
 
-  int getSerialId() const //TODO: Could be used for logging?
+  SerialId getSerialId() const
   {
-    return mDescriptor.serialId.getSerial();
+    return mDescriptor.serialId;
   }
 
   std::shared_ptr<Pda::PdaBar> getBar(int barIndex)