IBSocket: post small sends inline

src/common/net/ib/IBConnect.cc

@@ -89,6 +89,33 @@ monitor::CountRecorder acceptedFailed("common.ib.accept_failed");
 
 monitor::LatencyRecorder connectLatency("common.ib.connect_latency");
 monitor::LatencyRecorder acceptLatency("common.ib.accept_latency");
+
+// The verbs API cannot report the inline data limit. Devices whose kernel driver has a fixed limit try it first:
+// Intel irdma 216, 101 or 48 depending on the generation (101 on the E810, 48 on the X722;
+// drivers/infiniband/hw/irdma/ig3rdma_hw.h, user.h, i40iw_hw.h), Alibaba erdma 96
+// (drivers/infiniband/hw/erdma/erdma_verbs.h). Other devices start at the size asked for, and any refused size steps
+// down by kInlineDataStep until one is accepted. libfabric's verbs provider also probes the limit
+// (vrb_find_max_inline()).
+struct KnownInlineLimit {
+  uint32_t vendorId;
+  uint32_t maxInline;
+};
+constexpr KnownInlineLimit kKnownInlineLimits[] = {{0x8086, 216}, {0x8086, 101}, {0x8086, 48}, {0x1ded, 96}};
+constexpr uint32_t kInlineDataStep = 16;
+
+// Inline size to request: on the first try, size capped at the vendor's first known limit; after a refusal, the
+// vendor's next smaller known limit, else one step smaller.
+uint32_t inlineDataToRequest(uint32_t vendorId, uint32_t size, bool refused) {
+  for (const auto &known : kKnownInlineLimits) {
+    if (known.vendorId == vendorId && (!refused || known.maxInline < size)) {
+      return std::min(known.maxInline, size);
+    }
+  }
+  if (refused) {
+    size = size > kInlineDataStep ? size - kInlineDataStep : 0;
+  }
+  return size;
+}
 }  // namespace
 
 /* IBConnectService */
@@ -577,6 +604,11 @@ int IBSocket::qpCreate() {
     return -1;
   }
 
+  const uint32_t vendorId = device()->attr().vendor_id;
+  uint32_t inlineSize = device()->inlineDataRequest();
+  if (inlineSize == UINT32_MAX) {
+    inlineSize = inlineDataToRequest(vendorId, kBFMaxInlineSend, false);
+  }
   ibv_qp_init_attr attr;
   memset(&attr, 0, sizeof(attr));
   attr.qp_context = nullptr;
@@ -587,15 +619,29 @@ int IBSocket::qpCreate() {
   attr.cap.max_recv_wr = connectConfig_.qpMaxRecvWR();
   attr.cap.max_send_sge = connectConfig_.max_sge;
   attr.cap.max_recv_sge = 1;
-  attr.cap.max_inline_data = 0;
+  attr.cap.max_inline_data = inlineSize;
   attr.qp_type = IBV_QPT_RC;
   attr.sq_sig_all = 0;
 
   qp_.reset(ibv_create_qp(device()->pd(), &attr));
+  // A refused inline size fails with EINVAL; any other failure is reported below with its errno.
+  while (UNLIKELY(!qp_) && errno == EINVAL && inlineSize > 0) {
+    uint32_t next = inlineDataToRequest(vendorId, inlineSize, true);
+    XLOGF(DBG, "IBSocket {} ibv_create_qp refused {} bytes of inline data, trying {}", describe(), inlineSize, next);
+    inlineSize = next;
+    attr.cap.max_inline_data = inlineSize;
+    qp_.reset(ibv_create_qp(device()->pd(), &attr));
+  }
   if (UNLIKELY(!qp_)) {
     XLOGF(ERR, "IBSocket {} failed to create QP, errno {}", describe(), errno);
     return -1;
   }
+  device()->setInlineDataRequest(inlineSize);
+  // ibv_create_qp() writes the granted inline data size back into attr.
+  maxInlineData_ = attr.cap.max_inline_data;
+  if (vendorId == kMellanoxVendorId) {
+    maxInlineData_ = std::min(maxInlineData_, kBFMaxInlineSend);
+  }
 
   return initBufs();
 }

src/common/net/ib/IBDevice.h

@@ -136,6 +136,11 @@ class IBDevice : public std::enable_shared_from_this<IBDevice> {
   ibv_mr *regMemory(void *addr, size_t length, int access) const;
   int deregMemory(ibv_mr *mr) const;
 
+  // The inline size the last QP on this device was created with (UINT32_MAX: none yet). Later QPs start there, so a
+  // device that refuses the first size steps down once, not for every connection.
+  uint32_t inlineDataRequest() const { return inlineDataRequest_.load(std::memory_order_relaxed); }
+  void setInlineDataRequest(uint32_t size) const { inlineDataRequest_.store(size, std::memory_order_relaxed); }
+
  private:
   friend class IBManager;
   class BackgroundRunner;
@@ -167,6 +172,7 @@ class IBDevice : public std::enable_shared_from_this<IBDevice> {
   std::unique_ptr<ibv_pd, Deleter> pd_;
   ibv_device_attr attr_;
   std::map<uint8_t, Port> ports_;
+  mutable std::atomic<uint32_t> inlineDataRequest_ = UINT32_MAX;
 };
 
 class IBPort {

src/common/net/ib/IBSocket.cc

@@ -843,6 +843,11 @@ int IBSocket::postSend(uint32_t idx, size_t len, uint32_t flags) {
       .lkey = sendBufs_.getMr()->lkey,
   };
 
+  if (len <= maxInlineData_) {
+    // a small message goes in the WQE, so the NIC does not have to read it from memory
+    flags |= IBV_SEND_INLINE;
+  }
+
   uint32_t signal = 0;
   if (++sendNotSignaled_ >= connectConfig_.buf_signal_batch) {
     signal = sendNotSignaled_;

src/common/net/ib/IBSocket.h

@@ -232,6 +232,11 @@ class IBSocket : public Socket, folly::MoveOnly {
 
  private:
   static constexpr size_t kRDMAPostBatch = 8;
+  // Inline size asked for at QP creation, and on Mellanox/NVIDIA NICs the
+  // largest send posted inline: 256 - 16 (ctrl) - 4 (inline header), the most
+  // that fits the 256-byte BlueFlame buffer of mlx5 NICs.
+  static constexpr uint32_t kBFMaxInlineSend = 256 - 16 - 4;
+  static constexpr uint32_t kMellanoxVendorId = 0x02c9;
 
   enum class State {
     INIT,
@@ -538,6 +543,7 @@ class IBSocket : public Socket, folly::MoveOnly {
 
   // socket send
   SendBuffers sendBufs_;
+  uint32_t maxInlineData_ = 0;
   size_t sendNotSignaled_ = 0;
   size_t sendSignaled_ = 0;
   size_t sendAcked_ = 0;