From 125bcf16545271b405ce2a4c7905883aaf9e5d90 Mon Sep 17 00:00:00 2001 From: Andrei-Alexandru Bleortu <46914650+alxrxs@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:35:49 +0300 Subject: [PATCH 1/4] IBSocket: post small sends inline postSend() posts every send buffer by reference, so the NIC has to read a small RPC message from host memory before sending it. Request 236 bytes of inline data in qpCreate(), falling back to none if the device refuses, and set IBV_SEND_INLINE when a send fits the granted size. 236 bytes is the largest Send whose WQE fits the 256-byte BlueFlame buffer of current mlx5 NICs. Co-Authored-By: Claude Opus 5.5 --- src/common/net/ib/IBConnect.cc | 9 ++++++++- src/common/net/ib/IBSocket.cc | 5 +++++ src/common/net/ib/IBSocket.h | 5 +++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/common/net/ib/IBConnect.cc b/src/common/net/ib/IBConnect.cc index 0f63c423..8de71dd3 100644 --- a/src/common/net/ib/IBConnect.cc +++ b/src/common/net/ib/IBConnect.cc @@ -587,15 +587,22 @@ int IBSocket::qpCreate() { attr.cap.max_recv_wr = connectConfig_.qpMaxRecvWR(); attr.cap.max_send_sge = connectConfig_.max_sge; attr.cap.max_recv_sge = 1; - attr.cap.max_inline_data = 0; + attr.cap.max_inline_data = kBFMaxInlineSend; attr.qp_type = IBV_QPT_RC; attr.sq_sig_all = 0; qp_.reset(ibv_create_qp(device()->pd(), &attr)); + if (UNLIKELY(!qp_)) { + // the device may not support inline data, try again without it + attr.cap.max_inline_data = 0; + qp_.reset(ibv_create_qp(device()->pd(), &attr)); + } if (UNLIKELY(!qp_)) { XLOGF(ERR, "IBSocket {} failed to create QP, errno {}", describe(), errno); return -1; } + // ibv_create_qp() writes the granted inline data size back into attr. + maxInlineData_ = std::min(attr.cap.max_inline_data, kBFMaxInlineSend); return initBufs(); } diff --git a/src/common/net/ib/IBSocket.cc b/src/common/net/ib/IBSocket.cc index 28861571..41483824 100644 --- a/src/common/net/ib/IBSocket.cc +++ b/src/common/net/ib/IBSocket.cc @@ -843,6 +843,11 @@ int IBSocket::postSend(uint32_t idx, size_t len, uint32_t flags) { .lkey = sendBufs_.getMr()->lkey, }; + if (len <= maxInlineData_) { + // a small message goes in the WQE, so the NIC does not have to read it from memory + flags |= IBV_SEND_INLINE; + } + uint32_t signal = 0; if (++sendNotSignaled_ >= connectConfig_.buf_signal_batch) { signal = sendNotSignaled_; diff --git a/src/common/net/ib/IBSocket.h b/src/common/net/ib/IBSocket.h index 45ca2985..b7e986f1 100644 --- a/src/common/net/ib/IBSocket.h +++ b/src/common/net/ib/IBSocket.h @@ -232,6 +232,10 @@ class IBSocket : public Socket, folly::MoveOnly { private: static constexpr size_t kRDMAPostBatch = 8; + // Largest send posted inline, and the inline size asked for at QP creation: + // 256 - 16 (ctrl) - 4 (inline header), the most that fits the 256-byte + // BlueFlame buffer of current mlx5 NICs. + static constexpr uint32_t kBFMaxInlineSend = 256 - 16 - 4; enum class State { INIT, @@ -538,6 +542,7 @@ class IBSocket : public Socket, folly::MoveOnly { // socket send SendBuffers sendBufs_; + uint32_t maxInlineData_ = 0; size_t sendNotSignaled_ = 0; size_t sendSignaled_ = 0; size_t sendAcked_ = 0; From cf9f23740e929f50ee12983c49647c8b6a57cf76 Mon Sep 17 00:00:00 2001 From: Andrei-Alexandru Bleortu <46914650+alxrxs@users.noreply.github.com> Date: Sun, 4 Oct 2026 00:30:07 +0300 Subject: [PATCH 2/4] IBSocket: refine the inline size limit and the QP creation fallback The 236-byte limit keeps an inlined Send within the 256-byte BlueFlame buffer of mlx5 NICs. Apply it only on Mellanox/NVIDIA devices (vendor ID 0x02c9); on other devices the inline size the QP was granted is the limit. If QP creation with 236 bytes of inline data fails, try 64 bytes before falling back to none. irdma (Intel E810) rejects requests above 101 bytes, so on those NICs nothing was inlined. Co-Authored-By: Claude Opus 5.5 --- src/common/net/ib/IBConnect.cc | 10 +++++++++- src/common/net/ib/IBSocket.h | 7 ++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/common/net/ib/IBConnect.cc b/src/common/net/ib/IBConnect.cc index 8de71dd3..3b3bcf76 100644 --- a/src/common/net/ib/IBConnect.cc +++ b/src/common/net/ib/IBConnect.cc @@ -592,6 +592,11 @@ int IBSocket::qpCreate() { attr.sq_sig_all = 0; qp_.reset(ibv_create_qp(device()->pd(), &attr)); + if (UNLIKELY(!qp_)) { + // some devices take less inline data (irdma: 101 bytes), try 64 + attr.cap.max_inline_data = 64; + qp_.reset(ibv_create_qp(device()->pd(), &attr)); + } if (UNLIKELY(!qp_)) { // the device may not support inline data, try again without it attr.cap.max_inline_data = 0; @@ -602,7 +607,10 @@ int IBSocket::qpCreate() { return -1; } // ibv_create_qp() writes the granted inline data size back into attr. - maxInlineData_ = std::min(attr.cap.max_inline_data, kBFMaxInlineSend); + maxInlineData_ = attr.cap.max_inline_data; + if (device()->attr().vendor_id == kMellanoxVendorId) { + maxInlineData_ = std::min(maxInlineData_, kBFMaxInlineSend); + } return initBufs(); } diff --git a/src/common/net/ib/IBSocket.h b/src/common/net/ib/IBSocket.h index b7e986f1..7da554b6 100644 --- a/src/common/net/ib/IBSocket.h +++ b/src/common/net/ib/IBSocket.h @@ -232,10 +232,11 @@ class IBSocket : public Socket, folly::MoveOnly { private: static constexpr size_t kRDMAPostBatch = 8; - // Largest send posted inline, and the inline size asked for at QP creation: - // 256 - 16 (ctrl) - 4 (inline header), the most that fits the 256-byte - // BlueFlame buffer of current mlx5 NICs. + // Inline size asked for at QP creation, and on Mellanox/NVIDIA NICs the + // largest send posted inline: 256 - 16 (ctrl) - 4 (inline header), the most + // that fits the 256-byte BlueFlame buffer of mlx5 NICs. static constexpr uint32_t kBFMaxInlineSend = 256 - 16 - 4; + static constexpr uint32_t kMellanoxVendorId = 0x02c9; enum class State { INIT, From f8c36ec483e43cc9f0623d6f86a974d57dd943c5 Mon Sep 17 00:00:00 2001 From: Andrei-Alexandru Bleortu <46914650+alxrxs@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:52:56 +0300 Subject: [PATCH 3/4] IBSocket: try known inline limits, then step down After a refused 236-byte inline request, QP creation was retried with 64 bytes, then with none. Devices take less than 236 by different amounts: Intel's irdma refuses more than 216, 101 or 48 bytes depending on the generation (101 on an E810, 48 on an X722), Alibaba's erdma more than 96. With 64 as the only middle step, an E810 or erdma QP got 64 bytes, a newer irdma QP 64 instead of 216, and an X722 QP none. The verbs API cannot report the limit, so step down to it. A device whose kernel driver has a fixed limit, found by the vendor ID IBDevice already holds, starts there: 216, then 101, then 48 on Intel, 96 on Alibaba, each capped at 236. Other devices start at 236. Each refused size is followed by the next one 16 bytes smaller, down to 0, so a failure unrelated to inline data still ends as before, after a few more attempts at QP creation. Devices that accept 236 (mlx5, rxe) are unchanged. libfabric's verbs provider also probes the limit by trial QP creation (vrb_find_max_inline()). Co-Authored-By: Claude Opus 5.5 --- src/common/net/ib/IBConnect.cc | 44 ++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/common/net/ib/IBConnect.cc b/src/common/net/ib/IBConnect.cc index 3b3bcf76..fed0c6f6 100644 --- a/src/common/net/ib/IBConnect.cc +++ b/src/common/net/ib/IBConnect.cc @@ -89,6 +89,33 @@ monitor::CountRecorder acceptedFailed("common.ib.accept_failed"); monitor::LatencyRecorder connectLatency("common.ib.connect_latency"); monitor::LatencyRecorder acceptLatency("common.ib.accept_latency"); + +// The verbs API cannot report the inline data limit. Devices whose kernel driver has a fixed limit try it first: +// Intel irdma 216, 101 or 48 depending on the generation (101 on the E810, 48 on the X722; +// drivers/infiniband/hw/irdma/ig3rdma_hw.h, user.h, i40iw_hw.h), Alibaba erdma 96 +// (drivers/infiniband/hw/erdma/erdma_verbs.h). Other devices start at the size asked for, and any refused size steps +// down by kInlineDataStep until one is accepted. libfabric's verbs provider also probes the limit +// (vrb_find_max_inline()). +struct KnownInlineLimit { + uint32_t vendorId; + uint32_t maxInline; +}; +constexpr KnownInlineLimit kKnownInlineLimits[] = {{0x8086, 216}, {0x8086, 101}, {0x8086, 48}, {0x1ded, 96}}; +constexpr uint32_t kInlineDataStep = 16; + +// Inline size to request: on the first try, size capped at the vendor's first known limit; after a refusal, the +// vendor's next smaller known limit, else one step smaller. +uint32_t inlineDataToRequest(uint32_t vendorId, uint32_t size, bool refused) { + for (const auto &known : kKnownInlineLimits) { + if (known.vendorId == vendorId && (!refused || known.maxInline < size)) { + return std::min(known.maxInline, size); + } + } + if (refused) { + size = size > kInlineDataStep ? size - kInlineDataStep : 0; + } + return size; +} } // namespace /* IBConnectService */ @@ -577,6 +604,8 @@ int IBSocket::qpCreate() { return -1; } + const uint32_t vendorId = device()->attr().vendor_id; + uint32_t inlineSize = inlineDataToRequest(vendorId, kBFMaxInlineSend, false); ibv_qp_init_attr attr; memset(&attr, 0, sizeof(attr)); attr.qp_context = nullptr; @@ -587,19 +616,14 @@ int IBSocket::qpCreate() { attr.cap.max_recv_wr = connectConfig_.qpMaxRecvWR(); attr.cap.max_send_sge = connectConfig_.max_sge; attr.cap.max_recv_sge = 1; - attr.cap.max_inline_data = kBFMaxInlineSend; + attr.cap.max_inline_data = inlineSize; attr.qp_type = IBV_QPT_RC; attr.sq_sig_all = 0; qp_.reset(ibv_create_qp(device()->pd(), &attr)); - if (UNLIKELY(!qp_)) { - // some devices take less inline data (irdma: 101 bytes), try 64 - attr.cap.max_inline_data = 64; - qp_.reset(ibv_create_qp(device()->pd(), &attr)); - } - if (UNLIKELY(!qp_)) { - // the device may not support inline data, try again without it - attr.cap.max_inline_data = 0; + while (UNLIKELY(!qp_) && inlineSize > 0) { + inlineSize = inlineDataToRequest(vendorId, inlineSize, true); + attr.cap.max_inline_data = inlineSize; qp_.reset(ibv_create_qp(device()->pd(), &attr)); } if (UNLIKELY(!qp_)) { @@ -608,7 +632,7 @@ int IBSocket::qpCreate() { } // ibv_create_qp() writes the granted inline data size back into attr. maxInlineData_ = attr.cap.max_inline_data; - if (device()->attr().vendor_id == kMellanoxVendorId) { + if (vendorId == kMellanoxVendorId) { maxInlineData_ = std::min(maxInlineData_, kBFMaxInlineSend); } From 2fdfef555c2330e8ce629a172e92386e15bde0e4 Mon Sep 17 00:00:00 2001 From: Andrei-Alexandru Bleortu <46914650+alxrxs@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:38:39 +0300 Subject: [PATCH 4/4] IBSocket: step down only on EINVAL and reuse each device's inline size A refused inline size makes ibv_create_qp() fail with EINVAL. Any other failure (ENOMEM, for example) is now reported at once with its own errno, not retried at smaller inline sizes. Each step down is logged at DBG. Each IBDevice also remembers the inline size its last QP was created with, and new QPs on that device start there. A device that refuses the first size (an Intel device limited to 48 bytes goes 216, 101, 48) then steps down once rather than on every connection. If the remembered size is refused, the step-down continues from it. The size is kept per device because a host can have NICs from different vendors. Co-Authored-By: Claude Opus 5.5 --- src/common/net/ib/IBConnect.cc | 13 ++++++++++--- src/common/net/ib/IBDevice.h | 6 ++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/common/net/ib/IBConnect.cc b/src/common/net/ib/IBConnect.cc index fed0c6f6..11cef05b 100644 --- a/src/common/net/ib/IBConnect.cc +++ b/src/common/net/ib/IBConnect.cc @@ -605,7 +605,10 @@ int IBSocket::qpCreate() { } const uint32_t vendorId = device()->attr().vendor_id; - uint32_t inlineSize = inlineDataToRequest(vendorId, kBFMaxInlineSend, false); + uint32_t inlineSize = device()->inlineDataRequest(); + if (inlineSize == UINT32_MAX) { + inlineSize = inlineDataToRequest(vendorId, kBFMaxInlineSend, false); + } ibv_qp_init_attr attr; memset(&attr, 0, sizeof(attr)); attr.qp_context = nullptr; @@ -621,8 +624,11 @@ int IBSocket::qpCreate() { attr.sq_sig_all = 0; qp_.reset(ibv_create_qp(device()->pd(), &attr)); - while (UNLIKELY(!qp_) && inlineSize > 0) { - inlineSize = inlineDataToRequest(vendorId, inlineSize, true); + // A refused inline size fails with EINVAL; any other failure is reported below with its errno. + while (UNLIKELY(!qp_) && errno == EINVAL && inlineSize > 0) { + uint32_t next = inlineDataToRequest(vendorId, inlineSize, true); + XLOGF(DBG, "IBSocket {} ibv_create_qp refused {} bytes of inline data, trying {}", describe(), inlineSize, next); + inlineSize = next; attr.cap.max_inline_data = inlineSize; qp_.reset(ibv_create_qp(device()->pd(), &attr)); } @@ -630,6 +636,7 @@ int IBSocket::qpCreate() { XLOGF(ERR, "IBSocket {} failed to create QP, errno {}", describe(), errno); return -1; } + device()->setInlineDataRequest(inlineSize); // ibv_create_qp() writes the granted inline data size back into attr. maxInlineData_ = attr.cap.max_inline_data; if (vendorId == kMellanoxVendorId) { diff --git a/src/common/net/ib/IBDevice.h b/src/common/net/ib/IBDevice.h index 26b488c0..32d1f6f9 100644 --- a/src/common/net/ib/IBDevice.h +++ b/src/common/net/ib/IBDevice.h @@ -136,6 +136,11 @@ class IBDevice : public std::enable_shared_from_this { ibv_mr *regMemory(void *addr, size_t length, int access) const; int deregMemory(ibv_mr *mr) const; + // The inline size the last QP on this device was created with (UINT32_MAX: none yet). Later QPs start there, so a + // device that refuses the first size steps down once, not for every connection. + uint32_t inlineDataRequest() const { return inlineDataRequest_.load(std::memory_order_relaxed); } + void setInlineDataRequest(uint32_t size) const { inlineDataRequest_.store(size, std::memory_order_relaxed); } + private: friend class IBManager; class BackgroundRunner; @@ -167,6 +172,7 @@ class IBDevice : public std::enable_shared_from_this { std::unique_ptr pd_; ibv_device_attr attr_; std::map ports_; + mutable std::atomic inlineDataRequest_ = UINT32_MAX; }; class IBPort {