From 06ae5b490a442c0d5dff938071fd96d6eba4e62b Mon Sep 17 00:00:00 2001
From: ToLiveAndLove <59090766+ToLiveAndLove@users.noreply.github.com>
Date: Wed, 16 Sep 2026 11:42:34 +0800
Subject: [PATCH] Honor configured IB port when querying RoCE GID

---
 src/meson.build          | 32 +++++++++++++++-
 src/rdma.cpp             |  3 +-
 src/test_rdma_device.cpp | 81 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+), 2 deletions(-)
 create mode 100644 src/test_rdma_device.cpp

diff --git a/src/meson.build b/src/meson.build
index 490ed02..4ec0bf9 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -67,4 +67,34 @@ python3.extension_module('_infinistore',
   sources: ext_sources + generated_headers,
   dependencies: [pybind_dep, libuv_dep, fmt_dep, boost_stack_dep, ibverbs_dep],
   install: true
-)
\ No newline at end of file
+)
+
+rdma_test_wrappers = [
+  'ibv_get_device_list',
+  'ibv_get_device_name',
+  'ibv_open_device',
+  'ibv_query_port',
+  'ibv_query_gid',
+  'ibv_alloc_pd',
+  'ibv_dealloc_pd',
+  'ibv_close_device',
+]
+rdma_test_link_args = []
+foreach symbol : rdma_test_wrappers
+  rdma_test_link_args += '-Wl,--wrap=' + symbol
+endforeach
+
+rdma_device_test = executable('test_rdma_device',
+  sources: ['test_rdma_device.cpp', 'rdma.cpp', 'utils.cpp'] + generated_headers,
+  dependencies: [fmt_dep, boost_stack_dep, ibverbs_dep],
+  link_args: rdma_test_link_args,
+  build_by_default: false,
+)
+foreach port : ['1', '2']
+  foreach hint : ['-1', '3']
+    test('rdma-gid-port-' + port + '-hint-' + hint, rdma_device_test,
+      args: [port, hint],
+      suite: 'rdma-device',
+    )
+  endforeach
+endforeach
diff --git a/src/rdma.cpp b/src/rdma.cpp
index e03dbe2..2c0df0e 100644
--- a/src/rdma.cpp
+++ b/src/rdma.cpp
@@ -108,7 +108,8 @@ int open_rdma_device(std::string dev_name, int ib_port, std::string link_type, i
             }
         }
 
-        if (ibv_query_gid(rdma_dev->ib_ctx, 1, rdma_dev->gid_index, &rdma_dev->gid) < 0) {
+        if (ibv_query_gid(rdma_dev->ib_ctx, rdma_dev->ib_port, rdma_dev->gid_index,
+                          &rdma_dev->gid) < 0) {
             ERROR("Failed to get GID from index {}", rdma_dev->gid_index);
             return -1;
         }
diff --git a/src/test_rdma_device.cpp b/src/test_rdma_device.cpp
new file mode 100644
index 0000000..855ee49
--- /dev/null
+++ b/src/test_rdma_device.cpp
@@ -0,0 +1,81 @@
+#include <cstdio>
+#include <cstdlib>
+
+#include "log.h"
+#include "rdma.h"
+
+static ibv_device device = {};
+static ibv_context context = {};
+static ibv_pd pd = {};
+static int expected_port;
+static int expected_index;
+static int query_port_seen;
+static int find_port_seen;
+static int query_gid_port_seen;
+static int query_gid_index_seen;
+
+extern "C" {
+ibv_device **__wrap_ibv_get_device_list(int *count) {
+    static ibv_device *devices[] = {&device, nullptr};
+    *count = 1;
+    return devices;
+}
+
+const char *__wrap_ibv_get_device_name(ibv_device *) { return "mock"; }
+
+ibv_context *__wrap_ibv_open_device(ibv_device *) { return &context; }
+
+int __wrap_ibv_query_port(ibv_context *, uint8_t port, _compat_ibv_port_attr *attr) {
+    query_port_seen = port;
+    // verbs 的兼容入口接收实际的 ibv_port_attr 缓冲区。
+    auto *port_attr = reinterpret_cast<ibv_port_attr *>(attr);
+    port_attr->link_layer = IBV_LINK_LAYER_ETHERNET;
+    port_attr->active_mtu = IBV_MTU_1024;
+    return 0;
+}
+
+int __wrap_ibv_query_gid(ibv_context *, uint8_t port, int index, ibv_gid *gid) {
+    query_gid_port_seen = port;
+    query_gid_index_seen = index;
+    *gid = {};
+    gid->raw[15] = port;
+    return 0;
+}
+
+ibv_pd *__wrap_ibv_alloc_pd(ibv_context *) { return &pd; }
+
+int __wrap_ibv_dealloc_pd(ibv_pd *) { return 0; }
+
+int __wrap_ibv_close_device(ibv_context *) { return 0; }
+}
+
+int ibv_find_sgid_type(ibv_context *, uint8_t port, ibv_gid_type, int) {
+    find_port_seen = port;
+    return expected_index;
+}
+
+int main(int argc, char **argv) {
+    if (argc != 3) {
+        return 2;
+    }
+    expected_port = std::atoi(argv[1]);
+    int hint = std::atoi(argv[2]);
+    expected_index = hint < 0 ? 5 : hint;
+    spdlog::stdout_color_mt(APP_NAME);
+    spdlog::set_level(spdlog::level::off);
+
+    rdma_device rdma_dev;
+    int ret = open_rdma_device("mock", expected_port, "Ethernet", hint, &rdma_dev);
+    if (ret != 0 || query_port_seen != expected_port || query_gid_port_seen != expected_port ||
+        query_gid_index_seen != expected_index ||
+        find_port_seen != (hint < 0 ? expected_port : 0) || rdma_dev.ib_port != expected_port ||
+        rdma_dev.gid_index != expected_index || rdma_dev.gid.raw[15] != expected_port) {
+        std::fprintf(stderr,
+                     "port=%d hint=%d: ret=%d query_port=%d find_port=%d "
+                     "query_gid_port=%d query_gid_index=%d\n",
+                     expected_port, hint, ret, query_port_seen, find_port_seen, query_gid_port_seen,
+                     query_gid_index_seen);
+        return 1;
+    }
+    return close_rdma_device(&rdma_dev);
+}