diff --git a/Makefile.am b/Makefile.am index 79a737042..62ae5d592 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,8 +3,9 @@ pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = thapi.pc ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = utils xprof sampling backends/opencl backends/ze backends/cuda backends/omp backends/hip backends/mpi backends/cxi backends/itt backends/pytorch - +SUBDIRS = utils xprof sampling backends/opencl backends/ze backends/cuda backends/omp backends/hip backends/mpi backends/cxi backends/itt backends/pytorch backends/libfabric EXTRA_DIST = autogen.sh README.md .valgrind/dlopen.supp thapi.pc.in CLEANFILES = thapi.pc + +DIST_SUBDIRS = utils xprof sampling backends/opencl backends/ze backends/cuda backends/omp backends/hip backends/mpi backends/cxi backends/itt backends/pytorch backends/libfabric diff --git a/backends/libfabric/Makefile.am b/backends/libfabric/Makefile.am new file mode 100644 index 000000000..657fe9486 --- /dev/null +++ b/backends/libfabric/Makefile.am @@ -0,0 +1,169 @@ +if STRICT + WERROR = -Werror +else + WERROR = +endif + +# +# ---------------------------------------------------------------------- +# Tracepoint provider +# ---------------------------------------------------------------------- +# +# libfabric_events.yaml -> (gen_custom_probes.rb) -> .tp -> (lttng-gen-tp) +# -> libfabric_tracepoints.{c,h} +# +# No h2yaml and no _model.rb anywhere in this backend: the tracer is +# hand-written, so there is no header to parse. backends/pytorch and +# backends/cxi are the two other backends built this way. +# + +FI_STATIC_PROBES = libfabric_tracepoints + +FI_STATIC_PROBES_TP = $(FI_STATIC_PROBES:=.tp) + +FI_STATIC_PROBES_INCL = $(FI_STATIC_PROBES:=.h) + +FI_STATIC_PROBES_SRC = $(FI_STATIC_PROBES:=.c) + +$(FI_STATIC_PROBES_TP): %.tp: $(top_srcdir)/utils/gen_custom_probes.rb $(srcdir)/libfabric_events.yaml $(top_srcdir)/utils/gen_probe_base.rb + $(RUBY) $< $(srcdir)/libfabric_events.yaml lttng_ust_fi > $@ + +%.h %.c: %.tp + $(LTTNG_GEN_TP) $< -o $*.c -o $*.h + +noinst_LTLIBRARIES = libfitracepoints.la + +nodist_libfitracepoints_la_SOURCES = \ + $(FI_STATIC_PROBES_INCL) \ + $(FI_STATIC_PROBES_SRC) + +libfitracepoints_la_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I./ +libfitracepoints_la_CFLAGS = -fPIC -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Wno-sign-compare $(WERROR) $(LTTNG_UST_CFLAGS) +libfitracepoints_la_LDFLAGS = $(LTTNG_UST_LIBS) + +EXTRA_DIST = \ + libfabric_events.yaml \ + $(top_srcdir)/utils/gen_custom_probes.rb \ + include + +CLEANFILES = \ + $(FI_STATIC_PROBES_TP) \ + $(FI_STATIC_PROBES_INCL) \ + $(FI_STATIC_PROBES_SRC) + +BUILT_SOURCES = \ + $(FI_STATIC_PROBES_INCL) + +# +# ---------------------------------------------------------------------- +# Tracer +# ---------------------------------------------------------------------- +# +# Note what is absent compared to backends/pytorch: there is no dummy_libs +# stanza, because we never link against libfabric. Every provider entry point +# is reached either through dlvsym(RTLD_NEXT, "fi_fabric", ...) or through a +# function pointer in a vtable the provider handed us, so the only thing +# libfabric contributes at build time is its headers. +# +# The headers are vendored under include/rdma/ so the backend builds on a +# machine with no libfabric installed -- same reason backends/mpi vendors +# mpi.h. Update them with: +# cp -r /include/rdma backends/libfabric/include/ +# + +bin_SCRIPTS = \ + tracer_libfabric.sh + +lib_LTLIBRARIES = libTracerLibfabric.la + +nodist_libTracerLibfabric_la_SOURCES = \ + $(FI_STATIC_PROBES_INCL) + +libTracerLibfabric_la_SOURCES = \ + src/fit_hook.c \ + src/fit_addr.c \ + src/fit_lttng.c + +libTracerLibfabric_la_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I$(srcdir)/include -I$(srcdir)/src -I./ +libTracerLibfabric_la_CFLAGS = -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers $(WERROR) $(LTTNG_UST_CFLAGS) +libTracerLibfabric_la_LDFLAGS = $(LTTNG_UST_LIBS) -ldl -avoid-version -module +libTracerLibfabric_la_LIBADD = libfitracepoints.la + +EXTRA_DIST += \ + src/fi_trace.h + +# +# ---------------------------------------------------------------------- +# Analysis: raw lttng_ust_fi trace -> interval stream +# ---------------------------------------------------------------------- +# +# Three names here are load-bearing and fail confusingly rather than loudly if +# you change them: +# +# -p fiinterval -c interval utils/babeltrace_thapi.in:161 looks the +# component up as "filter.#{b}interval.interval" +# where b is the short backend name, "fi". +# libFIInterval.la utils/test_wrapper_thapi_text_pretty.sh.in:69 +# builds lib"${be^^}"Interval.so. +# $(pkglibdir)/bt2 babeltrace_thapi.in:151 scans exactly this one +# directory for plugins. +# + +BTX_FI_GENERATED = \ + btx_filter_fi/metababel/metababel.h \ + btx_filter_fi/metababel/btx_component.h \ + btx_filter_fi/metababel/btx_component.c \ + btx_filter_fi/metababel/btx_upstream.h \ + btx_filter_fi/metababel/btx_upstream.c \ + btx_filter_fi/metababel/btx_downstream.h \ + btx_filter_fi/metababel/btx_downstream.c \ + btx_filter_fi/btx_main.c + +EXTRA_DIST += \ + $(top_srcdir)/xprof/btx_interval_model.yaml \ + $(srcdir)/btx_libfabric_model.yaml + +$(BTX_FI_GENERATED) &: $(top_srcdir)/xprof/btx_interval_model.yaml $(srcdir)/btx_libfabric_model.yaml + $(METABABEL) -u $(srcdir)/btx_libfabric_model.yaml -d $(top_srcdir)/xprof/btx_interval_model.yaml -t FILTER -o btx_filter_fi -p fiinterval -c interval + +CLEANFILES += \ + $(BTX_FI_GENERATED) + +BUILT_SOURCES += \ + $(BTX_FI_GENERATED) + +nodist_libFIInterval_la_SOURCES = \ + $(BTX_FI_GENERATED) + +libFIInterval_la_SOURCES = \ + btx_fiinterval_callbacks.cpp + +libFIInterval_la_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I./ -I./btx_filter_fi +libFIInterval_la_CFLAGS = -Wall -Wextra -Wno-unused-parameter $(WERROR) $(BABELTRACE2_CFLAGS) +libFIInterval_la_CXXFLAGS = -std=c++17 -Wall -Wextra -Wno-unused-parameter $(WERROR) $(BABELTRACE2_CFLAGS) +libFIInterval_la_LDFLAGS = $(BABELTRACE2_LIBS) -avoid-version -module + +bt2dir = $(pkglibdir)/bt2 +bt2_LTLIBRARIES = libFIInterval.la + +# +# ---------------------------------------------------------------------- +# Enable pretty.print in babeltrace_thapi +# ---------------------------------------------------------------------- +# +# The library name uses the *directory* name (libfabric), not the short +# backend name (fi) -- babeltrace_thapi.in:167 does the same for OpenCL, +# requiring babeltrace_opencl_lib for backend key "cl". +# + +babeltrace_libfabric_lib.rb: $(srcdir)/gen_babeltrace_libfabric_lib.rb $(top_srcdir)/utils/gen_babeltrace_lib_helper.rb $(top_srcdir)/utils/gen_probe_base.rb $(srcdir)/btx_libfabric_model.yaml + SRC_DIR=$(srcdir) $(RUBY) $< > $@ + +EXTRA_DIST += \ + gen_babeltrace_libfabric_lib.rb + +CLEANFILES += \ + babeltrace_libfabric_lib.rb + +data_DATA = \ + babeltrace_libfabric_lib.rb \ No newline at end of file diff --git a/backends/libfabric/btx_fiinterval_callbacks.cpp b/backends/libfabric/btx_fiinterval_callbacks.cpp new file mode 100644 index 000000000..c8b82f583 --- /dev/null +++ b/backends/libfabric/btx_fiinterval_callbacks.cpp @@ -0,0 +1,328 @@ +/* + * Analysis side of the libfabric backend: turn the raw lttng_ust_fi trace into + * THAPI's "interval" stream. + * + * Three jobs, in increasing order of difficulty: + * + * 1. Pair op_entry with op_exit for a per-call duration. This is the usual + * THAPI entry/exit pairing and measures how long the libfabric call + * itself took -- i.e. how long the provider spent accepting the request. + * + * 2. Join a posting to its completion for a transfer duration. This is the + * part no other THAPI backend has to do. A libfabric transfer is not a + * function call with a return; it is a posting that the application later + * reaps from a completion queue, possibly from a different thread. The + * two are joined on op_context, which is the application's own pointer. + * + * 3. Attribute bytes to a peer, for the traffic stream. Only the completion + * knows how many bytes actually moved. + * + * ep_info events build a side table so the pretty-printer and the traffic + * metadata can name the NIC and its dragonfly coordinates; this replaces the + * sidecar CSV + fitrace-merge join of the standalone tracer. + */ + +#include "xprof_utils.hpp" +#include + +#include +#include +#include +#include +#include + +/* ------------------------------------------------------------------ */ +/* state */ +/* ------------------------------------------------------------------ */ + +struct posting_s { + int64_t ts; /* when the application posted */ + std::string op; /* "fi_send", "fi_trecv", ... */ + std::string peer; /* as known at posting time; a wildcard + receive will be corrected by the + completion, which knows the real sender */ + uint64_t posted_bytes; /* buffer length, not transferred length */ +}; + +struct ep_info_s { + std::string src_addr; + std::string domain_name; + bool topo_valid; + uint32_t group, switch_id, port; +}; + +struct data_s { + /* + * Per-thread LIFO of open entry timestamps. A wrapper cannot re-enter + * itself, but a layered provider (rxm over verbs, or the tcp provider + * implementing fi_cq_read in terms of fi_cq_readfrom) dispatches back + * through the public inline and lands in another wrapper, so nesting is + * real and a stack is required rather than a single slot. + */ + std::unordered_map>> op_stack; + std::unordered_map>> ctrl_stack; + + /* + * Outstanding postings keyed by {hostname, vpid, op_context}. + * + * Note what is NOT in the key: vtid. A progress thread reaping another + * thread's completions is a normal libfabric design, so keying on the + * posting thread would silently drop every join in such an application. + * The cost is that two threads in one process posting with the same context + * pointer at the same time become ambiguous -- but libfabric already + * requires op_context to be unique among outstanding operations on an + * endpoint, so a correct application cannot hit this. + */ + std::unordered_map outstanding; + + std::unordered_map endpoints; /* key: host/pid/ep_id */ +}; +typedef struct data_s data_t; + +static void btx_initialize_component(void **usr_data) { *usr_data = new data_t; } + +static void btx_finalize_component(void *usr_data) { delete static_cast(usr_data); } + +/* ------------------------------------------------------------------ */ +/* helpers */ +/* ------------------------------------------------------------------ */ + +static std::string ctx_key(const char *hostname, int64_t vpid, uintptr_t op_context) { + return std::string(hostname) + "/" + std::to_string(vpid) + "/" + std::to_string(op_context); +} + +static std::string ep_key(const char *hostname, int64_t vpid, uint32_t ep_id) { + return std::string(hostname) + "/" + std::to_string(vpid) + "/" + std::to_string(ep_id); +} + +/* + * Traffic metadata: who the bytes went to or came from, qualified by the local + * NIC when we know it. This is the string that shows up in iprof's traffic + * table, so it wants to be stable and greppable rather than pretty. + */ +static std::string traffic_metadata( + data_t *s, const char *hostname, int64_t vpid, uint32_t ep_id, const std::string &peer) { + auto it = s->endpoints.find(ep_key(hostname, vpid, ep_id)); + if (it == s->endpoints.end()) + return peer; + + std::string m = it->second.src_addr + "->" + peer; + if (it->second.topo_valid) + m += " [" + it->second.domain_name + " g" + std::to_string(it->second.group) + "s" + + std::to_string(it->second.switch_id) + "p" + std::to_string(it->second.port) + "]"; + return m; +} + +/* ------------------------------------------------------------------ */ +/* data path */ +/* ------------------------------------------------------------------ */ + +static void op_entry_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t ep_id, + uintptr_t op_context) { + auto *s = static_cast(usr_data); + s->op_stack[{hostname, vpid, vtid}].push_back({ts, op}); +} + +static void op_exit_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t ep_id, + char *src_addr, + char *peer_addr, + char *direction, + uint64_t iov_count, + uint64_t size_bytes, + int64_t ret, + uintptr_t op_context) { + auto *s = static_cast(usr_data); + auto &stack = s->op_stack[{hostname, vpid, vtid}]; + + /* + * An exit with no entry happens at the head of a trace that started while + * the application was already inside libfabric. Drop it rather than + * inventing a start timestamp. + */ + if (stack.empty()) + return; + + const int64_t entry_ts = stack.back().first; + const std::string entry_op = stack.back().second; + stack.pop_back(); + + /* + * Duration of the libfabric call itself. err_flag is set on any negative + * return, which includes -FI_EAGAIN; that is a legitimate "try later", not a + * failure, but it is also not work the application wanted, and flagging it + * makes it visible in the tally instead of inflating the success counts. + */ + btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, entry_ts, BACKEND_FI, + entry_op.c_str(), (ts - entry_ts), (ret < 0)); + + if (ret < 0) + return; /* nothing became outstanding; the tracer already nulled context */ + + /* + * A NULL op_context is legal and common -- fi_inject takes no context at + * all, and plenty of applications pass NULL to the others. Such a posting + * can never be joined to a completion, because nothing distinguishes it from + * any other contextless operation in flight. Record it as a call (done + * above) but not as an outstanding transfer. + */ + if (!op_context) + return; + + /* + * Insert overwrites by design. op_context is the application's pointer and + * is recycled the instant an operation completes, so at most one live entry + * per key is correct; an overwrite means we missed a completion. + */ + s->outstanding[ctx_key(hostname, vpid, op_context)] = {entry_ts, entry_op, peer_addr, size_bytes}; +} + +static void cq_completion_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t obj_id, + char *src_addr, + char *peer_addr, + uint64_t size_bytes, + int64_t ret, + uintptr_t op_context) { + auto *s = static_cast(usr_data); + + if (!op_context) + return; + + auto it = s->outstanding.find(ctx_key(hostname, vpid, op_context)); + if (it == s->outstanding.end()) + return; /* completion for a posting we never saw -- normal at trace start */ + + const posting_s post = it->second; + s->outstanding.erase(it); + + /* + * The duration the application actually experienced: post to reap. + * + * This is an upper bound on the transfer time, and deliberately so. An + * application that posts and then does ten milliseconds of compute before + * polling will show ten milliseconds here. That is the right number to + * report -- it is the latency the application lived with -- but it is not + * the NIC's transfer time, and nothing in a libfabric trace can give you + * that. Compare against the CXI counters (iprof --backends fi,cxi) when you + * need the hardware view. + */ + btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, post.ts, BACKEND_FI, + (post.op + ".transfer").c_str(), (ts - post.ts), (ret < 0)); + + if (ret < 0) + return; + + /* + * Bytes are attributed from the completion, not the posting. A receive + * posts a buffer and completes with the message length; using the posted + * length would overstate received traffic by whatever slack the application + * left. Likewise peer_addr: a wildcard receive posts FI_ADDR_UNSPEC and + * only fi_cq_readfrom() reveals who actually sent. + */ + const std::string peer = (peer_addr && peer_addr[0] && std::string(peer_addr) != "fi_addr_unspec") + ? peer_addr + : post.peer; + + btx_push_message_lttng_traffic(btx_handle, hostname, vpid, vtid, post.ts, BACKEND_FI, + post.op.c_str(), size_bytes, + traffic_metadata(s, hostname, vpid, obj_id, peer).c_str()); +} + +/* ------------------------------------------------------------------ */ +/* control path */ +/* ------------------------------------------------------------------ */ + +static void ctrl_entry_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t obj_id) { + auto *s = static_cast(usr_data); + s->ctrl_stack[{hostname, vpid, vtid}].push_back({ts, op}); +} + +static void ctrl_exit_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t obj_id, + int64_t ret) { + auto *s = static_cast(usr_data); + auto &stack = s->ctrl_stack[{hostname, vpid, vtid}]; + if (stack.empty()) + return; + + /* + * The control path nests for real: fi_domain() calls into the provider, + * which opens an endpoint, which binds a CQ. A separate stack from the data + * path keeps that nesting from interleaving with postings. + */ + const int64_t entry_ts = stack.back().first; + const std::string entry_op = stack.back().second; + stack.pop_back(); + + btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, entry_ts, BACKEND_FI, + entry_op.c_str(), (ts - entry_ts), (ret < 0)); +} + +/* ------------------------------------------------------------------ */ +/* endpoint description */ +/* ------------------------------------------------------------------ */ + +static void ep_info_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + uint32_t ep_id, + char *src_addr, + char *domain_name, + uint32_t topo_valid, + uint32_t group, + uint32_t switch_id, + uint32_t port) { + auto *s = static_cast(usr_data); + s->endpoints[ep_key(hostname, vpid, ep_id)] = {src_addr, domain_name, topo_valid != 0, + group, switch_id, port}; +} + +/* ------------------------------------------------------------------ */ + +void btx_register_usr_callbacks(void *btx_handle) { + btx_register_callbacks_initialize_component(btx_handle, &btx_initialize_component); + btx_register_callbacks_finalize_component(btx_handle, &btx_finalize_component); + + btx_register_callbacks_lttng_ust_fi_op_entry(btx_handle, &op_entry_callback); + btx_register_callbacks_lttng_ust_fi_op_exit(btx_handle, &op_exit_callback); + btx_register_callbacks_lttng_ust_fi_cq_completion(btx_handle, &cq_completion_callback); + btx_register_callbacks_lttng_ust_fi_ctrl_entry(btx_handle, &ctrl_entry_callback); + btx_register_callbacks_lttng_ust_fi_ctrl_exit(btx_handle, &ctrl_exit_callback); + btx_register_callbacks_lttng_ust_fi_ep_info(btx_handle, &ep_info_callback); +} diff --git a/backends/libfabric/btx_libfabric_model.yaml b/backends/libfabric/btx_libfabric_model.yaml new file mode 100644 index 000000000..59c346e1c --- /dev/null +++ b/backends/libfabric/btx_libfabric_model.yaml @@ -0,0 +1,199 @@ +:environment: + :entries: + - :name: hostname + :type: string +:stream_classes: +- :name: thapi_libfabric + :default_clock_class: {} + :packet_context_field_class: + :type: structure + :members: + - :name: cpu_id + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 32 + :event_common_context_field_class: + :type: structure + :members: + - :name: vpid + :field_class: + :type: integer_signed + :cast_type: int64_t + :field_value_range: 64 + - :name: vtid + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 64 + :event_classes: + - :name: lttng_ust_fi:op_entry + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: ep_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: op_context + :field_class: + :type: integer_unsigned + :cast_type: uintptr_t + :field_value_range: 64 + :preferred_display_base: 16 + - :name: lttng_ust_fi:op_exit + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: ep_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: src_addr + :field_class: + :cast_type: char * + :type: string + - :name: peer_addr + :field_class: + :cast_type: char * + :type: string + - :name: direction + :field_class: + :cast_type: char * + :type: string + - :name: iov_count + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 64 + - :name: size_bytes + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 64 + - :name: ret + :field_class: + :type: integer_signed + :cast_type: int64_t + :field_value_range: 64 + - :name: op_context + :field_class: + :type: integer_unsigned + :cast_type: uintptr_t + :field_value_range: 64 + :preferred_display_base: 16 + - :name: lttng_ust_fi:cq_completion + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: obj_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: src_addr + :field_class: + :cast_type: char * + :type: string + - :name: peer_addr + :field_class: + :cast_type: char * + :type: string + - :name: size_bytes + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 64 + - :name: ret + :field_class: + :type: integer_signed + :cast_type: int64_t + :field_value_range: 64 + - :name: op_context + :field_class: + :type: integer_unsigned + :cast_type: uintptr_t + :field_value_range: 64 + :preferred_display_base: 16 + - :name: lttng_ust_fi:ctrl_entry + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: obj_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: lttng_ust_fi:ctrl_exit + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: obj_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: ret + :field_class: + :type: integer_signed + :cast_type: int64_t + :field_value_range: 64 + - :name: lttng_ust_fi:ep_info + :payload_field_class: + :type: structure + :members: + - :name: ep_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: src_addr + :field_class: + :cast_type: char * + :type: string + - :name: domain_name + :field_class: + :cast_type: char * + :type: string + - :name: topo_valid + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: group + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: switch_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: port + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 diff --git a/backends/libfabric/gen_babeltrace_libfabric_lib.rb b/backends/libfabric/gen_babeltrace_libfabric_lib.rb new file mode 100644 index 000000000..3e6ba30ec --- /dev/null +++ b/backends/libfabric/gen_babeltrace_libfabric_lib.rb @@ -0,0 +1,8 @@ +require_relative '../../utils/gen_babeltrace_lib_helper' + +# Passing nil for the naming argument is legal only because no field in +# btx_libfabric_model.yaml carries ":metadata: :be_class:". If you later add +# an enum or bitmask field -- rendering fi_cq flags symbolically, say -- this +# backend will also need a libfabric_library.rb defining the value tables, and +# nil has to become that naming object. backends/mpi is the example to copy. +add_babeltrace_event_callbacks(nil, File.join(SRC_DIR, 'btx_libfabric_model.yaml')) diff --git a/backends/libfabric/include/rdma/fabric.h b/backends/libfabric/include/rdma/fabric.h new file mode 100644 index 000000000..319d09780 --- /dev/null +++ b/backends/libfabric/include/rdma/fabric.h @@ -0,0 +1,832 @@ +/* + * Copyright (c) 2013-2017 Intel Corporation. All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. + * (C) Copyright 2020 Hewlett Packard Enterprise Development LP + * Copyright (c) 2022 DataDirect Networks, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FABRIC_H +#define FABRIC_H + +#include +#include +#include +#include +#include + +#ifdef __GNUC__ +#define FI_DEPRECATED_FUNC __attribute__((deprecated)) +#define FI_DEPRECATED_FIELD __attribute__((deprecated)) +#define FI_FORMAT_PRINTF(string, first) \ + __attribute__ ((__format__ (__printf__, (string), (first)))) +#elif defined(_MSC_VER) +#define FI_DEPRECATED_FUNC __declspec(deprecated) +#define FI_DEPRECATED_FIELD +#define FI_FORMAT_PRINTF(string, first) +#else +#define FI_DEPRECATED_FUNC +#define FI_DEPRECATED_FIELD +#define FI_FORMAT_PRINTF(string, first) +#endif + +#if defined(__GNUC__) && !defined(__clang__) +#define EXTERNALLY_VISIBLE externally_visible +#else +#define EXTERNALLY_VISIBLE +#endif + +#if defined(_WIN32) +#include +typedef SSIZE_T ssize_t; +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define FI_MAJOR_VERSION 2 +#define FI_MINOR_VERSION 3 +#define FI_REVISION_VERSION 1 + +/* Removing these breaks the build for some apps. + * The use of FI_NAME_MAX is undefined. + * FI_ATOMIC_OP_LAST and FI_DATATYPE_LAST values cannot change + * (such as inserting new enum values that they are intended to be the + * last of) without breaking apps that recompile. So, they are hard-coded + * here. + */ +enum { + FI_NAME_MAX = 64, + FI_ATOMIC_OP_LAST = 19, + FI_DATATYPE_LAST = 14, /* not actual last datatype */ +}; + +#define FI_VERSION(major, minor) (((major) << 16) | (minor)) +#define FI_MAJOR(version) (version >> 16) +#define FI_MINOR(version) (version & 0xFFFF) +#define FI_VERSION_GE(v1, v2) (v1 >= v2) +#define FI_VERSION_LT(v1, v2) (v1 < v2) + +uint32_t fi_version(void); + +struct fid; +struct fid_fabric; +struct fid_domain; +struct fid_av; +struct fid_wait; +struct fid_poll; +struct fid_eq; +struct fid_cq; +struct fid_cntr; +struct fid_ep; +struct fid_pep; +struct fid_stx; +struct fid_mr; +struct fid_nic; + +typedef struct fid *fid_t; + +/* + * Flags + * The 64-bit flag field is used as follows: + * 1-grow up common (usable with multiple operations) + * 59-grow down operation specific (used for single call/class) + * 60 - 63 provider specific + */ + +#define FI_MSG (1ULL << 1) +#define FI_RMA (1ULL << 2) +#define FI_TAGGED (1ULL << 3) +#define FI_ATOMIC (1ULL << 4) +#define FI_ATOMICS FI_ATOMIC +#define FI_MULTICAST (1ULL << 5) +#define FI_COLLECTIVE (1ULL << 6) + +#define FI_READ (1ULL << 8) +#define FI_WRITE (1ULL << 9) +#define FI_RECV (1ULL << 10) +#define FI_SEND (1ULL << 11) +#define FI_TRANSMIT FI_SEND +#define FI_REMOTE_READ (1ULL << 12) +#define FI_REMOTE_WRITE (1ULL << 13) + +#define FI_MULTI_RECV (1ULL << 16) +#define FI_REMOTE_CQ_DATA (1ULL << 17) +#define FI_MORE (1ULL << 18) +#define FI_PEEK (1ULL << 19) +#define FI_TRIGGER (1ULL << 20) +#define FI_FENCE (1ULL << 21) +/* #define FI_PRIORITY (1ULL << 22) */ + +#define FI_COMPLETION (1ULL << 24) +#define FI_EVENT FI_COMPLETION +#define FI_INJECT (1ULL << 25) +#define FI_INJECT_COMPLETE (1ULL << 26) +#define FI_TRANSMIT_COMPLETE (1ULL << 27) +#define FI_DELIVERY_COMPLETE (1ULL << 28) +#define FI_AFFINITY (1ULL << 29) +#define FI_COMMIT_COMPLETE (1ULL << 30) +#define FI_MATCH_COMPLETE (1ULL << 31) + +#define FI_RESCAN (1ULL << 35) +#define FI_PEER_TRANSFER (1ULL << 36) +/* #define FI_MR_DMABUF (1ULL << 40) */ +#define FI_AV_USER_ID (1ULL << 41) +#define FI_FIREWALL_ADDR (1ULL << 42) +#define FI_PEER (1ULL << 43) +/* #define FI_XPU_TRIGGER (1ULL << 44) */ + +#define FI_TAGGED_DIRECTED_RECV (1ULL << 45) +#define FI_TAGGED_MULTI_RECV (1ULL << 46) +#define FI_HMEM (1ULL << 47) +#define FI_EXACT_DIRECTED_RECV (1ULL << 48) +#define FI_RMA_PMEM (1ULL << 49) +#define FI_SOURCE_ERR (1ULL << 50) +#define FI_LOCAL_COMM (1ULL << 51) +#define FI_REMOTE_COMM (1ULL << 52) +#define FI_SHARED_AV (1ULL << 53) +#define FI_PROV_ATTR_ONLY (1ULL << 54) +#define FI_NUMERICHOST (1ULL << 55) +#define FI_RMA_EVENT (1ULL << 56) +#define FI_SOURCE (1ULL << 57) +#define FI_NAMED_RX_CTX (1ULL << 58) +#define FI_DIRECTED_RECV (1ULL << 59) + + +/* Tagged messages, buffered receives, CQ flags */ +#define FI_CLAIM (1ULL << 59) +#define FI_DISCARD (1ULL << 58) +#define FI_AUTH_KEY (1ULL << 57) + +struct fi_ioc { + void *addr; + size_t count; +}; + +/* + * Format for transport addresses to insert into address vectors + */ +enum { + FI_FORMAT_UNSPEC, /* void * */ + FI_SOCKADDR, /* struct sockaddr */ + FI_SOCKADDR_IN, /* struct sockaddr_in */ + FI_SOCKADDR_IN6, /* struct sockaddr_in6 */ + FI_SOCKADDR_IB, /* struct sockaddr_ib */ + /* PSMX provider is deprecated. + * We will keep this value in order to save binary compatibility. + */ + FI_ADDR_PSMX, /* uint64_t */ + FI_ADDR_GNI, + FI_ADDR_BGQ, + FI_ADDR_MLX, + FI_ADDR_STR, /* formatted char * */ + FI_ADDR_PSMX2, /* uint64_t[2] */ + FI_ADDR_IB_UD, /* uint64_t[4] */ + FI_ADDR_EFA, + FI_ADDR_PSMX3, /* uint64_t[4] */ + FI_ADDR_OPX, + FI_ADDR_CXI, + FI_ADDR_UCX, + + FI_SOCKADDR_IP, /* FI_SOCKADDR_IN and FI_SOCKADDR_IN6 */ +}; + +#define FI_ADDR_UNSPEC ((uint64_t) -1) +#define FI_ADDR_NOTAVAIL ((uint64_t) -1) +#define FI_KEY_NOTAVAIL ((uint64_t) -1) +#define FI_SHARED_CONTEXT SIZE_MAX +#define FI_AV_AUTH_KEY SIZE_MAX +typedef uint64_t fi_addr_t; + +enum fi_av_type { + FI_AV_UNSPEC, + FI_AV_MAP, + FI_AV_TABLE +}; + +#define FI_MR_UNSPEC _Pragma("GCC warning \"'FI_MR_UNSPEC' is deprecated\"") (0) +#define FI_MR_BASIC _Pragma("GCC warning \"'FI_MR_BASIC' is deprecated\"") (1 << 0) +#define FI_MR_SCALABLE _Pragma("GCC warning \"'FI_MR_SCALABLE' is deprecated\"") (1 << 1) + +#define FI_MR_LOCAL (1 << 2) +#define FI_MR_RAW (1 << 3) +#define FI_MR_VIRT_ADDR (1 << 4) +#define FI_MR_ALLOCATED (1 << 5) +#define FI_MR_PROV_KEY (1 << 6) +#define FI_MR_MMU_NOTIFY (1 << 7) +#define FI_MR_RMA_EVENT (1 << 8) +#define FI_MR_ENDPOINT (1 << 9) +#define FI_MR_HMEM (1 << 10) +#define FI_MR_COLLECTIVE (1 << 11) + +enum fi_progress { + FI_PROGRESS_UNSPEC, + FI_PROGRESS_AUTO, + FI_PROGRESS_MANUAL, + FI_PROGRESS_CONTROL_UNIFIED, +}; + +enum fi_threading { + FI_THREAD_UNSPEC, + FI_THREAD_SAFE, + FI_THREAD_FID, + FI_THREAD_DOMAIN, + FI_THREAD_COMPLETION, + FI_THREAD_ENDPOINT, +}; + +enum fi_resource_mgmt { + FI_RM_UNSPEC, + FI_RM_DISABLED, + FI_RM_ENABLED +}; + +#define FI_ORDER_NONE _Pragma("GCC warning \"'FI_ORDER_NONE' is deprecated\"") 0ULL +#define FI_ORDER_RAR (1ULL << 0) +#define FI_ORDER_RAW (1ULL << 1) +#define FI_ORDER_RAS (1ULL << 2) +#define FI_ORDER_WAR (1ULL << 3) +#define FI_ORDER_WAW (1ULL << 4) +#define FI_ORDER_WAS (1ULL << 5) +#define FI_ORDER_SAR (1ULL << 6) +#define FI_ORDER_SAW (1ULL << 7) +#define FI_ORDER_SAS (1ULL << 8) +#define FI_ORDER_STRICT _Pragma("GCC warning \"'FI_ORDER_STRICT' is deprecated\"") 0x1FF + +#define FI_ORDER_RMA_RAR (1ULL << 32) +#define FI_ORDER_RMA_RAW (1ULL << 33) +#define FI_ORDER_RMA_WAR (1ULL << 34) +#define FI_ORDER_RMA_WAW (1ULL << 35) +#define FI_ORDER_ATOMIC_RAR (1ULL << 36) +#define FI_ORDER_ATOMIC_RAW (1ULL << 37) +#define FI_ORDER_ATOMIC_WAR (1ULL << 38) +#define FI_ORDER_ATOMIC_WAW (1ULL << 39) + +#define FI_ORDER_DATA _Pragma("GCC warning \"'FI_ORDER_DATA' is deprecated\"") (1ULL << 16) + +enum fi_ep_type { + FI_EP_UNSPEC, + FI_EP_MSG, + FI_EP_DGRAM, + FI_EP_RDM, + /* FI_EP_SOCK_STREAM, */ + /* FI_EP_SOCK_DGRAM, */ +}; + +/* Endpoint protocol + * If two providers support the same protocol, then they shall interoperate + * when the protocol capabilities match. + */ +enum { + FI_PROTO_UNSPEC, + FI_PROTO_RDMA_CM_IB_RC, + FI_PROTO_IWARP, + FI_PROTO_IB_UD, + /* PSMX provider is deprecated. + * We will keep this value in order to save binary compatibility. + */ + FI_PROTO_PSMX, + FI_PROTO_UDP, + FI_PROTO_SOCK_TCP, + /* MXM provider is deprecated. + * We will keep this value in order to save binary compatibility. + */ + FI_PROTO_MXM, + FI_PROTO_IWARP_RDM, + FI_PROTO_IB_RDM, + FI_PROTO_GNI, + FI_PROTO_RXM, + FI_PROTO_RXD, + FI_PROTO_MLX, + FI_PROTO_NETWORKDIRECT, + FI_PROTO_PSMX2, + FI_PROTO_SHM, + FI_PROTO_MRAIL, + FI_PROTO_RSTREAM, + FI_PROTO_RDMA_CM_IB_XRC, + FI_PROTO_EFA, + FI_PROTO_PSMX3, + FI_PROTO_RXM_TCP, + FI_PROTO_OPX, + FI_PROTO_CXI, + FI_PROTO_XNET, + FI_PROTO_COLL, + FI_PROTO_UCX, + FI_PROTO_SM2, + FI_PROTO_CXI_RNR, + FI_PROTO_LPP, + FI_PROTO_LNX, +}; + +enum { + FI_TAG_BITS, + FI_TAG_MPI, + FI_TAG_CCL, + FI_TAG_MAX_FORMAT = (1ULL << 16), +}; + +enum { + FI_TC_UNSPEC = 0, + FI_TC_DSCP = 0x100, + FI_TC_LABEL = 0x200, + FI_TC_BEST_EFFORT = FI_TC_LABEL, + FI_TC_LOW_LATENCY, + FI_TC_DEDICATED_ACCESS, + FI_TC_BULK_DATA, + FI_TC_SCAVENGER, + FI_TC_NETWORK_CTRL, +}; + +static inline uint32_t fi_tc_dscp_set(uint8_t dscp) +{ + return ((uint32_t) dscp) | FI_TC_DSCP; +} + +static inline uint8_t fi_tc_dscp_get(uint32_t tclass) +{ + return tclass & FI_TC_DSCP ? (uint8_t) tclass : 0; +} + +/* Mode bits */ +#define FI_CONTEXT (1ULL << 59) +#define FI_MSG_PREFIX (1ULL << 58) +#define FI_ASYNC_IOV (1ULL << 57) +#define FI_RX_CQ_DATA (1ULL << 56) +#define FI_LOCAL_MR _Pragma("GCC warning \"'FI_LOCAL_MR' is deprecated\"") (1ULL << 55) +/* #define FI_NOTIFY_FLAGS_ONLY (1ULL << 54) */ +/* #define FI_RESTRICTED_COMP (1ULL << 53) */ +#define FI_CONTEXT2 (1ULL << 52) +/* #define FI_BUFFERED_RECV (1ULL << 51) */ +/* #define FI_PEER_TRANSFER (1ULL << 36) */ + +struct fi_tx_attr { + uint64_t caps; + uint64_t mode; + uint64_t op_flags; + uint64_t msg_order; + uint64_t comp_order; + size_t inject_size; + size_t size; + size_t iov_limit; + size_t rma_iov_limit; + uint32_t tclass; +}; + +struct fi_rx_attr { + uint64_t caps; + uint64_t mode; + uint64_t op_flags; + uint64_t msg_order; + uint64_t comp_order; + size_t total_buffered_recv; + size_t size; + size_t iov_limit; +}; + +struct fi_ep_attr { + enum fi_ep_type type; + uint32_t protocol; + uint32_t protocol_version; + size_t max_msg_size; + size_t msg_prefix_size; + size_t max_order_raw_size; + size_t max_order_war_size; + size_t max_order_waw_size; + uint64_t mem_tag_format; + size_t tx_ctx_cnt; + size_t rx_ctx_cnt; + size_t auth_key_size; + uint8_t *auth_key; +}; + +struct fi_domain_attr { + struct fid_domain *domain; + char *name; + enum fi_threading threading; + enum fi_progress control_progress; + union { + enum fi_progress data_progress; + enum fi_progress progress; + }; + enum fi_resource_mgmt resource_mgmt; + enum fi_av_type av_type; + int mr_mode; + size_t mr_key_size; + size_t cq_data_size; + size_t cq_cnt; + size_t ep_cnt; + size_t tx_ctx_cnt; + size_t rx_ctx_cnt; + size_t max_ep_tx_ctx; + size_t max_ep_rx_ctx; + size_t max_ep_stx_ctx; + size_t max_ep_srx_ctx; + size_t cntr_cnt; + size_t mr_iov_limit; + uint64_t caps; + uint64_t mode; + uint8_t *auth_key; + size_t auth_key_size; + size_t max_err_data; + size_t mr_cnt; + uint32_t tclass; + size_t max_ep_auth_key; + uint32_t max_group_id; +}; + +struct fi_fabric_attr { + struct fid_fabric *fabric; + char *name; + char *prov_name; + uint32_t prov_version; + uint32_t api_version; +}; + +struct fi_info { + struct fi_info *next; + uint64_t caps; + uint64_t mode; + uint32_t addr_format; + size_t src_addrlen; + size_t dest_addrlen; + void *src_addr; + void *dest_addr; + fid_t handle; + struct fi_tx_attr *tx_attr; + struct fi_rx_attr *rx_attr; + struct fi_ep_attr *ep_attr; + struct fi_domain_attr *domain_attr; + struct fi_fabric_attr *fabric_attr; + struct fid_nic *nic; +}; + +struct fi_device_attr { + char *name; + char *device_id; + char *device_version; + char *vendor_id; + char *driver; + char *firmware; +}; + +enum fi_bus_type { + FI_BUS_UNSPEC, + FI_BUS_UNKNOWN = FI_BUS_UNSPEC, + FI_BUS_PCI, +}; + +struct fi_pci_attr { + uint16_t domain_id; + uint8_t bus_id; + uint8_t device_id; + uint8_t function_id; +}; + +struct fi_bus_attr { + enum fi_bus_type bus_type; + union { + struct fi_pci_attr pci; + } attr; +}; + +enum fi_link_state { + FI_LINK_UNKNOWN, + FI_LINK_DOWN, + FI_LINK_UP, +}; + +struct fi_link_attr { + char *address; + size_t mtu; + size_t speed; + enum fi_link_state state; + char *network_type; +}; + +enum { + FI_CLASS_UNSPEC, + FI_CLASS_FABRIC, + FI_CLASS_DOMAIN, + FI_CLASS_EP, + FI_CLASS_SEP, + FI_CLASS_RX_CTX, + FI_CLASS_SRX_CTX, + FI_CLASS_TX_CTX, + FI_CLASS_STX_CTX, + FI_CLASS_PEP, + FI_CLASS_INTERFACE, + FI_CLASS_AV, + FI_CLASS_MR, + FI_CLASS_EQ, + FI_CLASS_CQ, + FI_CLASS_CNTR, + FI_CLASS_WAIT, + FI_CLASS_POLL, + FI_CLASS_CONNREQ, + FI_CLASS_MC, + FI_CLASS_NIC, + FI_CLASS_AV_SET, + FI_CLASS_MR_CACHE, + FI_CLASS_MEM_MONITOR, + FI_CLASS_PEER_CQ, + FI_CLASS_PEER_SRX, + FI_CLASS_LOG, + FI_CLASS_PEER_AV, + FI_CLASS_PEER_AV_SET, + FI_CLASS_PEER_CNTR, + FI_CLASS_PROFILE, +}; + +struct fi_eq_attr; +struct fi_wait_attr; + +/* fi_bind()-specific flags */ +#define FI_SELECTIVE_COMPLETION (1ULL << 59) + +struct fi_ops { + size_t size; + int (*close)(struct fid *fid); + int (*bind)(struct fid *fid, struct fid *bfid, uint64_t flags); + int (*control)(struct fid *fid, int command, void *arg); + int (*ops_open)(struct fid *fid, const char *name, + uint64_t flags, void **ops, void *context); + int (*tostr)(const struct fid *fid, char *buf, size_t len); + int (*ops_set)(struct fid *fid, const char *name, uint64_t flags, + void *ops, void *context); +}; + +/* All fabric interface descriptors must start with this structure */ +struct fid { + size_t fclass; + void *context; + struct fi_ops *ops; +}; + +int fi_getinfo(uint32_t version, const char *node, const char *service, + uint64_t flags, const struct fi_info *hints, + struct fi_info **info); +void fi_freeinfo(struct fi_info *info); +struct fi_info *fi_dupinfo(const struct fi_info *info); + +static inline struct fi_info *fi_allocinfo(void) +{ + return fi_dupinfo(NULL); +} + +struct fi_ops_fabric { + size_t size; + int (*domain)(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **dom, void *context); + int (*passive_ep)(struct fid_fabric *fabric, struct fi_info *info, + struct fid_pep **pep, void *context); + int (*eq_open)(struct fid_fabric *fabric, struct fi_eq_attr *attr, + struct fid_eq **eq, void *context); + int (*wait_open)(struct fid_fabric *fabric, struct fi_wait_attr *attr, + struct fid_wait **waitset); + int (*trywait)(struct fid_fabric *fabric, struct fid **fids, + int count); + int (*domain2)(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **dom, uint64_t flags, void *context); +}; + +struct fid_fabric { + struct fid fid; + struct fi_ops_fabric *ops; + uint32_t api_version; +}; + +int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, + uint64_t flags, void *context); +int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, + void *context); +int fi_open(uint32_t version, const char *name, void *attr, size_t attr_len, + uint64_t flags, struct fid **fid, void *context); + +struct fid_nic { + struct fid fid; + struct fi_device_attr *device_attr; + struct fi_bus_attr *bus_attr; + struct fi_link_attr *link_attr; + void *prov_attr; +}; + +#define FI_CHECK_OP(ops, opstype, op) \ + (ops && (ops->size > offsetof(opstype, op)) && ops->op) + +static inline int fi_close(struct fid *fid) +{ + return fid->ops->close(fid); +} + +struct fi_alias { + struct fid **fid; + uint64_t flags; +}; + +struct fi_fid_var { + int name; + void *val; +}; + +struct fi_mr_raw_attr { + uint64_t flags; + uint64_t *base_addr; + uint8_t *raw_key; + size_t *key_size; +}; + +struct fi_mr_map_raw { + uint64_t flags; + uint64_t base_addr; + uint8_t *raw_key; + size_t key_size; + uint64_t *key; +}; + +/* control commands */ +enum { + FI_GETFIDFLAG, /* uint64_t flags */ + FI_SETFIDFLAG, /* uint64_t flags */ + FI_GETOPSFLAG, /* uint64_t flags */ + FI_SETOPSFLAG, /* uint64_t flags */ + FI_ALIAS, /* struct fi_alias * */ + FI_GETWAIT, /* void * wait object */ + FI_ENABLE, /* NULL */ + FI_BACKLOG, /* integer * */ + FI_GET_RAW_MR, /* fi_mr_raw_attr */ + FI_MAP_RAW_MR, /* fi_mr_map_raw */ + FI_UNMAP_KEY, /* uint64_t key */ + FI_QUEUE_WORK, /* struct fi_deferred_work */ + FI_CANCEL_WORK, /* struct fi_deferred_work */ + FI_FLUSH_WORK, /* NULL */ + FI_REFRESH, /* mr: fi_mr_modify */ + FI_DUP, /* struct fid ** */ + FI_GETWAITOBJ, /*enum fi_wait_obj * */ + FI_GET_VAL, /* struct fi_fid_var */ + FI_SET_VAL, /* struct fi_fid_var */ + FI_EXPORT_FID, /* struct fi_fid_export */ + FI_GET_FD, /* int */ +}; + +static inline int fi_control(struct fid *fid, int command, void *arg) +{ + return fid->ops->control(fid, command, arg); +} + +static inline int fi_alias(struct fid *fid, struct fid **alias_fid, uint64_t flags) +{ + struct fi_alias alias; + alias.fid = alias_fid; + alias.flags = flags; + return fi_control(fid, FI_ALIAS, &alias); +} + +/* Provider specific names should set the uppermost bit. */ + +static inline int fi_get_val(struct fid *fid, int name, void *val) +{ + struct fi_fid_var var; + var.name = name; + var.val = val; + return fi_control(fid, FI_GET_VAL, &var); +} + +static inline int fi_set_val(struct fid *fid, int name, void *val) +{ + struct fi_fid_var var; + var.name = name; + var.val = val; + return fi_control(fid, FI_SET_VAL, &var); +} + +static inline int +fi_open_ops(struct fid *fid, const char *name, uint64_t flags, + void **ops, void *context) +{ + return fid->ops->ops_open(fid, name, flags, ops, context); +} + +static inline int +fi_set_ops(struct fid *fid, const char *name, uint64_t flags, + void *ops, void *context) +{ + return FI_CHECK_OP(fid->ops, struct fi_ops, ops_set) ? + fid->ops->ops_set(fid, name, flags, ops, context) : -FI_ENOSYS; +} + +enum fi_type { + FI_TYPE_INFO, + FI_TYPE_EP_TYPE, + FI_TYPE_CAPS, + FI_TYPE_OP_FLAGS, + FI_TYPE_ADDR_FORMAT, + FI_TYPE_TX_ATTR, + FI_TYPE_RX_ATTR, + FI_TYPE_EP_ATTR, + FI_TYPE_DOMAIN_ATTR, + FI_TYPE_FABRIC_ATTR, + FI_TYPE_THREADING, + FI_TYPE_PROGRESS, + FI_TYPE_PROTOCOL, + FI_TYPE_MSG_ORDER, + FI_TYPE_MODE, + FI_TYPE_AV_TYPE, + FI_TYPE_ATOMIC_TYPE, + FI_TYPE_ATOMIC_OP, + FI_TYPE_VERSION, + FI_TYPE_EQ_EVENT, + FI_TYPE_CQ_EVENT_FLAGS, + FI_TYPE_MR_MODE, + FI_TYPE_OP_TYPE, + FI_TYPE_FID, + FI_TYPE_COLLECTIVE_OP, + FI_TYPE_HMEM_IFACE, + FI_TYPE_CQ_FORMAT, + FI_TYPE_LOG_LEVEL, + FI_TYPE_LOG_SUBSYS, + FI_TYPE_AV_ATTR, + FI_TYPE_CQ_ATTR, + FI_TYPE_MR_ATTR, + FI_TYPE_CNTR_ATTR, + FI_TYPE_CQ_ERR_ENTRY, +}; + +char *fi_tostr(const void *data, enum fi_type datatype); +char *fi_tostr_r(char *buf, size_t len, const void *data, + enum fi_type datatype); + +enum fi_param_type { + FI_PARAM_STRING, + FI_PARAM_INT, + FI_PARAM_BOOL, + FI_PARAM_SIZE_T, +}; + +struct fi_param { + const char *name; + enum fi_param_type type; + const char *help_string; + const char *value; +}; + +int fi_getparams(struct fi_param **params, int *count); +void fi_freeparams(struct fi_param *params); + +/* Dummy definitions for removed flags/caps/types. For compiling old fabtests */ +#define FI_VARIABLE_MSG 0ULL +#define FI_NOTIFY_FLAGS_ONLY 0ULL +#define FI_RESTRICTED_COMP 0ULL +#define FI_EP_SOCK_STREAM FI_EP_UNSPEC + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_ +struct fi_context { + void *internal[4]; +}; + +struct fi_context2 { + void *internal[8]; +}; +#endif + +struct fi_recv_context { + struct fid_ep *ep; + void *context; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* FABRIC_H */ diff --git a/backends/libfabric/include/rdma/fi_atomic.h b/backends/libfabric/include/rdma/fi_atomic.h new file mode 100644 index 000000000..cc8b1e520 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_atomic.h @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_ATOMIC_H +#define FI_ATOMIC_H + +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Atomic flags */ +#define FI_FETCH_ATOMIC (1ULL << 58) +#define FI_COMPARE_ATOMIC (1ULL << 59) + +struct fi_atomic_attr { + size_t count; + size_t size; +}; + +struct fi_msg_atomic { + const struct fi_ioc *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t addr; + const struct fi_rma_ioc *rma_iov; + size_t rma_iov_count; + enum fi_datatype datatype; + enum fi_op op; + void *context; + uint64_t data; +}; + +struct fi_msg_fetch { + struct fi_ioc *msg_iov; + void **desc; + size_t iov_count; +}; + +struct fi_msg_compare { + const struct fi_ioc *msg_iov; + void **desc; + size_t iov_count; +}; + +struct fi_ops_atomic { + size_t size; + ssize_t (*write)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*writev)(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*writemsg)(struct fid_ep *ep, + const struct fi_msg_atomic *msg, uint64_t flags); + ssize_t (*inject)(struct fid_ep *ep, const void *buf, size_t count, + fi_addr_t dest_addr, uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op); + + ssize_t (*readwrite)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*readwritev)(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*readwritemsg)(struct fid_ep *ep, + const struct fi_msg_atomic *msg, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + uint64_t flags); + + ssize_t (*compwrite)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + const void *compare, void *compare_desc, + void *result, void *result_desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*compwritev)(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + const struct fi_ioc *comparev, void **compare_desc, size_t compare_count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*compwritemsg)(struct fid_ep *ep, + const struct fi_msg_atomic *msg, + const struct fi_ioc *comparev, void **compare_desc, size_t compare_count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + uint64_t flags); + + int (*writevalid)(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count); + int (*readwritevalid)(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count); + int (*compwritevalid)(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count); +}; + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_ATOMIC + +static inline ssize_t +fi_atomic(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->write(ep, buf, count, desc, dest_addr, addr, key, + datatype, op, context); +} + +static inline ssize_t +fi_atomicv(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->writev(ep, iov, desc, count, dest_addr, addr, key, + datatype, op, context); +} + +static inline ssize_t +fi_atomicmsg(struct fid_ep *ep, + const struct fi_msg_atomic *msg, uint64_t flags) +{ + return ep->atomic->writemsg(ep, msg, flags); +} + +static inline ssize_t +fi_inject_atomic(struct fid_ep *ep, const void *buf, size_t count, + fi_addr_t dest_addr, uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op) +{ + return ep->atomic->inject(ep, buf, count, dest_addr, addr, + key, datatype, op); +} + +static inline ssize_t +fi_fetch_atomic(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->readwrite(ep, buf, count, desc, result, result_desc, + dest_addr, addr, key, datatype, op, context); +} + +static inline ssize_t +fi_fetch_atomicv(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->readwritev(ep, iov, desc, count, + resultv, result_desc, result_count, + dest_addr, addr, key, datatype, op, context); +} + +static inline ssize_t +fi_fetch_atomicmsg(struct fid_ep *ep, + const struct fi_msg_atomic *msg, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + uint64_t flags) +{ + return ep->atomic->readwritemsg(ep, msg, resultv, result_desc, + result_count, flags); +} + +static inline ssize_t +fi_compare_atomic(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + const void *compare, void *compare_desc, + void *result, void *result_desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->compwrite(ep, buf, count, desc, + compare, compare_desc, result, result_desc, + dest_addr, addr, key, datatype, op, context); +} + +static inline ssize_t +fi_compare_atomicv(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + const struct fi_ioc *comparev, void **compare_desc, size_t compare_count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->compwritev(ep, iov, desc, count, + comparev, compare_desc, compare_count, + resultv, result_desc, result_count, + dest_addr, addr, key, datatype, op, context); +} + +static inline ssize_t +fi_compare_atomicmsg(struct fid_ep *ep, + const struct fi_msg_atomic *msg, + const struct fi_ioc *comparev, void **compare_desc, size_t compare_count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + uint64_t flags) +{ + return ep->atomic->compwritemsg(ep, msg, + comparev, compare_desc, compare_count, + resultv, result_desc, result_count, flags); +} + +static inline int +fi_atomicvalid(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count) +{ + return ep->atomic->writevalid(ep, datatype, op, count); +} + +static inline int +fi_fetch_atomicvalid(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count) +{ + return ep->atomic->readwritevalid(ep, datatype, op, count); +} + +static inline int +fi_compare_atomicvalid(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count) +{ + return ep->atomic->compwritevalid(ep, datatype, op, count); +} + +static inline int +fi_query_atomic(struct fid_domain *domain, + enum fi_datatype datatype, enum fi_op op, + struct fi_atomic_attr *attr, uint64_t flags) +{ + return FI_CHECK_OP(domain->ops, struct fi_ops_domain, query_atomic) ? + domain->ops->query_atomic(domain, datatype, op, attr, flags) : + -FI_ENOSYS; +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_ATOMIC_H */ diff --git a/backends/libfabric/include/rdma/fi_cm.h b/backends/libfabric/include/rdma/fi_cm.h new file mode 100644 index 000000000..c0faf1f37 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_cm.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2013-2016 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_CM_H +#define FI_CM_H + +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +struct fid_mc { + struct fid fid; + fi_addr_t fi_addr; +}; + +struct fi_ops_cm { + size_t size; + int (*setname)(fid_t fid, void *addr, size_t addrlen); + int (*getname)(fid_t fid, void *addr, size_t *addrlen); + int (*getpeer)(struct fid_ep *ep, void *addr, size_t *addrlen); + int (*connect)(struct fid_ep *ep, const void *addr, + const void *param, size_t paramlen); + int (*listen)(struct fid_pep *pep); + int (*accept)(struct fid_ep *ep, const void *param, size_t paramlen); + int (*reject)(struct fid_pep *pep, fid_t handle, + const void *param, size_t paramlen); + int (*shutdown)(struct fid_ep *ep, uint64_t flags); + int (*join)(struct fid_ep *ep, const void *addr, uint64_t flags, + struct fid_mc **mc, void *context); +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_CM + +static inline int fi_setname(fid_t fid, void *addr, size_t addrlen) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->cm->setname(fid, addr, addrlen); +} + +static inline int fi_getname(fid_t fid, void *addr, size_t *addrlen) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->cm->getname(fid, addr, addrlen); +} + +static inline int fi_getpeer(struct fid_ep *ep, void *addr, size_t *addrlen) +{ + return ep->cm->getpeer(ep, addr, addrlen); +} + +static inline int fi_listen(struct fid_pep *pep) +{ + return pep->cm->listen(pep); +} + +static inline int +fi_connect(struct fid_ep *ep, const void *addr, + const void *param, size_t paramlen) +{ + return ep->cm->connect(ep, addr, param, paramlen); +} + +static inline int +fi_accept(struct fid_ep *ep, const void *param, size_t paramlen) +{ + return ep->cm->accept(ep, param, paramlen); +} + +static inline int +fi_reject(struct fid_pep *pep, fid_t handle, + const void *param, size_t paramlen) +{ + return pep->cm->reject(pep, handle, param, paramlen); +} + +static inline int fi_shutdown(struct fid_ep *ep, uint64_t flags) +{ + return ep->cm->shutdown(ep, flags); +} + +static inline int fi_join(struct fid_ep *ep, const void *addr, uint64_t flags, + struct fid_mc **mc, void *context) +{ + return FI_CHECK_OP(ep->cm, struct fi_ops_cm, join) ? + ep->cm->join(ep, addr, flags, mc, context) : -FI_ENOSYS; +} + +static inline fi_addr_t fi_mc_addr(struct fid_mc *mc) +{ + return mc->fi_addr; +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_CM_H */ diff --git a/backends/libfabric/include/rdma/fi_collective.h b/backends/libfabric/include/rdma/fi_collective.h new file mode 100644 index 000000000..9e7ac6297 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_collective.h @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2019 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_COLLECTIVE_H +#define FI_COLLECTIVE_H + +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + + +struct fi_ops_av_set { + size_t size; + int (*set_union)(struct fid_av_set *dst, + const struct fid_av_set *src); + int (*intersect)(struct fid_av_set *dst, + const struct fid_av_set *src); + int (*diff)(struct fid_av_set *dst, const struct fid_av_set *src); + int (*insert)(struct fid_av_set *set, fi_addr_t addr); + int (*remove)(struct fid_av_set *set, fi_addr_t addr); + int (*addr)(struct fid_av_set *set, fi_addr_t *coll_addr); +}; + +struct fid_av_set { + struct fid fid; + struct fi_ops_av_set *ops; +}; + +struct fi_collective_attr { + enum fi_op op; + enum fi_datatype datatype; + struct fi_atomic_attr datatype_attr; + size_t max_members; + uint64_t mode; +}; + +struct fi_collective_addr { + const struct fid_av_set *set; + fi_addr_t coll_addr; +}; + +struct fi_msg_collective { + const struct fi_ioc *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t coll_addr; + fi_addr_t root_addr; + enum fi_collective_op coll; + enum fi_datatype datatype; + enum fi_op op; + void *context; +}; + +struct fi_ops_collective { + size_t size; + + ssize_t (*barrier)(struct fid_ep *ep, fi_addr_t coll_addr, void *context); + ssize_t (*broadcast)(struct fid_ep *ep, + void *buf, size_t count, void *desc, + fi_addr_t coll_addr, fi_addr_t root_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*alltoall)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*allreduce)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context); + ssize_t (*allgather)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*reduce_scatter)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context); + ssize_t (*reduce)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + fi_addr_t root_addr, enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context); + ssize_t (*scatter)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t coll_addr, fi_addr_t root_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*gather)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t coll_addr, fi_addr_t root_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*msg)(struct fid_ep *ep, + const struct fi_msg_collective *msg, + struct fi_ioc *resultv, void **result_desc, + size_t result_count, uint64_t flags); + ssize_t (*barrier2)(struct fid_ep *ep, fi_addr_t coll_addr, uint64_t flags, + void *context); +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_COLLECTIVE + +static inline int +fi_av_set(struct fid_av *av, struct fi_av_set_attr *attr, + struct fid_av_set **set, void * context) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, av_set) ? + av->ops->av_set(av, attr, set, context) : -FI_ENOSYS; +} + +static inline int +fi_av_set_union(struct fid_av_set *dst, const struct fid_av_set *src) +{ + return dst->ops->set_union(dst, src); +} + +static inline int +fi_av_set_intersect(struct fid_av_set *dst, const struct fid_av_set *src) +{ + return dst->ops->intersect(dst, src); +} + +static inline int +fi_av_set_diff(struct fid_av_set *dst, const struct fid_av_set *src) +{ + return dst->ops->diff(dst, src); +} + +static inline int +fi_av_set_insert(struct fid_av_set *set, fi_addr_t addr) +{ + return set->ops->insert(set, addr); +} + +static inline int +fi_av_set_remove(struct fid_av_set *set, fi_addr_t addr) +{ + return set->ops->remove(set, addr); +} + +static inline int +fi_av_set_addr(struct fid_av_set *set, fi_addr_t *coll_addr) +{ + return set->ops->addr(set, coll_addr); +} + +static inline int +fi_join_collective(struct fid_ep *ep, fi_addr_t coll_addr, + const struct fid_av_set *set, + uint64_t flags, struct fid_mc **mc, void *context) +{ + struct fi_collective_addr addr; + + addr.set = set; + addr.coll_addr = coll_addr; + return fi_join(ep, &addr, flags | FI_COLLECTIVE, mc, context); +} + +static inline ssize_t +fi_barrier(struct fid_ep *ep, fi_addr_t coll_addr, void *context) +{ + return ep->collective->barrier(ep, coll_addr, context); +} + +static inline ssize_t +fi_barrier2(struct fid_ep *ep, fi_addr_t coll_addr, uint64_t flags, void *context) +{ + if (!flags) + return fi_barrier(ep, coll_addr, context); + + return FI_CHECK_OP(ep->collective, struct fi_ops_collective, barrier2) ? + ep->collective->barrier2(ep, coll_addr, flags, context) : + -FI_ENOSYS; +} + +static inline ssize_t +fi_broadcast(struct fid_ep *ep, void *buf, size_t count, void *desc, + fi_addr_t coll_addr, fi_addr_t root_addr, + enum fi_datatype datatype, uint64_t flags, void *context) +{ + return ep->collective->broadcast(ep, buf, count, desc, + coll_addr, root_addr, datatype, flags, context); +} + +static inline ssize_t +fi_alltoall(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t coll_addr, enum fi_datatype datatype, + uint64_t flags, void *context) +{ + return ep->collective->alltoall(ep, buf, count, desc, + result, result_desc, coll_addr, datatype, flags, context); +} + +static inline ssize_t +fi_allreduce(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context) +{ + return ep->collective->allreduce(ep, buf, count, desc, + result, result_desc, coll_addr, datatype, op, flags, context); +} + +static inline ssize_t +fi_allgather(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, uint64_t flags, void *context) +{ + return ep->collective->allgather(ep, buf, count, desc, + result, result_desc, coll_addr, datatype, flags, context); +} + +static inline ssize_t +fi_reduce_scatter(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context) +{ + return ep->collective->reduce_scatter(ep, buf, count, desc, + result, result_desc, coll_addr, datatype, op, flags, context); +} + +static inline ssize_t +fi_reduce(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + fi_addr_t root_addr, enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context) +{ + return ep->collective->reduce(ep, buf, count, desc, result, result_desc, + coll_addr, root_addr, datatype, op, flags, context); +} + + +static inline ssize_t +fi_scatter(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + fi_addr_t root_addr, enum fi_datatype datatype, + uint64_t flags, void *context) +{ + return ep->collective->scatter(ep, buf, count, desc, result, result_desc, + coll_addr, root_addr, datatype, flags, context); +} + + +static inline ssize_t +fi_gather(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + fi_addr_t root_addr, enum fi_datatype datatype, + uint64_t flags, void *context) +{ + return ep->collective->gather(ep, buf, count, desc, result, result_desc, + coll_addr, root_addr, datatype, flags, context); +} + +static inline +int fi_query_collective(struct fid_domain *domain, enum fi_collective_op coll, + struct fi_collective_attr *attr, uint64_t flags) +{ + return FI_CHECK_OP(domain->ops, struct fi_ops_domain, query_collective) ? + domain->ops->query_collective(domain, coll, attr, flags) : + -FI_ENOSYS; +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_COLLECTIVE_H */ diff --git a/backends/libfabric/include/rdma/fi_cxi_ext.h b/backends/libfabric/include/rdma/fi_cxi_ext.h new file mode 100644 index 000000000..b69bb0a20 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_cxi_ext.h @@ -0,0 +1,539 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only + * + * Copyright (c) 2020-2024 Hewlett Packard Enterprise Development LP + */ + +#ifndef _FI_CXI_EXT_H_ +#define _FI_CXI_EXT_H_ + +/* CXI provider specific NIC attributes. This information is returned in + * fi_info::nid::prov_attr. + + * Users can optionally modify some fields. Depending on the field adjusted, + * this can impact fi_domain() or other endpoint allocation behavior. + */ + +#define FI_CXI_NIC_ATTR_VER 1U + +struct cxip_nic_attr { + /* Version of NIC attr. Must remain at the top of this struct. */ + uint32_t version; + + /* NIC address. Should never be modified. */ + const unsigned int addr; + + /* On output from fi_getinfo(), rgroup_id will be set in the following + * order: + * 1. Resource group ID returned from SLINGSHOT_SVC_ID environment + * variable + * 2. First resource group ID with matching UID + * 3. First resource group ID with matching GID + * 4. First resource group ID with open permissions + */ + const unsigned int default_rgroup_id; + + /* Default VNI used with the rgroup ID. */ + const unsigned int default_vni; +}; + +/* + * TODO: The following should be integrated into the include/rdma/fi_ext.h + * and are use for provider specific fi_control() operations. + */ +#define FI_PROV_SPECIFIC_CXI (0xccc << 16) + +enum { + FI_OPT_CXI_SET_TCLASS = -FI_PROV_SPECIFIC_CXI, /* uint32_t */ + FI_OPT_CXI_SET_MSG_ORDER, /* uint64_t */ + + /* fid_nic control operation to refresh NIC attributes. */ + FI_OPT_CXI_NIC_REFRESH_ATTR, + + FI_OPT_CXI_SET_MR_MATCH_EVENTS, /* bool */ + FI_OPT_CXI_GET_MR_MATCH_EVENTS, /* bool */ + FI_OPT_CXI_SET_OPTIMIZED_MRS, /* bool */ + FI_OPT_CXI_GET_OPTIMIZED_MRS, /* bool */ + FI_OPT_CXI_SET_PROV_KEY_CACHE, /* bool */ + FI_OPT_CXI_GET_PROV_KEY_CACHE, /* bool */ + FI_OPT_CXI_SET_RNR_MAX_RETRY_TIME, /* uint64_t */ +}; + +/* + * Defines for compatibility between main branch features and release + * branches back porting of features required for use. Any value + * included here should map exactly to the value established in the + * main branch (enum or define) and this CXI equivalent will exist forever. + */ +#define FI_CXI_CNTR_EVENTS_BYTES 1 /* FI_CNTR_EVENTS_BYTES */ + +/* + * CXI provider specific counter flag to return current/cached counter value + * in host memory. A request to update the count is requested, but the routine + * does not wait for the update to complete. Subsequent reads will pick up + * the updated counter value. The normal behavior is to wait for a memory update + * to complete (or to use the domain ops counter routines). + */ +#define FI_CXI_CNTR_CACHED (1ULL << 32) + +/* + * TODO: Set this to the upstream value prior to releasing software. + * This flag returned in a completion and indicates that the message was + * truncated and that the length indicates the truncated message length. + */ +#define FI_CXI_TRUNC (1ULL << 56) + +/* + * Execute a given libfabric atomic memory operation as a PCIe operation as + * compared to a NIC operation. + * + * Note: Ordering between PCIe atomic operations and NIC atomic/RMA operations + * is undefined. + * + * Note: This flag overloads the bit used for FI_SOURCE. But, since FI_SOURCE + * is invalid for AMO operations, overloading this bit is not an issue. + */ +#define FI_CXI_PCIE_AMO (1ULL << 57) + +/* + * Flag an accelerated collective as pre-reduced. + * + * This can be passed to the accelerated collectives operations to indicate + * that the supplied data is a pre-reduced cxip_coll_accumulator structure. + * + * Note: This flag overloads FI_CXI_PCIE_AMO. Accelerated collectives do not + * use FI_CXI_PCIE_AMO or FI_SOURCE. + */ +#define FI_CXI_PRE_REDUCED (1ULL << 57) + +/* + * Use CXI High Rate Puts (HRP). Increases message rate performance. Applies to + * RMA and unreliable, non-fetching AMO operations. + */ +#define FI_CXI_HRP (1ULL << 60) + +/* + * Disable AMO reliability. Increases message rate performance. Applies to + * non-fetching AMOs. Required for HRP AMOs. + */ +#define FI_CXI_UNRELIABLE (1ULL << 61) + +/* Depreciated. */ +#define FI_CXI_WEAK_FENCE \ + _Pragma ("GCC warning \"'FI_CXI_WEAK_FENCE' macro is deprecated\"") \ + (1ULL << 63) + +/* + * Used in conjunction with the deferred work queue API. If a deferred work + * queue operation has this flag set, the CXI provider will ensure a counter + * writeback occurs once the deferred work queue operation completes. + * Note: Addition hardware resources will be used to ensure a counter writeback + * occurs at the completion of the deferred work queue operation. + */ +#define FI_CXI_CNTR_WB (1ULL << 62) +#define FI_CXI_COUNTER_OPS "cxi_counter_ops" + +struct fi_cxi_cntr_ops { + /* Set the counter writeback address to a client provided address. */ + int (*set_wb_buffer)(struct fid *fid, void *buf, size_t len); + + /* Get the counter MMIO region. */ + int (*get_mmio_addr)(struct fid *fid, void **addr, size_t *len); +}; + +/* Success values cannot exceed FI_CXI_CNTR_SUCCESS_MAX */ +#define FI_CXI_CNTR_SUCCESS_MAX ((1ULL << 48) - 1) + +/* Failure values cannot exceed FI_CXI_CNTR_FAILURE_MAX */ +#define FI_CXI_CNTR_FAILURE_MAX ((1ULL << 7) - 1) + +/* fi_cntr_read() equivalent but for the writeback buffer. */ +static inline uint64_t fi_cxi_cntr_wb_read(const void *wb_buf) +{ + return (*(uint64_t *)wb_buf) & FI_CXI_CNTR_SUCCESS_MAX; +}; + +/* fi_cntr_reader() equivalent but for the writeback buffer. */ +static inline uint64_t fi_cxi_cntr_wb_readerr(const void *wb_buf) +{ + return ((*(uint64_t *)wb_buf) >> 48) & FI_CXI_CNTR_FAILURE_MAX; +}; + +/* Generate a counter success value which can be polled on. */ +static inline int fi_cxi_gen_cntr_success(uint64_t value, uint64_t *cxi_value) +{ + if (value > FI_CXI_CNTR_SUCCESS_MAX) + return -FI_EINVAL; + + *cxi_value = (1ULL << 63) | value; + return FI_SUCCESS; +}; + +/* fi_cntr_add() equivalent but for the MMIO region. */ +static inline int fi_cxi_cntr_add(void *cntr_mmio, uint64_t value) +{ + /* Success counter is only 48 bits wide. */ + if (value > FI_CXI_CNTR_SUCCESS_MAX) + return -FI_EINVAL; + + *((uint64_t *)cntr_mmio) = value; + return FI_SUCCESS; +} + +/* fi_cntr_adderr() equivalent but for the MMIO region. */ +static inline int fi_cxi_cntr_adderr(void *cntr_mmio, uint64_t value) +{ + /* Error counter is only 7 bits wide. */ + if (value > FI_CXI_CNTR_FAILURE_MAX) + return -FI_EINVAL; + + *((uint64_t *)cntr_mmio + 8) = value; + return FI_SUCCESS; +} + +/* fi_cntr_set() equivalent but for the MMIO region. */ +static inline int fi_cxi_cntr_set(void *cntr_mmio, uint64_t value) +{ + /* Only set of zero is supported through MMIO region. */ + if (value > 0) + return -FI_EINVAL; + + *((uint64_t *)cntr_mmio + 16) = 0; + return FI_SUCCESS; +} + +/* fi_cntr_seterr() equivalent but for MMIO region. */ +static inline int fi_cxi_cntr_seterr(void *cntr_mmio, uint64_t value) +{ + /* Only set of zero is supported through MMIO region. */ + if (value > 0) + return -FI_EINVAL; + + *((uint64_t *)cntr_mmio + 24) = 0; + return FI_SUCCESS; +} + +/* fi_cntr_add() equivalent but for the MMIO region. */ +static inline void *fi_cxi_get_cntr_add_addr(void *cntr_mmio) +{ + return cntr_mmio; +} + +/* fi_cntr_adderr() equivalent but for the MMIO region. */ +static inline void *fi_cxi_get_cntr_adderr_addr(void *cntr_mmio) +{ + return (void *)((uint64_t *)cntr_mmio + 8); +} + +/* fi_cntr_set() equivalent but for the MMIO region reset. + * NOTE: CXI does not support set to counter MMIO region. Only reset. + */ +static inline void *fi_cxi_get_cntr_reset_addr(void *cntr_mmio) +{ + return (void *)((uint64_t *)cntr_mmio + 16); +} + +/* fi_cntr_seterr() equivalent but for MMIO region reset. + * NOTE: CXI does not support set to counter MMIO region. Only reset. + */ +static inline void *fi_cxi_get_cntr_reseterr_addr(void *cntr_mmio) +{ + return (void *)((uint64_t *)cntr_mmio + 24); +} + +#define FI_CXI_DOM_OPS_1 "dom_ops_v1" +#define FI_CXI_DOM_OPS_2 "dom_ops_v2" +#define FI_CXI_DOM_OPS_3 "dom_ops_v3" +#define FI_CXI_DOM_OPS_4 "dom_ops_v4" +#define FI_CXI_DOM_OPS_5 "dom_ops_v5" +#define FI_CXI_DOM_OPS_6 "dom_ops_v6" + +/* v1 to v6 can use the same struct since they only appended a routine */ +struct fi_cxi_dom_ops { + int (*cntr_read)(struct fid *fid, unsigned int cntr, uint64_t *value, + struct timespec *ts); + int (*topology)(struct fid *fid, unsigned int *group_id, + unsigned int *switch_id, unsigned int *port_id); + + /* Enable hybrid MR desc mode. Hybrid MR desc allows for libfabric users + * to optionally pass in a valid MR desc for local communication + * operations. + * + * When enabled, if the MR desc is NULL, the provider will + * perform internal memory registration. Else, the provider will assume + * the MR desc field is valid and skip internal memory registration. + * + * When disabled, the provider will ignore the MR desc field and always + * perform internal memory registration. This is the default behavior. + * + * All child endpoints will inherit the current domain status of hybrid + * MR desc only during endpoint creation. Dynamically changing the + * domain hybrid MR desc status with endpoint allocate may not propagate + * to child endpoints. Thus, it is recommended to set hybrid MR desc + * status prior to allocating endpoints. + */ + int (*enable_hybrid_mr_desc)(struct fid *fid, bool enable); + + /* Get unexpected message information. + * + * Obtain a list of unexpected messages associated with the endpoint. + * The list is returned as an array of CQ tagged entries. The following + * is how the fields in fi_cq_tagged_entry are used. + * + * op_context: NULL since this message has not matched a posted receive + * flags: A combination of FI_MSG, FI_TAGGED, FI_RECV, + * and/or FI_REMOTE_CQ_DATA + * len: Unexpected message request length + * data: Completion queue data (only valid if FI_REMOTE_CQ_DATA + * is set) + * tag: Unexpected message tag (only valid if FI_TAGGED is set) + * + * @ep: Endpoint FID to have unexpected messages returned to user. + * @entry: Tagged entry array to be filled in by the provider. If the + * entry is NULL, only ux_count will be set. + * @count: Number of entries in entry and src_addr array. If count is + * zero,then only the ux_count will be set on return. + * @src_addr: Source address array to be filled in by the provider. If + * the entry is NULL, only ux_count will be set. + * @ux_count: Output variable used to return the number of unexpected + * messages queued on the given endpoint. + * + * Return: On success, number of entries copied into the users entry + * and src_addr arrays. On error, -FI_ERRNO. + */ + size_t (*ep_get_unexp_msgs)(struct fid_ep *fid_ep, + struct fi_cq_tagged_entry *entry, + size_t count, fi_addr_t *src_addr, + size_t *ux_count); + + /* Get the depth of the deferred work queue. The depth is the number of + * of triggered operation commands which can be queued to hardware. The + * depth is not per fi_domain. The depth is across all processes using + * the same CXI service which usually maps to a job-step. + */ + int (*get_dwq_depth)(struct fid *fid, size_t *depth); + + /* The following two functions have been deprecated in favor of + * using the fi_control() standardized interface. They will be + * removed in a future software release, but are left here initially + * to allow early users to adjust their usage. + */ + int (*enable_mr_match_events)(struct fid *fid, bool enable); + int (*enable_optimized_mrs)(struct fid *fid, bool enable); +}; + +/* + * CXI Authorization Key + */ +struct cxi_auth_key { + /* The CXI service assigned to the Domain and Endpoints. A CXI service + * is associated with a set of local resource limits, VNIs, and Traffic + * Classes. + * + * The svc_id used by an OFI Domain must match all Endpoints belonging + * to the Domain. + */ + uint32_t svc_id; + + /* The Virtual Network ID (VNI) assigned to the Endpoint. Two Endpoints + * must use the same VNI in order to communicate. + * + * Note that while the CXI service may define one or more VNIs which a + * process can access, an Endpoint is assigned to only one. + */ + uint16_t vni; +}; + +/* + * CXI Collectives + */ + +/* + * AV Set communication key. + * + * For production: + * - Set cxip_comm_key.keytype = COMM_KEY_NONE. + * - Initialize cxip_comm_key structure to zeros. + * - Create one av_set on each node. + * - Initialize each av_set to contain the NIC addresses of all endpoints. + * - Call fi_join_collective() once on each endpoint. + * - dest_addr is a multicast address created by the join. + * - hwroot_nic is assigned by the join. + * - The PTE will receive at the multicast ID value, index extension of zero. + * - Sending to the multicast ID will cause delivery to nodes according to the + * tree topology. + * + * For testing with externally established multicast address: + * - NOT IMPLEMENTED. + * + * For testing on a multinode system without multicast: + * - Set cxip_comm_key.keytype = COMM_KEY_UNICAST. + * - Set cxip_comm_key.ucast.hwroot_idx to the desired hw_root index. + * - Create one av_set on each node. + * - Initialize each av_set to contain the NIC addresses of all endpoints. + * - Call fi_join_collective() once one each endpoint. + * - hwroot_nic is the NIC address of the node that serves as the emulated + * hardware root of the tree. + * - The PTE will use the EP source NIC address and process PID, with a + * PID_IDX of CXIP_PTL_IDX_COLL. + * - Sending to any (valid) node address with CXIP_PTL_IDX_COLL will target the + * collectives PTE on that node. + * - The root/leaf send routines will distribute one or more packets to all + * fi_addr_t in the av_set as appropriate. + * + * For testing under NETSIM on a single node: + * - Set cxip_comm_key.keytype = COMM_KEY_RANK. + * - Set cxip_comm_key.rank.hwroot_idx to the desired hw_root index. + * - Set cxip_comm_key.rank.rank to the simulated rank. + * - Create N av_set objects, one for each simulated rank. + * - Call fi_join_collective() once for each simulated endpoint. + * - dest_addr is the MC object index. + * - hwroot_nic is the MC object index for the MC object to serve as the + * simulated hardware root. + * - The PTE will use the EP source NIC address and process PID, with a PID_IDX + * of 16 + dest_addr (MC object index). + * - Sending to the node's own address with a PID_IDX of 16 + MC index will + * target the appropriate MC object. + * - Simulation is limited to 32 simulated endpoints. + */ +enum cxip_comm_key_type { + COMM_KEY_NONE = 0, + COMM_KEY_MULTICAST, + COMM_KEY_UNICAST, + COMM_KEY_RANK, + COMM_KEY_MAX +}; + +/* Extends C_RC_* driver errors for libfabric */ +/* Translated to strings by cxip_strerror() -- keep synchronized */ +enum cxip_coll_prov_errno { + /* C_RC_* from cxi-driver overlaps first 6 bits of space [0,63] */ + + /* collectives CQ reduction error codes + * highest number error predominates + */ + FI_CXI_ERRNO_RED_FIRST = 1024, + FI_CXI_ERRNO_RED_FLT_OVERFLOW = 1024, + /* double precision value overflow */ + FI_CXI_ERRNO_RED_FLT_INVALID = 1025, + /* double precision sNAN/inf value */ + FI_CXI_ERRNO_RED_INT_OVERFLOW = 1026, + /* reproducible sum overflow */ + FI_CXI_ERRNO_RED_CONTR_OVERFLOW = 1027, + /* reduction contribution overflow */ + FI_CXI_ERRNO_RED_OP_MISMATCH = 1028, + /* reduction opcode mismatch */ + FI_CXI_ERRNO_RED_MC_FAILURE = 1029, + /* unused */ + FI_CXI_COLL_RC_RDMA_FAILURE = 1030, + /* leaf rdma read error */ + FI_CXI_COLL_RC_RDMA_DATA_FAILURE = 1031, + /* leaf rdma read data miscompare, unexpected packet data */ + FI_CXI_ERRNO_RED_OTHER = 1032, + /* non-specific reduction error, fatal */ + FI_CXI_ERRNO_RED_LAST = 1033, + + /* collectives EQ join error codes + * highest number error predominates + */ + FI_CXI_ERRNO_JOIN_FIRST = 2048, + FI_CXI_ERRNO_JOIN_MCAST_INUSE = 2048, + /* endpoint already using mcast address */ + FI_CXI_ERRNO_JOIN_HWROOT_INUSE = 2049, + /* endpoint already serving as HWRoot */ + FI_CXI_ERRNO_JOIN_MCAST_INVALID = 2050, + /* mcast address from FM is invalid */ + FI_CXI_ERRNO_JOIN_HWROOT_INVALID = 2051, + /* HWRoot address from FM is invalid */ + FI_CXI_ERRNO_JOIN_CURL_FAILED = 2052, + /* libcurl initiation failed */ + FI_CXI_ERRNO_JOIN_CURL_TIMEOUT = 2053, + /* libcurl timed out */ + FI_CXI_ERRNO_JOIN_SERVER_ERR = 2054, + /* unhandled CURL response code */ + FI_CXI_ERRNO_JOIN_FAIL_PTE = 2055, + /* libfabric PTE allocation failed */ + FI_CXI_ERRNO_JOIN_OTHER = 2056, + /* non-specific JOIN error, fatal */ + FI_CXI_ERRNO_JOIN_FAIL_RDMA = 2057, + /* root or leaf rdma init failure */ + FI_CXI_ERRNO_JOIN_LAST = FI_CXI_ERRNO_JOIN_FIRST + 43, + /* LAST is determined by the 43-bit error mask . + * Result is the OR of all bits set by different endpoints. + * This reserves space for all 43 bits for new errors. + */ +}; + +typedef unsigned int cxip_coll_op_t; // CXI collective opcode + +struct cxip_coll_mcast_key { + uint32_t hwroot_idx; // index of hwroot in av_set list + uint32_t mcast_addr; // 13-bit multicast address id +}; + +struct cxip_coll_unicast_key { + uint32_t hwroot_idx; // index of hwroot in av_set list + uint32_t mcast_addr; // 13-bit simulated multcast address +}; + +struct cxip_coll_rank_key { + uint32_t hwroot_idx; // index of hwroot in av_set list + uint32_t rank; // rank of this object + bool rx_discard; // clear to report RX events +}; + +struct cxip_comm_key { + enum cxip_comm_key_type keytype; + union { + struct cxip_coll_mcast_key mcast; + struct cxip_coll_unicast_key ucast; + struct cxip_coll_rank_key rank; + }; +}; + +/* Extended reduction opcodes. + * + * Only the following standard FI_ATOMIC operations are supported: + * - FI_MIN : INT or FLT + * - FI_MAX : INT or FLT + * - FI_SUM : INT or FLT + * - FI_BOR : INT + * - FI_BAND : INT + * - FI_BXOR : INT + * + * The codes below extend this standard FI_ATOMIC set to explicitly take + * advantage of extended hardware operations. These can be used as opcodes for + * any of the collective operations, just like FI_MIN or FI_SUM. + * + * Note that the current FI_ATOMIC set ends at opcode == 19. We start this one + * at 32, to accommodate possible expansion of the FI_ATOMIC set, and check for + * overlap during initialization. + */ +enum cxip_coll_op { + FI_CXI_MINMAXLOC = 32, // FLT or INT + FI_CXI_REPSUM, // FLT only + FI_CXI_OP_LAST +}; + +/* Extended accelerated reduction structures. + */ +struct cxip_coll_intminmax { + int64_t minval; + uint64_t minidx; + int64_t maxval; + uint64_t maxidx; +}; + +struct cxip_coll_fltminmax { + double minval; + uint64_t minidx; + double maxval; + uint64_t maxidx; +}; + +/* opaque export of struct cxip_coll_data */ +struct cxip_coll_accumulator { + uint8_t accum[64]; +}; + +#endif /* _FI_CXI_EXT_H_ */ diff --git a/backends/libfabric/include/rdma/fi_domain.h b/backends/libfabric/include/rdma/fi_domain.h new file mode 100644 index 000000000..0e6d0acb6 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_domain.h @@ -0,0 +1,598 @@ +/* + * Copyright (c) 2013-2017 Intel Corporation. All rights reserved. + * (C) Copyright 2020 Hewlett Packard Enterprise Development LP + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_DOMAIN_H +#define FI_DOMAIN_H + +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* + * AV = Address Vector + * Maps and stores transport/network addresses. + */ + +#define FI_SYMMETRIC (1ULL << 59) +#define FI_SYNC_ERR (1ULL << 58) +#define FI_UNIVERSE (1ULL << 57) +#define FI_BARRIER_SET (1ULL << 40) +#define FI_BROADCAST_SET (1ULL << 41) +#define FI_ALLTOALL_SET (1ULL << 42) +#define FI_ALLREDUCE_SET (1ULL << 43) +#define FI_ALLGATHER_SET (1ULL << 44) +#define FI_REDUCE_SCATTER_SET (1ULL << 45) +#define FI_REDUCE_SET (1ULL << 46) +#define FI_SCATTER_SET (1ULL << 47) +#define FI_GATHER_SET (1ULL << 48) + +struct fi_av_attr { + enum fi_av_type type; + int rx_ctx_bits; + size_t count; + size_t ep_per_node; + const char *name; + void *map_addr; + uint64_t flags; +}; + +struct fi_av_set_attr { + size_t count; + fi_addr_t start_addr; + fi_addr_t end_addr; + uint64_t stride; + size_t comm_key_size; + uint8_t *comm_key; + uint64_t flags; +}; + +struct fid_av_set; + +struct fi_ops_av { + size_t size; + int (*insert)(struct fid_av *av, const void *addr, size_t count, + fi_addr_t *fi_addr, uint64_t flags, void *context); + int (*insertsvc)(struct fid_av *av, const char *node, + const char *service, fi_addr_t *fi_addr, + uint64_t flags, void *context); + int (*insertsym)(struct fid_av *av, const char *node, size_t nodecnt, + const char *service, size_t svccnt, fi_addr_t *fi_addr, + uint64_t flags, void *context); + int (*remove)(struct fid_av *av, fi_addr_t *fi_addr, size_t count, + uint64_t flags); + int (*lookup)(struct fid_av *av, fi_addr_t fi_addr, void *addr, + size_t *addrlen); + const char * (*straddr)(struct fid_av *av, const void *addr, + char *buf, size_t *len); + int (*av_set)(struct fid_av *av, struct fi_av_set_attr *attr, + struct fid_av_set **av_set, void *context); + int (*insert_auth_key)(struct fid_av *av, const void *auth_key, + size_t auth_key_size, fi_addr_t *fi_addr, + uint64_t flags); + int (*lookup_auth_key)(struct fid_av *av, fi_addr_t fi_addr, + void *auth_key, size_t *auth_key_size); + int (*set_user_id)(struct fid_av *av, fi_addr_t fi_addr, + fi_addr_t user_id, uint64_t flags); +}; + +struct fid_av { + struct fid fid; + struct fi_ops_av *ops; +}; + + +/* + * MR = Memory Region + * Tracks registered memory regions, primarily for remote access, + * but also for local access until we can remove that need. + */ + +#define FI_MR_DMABUF (1ULL << 40) +#define FI_MR_SINGLE_USE (1ULL << 41) +#define FI_HMEM_HOST_ALLOC (1ULL << 45) +#define FI_HMEM_DEVICE_ONLY (1ULL << 46) + +struct fid_mr { + struct fid fid; + void *mem_desc; + uint64_t key; +}; + +enum fi_hmem_iface { + FI_HMEM_SYSTEM = 0, + FI_HMEM_CUDA, + FI_HMEM_ROCR, + FI_HMEM_ZE, + FI_HMEM_NEURON, + FI_HMEM_SYNAPSEAI, +}; + +static inline int fi_hmem_ze_device(int driver_index, int device_index) +{ + return driver_index << 16 | device_index; +} + +struct fi_mr_dmabuf { + int fd; + uint64_t offset; + size_t len; + void *base_addr; +}; + +struct fi_mr_auth_key { + struct fid_av *av; + fi_addr_t src_addr; +}; + +struct fi_mr_attr { + union { + const struct iovec *mr_iov; + const struct fi_mr_dmabuf *dmabuf; + }; + size_t iov_count; + uint64_t access; + uint64_t offset; + uint64_t requested_key; + void *context; + size_t auth_key_size; + uint8_t *auth_key; + enum fi_hmem_iface iface; + union { + uint64_t reserved; + int cuda; + int ze; + int neuron; + int synapseai; + } device; + void *hmem_data; + size_t page_size; + const struct fid_mr *base_mr; + size_t sub_mr_cnt; +}; + +struct fi_mr_modify { + uint64_t flags; + struct fi_mr_attr attr; +}; + +#define FI_SET_OPS_HMEM_OVERRIDE "hmem_override_ops" + +struct fi_hmem_override_ops { + size_t size; + + ssize_t (*copy_from_hmem_iov)(void *dest, size_t size, + enum fi_hmem_iface iface, uint64_t device, + const struct iovec *hmem_iov, + size_t hmem_iov_count, + uint64_t hmem_iov_offset); + + ssize_t (*copy_to_hmem_iov)(enum fi_hmem_iface iface, uint64_t device, + const struct iovec *hmem_iov, + size_t hmem_iov_count, + uint64_t hmem_iov_offset, const void *src, + size_t size); +}; + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_ATOMIC_DEF + +#define FI_COLLECTIVE_OFFSET 256 + +enum fi_datatype { + FI_INT8, + FI_UINT8, + FI_INT16, + FI_UINT16, + FI_INT32, + FI_UINT32, + FI_INT64, + FI_UINT64, + FI_FLOAT, + FI_DOUBLE, + FI_FLOAT_COMPLEX, + FI_DOUBLE_COMPLEX, + FI_LONG_DOUBLE, + FI_LONG_DOUBLE_COMPLEX, + FI_INT128, + FI_UINT128, + FI_FLOAT16, + FI_BFLOAT16, + FI_FLOAT8_E4M3, + FI_FLOAT8_E5M2, + + /* Collective datatypes */ + FI_VOID = FI_COLLECTIVE_OFFSET, +}; + +enum fi_op { + FI_MIN, + FI_MAX, + FI_SUM, + FI_PROD, + FI_LOR, + FI_LAND, + FI_BOR, + FI_BAND, + FI_LXOR, + FI_BXOR, + FI_ATOMIC_READ, + FI_ATOMIC_WRITE, + FI_CSWAP, + FI_CSWAP_NE, + FI_CSWAP_LE, + FI_CSWAP_LT, + FI_CSWAP_GE, + FI_CSWAP_GT, + FI_MSWAP, + FI_DIFF, + + /* Collective datatypes */ + FI_NOOP = FI_COLLECTIVE_OFFSET, +}; + +#endif + +#ifndef FABRIC_DIRECT_COLLECTIVE_DEF + +enum fi_collective_op { + FI_BARRIER, + FI_BROADCAST, + FI_ALLTOALL, + FI_ALLREDUCE, + FI_ALLGATHER, + FI_REDUCE_SCATTER, + FI_REDUCE, + FI_SCATTER, + FI_GATHER, +}; + +#endif + + +struct fi_atomic_attr; +struct fi_cq_attr; +struct fi_cntr_attr; +struct fi_collective_attr; + +struct fi_ops_domain { + size_t size; + int (*av_open)(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context); + int (*cq_open)(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context); + int (*endpoint)(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context); + int (*scalable_ep)(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **sep, void *context); + int (*cntr_open)(struct fid_domain *domain, struct fi_cntr_attr *attr, + struct fid_cntr **cntr, void *context); + int (*poll_open)(struct fid_domain *domain, struct fi_poll_attr *attr, + struct fid_poll **pollset); + int (*stx_ctx)(struct fid_domain *domain, + struct fi_tx_attr *attr, struct fid_stx **stx, + void *context); + int (*srx_ctx)(struct fid_domain *domain, + struct fi_rx_attr *attr, struct fid_ep **rx_ep, + void *context); + int (*query_atomic)(struct fid_domain *domain, + enum fi_datatype datatype, enum fi_op op, + struct fi_atomic_attr *attr, uint64_t flags); + int (*query_collective)(struct fid_domain *domain, + enum fi_collective_op coll, + struct fi_collective_attr *attr, uint64_t flags); + int (*endpoint2)(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, uint64_t flags, void *context); +}; + +/* Memory registration flags */ +/* #define FI_RMA_EVENT (1ULL << 56) */ + +struct fi_ops_mr { + size_t size; + int (*reg)(struct fid *fid, const void *buf, size_t len, + uint64_t access, uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context); + int (*regv)(struct fid *fid, const struct iovec *iov, + size_t count, uint64_t access, + uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context); + int (*regattr)(struct fid *fid, const struct fi_mr_attr *attr, + uint64_t flags, struct fid_mr **mr); +}; + +/* Domain bind flags */ +#define FI_REG_MR _Pragma("GCC warning \"'FI_REG_MR' is deprecated\"") (1ULL << 59) + +struct fid_domain { + struct fid fid; + struct fi_ops_domain *ops; + struct fi_ops_mr *mr; +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_DOMAIN + +static inline int +fi_domain(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **domain, void *context) +{ + return fabric->ops->domain(fabric, info, domain, context); +} + +static inline int +fi_domain2(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **domain, uint64_t flags, void *context) +{ + if (!flags) + return fi_domain(fabric, info, domain, context); + + return FI_CHECK_OP(fabric->ops, struct fi_ops_fabric, domain2) ? + fabric->ops->domain2(fabric, info, domain, flags, context) : + -FI_ENOSYS; +} + +static inline int +fi_domain_bind(struct fid_domain *domain, struct fid *fid, uint64_t flags) +{ + return domain->fid.ops->bind(&domain->fid, fid, flags); +} + +static inline int +fi_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context) +{ + return domain->ops->cq_open(domain, attr, cq, context); +} + +static inline int +fi_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr, + struct fid_cntr **cntr, void *context) +{ + return domain->ops->cntr_open(domain, attr, cntr, context); +} + +static inline FI_DEPRECATED_FUNC int +fi_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr, + struct fid_wait **waitset) +{ + return fabric->ops->wait_open(fabric, attr, waitset); +} + +static inline FI_DEPRECATED_FUNC int +fi_poll_open(struct fid_domain *domain, struct fi_poll_attr *attr, + struct fid_poll **pollset) +{ + return domain->ops->poll_open(domain, attr, pollset); +} + +static inline int +fi_mr_reg(struct fid_domain *domain, const void *buf, size_t len, + uint64_t acs, uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context) +{ + return domain->mr->reg(&domain->fid, buf, len, acs, offset, + requested_key, flags, mr, context); +} + +static inline int +fi_mr_regv(struct fid_domain *domain, const struct iovec *iov, + size_t count, uint64_t acs, + uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context) +{ + return domain->mr->regv(&domain->fid, iov, count, acs, + offset, requested_key, flags, mr, context); +} + +static inline int +fi_mr_regattr(struct fid_domain *domain, const struct fi_mr_attr *attr, + uint64_t flags, struct fid_mr **mr) +{ + return domain->mr->regattr(&domain->fid, attr, flags, mr); +} + +static inline void *fi_mr_desc(struct fid_mr *mr) +{ + return mr->mem_desc; +} + +static inline uint64_t fi_mr_key(struct fid_mr *mr) +{ + return mr->key; +} + +static inline int +fi_mr_raw_attr(struct fid_mr *mr, uint64_t *base_addr, + uint8_t *raw_key, size_t *key_size, uint64_t flags) +{ + struct fi_mr_raw_attr attr; + attr.flags = flags; + attr.base_addr = base_addr; + attr.raw_key = raw_key; + attr.key_size = key_size; + return mr->fid.ops->control(&mr->fid, FI_GET_RAW_MR, &attr); +} + +static inline int +fi_mr_map_raw(struct fid_domain *domain, uint64_t base_addr, + uint8_t *raw_key, size_t key_size, uint64_t *key, uint64_t flags) +{ + struct fi_mr_map_raw map; + map.flags = flags; + map.base_addr = base_addr; + map.raw_key = raw_key; + map.key_size = key_size; + map.key = key; + return domain->fid.ops->control(&domain->fid, FI_MAP_RAW_MR, &map); +} + +static inline int +fi_mr_unmap_key(struct fid_domain *domain, uint64_t key) +{ + return domain->fid.ops->control(&domain->fid, FI_UNMAP_KEY, &key); +} + +static inline int fi_mr_bind(struct fid_mr *mr, struct fid *bfid, uint64_t flags) +{ + return mr->fid.ops->bind(&mr->fid, bfid, flags); +} + +static inline int +fi_mr_refresh(struct fid_mr *mr, const struct iovec *iov, size_t count, + uint64_t flags) +{ + struct fi_mr_modify modify; + memset(&modify, 0, sizeof(modify)); + modify.flags = flags; + modify.attr.mr_iov = iov; + modify.attr.iov_count = count; + return mr->fid.ops->control(&mr->fid, FI_REFRESH, &modify); +} + +static inline int fi_mr_enable(struct fid_mr *mr) +{ + return mr->fid.ops->control(&mr->fid, FI_ENABLE, NULL); +} + +static inline int +fi_av_open(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context) +{ + return domain->ops->av_open(domain, attr, av, context); +} + +static inline FI_DEPRECATED_FUNC int +fi_av_bind(struct fid_av *av, struct fid *fid, uint64_t flags) +{ + return av->fid.ops->bind(&av->fid, fid, flags); +} + +static inline int +fi_av_insert(struct fid_av *av, const void *addr, size_t count, + fi_addr_t *fi_addr, uint64_t flags, void *context) +{ + return av->ops->insert(av, addr, count, fi_addr, flags, context); +} + +static inline int +fi_av_insertsvc(struct fid_av *av, const char *node, const char *service, + fi_addr_t *fi_addr, uint64_t flags, void *context) +{ + return av->ops->insertsvc(av, node, service, fi_addr, flags, context); +} + +static inline int +fi_av_insertsym(struct fid_av *av, const char *node, size_t nodecnt, + const char *service, size_t svccnt, + fi_addr_t *fi_addr, uint64_t flags, void *context) +{ + return av->ops->insertsym(av, node, nodecnt, service, svccnt, + fi_addr, flags, context); +} + +static inline int +fi_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count, uint64_t flags) +{ + return av->ops->remove(av, fi_addr, count, flags); +} + +static inline int +fi_av_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr, size_t *addrlen) +{ + return av->ops->lookup(av, fi_addr, addr, addrlen); +} + +static inline const char * +fi_av_straddr(struct fid_av *av, const void *addr, char *buf, size_t *len) +{ + return av->ops->straddr(av, addr, buf, len); +} + +static inline int +fi_av_insert_auth_key(struct fid_av *av, const void *auth_key, + size_t auth_key_size, fi_addr_t *fi_addr, uint64_t flags) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, insert_auth_key) ? + av->ops->insert_auth_key(av, auth_key, auth_key_size, fi_addr, + flags) : -FI_ENOSYS; +} + +static inline int +fi_av_lookup_auth_key(struct fid_av *av, fi_addr_t addr, void *auth_key, + size_t *auth_key_size) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, lookup_auth_key) ? + av->ops->lookup_auth_key(av, addr, auth_key, auth_key_size) : + -FI_ENOSYS; +} + +static inline int +fi_av_set_user_id(struct fid_av *av, fi_addr_t fi_addr, fi_addr_t user_id, + uint64_t flags) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, set_user_id) ? + av->ops->set_user_id(av, fi_addr, user_id, flags) : -FI_ENOSYS; +} + +static inline fi_addr_t +fi_rx_addr(fi_addr_t fi_addr, int rx_index, int rx_ctx_bits) +{ + return (fi_addr_t) (((uint64_t) rx_index << (64 - rx_ctx_bits)) | fi_addr); +} + +static inline fi_addr_t +fi_group_addr(fi_addr_t fi_addr, uint32_t group_id) +{ + return (fi_addr_t) (((uint64_t) group_id << 32) | fi_addr); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_DOMAIN_H */ diff --git a/backends/libfabric/include/rdma/fi_endpoint.h b/backends/libfabric/include/rdma/fi_endpoint.h new file mode 100644 index 000000000..918972958 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_endpoint.h @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_ENDPOINT_H +#define FI_ENDPOINT_H + +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +struct fi_msg { + const struct iovec *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t addr; + void *context; + uint64_t data; +}; + +/* Endpoint option levels */ +enum { + FI_OPT_ENDPOINT +}; + +/* FI_OPT_ENDPOINT option names */ +enum { + FI_OPT_MIN_MULTI_RECV, /* size_t */ + FI_OPT_CM_DATA_SIZE, /* size_t */ + FI_OPT_BUFFERED_MIN, /* size_t */ + FI_OPT_BUFFERED_LIMIT, /* size_t */ + FI_OPT_SEND_BUF_SIZE, + FI_OPT_RECV_BUF_SIZE, + FI_OPT_TX_SIZE, + FI_OPT_RX_SIZE, + FI_OPT_FI_HMEM_P2P, /* int */ + FI_OPT_XPU_TRIGGER, /* reserved for compatibility */ + FI_OPT_CUDA_API_PERMITTED, /* bool */ + FI_OPT_SHARED_MEMORY_PERMITTED, /* bool */ + FI_OPT_MAX_MSG_SIZE, /* size_t */ + FI_OPT_MAX_TAGGED_SIZE, /* size_t */ + FI_OPT_MAX_RMA_SIZE, /* size_t */ + FI_OPT_MAX_ATOMIC_SIZE, /* size_t */ + FI_OPT_INJECT_MSG_SIZE, /* size_t */ + FI_OPT_INJECT_TAGGED_SIZE, /* size_t */ + FI_OPT_INJECT_RMA_SIZE, /* size_t */ + FI_OPT_INJECT_ATOMIC_SIZE, /* size_t */ + FI_OPT_FIREWALL_ADDR, /* bool */ +}; + +/* + * Parameters for FI_OPT_HMEM_P2P to allow endpoint control over peer to peer + * support and FI_HMEM. + */ +enum { + FI_HMEM_P2P_ENABLED, /* Provider decides when to use P2P, default. */ + FI_HMEM_P2P_REQUIRED, /* Must use P2P for all transfers */ + FI_HMEM_P2P_PREFERRED, /* Should use P2P for all transfers if available */ + FI_HMEM_P2P_DISABLED /* Do not use P2P */ +}; + +struct fi_ops_ep { + size_t size; + ssize_t (*cancel)(fid_t fid, void *context); + int (*getopt)(fid_t fid, int level, int optname, + void *optval, size_t *optlen); + int (*setopt)(fid_t fid, int level, int optname, + const void *optval, size_t optlen); + int (*tx_ctx)(struct fid_ep *sep, int index, + struct fi_tx_attr *attr, struct fid_ep **tx_ep, + void *context); + int (*rx_ctx)(struct fid_ep *sep, int index, + struct fi_rx_attr *attr, struct fid_ep **rx_ep, + void *context); + ssize_t (*rx_size_left)(struct fid_ep *ep); + ssize_t (*tx_size_left)(struct fid_ep *ep); +}; + +struct fi_ops_msg { + size_t size; + ssize_t (*recv)(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, void *context); + ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, void *context); + ssize_t (*recvmsg)(struct fid_ep *ep, const struct fi_msg *msg, + uint64_t flags); + ssize_t (*send)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, void *context); + ssize_t (*sendv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, void *context); + ssize_t (*sendmsg)(struct fid_ep *ep, const struct fi_msg *msg, + uint64_t flags); + ssize_t (*inject)(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr); + ssize_t (*senddata)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, void *context); + ssize_t (*injectdata)(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr); +}; + +struct fi_ops_cm; +struct fi_ops_rma; +struct fi_ops_tagged; +struct fi_ops_atomic; +struct fi_ops_collective; + +/* + * Calls which modify the properties of a endpoint (control, setopt, bind, ...) + * must be serialized against all other operations. Those calls may modify the + * operations referenced by a endpoint in order to optimize the data transfer code + * paths. + * + * A provider may allocate the minimal size structure needed to support the + * ops requested by the user. + */ +struct fid_ep { + struct fid fid; + struct fi_ops_ep *ops; + struct fi_ops_cm *cm; + struct fi_ops_msg *msg; + struct fi_ops_rma *rma; + struct fi_ops_tagged *tagged; + struct fi_ops_atomic *atomic; + struct fi_ops_collective *collective; +}; + +struct fid_pep { + struct fid fid; + struct fi_ops_ep *ops; + struct fi_ops_cm *cm; +}; + +struct fid_stx { + struct fid fid; + struct fi_ops_ep *ops; +}; + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_ENDPOINT + +static inline int +fi_passive_ep(struct fid_fabric *fabric, struct fi_info *info, + struct fid_pep **pep, void *context) +{ + return fabric->ops->passive_ep(fabric, info, pep, context); +} + +static inline int +fi_endpoint(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context) +{ + return domain->ops->endpoint(domain, info, ep, context); +} + +static inline int +fi_endpoint2(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, uint64_t flags, void *context) +{ + if (!flags) + return fi_endpoint(domain, info, ep, context); + + return FI_CHECK_OP(domain->ops, struct fi_ops_domain, endpoint2) ? + domain->ops->endpoint2(domain, info, ep, flags, context) : + -FI_ENOSYS; +} + +static inline int +fi_scalable_ep(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **sep, void *context) +{ + return domain->ops->scalable_ep(domain, info, sep, context); +} + +static inline int fi_ep_bind(struct fid_ep *ep, struct fid *bfid, uint64_t flags) +{ + return ep->fid.ops->bind(&ep->fid, bfid, flags); +} + +static inline int fi_pep_bind(struct fid_pep *pep, struct fid *bfid, uint64_t flags) +{ + return pep->fid.ops->bind(&pep->fid, bfid, flags); +} + +static inline int fi_scalable_ep_bind(struct fid_ep *sep, struct fid *bfid, uint64_t flags) +{ + return sep->fid.ops->bind(&sep->fid, bfid, flags); +} + +static inline int fi_enable(struct fid_ep *ep) +{ + return ep->fid.ops->control(&ep->fid, FI_ENABLE, NULL); +} + +static inline ssize_t fi_cancel(fid_t fid, void *context) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->ops->cancel(fid, context); +} + +static inline int +fi_setopt(fid_t fid, int level, int optname, + const void *optval, size_t optlen) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->ops->setopt(fid, level, optname, optval, optlen); +} + +static inline int +fi_getopt(fid_t fid, int level, int optname, + void *optval, size_t *optlen) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->ops->getopt(fid, level, optname, optval, optlen); +} + +static inline int fi_ep_alias(struct fid_ep *ep, struct fid_ep **alias_ep, + uint64_t flags) +{ + int ret; + struct fid *fid; + ret = fi_alias(&ep->fid, &fid, flags); + if (!ret) + *alias_ep = (struct fid_ep *) fid; + return ret; +} + +static inline int +fi_tx_context(struct fid_ep *ep, int idx, struct fi_tx_attr *attr, + struct fid_ep **tx_ep, void *context) +{ + return ep->ops->tx_ctx(ep, idx, attr, tx_ep, context); +} + +static inline int +fi_rx_context(struct fid_ep *ep, int idx, struct fi_rx_attr *attr, + struct fid_ep **rx_ep, void *context) +{ + return ep->ops->rx_ctx(ep, idx, attr, rx_ep, context); +} + +static inline FI_DEPRECATED_FUNC ssize_t +fi_rx_size_left(struct fid_ep *ep) +{ + return ep->ops->rx_size_left(ep); +} + +static inline FI_DEPRECATED_FUNC ssize_t +fi_tx_size_left(struct fid_ep *ep) +{ + return ep->ops->tx_size_left(ep); +} + +static inline int +fi_stx_context(struct fid_domain *domain, struct fi_tx_attr *attr, + struct fid_stx **stx, void *context) +{ + return domain->ops->stx_ctx(domain, attr, stx, context); +} + +static inline int +fi_srx_context(struct fid_domain *domain, struct fi_rx_attr *attr, + struct fid_ep **rx_ep, void *context) +{ + return domain->ops->srx_ctx(domain, attr, rx_ep, context); +} + +static inline ssize_t +fi_recv(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, + void *context) +{ + return ep->msg->recv(ep, buf, len, desc, src_addr, context); +} + +static inline ssize_t +fi_recvv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, void *context) +{ + return ep->msg->recvv(ep, iov, desc, count, src_addr, context); +} + +static inline ssize_t +fi_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) +{ + return ep->msg->recvmsg(ep, msg, flags); +} + +static inline ssize_t +fi_send(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, void *context) +{ + return ep->msg->send(ep, buf, len, desc, dest_addr, context); +} + +static inline ssize_t +fi_sendv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, void *context) +{ + return ep->msg->sendv(ep, iov, desc, count, dest_addr, context); +} + +static inline ssize_t +fi_sendmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) +{ + return ep->msg->sendmsg(ep, msg, flags); +} + +static inline ssize_t +fi_inject(struct fid_ep *ep, const void *buf, size_t len, fi_addr_t dest_addr) +{ + return ep->msg->inject(ep, buf, len, dest_addr); +} + +static inline ssize_t +fi_senddata(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, void *context) +{ + return ep->msg->senddata(ep, buf, len, desc, data, dest_addr, context); +} + +static inline ssize_t +fi_injectdata(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr) +{ + return ep->msg->injectdata(ep, buf, len, data, dest_addr); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_ENDPOINT_H */ diff --git a/backends/libfabric/include/rdma/fi_eq.h b/backends/libfabric/include/rdma/fi_eq.h new file mode 100644 index 000000000..ee213986b --- /dev/null +++ b/backends/libfabric/include/rdma/fi_eq.h @@ -0,0 +1,488 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_EQ_H +#define FI_EQ_H + +#include + +#ifndef _WIN32 +#include +#endif /* _WIN32 */ + +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + + +/* + * Wait Set + * Allows associating multiple EQs and counters with a single wait object. + */ + +/* Use fi_control GETWAIT to get underlying wait object(s) */ +enum fi_wait_obj { + FI_WAIT_NONE, + FI_WAIT_UNSPEC, + FI_WAIT_SET, + FI_WAIT_FD, + FI_WAIT_MUTEX_COND, /* pthread mutex & cond, deprecated */ + FI_WAIT_YIELD, + FI_WAIT_POLLFD, +}; + +struct fi_wait_attr { + enum fi_wait_obj wait_obj; + uint64_t flags; +}; + +/* deprecated */ +struct fi_ops_wait { + size_t size; + int (*wait)(struct fid_wait *waitset, int timeout); +}; + +/* deprecated */ +struct fid_wait { + struct fid fid; + struct fi_ops_wait *ops; +}; + +#ifndef _WIN32 +struct fi_mutex_cond { + pthread_mutex_t *mutex; + pthread_cond_t *cond; +}; +#endif /* _WIN32 */ + +struct fi_wait_pollfd { + uint64_t change_index; + size_t nfds; + struct pollfd *fd; +}; + +/* + * Poll Set + * Allows polling multiple event queues and counters for progress + */ + +struct fi_poll_attr { + uint64_t flags; +}; + +/* deprecated */ +struct fi_ops_poll { + size_t size; + int (*poll)(struct fid_poll *pollset, void **context, int count); + int (*poll_add)(struct fid_poll *pollset, struct fid *event_fid, + uint64_t flags); + int (*poll_del)(struct fid_poll *pollset, struct fid *event_fid, + uint64_t flags); +}; + +/* deprecated */ +struct fid_poll { + struct fid fid; + struct fi_ops_poll *ops; +}; + +/* + * EQ = Event Queue + * Used to report various control (not data transfer) events and operations. + */ + +struct fi_eq_attr { + size_t size; + uint64_t flags; + enum fi_wait_obj wait_obj; + int signaling_vector; + struct fid_wait *wait_set; /* deprecated */ +}; + +/* Standard EQ events */ +enum { + FI_NOTIFY, + FI_CONNREQ, + FI_CONNECTED, + FI_SHUTDOWN, + FI_MR_COMPLETE, + FI_AV_COMPLETE, + FI_JOIN_COMPLETE, +}; + +struct fi_eq_entry { + fid_t fid; + void *context; + uint64_t data; +}; + +struct fi_eq_err_entry { + fid_t fid; + void *context; + uint64_t data; + int err; + int prov_errno; + /* err_data is available until the next time the EQ is read */ + void *err_data; + size_t err_data_size; +}; + +struct fi_eq_cm_entry { + fid_t fid; + /* user must call fi_freeinfo to release info */ + struct fi_info *info; + /* connection data placed here, up to space provided */ + uint8_t data[]; +}; + +struct fi_ops_eq { + size_t size; + ssize_t (*read)(struct fid_eq *eq, uint32_t *event, + void *buf, size_t len, uint64_t flags); + ssize_t (*readerr)(struct fid_eq *eq, struct fi_eq_err_entry *buf, + uint64_t flags); + ssize_t (*write)(struct fid_eq *eq, uint32_t event, + const void *buf, size_t len, uint64_t flags); + ssize_t (*sread)(struct fid_eq *eq, uint32_t *event, + void *buf, size_t len, int timeout, uint64_t flags); + const char * (*strerror)(struct fid_eq *eq, int prov_errno, + const void *err_data, char *buf, size_t len); +}; + +struct fid_eq { + struct fid fid; + struct fi_ops_eq *ops; +}; + + +/* + * CQ = Complete Queue + * Used to report the completion of data transfer operations. + */ + +enum fi_cq_format { + FI_CQ_FORMAT_UNSPEC, + FI_CQ_FORMAT_CONTEXT, + FI_CQ_FORMAT_MSG, + FI_CQ_FORMAT_DATA, + FI_CQ_FORMAT_TAGGED, +}; + +struct fi_cq_entry { + void *op_context; +}; + +struct fi_cq_msg_entry { + void *op_context; + uint64_t flags; + size_t len; +}; + +struct fi_cq_data_entry { + void *op_context; + uint64_t flags; + size_t len; + void *buf; + /* data depends on operation and/or flags - e.g. remote EQ data */ + uint64_t data; +}; + +struct fi_cq_tagged_entry { + void *op_context; + uint64_t flags; + size_t len; + void *buf; + uint64_t data; + uint64_t tag; +}; + +struct fi_cq_err_entry { + void *op_context; + uint64_t flags; + size_t len; + void *buf; + uint64_t data; + uint64_t tag; + size_t olen; + int err; + int prov_errno; + /* err_data is available until the next time the CQ is read */ + void *err_data; + size_t err_data_size; + fi_addr_t src_addr; +}; + +enum fi_cq_wait_cond { + FI_CQ_COND_NONE, + FI_CQ_COND_THRESHOLD /* size_t threshold */ +}; + +struct fi_cq_attr { + size_t size; + uint64_t flags; + enum fi_cq_format format; + enum fi_wait_obj wait_obj; + int signaling_vector; + enum fi_cq_wait_cond wait_cond; + struct fid_wait *wait_set; /* deprecated */ +}; + +struct fi_ops_cq { + size_t size; + ssize_t (*read)(struct fid_cq *cq, void *buf, size_t count); + ssize_t (*readfrom)(struct fid_cq *cq, void *buf, size_t count, + fi_addr_t *src_addr); + ssize_t (*readerr)(struct fid_cq *cq, struct fi_cq_err_entry *buf, + uint64_t flags); + ssize_t (*sread)(struct fid_cq *cq, void *buf, size_t count, + const void *cond, int timeout); + ssize_t (*sreadfrom)(struct fid_cq *cq, void *buf, size_t count, + fi_addr_t *src_addr, const void *cond, int timeout); + int (*signal)(struct fid_cq *cq); + const char * (*strerror)(struct fid_cq *cq, int prov_errno, + const void *err_data, char *buf, size_t len); +}; + +struct fid_cq { + struct fid fid; + struct fi_ops_cq *ops; +}; + + +/* + * CNTR = Counter + * Used to report the number of completed of asynchronous operations. + */ + +enum fi_cntr_events { + FI_CNTR_EVENTS_COMP, + FI_CNTR_EVENTS_BYTES /* count bytes not completeion events */ +}; + +struct fi_cntr_attr { + enum fi_cntr_events events; + enum fi_wait_obj wait_obj; + struct fid_wait *wait_set; /* deprecated */ + uint64_t flags; +}; + +struct fi_ops_cntr { + size_t size; + uint64_t (*read)(struct fid_cntr *cntr); + uint64_t (*readerr)(struct fid_cntr *cntr); + int (*add)(struct fid_cntr *cntr, uint64_t value); + int (*set)(struct fid_cntr *cntr, uint64_t value); + int (*wait)(struct fid_cntr *cntr, uint64_t threshold, int timeout); + int (*adderr)(struct fid_cntr *cntr, uint64_t value); + int (*seterr)(struct fid_cntr *cntr, uint64_t value); +}; + +struct fid_cntr { + struct fid fid; + struct fi_ops_cntr *ops; +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_EQ + +static inline int +fi_trywait(struct fid_fabric *fabric, struct fid **fids, int count) +{ + return fabric->ops->trywait(fabric, fids, count); +} + +static inline FI_DEPRECATED_FUNC int +fi_wait(struct fid_wait *waitset, int timeout) +{ + return waitset->ops->wait(waitset, timeout); +} + +static inline FI_DEPRECATED_FUNC int +fi_poll(struct fid_poll *pollset, void **context, int count) +{ + return pollset->ops->poll(pollset, context, count); +} + +static inline FI_DEPRECATED_FUNC int +fi_poll_add(struct fid_poll *pollset, struct fid *event_fid, uint64_t flags) +{ + return pollset->ops->poll_add(pollset, event_fid, flags); +} + +static inline FI_DEPRECATED_FUNC int +fi_poll_del(struct fid_poll *pollset, struct fid *event_fid, uint64_t flags) +{ + return pollset->ops->poll_del(pollset, event_fid, flags); +} + +static inline int +fi_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr, + struct fid_eq **eq, void *context) +{ + return fabric->ops->eq_open(fabric, attr, eq, context); +} + +static inline ssize_t +fi_eq_read(struct fid_eq *eq, uint32_t *event, void *buf, + size_t len, uint64_t flags) +{ + return eq->ops->read(eq, event, buf, len, flags); +} + +static inline ssize_t +fi_eq_readerr(struct fid_eq *eq, struct fi_eq_err_entry *buf, uint64_t flags) +{ + return eq->ops->readerr(eq, buf, flags); +} + +static inline ssize_t +fi_eq_write(struct fid_eq *eq, uint32_t event, const void *buf, + size_t len, uint64_t flags) +{ + return eq->ops->write(eq, event, buf, len, flags); +} + +static inline ssize_t +fi_eq_sread(struct fid_eq *eq, uint32_t *event, void *buf, size_t len, + int timeout, uint64_t flags) +{ + return eq->ops->sread(eq, event, buf, len, timeout, flags); +} + +static inline const char * +fi_eq_strerror(struct fid_eq *eq, int prov_errno, const void *err_data, + char *buf, size_t len) +{ + return eq->ops->strerror(eq, prov_errno, err_data, buf, len); +} + + +static inline ssize_t fi_cq_read(struct fid_cq *cq, void *buf, size_t count) +{ + return cq->ops->read(cq, buf, count); +} + +static inline ssize_t +fi_cq_readfrom(struct fid_cq *cq, void *buf, size_t count, fi_addr_t *src_addr) +{ + return cq->ops->readfrom(cq, buf, count, src_addr); +} + +static inline ssize_t +fi_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf, uint64_t flags) +{ + /* For compatibility with older providers. */ + if (buf) + buf->src_addr = FI_ADDR_NOTAVAIL; + return cq->ops->readerr(cq, buf, flags); +} + +static inline ssize_t +fi_cq_sread(struct fid_cq *cq, void *buf, size_t count, const void *cond, int timeout) +{ + return cq->ops->sread(cq, buf, count, cond, timeout); +} + +static inline ssize_t +fi_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count, + fi_addr_t *src_addr, const void *cond, int timeout) +{ + return cq->ops->sreadfrom(cq, buf, count, src_addr, cond, timeout); +} + +static inline int fi_cq_signal(struct fid_cq *cq) +{ + return cq->ops->signal(cq); +} + +static inline const char * +fi_cq_strerror(struct fid_cq *cq, int prov_errno, const void *err_data, + char *buf, size_t len) +{ + return cq->ops->strerror(cq, prov_errno, err_data, buf, len); +} + + +static inline uint64_t fi_cntr_read(struct fid_cntr *cntr) +{ + return cntr->ops->read(cntr); +} + +static inline uint64_t fi_cntr_readerr(struct fid_cntr *cntr) +{ + return cntr->ops->readerr(cntr); +} + +static inline int fi_cntr_add(struct fid_cntr *cntr, uint64_t value) +{ + return cntr->ops->add(cntr, value); +} + +static inline int fi_cntr_adderr(struct fid_cntr *cntr, uint64_t value) +{ + return FI_CHECK_OP(cntr->ops, struct fi_ops_cntr, adderr) ? + cntr->ops->adderr(cntr, value) : -FI_ENOSYS; +} + +static inline int fi_cntr_set(struct fid_cntr *cntr, uint64_t value) +{ + return cntr->ops->set(cntr, value); +} + +static inline int fi_cntr_seterr(struct fid_cntr *cntr, uint64_t value) +{ + return FI_CHECK_OP(cntr->ops, struct fi_ops_cntr, seterr) ? + cntr->ops->seterr(cntr, value) : -FI_ENOSYS; +} + +static inline int +fi_cntr_wait(struct fid_cntr *cntr, uint64_t threshold, int timeout) +{ + return cntr->ops->wait(cntr, threshold, timeout); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_EQ_H */ diff --git a/backends/libfabric/include/rdma/fi_errno.h b/backends/libfabric/include/rdma/fi_errno.h new file mode 100644 index 000000000..11286ab5c --- /dev/null +++ b/backends/libfabric/include/rdma/fi_errno.h @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_ERRNO_H +#define FI_ERRNO_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* FI directly mapped errno values */ + +#define FI_SUCCESS 0 + +#define FI_EPERM EPERM /* Operation not permitted */ +#define FI_ENOENT ENOENT /* No such file or directory */ +//#define FI_ESRCH ESRCH /* No such process */ +#define FI_EINTR EINTR /* Interrupted system call */ +#define FI_EIO EIO /* I/O error */ +//#define FI_ENXIO ENXIO /* No such device or address */ +#define FI_E2BIG E2BIG /* Argument list too long */ +//#define FI_ENOEXEC ENOEXEC /* Exec format error */ +#define FI_EBADF EBADF /* Bad file number */ +//#define FI_ECHILD ECHILD /* No child processes */ +#define FI_EAGAIN EAGAIN /* Try again */ +#define FI_ENOMEM ENOMEM /* Out of memory */ +#define FI_EACCES EACCES /* Permission denied */ +#define FI_EFAULT EFAULT /* Bad address */ +//#define FI_ENOTBLK ENOTBLK /* Block device required */ +#define FI_EBUSY EBUSY /* Device or resource busy */ +//#define FI_EEXIST EEXIST /* File exists */ +//#define FI_EXDEV EXDEV /* Cross-device link */ +#define FI_ENODEV ENODEV /* No such device */ +//#define FI_ENOTDIR ENOTDIR /* Not a directory */ +//#define FI_EISDIR EISDIR /* Is a directory */ +#define FI_EINVAL EINVAL /* Invalid argument */ +//#define FI_ENFILE ENFILE /* File table overflow */ +#define FI_EMFILE EMFILE /* Too many open files */ +//#define FI_ENOTTY ENOTTY /* Not a typewriter */ +//#define FI_ETXTBSY ETXTBSY /* Text file busy */ +//#define FI_EFBIG EFBIG /* File too large */ +#define FI_ENOSPC ENOSPC /* No space left on device */ +//#define FI_ESPIPE ESPIPE /* Illegal seek */ +//#define FI_EROFS EROFS /* Read-only file system */ +//#define FI_EMLINK EMLINK /* Too many links */ +//#define FI_EPIPE EPIPE /* Broken pipe */ +//#define FI_EDOM EDOM /* Math argument out of domain of func */ +//#define FI_ERANGE ERANGE /* Math result not representable */ +//#define FI_EDEADLK EDEADLK /* Resource deadlock would occur */ +//#define FI_ENAMETOOLONG ENAMETOLONG /* File name too long */ +//#define FI_ENOLCK ENOLCK /* No record locks available */ +#define FI_ENOSYS ENOSYS /* Function not implemented */ +//#define FI_ENOTEMPTY ENOTEMPTY /* Directory not empty */ +//#define FI_ELOOP ELOOP /* Too many symbolic links encountered */ +#define FI_EWOULDBLOCK EWOULDBLOCK /* Operation would block */ +#define FI_ENOMSG ENOMSG /* No message of desired type */ +//#define FI_EIDRM EIDRM /* Identifier removed */ +//#define FI_ECHRNG ECHRNG /* Channel number out of range */ +//#define FI_EL2NSYNC EL2NSYCN /* Level 2 not synchronized */ +//#define FI_EL3HLT EL3HLT /* Level 3 halted */ +//#define FI_EL3RST EL3RST /* Level 3 reset */ +//#define FI_ELNRNG ELNRNG /* Link number out of range */ +//#define FI_EUNATCH EUNATCH /* Protocol driver not attached */ +//#define FI_ENOCSI ENOCSI /* No CSI structure available */ +//#define FI_EL2HLT EL2HLT /* Level 2 halted */ +//#define FI_EBADE EBADE /* Invalid exchange */ +//#define FI_EBADR EBADDR /* Invalid request descriptor */ +//#define FI_EXFULL EXFULL /* Exchange full */ +//#define FI_ENOANO ENOANO /* No anode */ +//#define FI_EBADRQC EBADRQC /* Invalid request code */ +//#define FI_EBADSLT EBADSLT /* Invalid slot */ +//#define FI_EDEADLOCK EDEADLOCK /* Resource deadlock would occur */ +//#define FI_EBFONT EBFONT /* Bad font file format */ +//#define FI_ENOSTR ENOSTR /* Device not a stream */ +#define FI_ENODATA ENODATA /* No data available */ +//#define FI_ETIME ETIME /* Timer expired */ +//#define FI_ENOSR ENOSR /* Out of streams resources */ +//#define FI_ENONET ENONET /* Machine is not on the network */ +//#define FI_ENOPKG ENOPKG /* Package not installed */ +//#define FI_EREMOTE EREMOTE /* Object is remote */ +//#define FI_ENOLINK ENOLINK /* Link has been severed */ +//#define FI_EADV EADV /* Advertise error */ +//#define FI_ESRMNT ESRMNT /* Srmount error */ +//#define FI_ECOMM ECOMM /* Communication error on send */ +#define FI_EPROTO EPROTO /* Protocol error */ +//#define FI_EMULTIHOP EMULTIHOP /* Multihop attempted */ +//#define FI_EDOTDOT EDOTDOT /* RFS specific error */ +//#define FI_EBADMSG EBADMSG /* Not a data message */ +#define FI_EOVERFLOW EOVERFLOW /* Value too large for defined data type */ +//#define FI_ENOTUNIQ ENOTUNIQ /* Name not unique on network */ +//#define FI_EBADFD EBADFD /* File descriptor in bad state */ +//#define FI_EREMCHG EREMCHG /* Remote address changed */ +//#define FI_ELIBACC ELIBACC /* Can not access a needed shared library */ +//#define FI_ELIBBAD ELIBBAD /* Accessing a corrupted shared library */ +//#define FI_ELIBSCN ELIBSCN /* .lib section in a.out corrupted */ +//#define FI_ELIBMAX ELIBMAX /* Attempting to link in too many shared libraries */ +//#define FI_ELIBEXEC ELIBEXEC /* Cannot exec a shared library directly */ +//#define FI_EILSEQ EILSEQ /* Illegal byte sequence */ +//#define FI_ERESTART ERESTART /* Interrupted system call should be restarted */ +//#define FI_ESTRPIPE ESTRPIPE /* Streams pipe error */ +//#define FI_EUSERS EUSERS /* Too many users */ +//#define FI_ENOTSOCK ENOTSOCK /* Socket operation on non-socket */ +//#define FI_EDESTADDRREQ EDESTADDRREQ /* Destination address required */ +#define FI_EMSGSIZE EMSGSIZE /* Message too long */ +//#define FI_EPROTOTYPE EPROTOTYPE /* Protocol wrong type for endpoint */ +#define FI_ENOPROTOOPT ENOPROTOOPT /* Protocol not available */ +//#define FI_EPROTONOSUPPORT EPROTONOSUPPORT /* Protocol not supported */ +//#define FI_ESOCKTNOSUPPORT ESOCKTNOSUPPORT /* Socket type not supported */ +#define FI_EOPNOTSUPP EOPNOTSUPP /* Operation not supported on transport endpoint */ +//#define FI_EPFNOSUPPORT EPFNOSUPPORT /* Protocol family not supported */ +//#define FI_EAFNOSUPPORT EAFNOSUPPORT /* Address family not supported by protocol */ +#define FI_EADDRINUSE EADDRINUSE /* Address already in use */ +#define FI_EADDRNOTAVAIL EADDRNOTAVAIL /* Cannot assign requested address */ +#define FI_ENETDOWN ENETDOWN /* Network is down */ +#define FI_ENETUNREACH ENETUNREACH /* Network is unreachable */ +//#define FI_ENETRESET ENETRESET /* Network dropped connection because of reset */ +#define FI_ECONNABORTED ECONNABORTED /* Software caused connection abort */ +#define FI_ECONNRESET ECONNRESET /* Connection reset by peer */ +#define FI_ENOBUFS ENOBUFS /* No buffer space available */ +#define FI_EISCONN EISCONN /* Transport endpoint is already connected */ +#define FI_ENOTCONN ENOTCONN /* Transport endpoint is not connected */ +#define FI_ESHUTDOWN ESHUTDOWN /* Cannot send after transport endpoint shutdown */ +//#define FI_ETOOMANYREFS ETOOMANYREFS /* Too many references: cannot splice */ +#define FI_ETIMEDOUT ETIMEDOUT /* Connection timed out */ +#define FI_ECONNREFUSED ECONNREFUSED /* Connection refused */ +#define FI_EHOSTDOWN EHOSTDOWN /* Host is down */ +#define FI_EHOSTUNREACH EHOSTUNREACH /* No route to host */ +#define FI_EALREADY EALREADY /* Operation already in progress */ +#define FI_EINPROGRESS EINPROGRESS /* Operation now in progress */ +//#define FI_ESTALE ESTALE /* Stale NFS file handle */ +//#define FI_EUCLEAN EUNCLEAN /* Structure needs cleaning */ +//#define FI_ENOTNAM ENOTNAM /* Not a XENIX named type file */ +//#define FI_ENAVAIL ENAVAIL /* No XENIX semaphores available */ +//#define FI_EISNAM EISNAM /* Is a named type file */ +#define FI_EREMOTEIO EREMOTEIO /* Remote I/O error */ +//#define FI_EDQUOT EDQUOT /* Quota exceeded */ +//#define FI_ENOMEDIUM ENOMEDIUM /* No medium found */ +//#define FI_EMEDIUMTYPE EMEDIUMTYPE /* Wrong medium type */ +#define FI_ECANCELED ECANCELED /* Operation Canceled */ + +//#define FI_EKEYEXPIRED EKEYEXPIRED /* Key has expired */ +//#define FI_EKEYREVOKED EKEYREVOKED /* Key has been revoked */ +#define FI_EKEYREJECTED EKEYREJECTED /* Key was rejected by service */ +//#define FI_EOWNERDEAD EOWNERDEAD /* Owner died */ +//#define FI_ENOTRECOVERABLE ENOTRECOVERABLE /* State not recoverable */ + +/* FI specific return values: >= 256 */ +#define FI_ERRNO_OFFSET 256 + +enum { + FI_EOTHER = FI_ERRNO_OFFSET, /* Unspecified error */ + FI_ETOOSMALL = 257, /* Provided buffer is too small */ + FI_EOPBADSTATE = 258, /* Operation not permitted in current state */ + FI_EAVAIL = 259, /* Error available */ + FI_EBADFLAGS = 260, /* Flags not supported */ + FI_ENOEQ = 261, /* Missing or unavailable event queue */ + FI_EDOMAIN = 262, /* Invalid resource domain */ + FI_ENOCQ = 263, /* Missing or unavailable completion queue */ + FI_ECRC = 264, /* CRC error */ + FI_ETRUNC = 265, /* Truncation error */ + FI_ENOKEY = 266, /* Required key not available */ + FI_ENOAV = 267, /* Missing or unavailable address vector */ + FI_EOVERRUN = 268, /* Queue has been overrun */ + FI_ENORX = 269, /* Receiver not ready, no receive buffers available */ + FI_ENOMR = 270, /* No more memory registrations available */ + FI_EFIREWALLADDR = 271, /* Host unreachable because it is behind firewall */ + FI_ERRNO_MAX +}; + +const char *fi_strerror(int errnum); + +#ifdef __cplusplus +} +#endif + +#endif /* FI_ERRNO_H */ diff --git a/backends/libfabric/include/rdma/fi_ext.h b/backends/libfabric/include/rdma/fi_ext.h new file mode 100644 index 000000000..9932ba0cb --- /dev/null +++ b/backends/libfabric/include/rdma/fi_ext.h @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2021-2023 Intel Corporation. All rights reserved. + * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright (c) 2022 DataDirect Networks, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_EXT_H +#define FI_EXT_H + +#include +#include +#include +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Each provider needs to define an unique 12-bit provider + * specific code to avoid overlapping with other providers, + * then bit left shift the code 16 bits. Note that the + * highest 4 bits are not touched, so they are still left + * to 0. The lowest 16 bits can be used to define provider + * specific values. E.g., + * + * define FI_PROV_SPECIFIC_XXX (0xabc << 16) + * + * enum { + * FI_PROV_XXX_FOO = -(FI_PROV_SPECIFIC_XXX), + * FI_PROV_XXX_BAR, + * } + */ + +#define FI_PROV_SPECIFIC_EFA (0xefa << 16) +#define FI_PROV_SPECIFIC_TCP (0x7cb << 16) + + +/* negative options are provider specific */ +enum { + FI_OPT_EFA_RNR_RETRY = -FI_PROV_SPECIFIC_EFA, + FI_OPT_EFA_EMULATED_READ, /* bool */ + FI_OPT_EFA_EMULATED_WRITE, /* bool */ + FI_OPT_EFA_EMULATED_ATOMICS, /* bool */ + FI_OPT_EFA_USE_DEVICE_RDMA, /* bool */ + FI_OPT_EFA_SENDRECV_IN_ORDER_ALIGNED_128_BYTES, /* bool */ + FI_OPT_EFA_WRITE_IN_ORDER_ALIGNED_128_BYTES, /* bool */ + FI_OPT_EFA_HOMOGENEOUS_PEERS, /* bool */ +}; + +struct fi_fid_export { + struct fid **fid; + uint64_t flags; + void *context; +}; + +static inline int +fi_export_fid(struct fid *fid, uint64_t flags, + struct fid **expfid, void *context) +{ + struct fi_fid_export exp; + + exp.fid = expfid; + exp.flags = flags; + exp.context = context; + return fi_control(fid, FI_EXPORT_FID, &exp); +} + +static inline int +fi_import_fid(struct fid *fid, struct fid *expfid, uint64_t flags) +{ + return fid->ops->bind(fid, expfid, flags); +} + + +/* + * System memory monitor import extension: + * To use, open mr_cache fid and import. + */ + +struct fid_mem_monitor; + +struct fi_ops_mem_monitor { + size_t size; + int (*start)(struct fid_mem_monitor *monitor); + void (*stop)(struct fid_mem_monitor *monitor); + int (*subscribe)(struct fid_mem_monitor *monitor, + const void *addr, size_t len); + void (*unsubscribe)(struct fid_mem_monitor *monitor, + const void *addr, size_t len); + bool (*valid)(struct fid_mem_monitor *monitor, + const void *addr, size_t len); +}; + +struct fi_ops_mem_notify { + size_t size; + void (*notify)(struct fid_mem_monitor *monitor, const void *addr, + size_t len); +}; + +struct fid_mem_monitor { + struct fid fid; + struct fi_ops_mem_monitor *export_ops; + struct fi_ops_mem_notify *import_ops; +}; + + +/* + * System logging import extension: + * To use, open logging fid and import. + */ + +#define FI_LOG_PROV_FILTERED (1ULL << 0) /* Filter provider */ + +struct fi_ops_log { + size_t size; + int (*enabled)(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, uint64_t flags); + int (*ready)(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, uint64_t flags, uint64_t *showtime); + void (*log)(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, const char *func, int line, + const char *msg); +}; + +struct fid_logging { + struct fid fid; + struct fi_ops_log *ops; +}; + +static inline int fi_import(uint32_t version, const char *name, void *attr, + size_t attr_len, uint64_t flags, struct fid *fid, + void *context) +{ + struct fid *open_fid; + int ret; + + ret = fi_open(version, name, attr, attr_len, flags, &open_fid, context); + if (ret != FI_SUCCESS) + return ret; + + ret = fi_import_fid(open_fid, fid, flags); + fi_close(open_fid); + return ret; +} + +static inline int fi_import_log(uint32_t version, uint64_t flags, + struct fid_logging *log_fid) +{ + log_fid->fid.fclass = FI_CLASS_LOG; + log_fid->ops->size = sizeof(struct fi_ops_log); + + return fi_import(version, "logging", NULL, 0, flags, &log_fid->fid, + log_fid); +} + +#ifdef __cplusplus +} +#endif + +#endif /* FI_EXT_H */ diff --git a/backends/libfabric/include/rdma/fi_profile.h b/backends/libfabric/include/rdma/fi_profile.h new file mode 100644 index 000000000..cc8d8918b --- /dev/null +++ b/backends/libfabric/include/rdma/fi_profile.h @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2023 Intel Corporation, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_PROFILE_H +#define FI_PROFILE_H + +#include +#include +#include + +#include + +/* + * pre-defined profiling variables + */ +enum { + FI_VAR_UNEXP_MSG_CNT, // datatype: FI_UINT64 + FI_VAR_UNEXP_MSG_QUEUE, // datatype: FI_TYPE_CQ_ERR_ENTRY + FI_VAR_MSG_QUEUE_CNT, // datatype: FI_UNIT64 + FI_VAR_CONNECTION_CNT, // datatype: FI_UNIT64 + FI_VAR_CONN_REQUEST, // datatype: FI_UNIT64 + FI_VAR_CONN_ACCEPT, // datatype: FI_UNIT64 + FI_VAR_CONN_REJECT, // datatype: FI_UNIT64 + FI_VAR_OFI_MEM, // datatype: FI_UINT64 +}; + +/* + * pre-defined events which can change the profiling variables + */ +enum { + FI_EVENT_UNEXP_MSG_RECVD, // var = FI_VAR_UNEXP_MSG_CNT + FI_EVENT_UNEXP_MSG_MATCHED, // var = FI_VAR_UNEXP_MSG_CNT +}; + +enum fi_profile_type { + fi_primitive_type, + fi_defined_type, +}; + +struct fi_profile_desc { + uint32_t id; + enum fi_profile_type datatype_sel; + union { + enum fi_datatype primitive; + enum fi_type defined; + } datatype; + uint64_t flags; + size_t size; + const char *name; + const char *desc; +}; + +struct fid_profile; + +struct fi_profile_ops { + size_t size; + void (*reset)(struct fid_profile *prof_fid, uint64_t flags); + ssize_t (*query_vars)(struct fid_profile *prof_fid, + struct fi_profile_desc *varlist, size_t *count); + ssize_t (*query_events)(struct fid_profile *prof_fid, + struct fi_profile_desc *eventlist, size_t *count); + ssize_t (*read_var)(struct fid_profile *prof_fid, uint32_t var_id, + void *data, size_t *size); + int (*reg_callback)(struct fid_profile *prof_fid, uint32_t event_id, + int (*callback)(struct fid_profile *prof_fid, + struct fi_profile_desc *event, void *param, size_t size, + void *context), + void *context); + void (*start_reads)(struct fid_profile *prof_fid, uint64_t flags); + void (*end_reads)(struct fid_profile *prof_fid, uint64_t flags); +}; + +struct fid_profile { + struct fid fid; + struct fi_profile_ops *ops; +}; + +static inline void +fi_profile_reset(struct fid_profile *prof_fid, uint64_t flags) +{ + return prof_fid->ops->reset(prof_fid, flags); +} + + +static inline ssize_t +fi_profile_query_vars(struct fid_profile *prof_fid, + struct fi_profile_desc *varlist, size_t *count) +{ + return prof_fid->ops->query_vars(prof_fid, varlist, count); +} + +static inline ssize_t +fi_profile_query_events(struct fid_profile *prof_fid, + struct fi_profile_desc *eventlist, size_t *count) +{ + return prof_fid->ops->query_events(prof_fid, eventlist, count); +} + +static inline ssize_t +fi_profile_read_u64(struct fid_profile *prof_fid, uint32_t var_id, + uint64_t *data) +{ + size_t size = sizeof(uint64_t); + ssize_t ret = prof_fid->ops->read_var(prof_fid, var_id, + (void *)data, &size); + return (ret > 0) ? 0 : ret; +} + +static inline int +fi_profile_register_callback(struct fid_profile *prof_fid, uint32_t event_id, + int (*callback)(struct fid_profile *prof_fid, + struct fi_profile_desc *event, void *param, + size_t size, void *context), + void *context) +{ + return prof_fid->ops->reg_callback(prof_fid, event_id, callback, context); +} + +static inline void +fi_profile_start_reads(struct fid_profile *prof_fid, uint64_t flags) +{ + return prof_fid->ops->start_reads(prof_fid, flags); +} + +static inline void +fi_profile_end_reads(struct fid_profile *prof_fid, uint64_t flags) +{ + return prof_fid->ops->end_reads(prof_fid, flags); +} + +static inline int +fi_profile_open(struct fid *fid, uint64_t flags, + struct fid_profile **prof_fid, void *context) +{ + struct fi_profile_ops *ops; + int ret = fi_open_ops(fid, "fi_profile_ops", flags, + (void **)&ops, context); + if (!ret) + *prof_fid = container_of(ops, struct fid_profile, ops); + + return ret; +} + +static inline int +fi_profile_close(struct fid_profile *prof_fid) +{ + return prof_fid->fid.ops->close(&(prof_fid->fid)); +} + +#endif /* FI_PROFILE_H */ diff --git a/backends/libfabric/include/rdma/fi_rma.h b/backends/libfabric/include/rdma/fi_rma.h new file mode 100644 index 000000000..003c10fd9 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_rma.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_RMA_H +#define FI_RMA_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct fi_rma_iov { + uint64_t addr; + size_t len; + uint64_t key; +}; + +struct fi_rma_ioc { + uint64_t addr; + size_t count; + uint64_t key; +}; + +struct fi_msg_rma { + const struct iovec *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t addr; + const struct fi_rma_iov *rma_iov; + size_t rma_iov_count; + void *context; + uint64_t data; +}; + +struct fi_ops_rma { + size_t size; + ssize_t (*read)(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context); + ssize_t (*readv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key, + void *context); + ssize_t (*readmsg)(struct fid_ep *ep, const struct fi_msg_rma *msg, + uint64_t flags); + ssize_t (*write)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context); + ssize_t (*writev)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key, + void *context); + ssize_t (*writemsg)(struct fid_ep *ep, const struct fi_msg_rma *msg, + uint64_t flags); + ssize_t (*inject)(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t addr, uint64_t key); + ssize_t (*writedata)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, + void *context); + ssize_t (*injectdata)(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key); +}; + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_RMA + +static inline ssize_t +fi_read(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context) +{ + return ep->rma->read(ep, buf, len, desc, src_addr, addr, key, context); +} + +static inline ssize_t +fi_readv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key, + void *context) +{ + return ep->rma->readv(ep, iov, desc, count, src_addr, addr, key, context); +} + +static inline ssize_t +fi_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) +{ + return ep->rma->readmsg(ep, msg, flags); +} + +static inline ssize_t +fi_write(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) +{ + return ep->rma->write(ep, buf, len, desc, dest_addr, addr, key, context); +} + +static inline ssize_t +fi_writev(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key, + void *context) +{ + return ep->rma->writev(ep, iov, desc, count, dest_addr, addr, key, context); +} + +static inline ssize_t +fi_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) +{ + return ep->rma->writemsg(ep, msg, flags); +} + +static inline ssize_t +fi_inject_write(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t addr, uint64_t key) +{ + return ep->rma->inject(ep, buf, len, dest_addr, addr, key); +} + +static inline ssize_t +fi_writedata(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, + void *context) +{ + return ep->rma->writedata(ep, buf, len, desc,data, dest_addr, + addr, key, context); +} + +static inline ssize_t +fi_inject_writedata(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key) +{ + return ep->rma->injectdata(ep, buf, len, data, dest_addr, addr, key); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_RMA_H */ diff --git a/backends/libfabric/include/rdma/fi_tagged.h b/backends/libfabric/include/rdma/fi_tagged.h new file mode 100644 index 000000000..9230c34df --- /dev/null +++ b/backends/libfabric/include/rdma/fi_tagged.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_TAGGED_H +#define FI_TAGGED_H + +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +#define FI_MPI_IGNORE_TAG ((uint64_t) UINT32_MAX) +#define FI_MPI_IGNORE_PAYLOAD (((uint64_t) UINT8_MAX) << 32) + + +static inline uint64_t +fi_tag_mpi(int tag, uint8_t payload_id) +{ + return (((uint64_t) payload_id) << 32) | ((uint64_t) (uint32_t) tag); +} + +struct fi_msg_tagged { + const struct iovec *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t addr; + uint64_t tag; + uint64_t ignore; + void *context; + uint64_t data; +}; + +struct fi_ops_tagged { + size_t size; + ssize_t (*recv)(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, + uint64_t tag, uint64_t ignore, void *context); + ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, + uint64_t tag, uint64_t ignore, void *context); + ssize_t (*recvmsg)(struct fid_ep *ep, const struct fi_msg_tagged *msg, + uint64_t flags); + ssize_t (*send)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, uint64_t tag, void *context); + ssize_t (*sendv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, uint64_t tag, void *context); + ssize_t (*sendmsg)(struct fid_ep *ep, const struct fi_msg_tagged *msg, + uint64_t flags); + ssize_t (*inject)(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t tag); + ssize_t (*senddata)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, uint64_t tag, void *context); + ssize_t (*injectdata)(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr, uint64_t tag); +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_TAGGED + +static inline ssize_t +fi_trecv(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, uint64_t tag, uint64_t ignore, void *context) +{ + return ep->tagged->recv(ep, buf, len, desc, src_addr, tag, ignore, + context); +} + +static inline ssize_t +fi_trecvv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, uint64_t tag, uint64_t ignore, + void *context) +{ + return ep->tagged->recvv(ep, iov, desc, count, src_addr, tag, ignore, + context); +} + +static inline ssize_t +fi_trecvmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) +{ + return ep->tagged->recvmsg(ep, msg, flags); +} + +static inline ssize_t +fi_tsend(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, uint64_t tag, void *context) +{ + return ep->tagged->send(ep, buf, len, desc, dest_addr, tag, context); +} + +static inline ssize_t +fi_tsendv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, uint64_t tag, void *context) +{ + return ep->tagged->sendv(ep, iov, desc, count, dest_addr,tag, context); +} + +static inline ssize_t +fi_tsendmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) +{ + return ep->tagged->sendmsg(ep, msg, flags); +} + +static inline ssize_t +fi_tinject(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t tag) +{ + return ep->tagged->inject(ep, buf, len, dest_addr, tag); +} + +static inline ssize_t +fi_tsenddata(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, uint64_t tag, void *context) +{ + return ep->tagged->senddata(ep, buf, len, desc, data, + dest_addr, tag, context); +} + +static inline ssize_t +fi_tinjectdata(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr, uint64_t tag) +{ + return ep->tagged->injectdata(ep, buf, len, data, dest_addr, tag); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_TAGGED_H */ diff --git a/backends/libfabric/include/rdma/fi_trigger.h b/backends/libfabric/include/rdma/fi_trigger.h new file mode 100644 index 000000000..80f7d648b --- /dev/null +++ b/backends/libfabric/include/rdma/fi_trigger.h @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_TRIGGER_H +#define FI_TRIGGER_H + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum fi_trigger_event { + FI_TRIGGER_THRESHOLD, + FI_TRIGGER_XPU, +}; + +enum fi_op_type { + FI_OP_RECV, + FI_OP_SEND, + FI_OP_TRECV, + FI_OP_TSEND, + FI_OP_READ, + FI_OP_WRITE, + FI_OP_ATOMIC, + FI_OP_FETCH_ATOMIC, + FI_OP_COMPARE_ATOMIC, + FI_OP_CNTR_SET, + FI_OP_CNTR_ADD +}; + +struct fi_trigger_threshold { + struct fid_cntr *cntr; + size_t threshold; +}; + +struct fi_trigger_var { + enum fi_datatype datatype; + int count; + void *addr; + union { + uint8_t val8; + uint16_t val16; + uint32_t val32; + uint64_t val64; + uint8_t *data; + } value; +}; + +struct fi_trigger_xpu { + int count; + enum fi_hmem_iface iface; + union { + uint64_t reserved; + int cuda; + int ze; + } device; + struct fi_trigger_var *var; +}; + +struct fi_op_msg { + struct fid_ep *ep; + struct fi_msg msg; + uint64_t flags; +}; + +struct fi_op_tagged { + struct fid_ep *ep; + struct fi_msg_tagged msg; + uint64_t flags; +}; + +struct fi_op_rma { + struct fid_ep *ep; + struct fi_msg_rma msg; + uint64_t flags; +}; + +struct fi_op_atomic { + struct fid_ep *ep; + struct fi_msg_atomic msg; + uint64_t flags; +}; + +struct fi_op_fetch_atomic { + struct fid_ep *ep; + struct fi_msg_atomic msg; + struct fi_msg_fetch fetch; + uint64_t flags; +}; + +struct fi_op_compare_atomic { + struct fid_ep *ep; + struct fi_msg_atomic msg; + struct fi_msg_fetch fetch; + struct fi_msg_compare compare; + uint64_t flags; +}; + +struct fi_op_cntr { + struct fid_cntr *cntr; + uint64_t value; +}; + +#ifdef FABRIC_DIRECT +#include +#endif + +#ifndef FABRIC_DIRECT_TRIGGER + +/* Size must match struct fi_context */ +struct fi_triggered_context { + enum fi_trigger_event event_type; + union { + struct fi_trigger_threshold threshold; + struct fi_trigger_xpu xpu; + void *internal[3]; + } trigger; +}; + +/* Size must match struct fi_context2 */ +struct fi_triggered_context2 { + enum fi_trigger_event event_type; + union { + struct fi_trigger_threshold threshold; + struct fi_trigger_xpu xpu; + void *internal[7]; + } trigger; +}; + +struct fi_deferred_work { + struct fi_context2 context; + + uint64_t threshold; + struct fid_cntr *triggering_cntr; + struct fid_cntr *completion_cntr; + + enum fi_op_type op_type; + + union { + struct fi_op_msg *msg; + struct fi_op_tagged *tagged; + struct fi_op_rma *rma; + struct fi_op_atomic *atomic; + struct fi_op_fetch_atomic *fetch_atomic; + struct fi_op_compare_atomic *compare_atomic; + struct fi_op_cntr *cntr; + } op; +}; + +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* FI_TRIGGER_H */ diff --git a/backends/libfabric/include/rdma/providers/fi_log.h b/backends/libfabric/include/rdma/providers/fi_log.h new file mode 100644 index 000000000..9268a2ee8 --- /dev/null +++ b/backends/libfabric/include/rdma/providers/fi_log.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2015-2016, Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015, Intel Corp., Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#ifndef FI_LOG_H +#define FI_LOG_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum fi_log_subsys { + FI_LOG_CORE, + FI_LOG_FABRIC, + FI_LOG_DOMAIN, + FI_LOG_EP_CTRL, + FI_LOG_EP_DATA, + FI_LOG_AV, + FI_LOG_CQ, + FI_LOG_EQ, + FI_LOG_MR, + FI_LOG_CNTR, +}; + +enum fi_log_level { + FI_LOG_WARN, + FI_LOG_TRACE, + FI_LOG_INFO, + FI_LOG_DEBUG, +}; + +int fi_log_enabled(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys); +int fi_log_ready(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, uint64_t *showtime); +void fi_log(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, const char *func, int line, + const char *fmt, ...) FI_FORMAT_PRINTF(6, 7); + +#define FI_LOG(prov, level, subsystem, ...) \ + do { \ + if (fi_log_enabled(prov, level, subsystem)) { \ + int saved_errno = errno; \ + fi_log(prov, level, subsystem, \ + __func__, __LINE__, __VA_ARGS__); \ + errno = saved_errno; \ + } \ + } while (0) + +#define FI_LOG_SPARSE(prov, level, subsystem, ...) \ + do { \ + static uint64_t showtime; \ + if (fi_log_ready(prov, level, subsystem, &showtime)) { \ + int saved_errno = errno; \ + fi_log(prov, level, subsystem, \ + __func__, __LINE__, __VA_ARGS__); \ + errno = saved_errno; \ + } \ + } while (0) + +#define FI_WARN(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_WARN, subsystem, __VA_ARGS__) +#define FI_WARN_SPARSE(prov, subsystem, ...) \ + FI_LOG_SPARSE(prov, FI_LOG_WARN, subsystem, __VA_ARGS__) + +#define FI_TRACE(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_TRACE, subsystem, __VA_ARGS__) + +#define FI_INFO(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_INFO, subsystem, __VA_ARGS__) + +#if defined(ENABLE_DEBUG) && ENABLE_DEBUG +#define FI_DBG(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_DEBUG, subsystem, __VA_ARGS__) +#define FI_DBG_TRACE(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_TRACE, subsystem, __VA_ARGS__) +#else +#define FI_DBG(prov_name, subsystem, ...) \ + do {} while (0) +#define FI_DBG_TRACE(prov, subsystem, ...) \ + do {} while (0) +#endif + +#define FI_WARN_ONCE(prov, subsystem, ...) \ + do { \ + static int warned = 0; \ + if (!warned && \ + fi_log_enabled(prov, FI_LOG_WARN, subsystem)) { \ + int saved_errno = errno; \ + fi_log(prov, FI_LOG_WARN, subsystem, \ + __func__, __LINE__, __VA_ARGS__); \ + warned = 1; \ + errno = saved_errno; \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* FI_LOG_H */ diff --git a/backends/libfabric/include/rdma/providers/fi_peer.h b/backends/libfabric/include/rdma/providers/fi_peer.h new file mode 100644 index 000000000..63ebce01f --- /dev/null +++ b/backends/libfabric/include/rdma/providers/fi_peer.h @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2021-2023 Intel Corporation. All rights reserved. + * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright (c) 2022 DataDirect Networks, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_PEER_H +#define FI_PEER_H + +#include +#include +#include +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Peer provider AV support. + */ +struct fid_peer_av; + +struct fi_ops_av_owner { + size_t size; + int (*query)(struct fid_peer_av *av, struct fi_av_attr *attr); + fi_addr_t (*ep_addr)(struct fid_peer_av *av, struct fid_ep *ep); +}; + +struct fid_peer_av { + struct fid fid; + struct fi_ops_av_owner *owner_ops; +}; + +struct fi_peer_av_context { + size_t size; + struct fid_peer_av *av; +}; + + +/* + * Peer provider AV set support. + */ +struct fid_peer_av_set; + +struct fi_ops_av_set_owner { + size_t size; + int (*members)(struct fid_peer_av_set *av, fi_addr_t *addr, + size_t *count); +}; + +struct fid_peer_av_set { + struct fid fid; + struct fi_ops_av_set_owner *owner_ops; +}; + +struct fi_peer_av_set_context { + size_t size; + struct fi_peer_av_set *av_set; +}; + + +/* + * Peer provider CQ support. + */ +struct fid_peer_cq; + +struct fi_ops_cq_owner { + size_t size; + ssize_t (*write)(struct fid_peer_cq *cq, void *context, uint64_t flags, + size_t len, void *buf, uint64_t data, uint64_t tag, + fi_addr_t src); + ssize_t (*writeerr)(struct fid_peer_cq *cq, + const struct fi_cq_err_entry *err_entry); +}; + +struct fid_peer_cq { + struct fid fid; + struct fi_ops_cq_owner *owner_ops; +}; + +struct fi_peer_cq_context { + size_t size; + struct fid_peer_cq *cq; +}; + +/* + * Peer provider counter support. + */ +struct fid_peer_cntr; + +struct fi_ops_cntr_owner { + size_t size; + void (*inc)(struct fid_peer_cntr *cntr); + void (*incerr)(struct fid_peer_cntr *cntr); +}; + +struct fid_peer_cntr { + struct fid fid; + struct fi_ops_cntr_owner *owner_ops; +}; + +struct fi_peer_cntr_context { + size_t size; + struct fid_peer_cntr *cntr; +}; + +/* + * Peer provider domain support. + */ +struct fi_peer_domain_context { + size_t size; + struct fid_domain *domain; +}; + + +/* + * Peer provider EQ support. + */ +struct fi_peer_eq_context { + size_t size; + struct fid_eq *eq; +}; + + +/* + * Peer shared rx context + */ +struct fid_peer_srx; + +struct fi_peer_rx_entry { + struct fid_peer_srx *srx; + fi_addr_t addr; + size_t msg_size; + uint64_t tag; + uint64_t cq_data; + uint64_t flags; + void *context; + size_t count; + void **desc; + void *peer_context; + void *owner_context; + struct iovec *iov; +}; + +struct fi_peer_match_attr { + fi_addr_t addr; + size_t msg_size; + uint64_t tag; +}; + +struct fi_ops_srx_owner { + size_t size; + int (*get_msg)(struct fid_peer_srx *srx, + struct fi_peer_match_attr *attr, + struct fi_peer_rx_entry **entry); + int (*get_tag)(struct fid_peer_srx *srx, + struct fi_peer_match_attr *attr, + struct fi_peer_rx_entry **entry); + int (*queue_msg)(struct fi_peer_rx_entry *entry); + int (*queue_tag)(struct fi_peer_rx_entry *entry); + void (*foreach_unspec_addr)(struct fid_peer_srx *srx, + fi_addr_t (*get_addr)(struct fi_peer_rx_entry *)); + + void (*free_entry)(struct fi_peer_rx_entry *entry); +}; + +struct fi_ops_srx_peer { + size_t size; + int (*start_msg)(struct fi_peer_rx_entry *entry); + int (*start_tag)(struct fi_peer_rx_entry *entry); + int (*discard_msg)(struct fi_peer_rx_entry *entry); + int (*discard_tag)(struct fi_peer_rx_entry *entry); +}; + +struct fid_peer_srx { + struct fid_ep ep_fid; + struct fi_ops_srx_owner *owner_ops; + struct fi_ops_srx_peer *peer_ops; +}; + +struct fi_peer_srx_context { + size_t size; + struct fid_peer_srx *srx; +}; + + +/* + * Peer transfers + */ +struct fi_peer_transfer_context; + +struct fi_ops_transfer_peer { + size_t size; + ssize_t (*complete)(struct fid_ep *ep, struct fi_cq_tagged_entry *buf, + fi_addr_t src_addr); + ssize_t (*comperr)(struct fid_ep *ep, struct fi_cq_err_entry *buf); +}; + +struct fi_peer_transfer_context { + size_t size; + struct fi_info *info; + struct fid_ep *ep; + struct fi_ops_transfer_peer *peer_ops; +}; + + +#ifdef __cplusplus +} +#endif + +#endif /* FI_PEER_H */ diff --git a/backends/libfabric/include/rdma/providers/fi_prov.h b/backends/libfabric/include/rdma/providers/fi_prov.h new file mode 100644 index 000000000..ab8858d8f --- /dev/null +++ b/backends/libfabric/include/rdma/providers/fi_prov.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_PROV_H +#define FI_PROV_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Extension that dl-loaded providers should add to their .so filename + * (probably via libtool "-release" option). For example a provider + * driver named "foo" should build a plug-in named "libfoo-fi.so", and + * place it in $prefix/$libdir/libfabric/ + */ +#define FI_LIB_EXTENSION "fi" +#define FI_LIB_SUFFIX FI_LIB_EXTENSION ".so" + +/* + * Dynamically loaded providers must export the following entry point. + * This is invoked by the libfabric framework when the provider library + * is loaded. + */ +#define FI_EXT_INI \ + __attribute__((visibility ("default"),EXTERNALLY_VISIBLE)) \ + struct fi_provider* fi_prov_ini(void) + +struct fi_provider { + uint32_t version; + uint32_t fi_version; + struct fi_context context; + const char *name; + int (*getinfo)(uint32_t version, const char *node, const char *service, + uint64_t flags, const struct fi_info *hints, + struct fi_info **info); + int (*fabric)(struct fi_fabric_attr *attr, struct fid_fabric **fabric, + void *context); + void (*cleanup)(void); +}; + + +/* + * Defines a configuration parameter for use with libfabric. + */ +int fi_param_define(const struct fi_provider *provider, const char *param_name, + enum fi_param_type type, const char *help_string_fmt, ...); + +/* + * Get the value of a configuration variable. + * + * Currently, configuration parameter will only be read from the + * environment. Someday this call could be expanded to also check + * config files. + */ +int fi_param_get(struct fi_provider *provider, const char *param_name, + void *value); + +static inline int +fi_param_get_str(struct fi_provider *provider, const char *param_name, char **value) +{ + return fi_param_get(provider, param_name, value); +} + +static inline int +fi_param_get_int(struct fi_provider *provider, const char *param_name, int *value) +{ + return fi_param_get(provider, param_name, value); +} + +static inline int +fi_param_get_bool(struct fi_provider *provider, const char *param_name, int *value) +{ + return fi_param_get(provider, param_name, value); +} + +static inline int +fi_param_get_size_t(struct fi_provider *provider, const char *param_name, size_t *value) +{ + return fi_param_get(provider, param_name, value); +} + +#ifdef __cplusplus +} +#endif + +#endif /* FI_PROV_H */ diff --git a/backends/libfabric/libfabric_events.yaml b/backends/libfabric/libfabric_events.yaml new file mode 100644 index 000000000..0570b9e05 --- /dev/null +++ b/backends/libfabric/libfabric_events.yaml @@ -0,0 +1,82 @@ +lttng_ust_fi: + events: + - name: op_entry + args: + - ["const char *", op] + - [uint32_t, ep_id] + - ["const void *", op_context] + fields: + - [ctf_string, op, op] + - [ctf_integer, uint32_t, ep_id, ep_id] + - [ctf_integer_hex, uintptr_t, op_context, "(uintptr_t)op_context"] + - name: op_exit + args: + - ["const char *", op] + - [uint32_t, ep_id] + - ["const char *", src_addr] + - ["const char *", peer_addr] + - ["const char *", direction] + - [uint64_t, iov_count] + - [uint64_t, size_bytes] + - [int64_t, ret] + - ["const void *", op_context] + fields: + - [ctf_string, op, op] + - [ctf_integer, uint32_t, ep_id, ep_id] + - [ctf_string, src_addr, src_addr] + - [ctf_string, peer_addr, peer_addr] + - [ctf_string, direction, direction] + - [ctf_integer, uint64_t, iov_count, iov_count] + - [ctf_integer, uint64_t, size_bytes, size_bytes] + - [ctf_integer, int64_t, ret, ret] + - [ctf_integer_hex, uintptr_t, op_context, "(uintptr_t)op_context"] + - name: cq_completion + args: + - ["const char *", op] + - [uint32_t, obj_id] + - ["const char *", src_addr] + - ["const char *", peer_addr] + - [uint64_t, size_bytes] + - [int64_t, ret] + - ["const void *", op_context] + fields: + - [ctf_string, op, op] + - [ctf_integer, uint32_t, obj_id, obj_id] + - [ctf_string, src_addr, src_addr] + - [ctf_string, peer_addr, peer_addr] + - [ctf_integer, uint64_t, size_bytes, size_bytes] + - [ctf_integer, int64_t, ret, ret] + - [ctf_integer_hex, uintptr_t, op_context, "(uintptr_t)op_context"] + - name: ctrl_entry + args: + - ["const char *", op] + - [uint32_t, obj_id] + fields: + - [ctf_string, op, op] + - [ctf_integer, uint32_t, obj_id, obj_id] + - name: ctrl_exit + args: + - ["const char *", op] + - [uint32_t, obj_id] + - [int64_t, ret] + fields: + - [ctf_string, op, op] + - [ctf_integer, uint32_t, obj_id, obj_id] + - [ctf_integer, int64_t, ret, ret] + - name: ep_info + args: + - [uint32_t, ep_id] + - ["const char *", src_addr] + - ["const char *", domain_name] + - [uint32_t, topo_valid] + - [uint32_t, group] + - [uint32_t, switch_id] + - [uint32_t, port] + fields: + - [ctf_integer, uint32_t, ep_id, ep_id] + - [ctf_string, src_addr, src_addr] + - [ctf_string, domain_name, domain_name] + - [ctf_integer, uint32_t, topo_valid, topo_valid] + - [ctf_integer, uint32_t, group, group] + - [ctf_integer, uint32_t, switch_id, switch_id] + - [ctf_integer, uint32_t, port, port] diff --git a/backends/libfabric/src/fi_trace.h b/backends/libfabric/src/fi_trace.h new file mode 100644 index 000000000..b00e62d43 --- /dev/null +++ b/backends/libfabric/src/fi_trace.h @@ -0,0 +1,312 @@ +/* + * fi_trace -- LD_PRELOAD per-operation tracer for libfabric + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef FI_TRACE_H +#define FI_TRACE_H + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#define FIT_ADDR_MAX 96 /* "fi_addr_cxi://0x04656600" and friends */ +#define FIT_RAW_ADDR_MAX 64 +#define FIT_REC_MAX 1024 /* worst-case bytes of one CSV line */ + +#ifndef fit_container_of +#define fit_container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) +#endif + +struct fit_ep; +struct fit_cq; +struct fit_topo; + +void fit_record_entry(struct fit_ep *h, const char *op, char dir, const void *context); +void fit_record_ep_info(unsigned int ep_id, + const char *src, + size_t src_len, + const struct fit_topo *t); +void fit_record_ctrl_entry(const char *op, unsigned int obj_id); +void fit_record_ctrl_exit(const char *op, unsigned int obj_id, int ret); + +/* ------------------------------------------------------------------ */ +/* configuration (read once from the environment at load time) */ +/* ------------------------------------------------------------------ */ + +struct fit_config { + int enabled; /* FI_TRACE_ENABLE */ + int resolve; /* FI_TRACE_RESOLVE */ + int trace_msg; /* FI_TRACE_OPS */ + int trace_tagged; + int trace_rma; + int trace_rx; /* FI_TRACE_RX */ + int trace_srx; /* FI_TRACE_SRX */ + int trace_eagain; /* FI_TRACE_EAGAIN */ + int trace_cq; /* FI_TRACE_CQ */ + int verbose; /* FI_TRACE_VERBOSE */ + size_t bufsz; /* FI_TRACE_BUFSZ */ + uint64_t max_records; /* FI_TRACE_MAX_RECORDS */ + char dir[4096]; /* FI_TRACE_DIR */ + char host[256]; + size_t host_len; +}; + +/* CSV fields are sanitised and measured once, where they are produced, + * so the hot path only has to memcpy them. */ +void fit_sanitize(char *s, size_t *len_out); + +extern struct fit_config fit_cfg; +extern int fit_armed; + +void fit_config_init(void); +void fit_log(const char *fmt, ...); + +/* ------------------------------------------------------------------ */ +/* peer (fi_addr_t -> NIC address string) cache */ +/* */ +/* Entries are individually allocated and never moved or freed while */ +/* the endpoint lives, so a lookup can hand back a stable pointer and */ +/* the hot path neither copies the string nor takes a lock. Only */ +/* inserts (one per peer, ever) serialise on the endpoint mutex. */ +/* ------------------------------------------------------------------ */ + +struct fit_peer { + fi_addr_t key; + size_t len; /* strlen(str), precomputed */ + char str[FIT_ADDR_MAX]; +}; + +struct fit_peer_tab { + size_t cap; /* power of two */ + struct fit_peer_tab *older; /* retired tables, freed at ep close */ + _Atomic(struct fit_peer *) slot[]; +}; + +/* ------------------------------------------------------------------ */ +/* where a NIC sits in the fabric */ +/* */ +/* A dragonfly address is the part of "which NIC talked to which NIC" */ +/* that the NIC address itself cannot tell you: two endpoints whose */ +/* addresses look equally far apart may be on the same switch or */ +/* three hops away. CXI exposes it through a domain extension, so it */ +/* is resolved once per domain and inherited by every endpoint on it. */ +/* */ +/* This is deliberately not a per-record column. It never changes */ +/* for the life of a domain, so repeating it on millions of rows */ +/* would cost bandwidth and tell you nothing new; it goes in a */ +/* sidecar file keyed by NIC address, and the merge tool joins it. */ +/* ------------------------------------------------------------------ */ + +struct fit_topo { + int valid; /* 0 if the provider would not say */ + unsigned group; /* dragonfly group */ + unsigned sw; /* switch within the group */ + unsigned port; /* port on that switch */ + char name[64]; /* domain name, e.g. "cxi2" */ + size_t name_len; +}; + +/* Ask the provider where this domain's NIC is. Leaves t->valid 0 on + * anything that is not CXI, or a CXI too old to answer. */ +void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, const struct fi_info *info); + +/* Record one endpoint's location, once, when its address is known. */ + +/* ------------------------------------------------------------------ */ +/* per-object hook state */ +/* */ +/* Each wrapped object gets a private *copy* of the provider vtable */ +/* embedded in the hook struct. The object's vtable pointer is then */ +/* retargeted at that copy, so a wrapper can recover its own state */ +/* with container_of() -- no hash lookup on the hot path. */ +/* ------------------------------------------------------------------ */ + +struct fit_ep { + /* our vtables -- must stay first-class members for container_of */ + struct fi_ops_msg msg; + struct fi_ops_tagged tagged; + struct fi_ops_rma rma; + struct fi_ops fid_ops; + struct fi_ops_ep ep_ops; + + /* the provider's originals */ + const struct fi_ops_msg *o_msg; + const struct fi_ops_tagged *o_tagged; + const struct fi_ops_rma *o_rma; + const struct fi_ops *o_fid; + const struct fi_ops_ep *o_ep; + + struct fid_ep *ep; + struct fid_av *av; /* bound AV, used to render addresses */ + unsigned int ep_id; + + /* Points into the owning domain's hook state, which outlives every + * endpoint on it. NULL for an endpoint we met without its domain. */ + const struct fit_topo *topo; + int topo_written; + + char src[FIT_ADDR_MAX]; + size_t src_len; + int src_valid; + + pthread_mutex_t lock; /* serialises inserts + src resolution */ + _Atomic(struct fit_peer_tab *) peers; + size_t peer_cnt; /* written under lock */ + unsigned int peer_gen; /* fit_av_gen when the table was built */ +}; + +/* + * An fi_addr_t is an index into the address vector, and fi_av_remove() + * frees that index for reuse. A cached fi_addr_t -> NIC address mapping + * is therefore only valid until the next removal. Removals are rare + * (most applications never call it), so rather than track which entries + * died, bump a global counter and let each endpoint retire its whole + * table the next time it looks something up. + */ +extern _Atomic unsigned int fit_av_gen; + +void fit_ep_peers_free(struct fit_ep *h); + +struct fit_domain { + struct fi_ops_domain ops; + struct fi_ops fid_ops; + const struct fi_ops_domain *orig; + const struct fi_ops *o_fid; + + /* Resolved once at hook time and handed to every endpoint this + * domain creates. The address of this member is what endpoints + * hold, so the struct must not move. */ + struct fit_topo topo; +}; + +struct fit_fabric { + struct fi_ops_fabric ops; + struct fi_ops fid_ops; + const struct fi_ops_fabric *orig; + const struct fi_ops *o_fid; +}; + +struct fit_av { + struct fi_ops_av ops; + struct fi_ops fid_ops; + const struct fi_ops_av *orig; + const struct fi_ops *o_fid; +}; + +/* + * Completion queue. Hooking this is what turns a log of *submitted* + * operations into a log of transfers with a measurable duration: the + * posting record and the completion record carry the same op_context, so + * subtracting their timestamps gives the latency the application saw. + */ +struct fit_cq { + struct fi_ops_cq ops; + struct fi_ops fid_ops; + const struct fi_ops_cq *orig; + const struct fi_ops *o_fid; + + struct fid_cq *cq; + enum fi_cq_format format; + size_t entry_sz; /* 0 if the format is unknown */ + + /* The endpoint this CQ was bound to, so a completion can be + * attributed to a local NIC address. Left NULL if more than one + * endpoint binds the same CQ, because then it would be a guess. */ + struct fit_ep *ep; + int ep_ambiguous; + unsigned int cq_id; +}; + +/* ------------------------------------------------------------------ */ +/* writer */ +/* ------------------------------------------------------------------ */ + +// void fit_writer_init(void); +void fit_writer_flush_all(void); + +/* Emit one CSV record. dir is 'T' (local initiates) or 'R' (local posts + * a receive). peer is the fi_addr_t supplied by the caller; it is + * rendered through the endpoint's AV when possible. context is the + * application's op_context, recorded so the completion can be matched to + * this record; it is dropped when the provider refused the operation, + * since nothing became outstanding. */ +void fit_record(struct fit_ep *h, + const char *op, + char dir, + fi_addr_t peer, + size_t bytes, + size_t iov_count, + ssize_t ret, + const void *context); + +/* Emit one completion record. now is the time the batch was reaped. */ +void fit_record_cq(struct fit_cq *c, + const char *op, + fi_addr_t peer, + size_t bytes, + ssize_t ret, + const void *context, + uint64_t now); + +uint64_t fit_now_us(void); + +/* ------------------------------------------------------------------ */ +/* address rendering */ +/* ------------------------------------------------------------------ */ + +void fit_ep_resolve_src(struct fit_ep *h); + +/* Returns a sanitised, NUL-terminated peer string and its length. The + * result may point into the endpoint's peer cache (stable until the + * endpoint closes) or into the caller's scratch buffer. */ +const char *fit_ep_peer_str( + struct fit_ep *h, fi_addr_t peer, char *scratch, size_t scratch_len, size_t *len_out); + +/* ------------------------------------------------------------------ */ +/* installers */ +/* ------------------------------------------------------------------ */ + +void fit_hook_fabric(struct fid_fabric *fabric); +void fit_hook_domain(struct fid_domain *domain, const struct fi_info *info); +/* topo may be NULL; it is borrowed from the owning domain and outlives + * the endpoint. */ +void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo); +void fit_hook_av(struct fid_av *av); +void fit_hook_cq(struct fid_cq *cq, enum fi_cq_format format); + +static inline size_t fit_iov_len(const struct iovec *iov, size_t count) { + size_t i, total = 0; + + if (!iov) + return 0; + for (i = 0; i < count; i++) + total += iov[i].iov_len; + return total; +} + +static inline size_t fit_rma_iov_len(const struct fi_rma_iov *iov, size_t count) { + size_t i, total = 0; + + if (!iov) + return 0; + for (i = 0; i < count; i++) + total += iov[i].len; + return total; +} + +#endif /* FI_TRACE_H */ diff --git a/backends/libfabric/src/fit_addr.c b/backends/libfabric/src/fit_addr.c new file mode 100644 index 000000000..58ccb250a --- /dev/null +++ b/backends/libfabric/src/fit_addr.c @@ -0,0 +1,393 @@ +/* + * fi_trace -- NIC address rendering + * + * The local address comes from fi_getname() on the endpoint. A remote + * peer is identified in the API only by an opaque fi_addr_t, so it is + * reverse-mapped through the endpoint's address vector with + * fi_av_lookup() and then formatted with fi_av_straddr() -- which is + * what produces the "fi_addr_cxi://0x04656600" form. + * + * Both are cached: the local address once per endpoint, remote ones in + * a small open-addressing table keyed by fi_addr_t. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "fi_trace.h" + +#include +#include +#include +#include +#include +#include + +#define FIT_PEER_CAP_INIT 64 + +static void raw_to_hex(const void *addr, size_t len, char *out, size_t out_len) { + static const char hex[] = "0123456789abcdef"; + const unsigned char *b = addr; + size_t i, n = 0; + + if (out_len < 4) { + if (out_len) + out[0] = '\0'; + return; + } + out[n++] = '0'; + out[n++] = 'x'; + for (i = 0; i < len && n + 2 < out_len; i++) { + out[n++] = hex[b[i] >> 4]; + out[n++] = hex[b[i] & 0xf]; + } + out[n] = '\0'; +} + +/* Format a raw provider address, preferring the provider's own + * stringifier so the output matches fi_info / existing telemetry. */ +static void +format_raw(struct fid_av *av, const void *raw, size_t raw_len, char *out, size_t out_len) { + if (av && av->ops && av->ops->straddr) { + size_t len = out_len; + const char *s = fi_av_straddr(av, raw, out, &len); + + if (s) { + out[out_len - 1] = '\0'; + return; + } + } + raw_to_hex(raw, raw_len, out, out_len); +} + +void fit_ep_resolve_src(struct fit_ep *h) { + char raw[FIT_RAW_ADDR_MAX]; + size_t raw_len = sizeof(raw); + int ret; + + pthread_mutex_lock(&h->lock); + if (h->src_valid) + goto out; + + /* Mark resolved up front: if the provider cannot answer we do not + * want to retry fi_getname() on every single operation. */ + h->src_valid = 1; + snprintf(h->src, sizeof(h->src), "unknown"); + + if (!fit_cfg.resolve || !h->ep || !h->ep->cm || !h->ep->cm->getname) + goto out; + + ret = fi_getname(&h->ep->fid, raw, &raw_len); + if (ret) { + /* Not enabled yet -- allow a later retry. */ + if (ret == -FI_EOPBADSTATE || ret == -FI_ENOENT) + h->src_valid = 0; + goto out; + } + if (raw_len > sizeof(raw)) + goto out; + + format_raw(h->av, raw, raw_len, h->src, sizeof(h->src)); +out: + fit_sanitize(h->src, &h->src_len); + + /* The two halves of "where is this endpoint" become available at + * different moments: the dragonfly coordinates at domain creation, + * the NIC address only after FI_ENABLE. This is the first point + * where both are known, so it is where the pair gets written -- + * once per endpoint, never on the hot path. */ + if (h->src_valid && !h->topo_written) { + h->topo_written = 1; + fit_record_ep_info(h->ep_id, h->src, h->src_len, h->topo); + } + + pthread_mutex_unlock(&h->lock); +} + +/* ------------------------------------------------------------------ */ +/* dragonfly location */ +/* */ +/* CXI will say which group, switch and port a NIC is wired to, but */ +/* not through the core API -- it is a provider extension, reached by */ +/* opening a named ops structure on the domain. The name carries a */ +/* version, and ->topology() does not exist in the oldest ones, so try */ +/* newest first and stop at the first that answers. */ +/* ------------------------------------------------------------------ */ + +#ifdef HAVE_FI_CXI_TOPOLOGY +#include +#include +#endif + +void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, const struct fi_info *info) { + memset(t, 0, sizeof(*t)); + + /* The NIC's name ("cxi3") is worth keeping even when the provider + * will not give coordinates: it is how the node's own tooling and + * the counter telemetry refer to the device. */ + if (info && info->domain_attr && info->domain_attr->name) + snprintf(t->name, sizeof(t->name), "%s", info->domain_attr->name); + else + snprintf(t->name, sizeof(t->name), "unknown"); + fit_sanitize(t->name, &t->name_len); + +#ifdef HAVE_FI_CXI_TOPOLOGY + { + static const char *const versions[] = {"dom_ops_v6", "dom_ops_v5", "dom_ops_v4", NULL}; + int i; + + if (!domain || !domain->fid.ops || !domain->fid.ops->ops_open) { + fit_log("topology %s: domain has no ops_open", t->name); + return; + } + + /* Keep the table local to this domain; never reuse another NIC's + * extension pointer. Try the same versions as bidir_intra.cpp. */ + for (i = 0; versions[i]; i++) { + struct fi_cxi_dom_ops *ops = NULL; + unsigned grp = 0, sw = 0, port = 0; + int ret = fi_open_ops(&domain->fid, versions[i], 0, (void **)&ops, NULL); + + if (ret || !ops || !ops->topology) { + fit_log("topology %s: %s unavailable (ret=%d)", t->name, versions[i], ret); + continue; + } + + ret = ops->topology(&domain->fid, &grp, &sw, &port); + if (ret) { + fit_log("topology %s: %s query failed (ret=%d)", t->name, versions[i], ret); + continue; + } + + t->group = grp; + t->sw = sw; + t->port = port; + t->valid = 1; + fit_log("topology %s: %s group=%u switch=%u port=%u", t->name, versions[i], grp, sw, port); + return; + } + fit_log("topology %s: no supported query succeeded", t->name); + } +#else + (void)domain; + fit_log("topology %s: disabled at build time; run make check-topology " + "with the target libfabric headers, then rebuild", + t->name); +#endif +} + +/* ------------------------------------------------------------------ */ +/* the topology sidecar */ +/* */ +/* One row per endpoint, in its own small file next to the traces. */ +/* The alternative -- four more columns on every record -- would repeat*/ +/* a constant tens of millions of times to say something that changes */ +/* never. The merge tool reads the union of these files and joins on */ +/* the NIC address, which is enough to place both ends of every flow: */ +/* a peer this rank only ever sent to is some other rank's local */ +/* endpoint, and that rank wrote the row. */ +/* */ +/* Writing is rare -- once per endpoint for the life of the process -- */ +/* so it takes a lock and writes straight through, rather than paying */ +/* for the buffered per-thread machinery the hot path needs. */ +/* ------------------------------------------------------------------ */ + +/* ------------------------------------------------------------------ */ +/* peer cache */ +/* ------------------------------------------------------------------ */ + +static inline size_t peer_hash(fi_addr_t key, size_t cap) { + uint64_t h = (uint64_t)key; + + h ^= h >> 33; + h *= 0xff51afd7ed558ccdull; + h ^= h >> 33; + return (size_t)(h & (cap - 1)); +} + +/* Lock-free lookup. Entry pointers are stable for the life of the + * endpoint, so the caller may keep the returned string. */ +static const char *peer_find(struct fit_ep *h, fi_addr_t key, size_t *len_out) { + struct fit_peer_tab *tab; + size_t i, slot, cap; + + tab = atomic_load_explicit(&h->peers, memory_order_acquire); + if (!tab) + return NULL; + + cap = tab->cap; + slot = peer_hash(key, cap); + for (i = 0; i < cap; i++) { + struct fit_peer *e = + atomic_load_explicit(&tab->slot[(slot + i) & (cap - 1)], memory_order_acquire); + + if (!e) + return NULL; + if (e->key == key) { + *len_out = e->len; + return e->str; + } + } + return NULL; +} + +/* Caller holds h->lock. */ +static void tab_place(struct fit_peer_tab *tab, struct fit_peer *e) { + size_t slot = peer_hash(e->key, tab->cap); + size_t i; + + for (i = 0; i < tab->cap; i++) { + _Atomic(struct fit_peer *) *s = &tab->slot[(slot + i) & (tab->cap - 1)]; + + if (!atomic_load_explicit(s, memory_order_relaxed)) { + atomic_store_explicit(s, e, memory_order_release); + return; + } + } +} + +/* Caller holds h->lock. The old table is retired rather than freed so + * that a concurrent reader still walking it stays safe. */ +static int peer_grow(struct fit_ep *h) { + struct fit_peer_tab *old = atomic_load_explicit(&h->peers, memory_order_relaxed); + size_t new_cap = old ? old->cap * 2 : FIT_PEER_CAP_INIT; + struct fit_peer_tab *fresh; + size_t i; + + fresh = calloc(1, sizeof(*fresh) + new_cap * sizeof(fresh->slot[0])); + if (!fresh) + return -1; + + fresh->cap = new_cap; + fresh->older = old; + + if (old) { + for (i = 0; i < old->cap; i++) { + struct fit_peer *e = atomic_load_explicit(&old->slot[i], memory_order_relaxed); + + if (e) + tab_place(fresh, e); + } + } + + atomic_store_explicit(&h->peers, fresh, memory_order_release); + return 0; +} + +/* Caller holds h->lock. Returns the stable string, or NULL on OOM. */ +static const char *peer_insert(struct fit_ep *h, fi_addr_t key, const char *str, size_t *len_out) { + struct fit_peer_tab *tab = atomic_load_explicit(&h->peers, memory_order_relaxed); + struct fit_peer *e; + + /* Keep the load factor at or below 1/2 so probing stays short. */ + if (!tab || (h->peer_cnt + 1) * 2 > tab->cap) { + if (peer_grow(h)) + return NULL; + tab = atomic_load_explicit(&h->peers, memory_order_relaxed); + } + + e = calloc(1, sizeof(*e)); + if (!e) + return NULL; + + e->key = key; + snprintf(e->str, sizeof(e->str), "%s", str); + fit_sanitize(e->str, &e->len); + + tab_place(tab, e); + h->peer_cnt++; + *len_out = e->len; + return e->str; +} + +void fit_ep_peers_free(struct fit_ep *h) { + struct fit_peer_tab *tab = atomic_load_explicit(&h->peers, memory_order_relaxed); + size_t i; + + if (tab) { + for (i = 0; i < tab->cap; i++) + free(atomic_load_explicit(&tab->slot[i], memory_order_relaxed)); + } + while (tab) { + struct fit_peer_tab *older = tab->older; + + free(tab); + tab = older; + } + atomic_store_explicit(&h->peers, NULL, memory_order_relaxed); +} + +/* + * fi_av_remove() frees an fi_addr_t for reuse, so every cached mapping + * becomes suspect the moment one happens. Retire the whole table rather + * than try to work out which entries died: removals are rare, and the + * retired table stays reachable through ->older so a concurrent reader + * walking it is still safe. Caller holds h->lock. + */ +static void peer_retire_stale(struct fit_ep *h, unsigned int gen) { + struct fit_peer_tab *old, *fresh; + + if (h->peer_gen == gen) + return; + + old = atomic_load_explicit(&h->peers, memory_order_relaxed); + if (old) { + fresh = calloc(1, sizeof(*fresh) + FIT_PEER_CAP_INIT * sizeof(fresh->slot[0])); + if (!fresh) + return; /* keep the stale table over none */ + fresh->cap = FIT_PEER_CAP_INIT; + fresh->older = old; + atomic_store_explicit(&h->peers, fresh, memory_order_release); + fit_log("address vector changed; dropped %u cached peers", (unsigned)h->peer_cnt); + h->peer_cnt = 0; + } + h->peer_gen = gen; +} + +const char *fit_ep_peer_str( + struct fit_ep *h, fi_addr_t peer, char *scratch, size_t scratch_len, size_t *len_out) { + char raw[FIT_RAW_ADDR_MAX]; + size_t raw_len = sizeof(raw); + const char *cached; + unsigned int gen; + int resolvable; + + if (peer == FI_ADDR_UNSPEC) { + *len_out = sizeof("fi_addr_unspec") - 1; + return "fi_addr_unspec"; + } + if (peer == FI_ADDR_NOTAVAIL) { + *len_out = sizeof("fi_addr_notavail") - 1; + return "fi_addr_notavail"; + } + + /* The common case: already seen this peer, and no fi_av_remove() + * has invalidated the mapping since. Both resolved and unresolved + * renderings are cached, so neither path formats a string more than + * once per peer. */ + gen = atomic_load_explicit(&fit_av_gen, memory_order_acquire); + if (gen == h->peer_gen) { + cached = peer_find(h, peer, len_out); + if (cached) + return cached; + } + + resolvable = fit_cfg.resolve && h->av && h->av->ops && h->av->ops->lookup; + + if (resolvable && !fi_av_lookup(h->av, peer, raw, &raw_len) && raw_len <= sizeof(raw)) + format_raw(h->av, raw, raw_len, scratch, scratch_len); + else + snprintf(scratch, scratch_len, "fi_addr://0x%016llx", (unsigned long long)peer); + + pthread_mutex_lock(&h->lock); + peer_retire_stale(h, gen); + cached = peer_find(h, peer, len_out); /* lost a race? */ + if (!cached) + cached = peer_insert(h, peer, scratch, len_out); + pthread_mutex_unlock(&h->lock); + + if (cached) + return cached; + + fit_sanitize(scratch, len_out); + return scratch; +} diff --git a/backends/libfabric/src/fit_hook.c b/backends/libfabric/src/fit_hook.c new file mode 100644 index 000000000..84b0b7a2f --- /dev/null +++ b/backends/libfabric/src/fit_hook.c @@ -0,0 +1,1521 @@ +/* + * fi_trace -- LD_PRELOAD interposition and vtable wrapping + * + * Why this is not a plain dlsym() shim + * ------------------------------------ + * fi_send(), fi_recv(), fi_tsend(), fi_write() and friends are *static + * inline* functions in that dispatch straight + * through the provider vtables (ep->msg->send, ep->tagged->recv, ...). + * They are not exported symbols, so dlsym(RTLD_NEXT, "fi_send") cannot + * find them and an interposed definition would never be called. + * + * The only data-path-reaching entry points libfabric actually exports + * are fi_fabric() and fi_fabric2(). So we interpose those, and from + * the fabric we walk down the object tree -- fabric -> domain -> + * endpoint -- retargeting each object's vtable pointer at a private + * copy whose data-path slots are our wrappers. The provider's own + * objects are never replaced, only the pointers the application + * dispatches through, so provider-internal container_of() stays valid. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "fi_trace.h" + +#include +#include +#include +#include +#include + +static atomic_uint fit_ep_counter; + +static void fit_cq_attach_ep(struct fid *bfid, struct fit_ep *h); + +/* ------------------------------------------------------------------ */ +/* recovering hook state from a vtable pointer */ +/* ------------------------------------------------------------------ */ + +#define EP_FROM_MSG(ep) fit_container_of((ep)->msg, struct fit_ep, msg) +#define EP_FROM_TAGGED(ep) fit_container_of((ep)->tagged, struct fit_ep, tagged) +#define EP_FROM_RMA(ep) fit_container_of((ep)->rma, struct fit_ep, rma) +#define EP_FROM_FID(fid) fit_container_of((fid)->ops, struct fit_ep, fid_ops) +#define EP_FROM_EPOPS(ep) fit_container_of((ep)->ops, struct fit_ep, ep_ops) + +#define CALL_OR_ENOSYS(fn, args) ((fn) ? (fn)args : -FI_ENOSYS) + +/* ================================================================== */ +/* message operations */ +/* ================================================================== */ + +static ssize_t fit_msg_send(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + fi_addr_t dest_addr, + void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_send", 'T', context); + ret = CALL_OR_ENOSYS(h->o_msg->send, (ep, buf, len, desc, dest_addr, context)); + fit_record(h, "fi_send", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_msg_sendv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t dest_addr, + void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_sendv", 'T', context); + ret = CALL_OR_ENOSYS(h->o_msg->sendv, (ep, iov, desc, count, dest_addr, context)); + fit_record(h, "fi_sendv", 'T', dest_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_msg_sendmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_sendmsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_msg->sendmsg, (ep, msg, flags)); + fit_record(h, "fi_sendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; +} + +static ssize_t fit_msg_inject(struct fid_ep *ep, const void *buf, size_t len, fi_addr_t dest_addr) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_inject", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_msg->inject, (ep, buf, len, dest_addr)); + fit_record(h, "fi_inject", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_msg_senddata(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + uint64_t data, + fi_addr_t dest_addr, + void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_senddata", 'T', context); + ret = CALL_OR_ENOSYS(h->o_msg->senddata, (ep, buf, len, desc, data, dest_addr, context)); + fit_record(h, "fi_senddata", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_msg_injectdata( + struct fid_ep *ep, const void *buf, size_t len, uint64_t data, fi_addr_t dest_addr) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_injectdata", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_msg->injectdata, (ep, buf, len, data, dest_addr)); + fit_record(h, "fi_injectdata", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_msg_recv( + struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_recv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_msg->recv, (ep, buf, len, desc, src_addr, context)); + fit_record(h, "fi_recv", 'R', src_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_msg_recvv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t src_addr, + void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_recvv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_msg->recvv, (ep, iov, desc, count, src_addr, context)); + fit_record(h, "fi_recvv", 'R', src_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_msg_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_recvmsg", 'R', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_msg->recvmsg, (ep, msg, flags)); + fit_record(h, "fi_recvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; +} + +/* ================================================================== */ +/* tagged operations */ +/* ================================================================== */ + +static ssize_t fit_tag_send(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + fi_addr_t dest_addr, + uint64_t tag, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tsend", 'T', context); + ret = CALL_OR_ENOSYS(h->o_tagged->send, (ep, buf, len, desc, dest_addr, tag, context)); + fit_record(h, "fi_tsend", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_tag_sendv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t dest_addr, + uint64_t tag, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tsendv", 'T', context); + ret = CALL_OR_ENOSYS(h->o_tagged->sendv, (ep, iov, desc, count, dest_addr, tag, context)); + fit_record(h, "fi_tsendv", 'T', dest_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_tag_sendmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tsendmsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->sendmsg, (ep, msg, flags)); + fit_record(h, "fi_tsendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; +} + +static ssize_t +fit_tag_inject(struct fid_ep *ep, const void *buf, size_t len, fi_addr_t dest_addr, uint64_t tag) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tinject", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->inject, (ep, buf, len, dest_addr, tag)); + fit_record(h, "fi_tinject", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_tag_senddata(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + uint64_t data, + fi_addr_t dest_addr, + uint64_t tag, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tsenddata", 'T', context); + ret = CALL_OR_ENOSYS(h->o_tagged->senddata, (ep, buf, len, desc, data, dest_addr, tag, context)); + fit_record(h, "fi_tsenddata", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_tag_injectdata(struct fid_ep *ep, + const void *buf, + size_t len, + uint64_t data, + fi_addr_t dest_addr, + uint64_t tag) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tinjectdata", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->injectdata, (ep, buf, len, data, dest_addr, tag)); + fit_record(h, "fi_tinjectdata", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_tag_recv(struct fid_ep *ep, + void *buf, + size_t len, + void *desc, + fi_addr_t src_addr, + uint64_t tag, + uint64_t ignore, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_trecv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_tagged->recv, (ep, buf, len, desc, src_addr, tag, ignore, context)); + fit_record(h, "fi_trecv", 'R', src_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_tag_recvv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t src_addr, + uint64_t tag, + uint64_t ignore, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_trecvv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_tagged->recvv, (ep, iov, desc, count, src_addr, tag, ignore, context)); + fit_record(h, "fi_trecvv", 'R', src_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_tag_recvmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_trecvmsg", 'R', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->recvmsg, (ep, msg, flags)); + fit_record(h, "fi_trecvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; +} + +/* ================================================================== */ +/* RMA operations */ +/* ================================================================== */ + +static ssize_t fit_rma_write(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_write", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->write, (ep, buf, len, desc, dest_addr, addr, key, context)); + fit_record(h, "fi_write", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_rma_writev(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_writev", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->writev, (ep, iov, desc, count, dest_addr, addr, key, context)); + fit_record(h, "fi_writev", 'T', dest_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_rma_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_writemsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_rma->writemsg, (ep, msg, flags)); + fit_record(h, "fi_writemsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; +} + +static ssize_t fit_rma_inject(struct fid_ep *ep, + const void *buf, + size_t len, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_inject_write", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_rma->inject, (ep, buf, len, dest_addr, addr, key)); + fit_record(h, "fi_inject_write", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_rma_writedata(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + uint64_t data, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_writedata", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->writedata, + (ep, buf, len, desc, data, dest_addr, addr, key, context)); + fit_record(h, "fi_writedata", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_rma_injectdata(struct fid_ep *ep, + const void *buf, + size_t len, + uint64_t data, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_inject_writedata", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_rma->injectdata, (ep, buf, len, data, dest_addr, addr, key)); + fit_record(h, "fi_inject_writedata", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_rma_read(struct fid_ep *ep, + void *buf, + size_t len, + void *desc, + fi_addr_t src_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_read", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->read, (ep, buf, len, desc, src_addr, addr, key, context)); + fit_record(h, "fi_read", 'T', src_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_rma_readv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t src_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_readv", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->readv, (ep, iov, desc, count, src_addr, addr, key, context)); + fit_record(h, "fi_readv", 'T', src_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_readmsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_rma->readmsg, (ep, msg, flags)); + fit_record(h, "fi_readmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; +} + +/* ================================================================== */ +/* endpoint lifecycle */ +/* ================================================================== */ + +/* + * Install our copies of the endpoint's data-path vtables. + * + * Idempotent, and deliberately re-run after every bind and after + * FI_ENABLE. An endpoint is not finished at the moment fi_endpoint() + * returns: a provider may hand back placeholder ops and swap in the real + * ones later, once it knows which capabilities were negotiated and the + * transmit and receive contexts exist. Patching only at creation then + * loses the race -- the provider overwrites our pointers, every + * subsequent operation goes straight to the provider, and nothing + * reports an error. The symptom is a trace with a full set of "hooked" + * lines and not one data record, which is precisely what CXI produced. + * + * Re-arming is cheap (three pointer compares when nothing changed) and + * happens only on the control path, never per operation. + */ +static void fit_ep_arm_ops(struct fit_ep *h, const char *when) { + struct fid_ep *ep = h->ep; + + if (fit_cfg.trace_msg && ep->msg && ep->msg != &h->msg) { + if (h->o_msg) + fit_log("endpoint %u: provider replaced msg ops at %s;" + " re-arming", + h->ep_id, when); + h->o_msg = ep->msg; + h->msg = *ep->msg; + h->msg.send = fit_msg_send; + h->msg.sendv = fit_msg_sendv; + h->msg.sendmsg = fit_msg_sendmsg; + h->msg.inject = fit_msg_inject; + h->msg.senddata = fit_msg_senddata; + h->msg.injectdata = fit_msg_injectdata; + h->msg.recv = fit_msg_recv; + h->msg.recvv = fit_msg_recvv; + h->msg.recvmsg = fit_msg_recvmsg; + ep->msg = &h->msg; + } + + if (fit_cfg.trace_tagged && ep->tagged && ep->tagged != &h->tagged) { + if (h->o_tagged) + fit_log("endpoint %u: provider replaced tagged ops at" + " %s; re-arming", + h->ep_id, when); + h->o_tagged = ep->tagged; + h->tagged = *ep->tagged; + h->tagged.send = fit_tag_send; + h->tagged.sendv = fit_tag_sendv; + h->tagged.sendmsg = fit_tag_sendmsg; + h->tagged.inject = fit_tag_inject; + h->tagged.senddata = fit_tag_senddata; + h->tagged.injectdata = fit_tag_injectdata; + h->tagged.recv = fit_tag_recv; + h->tagged.recvv = fit_tag_recvv; + h->tagged.recvmsg = fit_tag_recvmsg; + ep->tagged = &h->tagged; + } + + if (fit_cfg.trace_rma && ep->rma && ep->rma != &h->rma) { + if (h->o_rma) + fit_log("endpoint %u: provider replaced rma ops at %s;" + " re-arming", + h->ep_id, when); + h->o_rma = ep->rma; + h->rma = *ep->rma; + h->rma.write = fit_rma_write; + h->rma.writev = fit_rma_writev; + h->rma.writemsg = fit_rma_writemsg; + h->rma.inject = fit_rma_inject; + h->rma.writedata = fit_rma_writedata; + h->rma.injectdata = fit_rma_injectdata; + h->rma.read = fit_rma_read; + h->rma.readv = fit_rma_readv; + h->rma.readmsg = fit_rma_readmsg; + ep->rma = &h->rma; + } +} + +static int fit_ep_bind_inner(struct fid *fid, struct fid *bfid, uint64_t flags) { + struct fit_ep *h = EP_FROM_FID(fid); + int ret = h->o_fid->bind ? h->o_fid->bind(fid, bfid, flags) : -FI_ENOSYS; + + /* Binding a CQ or a counter is exactly the sort of event that tells + * a provider which data path this endpoint will use. */ + if (!ret) + fit_ep_arm_ops(h, "bind"); + + if (ret || !bfid) + return ret; + + /* Remember the AV so remote fi_addr_t values can be reverse-mapped. */ + if (bfid->fclass == FI_CLASS_AV) + h->av = (struct fid_av *)bfid; + + /* And note which endpoint owns the CQ, so its completions can be + * attributed to a local NIC address. */ + if (bfid->fclass == FI_CLASS_CQ) + fit_cq_attach_ep(bfid, h); + + return ret; +} + +static int fit_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_ep_bind", fit_obj); + fit_ret = fit_ep_bind_inner(fid, bfid, flags); + fit_record_ctrl_exit("fi_ep_bind", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_control_inner(struct fid *fid, int command, void *arg) { + struct fit_ep *h = EP_FROM_FID(fid); + int ret = h->o_fid->control ? h->o_fid->control(fid, command, arg) : -FI_ENOSYS; + + /* The local address only becomes available once enabled -- and so, + * for some providers, does the real data path. */ + if (!ret && command == FI_ENABLE) { + fit_ep_arm_ops(h, "enable"); + fit_ep_resolve_src(h); + } + + return ret; +} + +static int fit_ep_control(struct fid *fid, int command, void *arg) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_ep_control", fit_obj); + fit_ret = fit_ep_control_inner(fid, command, arg); + fit_record_ctrl_exit("fi_ep_control", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_close_inner(struct fid *fid) { + struct fit_ep *h = EP_FROM_FID(fid); + int ret; + + ret = h->o_fid->close ? h->o_fid->close(fid) : -FI_ENOSYS; + if (ret) + return ret; + + /* The provider object is gone; our side state can go too. */ + fit_ep_peers_free(h); + pthread_mutex_destroy(&h->lock); + free(h); + return 0; +} + +static int fit_ep_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_close_ep", fit_obj); + fit_ret = fit_ep_close_inner(fid); + fit_record_ctrl_exit("fi_close_ep", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_tx_ctx_inner( + struct fid_ep *sep, int index, struct fi_tx_attr *attr, struct fid_ep **tx_ep, void *context) { + struct fit_ep *h = EP_FROM_EPOPS(sep); + int ret = h->o_ep->tx_ctx ? h->o_ep->tx_ctx(sep, index, attr, tx_ep, context) : -FI_ENOSYS; + + if (!ret && tx_ep && *tx_ep) + fit_hook_ep(*tx_ep, h->topo); + return ret; +} + +static int fit_ep_tx_ctx( + struct fid_ep *sep, int index, struct fi_tx_attr *attr, struct fid_ep **tx_ep, void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_EPOPS(sep)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_tx_context", fit_obj); + fit_ret = fit_ep_tx_ctx_inner(sep, index, attr, tx_ep, context); + fit_record_ctrl_exit("fi_tx_context", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_rx_ctx_inner( + struct fid_ep *sep, int index, struct fi_rx_attr *attr, struct fid_ep **rx_ep, void *context) { + struct fit_ep *h = EP_FROM_EPOPS(sep); + int ret = h->o_ep->rx_ctx ? h->o_ep->rx_ctx(sep, index, attr, rx_ep, context) : -FI_ENOSYS; + + if (!ret && rx_ep && *rx_ep) + fit_hook_ep(*rx_ep, h->topo); + return ret; +} + +static int fit_ep_rx_ctx( + struct fid_ep *sep, int index, struct fi_rx_attr *attr, struct fid_ep **rx_ep, void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_EPOPS(sep)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_rx_context", fit_obj); + fit_ret = fit_ep_rx_ctx_inner(sep, index, attr, rx_ep, context); + fit_record_ctrl_exit("fi_rx_context", fit_obj, fit_ret); + return fit_ret; +} + +void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo) { + struct fit_ep *h; + + if (!ep) + return; + + h = calloc(1, sizeof(*h)); + if (!h) { + fit_log("out of memory hooking endpoint"); + return; + } + + pthread_mutex_init(&h->lock, NULL); + h->ep = ep; + h->ep_id = atomic_fetch_add(&fit_ep_counter, 1); + /* Borrowed from the domain, which outlives every endpoint on it. */ + h->topo = topo; + snprintf(h->src, sizeof(h->src), "unknown"); + fit_sanitize(h->src, &h->src_len); + + fit_ep_arm_ops(h, "create"); + + if (ep->fid.ops) { + h->o_fid = ep->fid.ops; + h->fid_ops = *ep->fid.ops; + h->fid_ops.bind = fit_ep_bind; + h->fid_ops.control = fit_ep_control; + h->fid_ops.close = fit_ep_close; + ep->fid.ops = &h->fid_ops; + } + + if (ep->ops) { + h->o_ep = ep->ops; + h->ep_ops = *ep->ops; + h->ep_ops.tx_ctx = fit_ep_tx_ctx; + h->ep_ops.rx_ctx = fit_ep_rx_ctx; + ep->ops = &h->ep_ops; + } + + fit_log("hooked endpoint %u (%p)", h->ep_id, (void *)ep); +} + +/* ================================================================== */ +/* address vector */ +/* */ +/* Wrapped for one reason only: fi_av_remove() frees an fi_addr_t for */ +/* reuse, which silently invalidates every cached fi_addr_t -> NIC */ +/* address mapping. Bump a generation counter so the caches notice. */ +/* ================================================================== */ + +#define AV_FROM_OPS(av) fit_container_of((av)->ops, struct fit_av, ops) + +static int +fit_av_remove_inner(struct fid_av *av, fi_addr_t *fi_addr, size_t count, uint64_t flags) { + struct fit_av *a = AV_FROM_OPS(av); + int ret = a->orig->remove ? a->orig->remove(av, fi_addr, count, flags) : -FI_ENOSYS; + + if (!ret) + atomic_fetch_add_explicit(&fit_av_gen, 1, memory_order_release); + return ret; +} + +static int fit_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count, uint64_t flags) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_av_remove", fit_obj); + fit_ret = fit_av_remove_inner(av, fi_addr, count, flags); + fit_record_ctrl_exit("fi_av_remove", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_av_close_inner(struct fid *fid) { + struct fit_av *a = fit_container_of(fid->ops, struct fit_av, fid_ops); + int ret = a->o_fid->close ? a->o_fid->close(fid) : -FI_ENOSYS; + + if (!ret) { + atomic_fetch_add_explicit(&fit_av_gen, 1, memory_order_release); + free(a); + } + return ret; +} + +static int fit_av_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_close_av", fit_obj); + fit_ret = fit_av_close_inner(fid); + fit_record_ctrl_exit("fi_close_av", fit_obj, fit_ret); + return fit_ret; +} + +void fit_hook_av(struct fid_av *av) { + struct fit_av *a; + + if (!av || !av->ops || av->ops->remove == fit_av_remove) + return; + + a = calloc(1, sizeof(*a)); + if (!a) + return; + + if (av->fid.ops) { + a->o_fid = av->fid.ops; + a->fid_ops = *av->fid.ops; + a->fid_ops.close = fit_av_close; + av->fid.ops = &a->fid_ops; + } + + a->orig = av->ops; + a->ops = *av->ops; + a->ops.remove = fit_av_remove; + av->ops = &a->ops; + + fit_log("hooked address vector %p", (void *)av); +} + +/* ================================================================== */ +/* completion queue */ +/* */ +/* This is where an operation acquires a duration. The posting record */ +/* says when the application handed the transfer to the provider; the */ +/* completion record says when the application learned it was done. */ +/* They are joined on op_context. */ +/* */ +/* The completion also carries two things the posting record cannot: */ +/* the length the provider actually transferred (a receive posts a */ +/* buffer, not a message size), and, via fi_cq_readfrom(), the address */ +/* the data really came from. */ +/* ================================================================== */ + +#define CQ_FROM_OPS(cq) fit_container_of((cq)->ops, struct fit_cq, ops) + +static atomic_uint fit_cq_counter; + +/* Every fi_cq_*_entry layout begins with these fields, in this order, + * so one accessor serves all of them. FI_CQ_FORMAT_CONTEXT stops after + * op_context, which is why entry_sz decides how much may be read. */ +struct fit_cqe_head { + void *op_context; + uint64_t flags; + size_t len; +}; + +static size_t fit_cq_entry_size(enum fi_cq_format format) { + switch (format) { + case FI_CQ_FORMAT_CONTEXT: + return sizeof(struct fi_cq_entry); + case FI_CQ_FORMAT_MSG: + return sizeof(struct fi_cq_msg_entry); + case FI_CQ_FORMAT_DATA: + return sizeof(struct fi_cq_data_entry); + case FI_CQ_FORMAT_TAGGED: + return sizeof(struct fi_cq_tagged_entry); + default: + return 0; + } +} + +/* + * Name the operation the completion refers to. Flags are only present + * from FI_CQ_FORMAT_MSG upwards; with FI_CQ_FORMAT_CONTEXT the provider + * tells us nothing but the context, and the posting record it joins to + * has to supply the operation type. + */ +static const char *fit_cq_opname(uint64_t flags, int have_flags) { + if (!have_flags) + return "cq_completion"; + + if (flags & FI_ATOMIC) + return (flags & FI_REMOTE_WRITE) ? "cq_remote_atomic" : "cq_atomic"; + if (flags & FI_REMOTE_WRITE) + return "cq_remote_write"; + if (flags & FI_REMOTE_READ) + return "cq_remote_read"; + if (flags & FI_WRITE) + return "cq_write"; + if (flags & FI_READ) + return "cq_read"; + if (flags & FI_SEND) + return (flags & FI_TAGGED) ? "cq_tsend" : "cq_send"; + if (flags & FI_RECV) + return (flags & FI_TAGGED) ? "cq_trecv" : "cq_recv"; + return "cq_completion"; +} + +/* Walk a batch the provider has just filled in and emit one record per + * entry. src is the parallel array fi_cq_readfrom() fills, or NULL. */ +/* + * A provider may implement one read entry point in terms of another, and + * it does so through the *public* inline wrapper -- the tcp provider's + * ofi_cq_read() is literally "return fi_cq_readfrom(cq, buf, count, + * NULL)". That inline dispatches through the vtable, so it lands back in + * here and one application poll would be recorded twice. + * + * Count the nesting and report only from the outermost frame: that is the + * call the application actually made, and it is the one carrying the src + * array if there is one, since an internal re-dispatch can only drop + * information, never add it. + */ +static __thread unsigned fit_cq_depth; + +static void fit_cq_report(struct fit_cq *c, const void *buf, ssize_t n, const fi_addr_t *src) { + int have_flags = c->entry_sz >= sizeof(struct fi_cq_msg_entry); + uint64_t now = fit_now_us(); + ssize_t i; + + if (!c->entry_sz) + return; + + for (i = 0; i < n; i++) { + const struct fit_cqe_head *e = + (const struct fit_cqe_head *)((const char *)buf + (size_t)i * c->entry_sz); + uint64_t flags = have_flags ? e->flags : 0; + size_t len = have_flags ? e->len : 0; + + fit_record_cq(c, fit_cq_opname(flags, have_flags), src ? src[i] : FI_ADDR_UNSPEC, len, 0, + e->op_context, now); + } +} + +static ssize_t fit_cq_read(struct fid_cq *cq, void *buf, size_t count) { + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; + + ret = CALL_OR_ENOSYS(c->orig->read, (cq, buf, count)); + fit_cq_depth = outer; + + /* A polling loop calls this constantly and almost always gets + * -FI_EAGAIN. Nothing happened, so nothing is recorded. */ + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, NULL); + return ret; +} + +static ssize_t fit_cq_readfrom(struct fid_cq *cq, void *buf, size_t count, fi_addr_t *src_addr) { + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; + + ret = CALL_OR_ENOSYS(c->orig->readfrom, (cq, buf, count, src_addr)); + fit_cq_depth = outer; + + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, src_addr); + return ret; +} + +static ssize_t +fit_cq_sread(struct fid_cq *cq, void *buf, size_t count, const void *cond, int timeout) { + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; + + ret = CALL_OR_ENOSYS(c->orig->sread, (cq, buf, count, cond, timeout)); + fit_cq_depth = outer; + + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, NULL); + return ret; +} + +static ssize_t fit_cq_sreadfrom(struct fid_cq *cq, + void *buf, + size_t count, + fi_addr_t *src_addr, + const void *cond, + int timeout) { + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; + + ret = CALL_OR_ENOSYS(c->orig->sreadfrom, (cq, buf, count, src_addr, cond, timeout)); + fit_cq_depth = outer; + + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, src_addr); + return ret; +} + +/* + * A failed operation never appears in the normal completion stream, so + * without this a transfer that errored would show a posting record and + * no completion, indistinguishable from one still in flight. + */ +static ssize_t fit_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf, uint64_t flags) { + struct fit_cq *c = CQ_FROM_OPS(cq); + ssize_t ret = CALL_OR_ENOSYS(c->orig->readerr, (cq, buf, flags)); + + if (ret > 0 && buf) + fit_record_cq(c, fit_cq_opname(buf->flags, 1), +#ifdef HAVE_FI_CQ_ERR_SRC_ADDR + buf->src_addr, +#else + /* Older libfabric has no src_addr here. */ + FI_ADDR_UNSPEC, +#endif + buf->len, -buf->err, buf->op_context, fit_now_us()); + return ret; +} + +static int fit_cq_close_inner(struct fid *fid) { + struct fit_cq *c = fit_container_of(fid->ops, struct fit_cq, fid_ops); + int ret; + + ret = c->o_fid->close ? c->o_fid->close(fid) : -FI_ENOSYS; + if (!ret) + free(c); + return ret; +} + +static int fit_cq_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(fit_container_of(fid->ops, struct fit_cq, fid_ops)->cq_id); + int fit_ret; + + fit_record_ctrl_entry("fi_close_cq", fit_obj); + fit_ret = fit_cq_close_inner(fid); + fit_record_ctrl_exit("fi_close_cq", fit_obj, fit_ret); + return fit_ret; +} + +void fit_hook_cq(struct fid_cq *cq, enum fi_cq_format format) { + struct fit_cq *c; + + if (!cq || !cq->ops || cq->ops->read == fit_cq_read) + return; + + c = calloc(1, sizeof(*c)); + if (!c) + return; + + c->cq = cq; + c->cq_id = atomic_fetch_add(&fit_cq_counter, 1); + c->format = format; + c->entry_sz = fit_cq_entry_size(format); + + /* + * FI_CQ_FORMAT_UNSPEC leaves the choice to the provider and there + * is no way to ask what it picked. Every layout starts with + * op_context, so fall back to that much: completions are still + * timestamped and still join to their posting record, they just + * carry no length of their own. + */ + if (!c->entry_sz) { + c->entry_sz = sizeof(struct fi_cq_entry); + fit_log("cq %u opened with FI_CQ_FORMAT_UNSPEC; " + "completion lengths unavailable", + c->cq_id); + } + + if (cq->fid.ops) { + c->o_fid = cq->fid.ops; + c->fid_ops = *cq->fid.ops; + c->fid_ops.close = fit_cq_close; + cq->fid.ops = &c->fid_ops; + } + + c->orig = cq->ops; + c->ops = *cq->ops; + c->ops.read = fit_cq_read; + c->ops.readfrom = fit_cq_readfrom; + c->ops.readerr = fit_cq_readerr; + c->ops.sread = fit_cq_sread; + c->ops.sreadfrom = fit_cq_sreadfrom; + cq->ops = &c->ops; + + fit_log("hooked cq %u (%p) format %d", c->cq_id, (void *)cq, (int)format); +} + +/* Called when an endpoint binds a CQ, so a completion can be attributed + * to a local NIC address. Only safe if we hooked that CQ ourselves -- + * the read slot is the tell. */ +static void fit_cq_attach_ep(struct fid *bfid, struct fit_ep *h) { + struct fid_cq *cq = (struct fid_cq *)bfid; + struct fit_cq *c; + + if (!cq->ops || cq->ops->read != fit_cq_read) + return; + + c = CQ_FROM_OPS(cq); + if (c->ep && c->ep != h) { + /* Shared across endpoints; guessing one would be wrong. */ + c->ep_ambiguous = 1; + c->ep = NULL; + return; + } + if (!c->ep_ambiguous) + c->ep = h; +} + +/* ================================================================== */ +/* domain */ +/* ================================================================== */ + +#define DOM_FROM_OPS(dom) fit_container_of((dom)->ops, struct fit_domain, ops) + +static int fit_dom_endpoint_inner(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **ep, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->endpoint ? d->orig->endpoint(domain, info, ep, context) : -FI_ENOSYS; + + if (!ret && ep && *ep) + fit_hook_ep(*ep, &d->topo); + return ret; +} + +static int fit_dom_endpoint(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **ep, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_endpoint", fit_obj); + fit_ret = fit_dom_endpoint_inner(domain, info, ep, context); + fit_record_ctrl_exit("fi_endpoint", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_endpoint2_inner(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **ep, + uint64_t flags, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->endpoint2 ? d->orig->endpoint2(domain, info, ep, flags, context) : -FI_ENOSYS; + + if (!ret && ep && *ep) + fit_hook_ep(*ep, &d->topo); + return ret; +} + +static int fit_dom_endpoint2(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **ep, + uint64_t flags, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_endpoint2", fit_obj); + fit_ret = fit_dom_endpoint2_inner(domain, info, ep, flags, context); + fit_record_ctrl_exit("fi_endpoint2", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_scalable_ep_inner(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **sep, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->scalable_ep ? d->orig->scalable_ep(domain, info, sep, context) : -FI_ENOSYS; + + if (!ret && sep && *sep) + fit_hook_ep(*sep, &d->topo); + return ret; +} + +static int fit_dom_scalable_ep(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **sep, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_scalable_ep", fit_obj); + fit_ret = fit_dom_scalable_ep_inner(domain, info, sep, context); + fit_record_ctrl_exit("fi_scalable_ep", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_srx_ctx_inner(struct fid_domain *domain, + struct fi_rx_attr *attr, + struct fid_ep **rx_ep, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->srx_ctx ? d->orig->srx_ctx(domain, attr, rx_ep, context) : -FI_ENOSYS; + + /* Off by default. Several providers (tcp among them) route an + * endpoint's receives through an internal shared receive context, so + * hooking both records every fi_recv twice -- once on the endpoint + * the application called, once on the srx underneath it. Set + * FI_TRACE_SRX=1 if the application posts to an srx directly. */ + if (!ret && rx_ep && *rx_ep && fit_cfg.trace_srx) + fit_hook_ep(*rx_ep, &d->topo); + return ret; +} + +static int fit_dom_srx_ctx(struct fid_domain *domain, + struct fi_rx_attr *attr, + struct fid_ep **rx_ep, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_srx_context", fit_obj); + fit_ret = fit_dom_srx_ctx_inner(domain, attr, rx_ep, context); + fit_record_ctrl_exit("fi_srx_context", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_cq_open_inner(struct fid_domain *domain, + struct fi_cq_attr *attr, + struct fid_cq **cq, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->cq_open ? d->orig->cq_open(domain, attr, cq, context) : -FI_ENOSYS; + + if (!ret && cq && *cq && fit_cfg.trace_cq) + fit_hook_cq(*cq, attr ? attr->format : FI_CQ_FORMAT_UNSPEC); + return ret; +} + +static int fit_dom_cq_open(struct fid_domain *domain, + struct fi_cq_attr *attr, + struct fid_cq **cq, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_cq_open", fit_obj); + fit_ret = fit_dom_cq_open_inner(domain, attr, cq, context); + fit_record_ctrl_exit("fi_cq_open", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_av_open_inner(struct fid_domain *domain, + struct fi_av_attr *attr, + struct fid_av **av, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->av_open ? d->orig->av_open(domain, attr, av, context) : -FI_ENOSYS; + + if (!ret && av && *av) + fit_hook_av(*av); + return ret; +} + +static int fit_dom_av_open(struct fid_domain *domain, + struct fi_av_attr *attr, + struct fid_av **av, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_av_open", fit_obj); + fit_ret = fit_dom_av_open_inner(domain, attr, av, context); + fit_record_ctrl_exit("fi_av_open", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_close_inner(struct fid *fid) { + struct fit_domain *d = fit_container_of(fid->ops, struct fit_domain, fid_ops); + int ret = d->o_fid->close ? d->o_fid->close(fid) : -FI_ENOSYS; + + if (!ret) + free(d); + return ret; +} + +static int fit_dom_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_close_domain", fit_obj); + fit_ret = fit_dom_close_inner(fid); + fit_record_ctrl_exit("fi_close_domain", fit_obj, fit_ret); + return fit_ret; +} + +void fit_hook_domain(struct fid_domain *domain, const struct fi_info *info) { + struct fit_domain *d; + + if (!domain || !domain->ops) + return; + + d = calloc(1, sizeof(*d)); + if (!d) + return; + + /* Before patching anything: the extension interface is reached + * through the domain's own ops, and asking while they are still + * entirely the provider's keeps this independent of what we are + * about to change. */ + fit_topo_resolve(&d->topo, domain, info); + + if (domain->fid.ops) { + d->o_fid = domain->fid.ops; + d->fid_ops = *domain->fid.ops; + d->fid_ops.close = fit_dom_close; + domain->fid.ops = &d->fid_ops; + } + + d->orig = domain->ops; + d->ops = *domain->ops; + d->ops.endpoint = fit_dom_endpoint; + d->ops.endpoint2 = fit_dom_endpoint2; + d->ops.scalable_ep = fit_dom_scalable_ep; + d->ops.srx_ctx = fit_dom_srx_ctx; + d->ops.cq_open = fit_dom_cq_open; + d->ops.av_open = fit_dom_av_open; + domain->ops = &d->ops; + + if (d->topo.valid) + fit_log("hooked domain %p (%s: group %u switch %u port %u)", (void *)domain, d->topo.name, + d->topo.group, d->topo.sw, d->topo.port); + else + fit_log("hooked domain %p", (void *)domain); +} + +/* ================================================================== */ +/* fabric */ +/* ================================================================== */ + +#define FAB_FROM_OPS(fab) fit_container_of((fab)->ops, struct fit_fabric, ops) + +static int fit_fab_domain_inner(struct fid_fabric *fabric, + struct fi_info *info, + struct fid_domain **dom, + void *context) { + struct fit_fabric *f = FAB_FROM_OPS(fabric); + int ret = f->orig->domain ? f->orig->domain(fabric, info, dom, context) : -FI_ENOSYS; + + if (!ret && dom && *dom) + fit_hook_domain(*dom, info); + return ret; +} + +static int fit_fab_domain(struct fid_fabric *fabric, + struct fi_info *info, + struct fid_domain **dom, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_domain", fit_obj); + fit_ret = fit_fab_domain_inner(fabric, info, dom, context); + fit_record_ctrl_exit("fi_domain", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_fab_domain2_inner(struct fid_fabric *fabric, + struct fi_info *info, + struct fid_domain **dom, + uint64_t flags, + void *context) { + struct fit_fabric *f = FAB_FROM_OPS(fabric); + int ret = f->orig->domain2 ? f->orig->domain2(fabric, info, dom, flags, context) : -FI_ENOSYS; + + if (!ret && dom && *dom) + fit_hook_domain(*dom, info); + return ret; +} + +static int fit_fab_domain2(struct fid_fabric *fabric, + struct fi_info *info, + struct fid_domain **dom, + uint64_t flags, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_domain2", fit_obj); + fit_ret = fit_fab_domain2_inner(fabric, info, dom, flags, context); + fit_record_ctrl_exit("fi_domain2", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_fab_close_inner(struct fid *fid) { + struct fit_fabric *f = fit_container_of(fid->ops, struct fit_fabric, fid_ops); + int ret; + + ret = f->o_fid->close ? f->o_fid->close(fid) : -FI_ENOSYS; + if (!ret) + free(f); + return ret; +} + +static int fit_fab_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_close_fabric", fit_obj); + fit_ret = fit_fab_close_inner(fid); + fit_record_ctrl_exit("fi_close_fabric", fit_obj, fit_ret); + return fit_ret; +} + +void fit_hook_fabric(struct fid_fabric *fabric) { + struct fit_fabric *f; + + if (!fabric || !fabric->ops) + return; + + f = calloc(1, sizeof(*f)); + if (!f) + return; + + if (fabric->fid.ops) { + f->o_fid = fabric->fid.ops; + f->fid_ops = *fabric->fid.ops; + f->fid_ops.close = fit_fab_close; + fabric->fid.ops = &f->fid_ops; + } + + f->orig = fabric->ops; + f->ops = *fabric->ops; + f->ops.domain = fit_fab_domain; + f->ops.domain2 = fit_fab_domain2; + fabric->ops = &f->ops; + + fit_log("hooked fabric %p", (void *)fabric); +} + +/* ================================================================== */ +/* interposed exported symbols */ +/* ================================================================== */ + +typedef int (*fi_fabric_fn)(struct fi_fabric_attr *, struct fid_fabric **, void *); +typedef int (*fi_fabric2_fn)(struct fi_info *, struct fid_fabric **, uint64_t, void *); + +static fi_fabric_fn real_fi_fabric; +static fi_fabric2_fn real_fi_fabric2; + +/* + * libfabric exports several versions of the same name: fi_fabric exists as + * both fi_fabric@FABRIC_1.0 and the default fi_fabric@@FABRIC_1.1. A plain + * dlsym() is free to return either, and on glibc it returns the 1.0 compat + * stub -- which immediately calls fi_fabric() again, lands back in this + * library, and recurses until the stack runs out. + * + * So ask for an explicit version, newest first, and take the first hit. + * The newest defined version of a symbol is by convention the default one. + * A libfabric built without symbol versioning has no versions to match, and + * the plain dlsym() fallback is then unambiguous. + */ +#ifndef __GLIBC__ +#error "fi_trace is Linux/glibc only: it needs LD_PRELOAD symbol interposition, \ +dlvsym() to pick the right versioned fi_fabric, and gettid(2). \ +On macOS or elsewhere, run it in a container: ./build/docker-demo.sh" +#endif + +static void *fit_dlnext(const char *name) { + static const char *const vers[] = { + "FABRIC_1.9", "FABRIC_1.8", "FABRIC_1.7", "FABRIC_1.6", "FABRIC_1.5", + "FABRIC_1.4", "FABRIC_1.3", "FABRIC_1.2", "FABRIC_1.1", "FABRIC_1.0", + }; + size_t i; + void *sym; + + for (i = 0; i < sizeof(vers) / sizeof(vers[0]); i++) { + sym = dlvsym(RTLD_NEXT, name, vers[i]); + if (sym) + return sym; + } + + (void)dlerror(); + return dlsym(RTLD_NEXT, name); +} + +__attribute__((constructor)) static void fit_init(void) { + fit_config_init(); + + if (!fit_cfg.enabled) { + fit_log("disabled via FI_TRACE_ENABLE"); + return; + } + + real_fi_fabric = (fi_fabric_fn)fit_dlnext("fi_fabric"); + real_fi_fabric2 = (fi_fabric2_fn)fit_dlnext("fi_fabric2"); + + if (!real_fi_fabric && !real_fi_fabric2) { + fit_log("cannot find libfabric fi_fabric(); tracing off"); + return; + } + + // fit_writer_init(); + fit_armed = 1; + fit_log("armed: dir=%s ops=%s%s%s rx=%d resolve=%d", fit_cfg.dir, fit_cfg.trace_msg ? "msg " : "", + fit_cfg.trace_tagged ? "tagged " : "", fit_cfg.trace_rma ? "rma" : "", fit_cfg.trace_rx, + fit_cfg.resolve); +} + +/* These must be exported even though the library is built with + * -fvisibility=hidden -- they are the whole point of the preload. */ +#define FIT_PUBLIC __attribute__((visibility("default"))) + +FIT_PUBLIC int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, void *context); +FIT_PUBLIC int +fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, void *context); + +static int fi_fabric_inner(struct fi_fabric_attr *attr, struct fid_fabric **fabric, void *context) { + int ret; + + if (!real_fi_fabric) + real_fi_fabric = (fi_fabric_fn)fit_dlnext("fi_fabric"); + if (!real_fi_fabric) + return -FI_ENOSYS; + + ret = real_fi_fabric(attr, fabric, context); + if (!ret && fit_armed && fabric && *fabric) + fit_hook_fabric(*fabric); + return ret; +} + +FIT_PUBLIC +int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_fabric", fit_obj); + fit_ret = fi_fabric_inner(attr, fabric, context); + fit_record_ctrl_exit("fi_fabric", fit_obj, fit_ret); + return fit_ret; +} + +static int +fi_fabric2_inner(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, void *context) { + int ret; + + if (!real_fi_fabric2) + real_fi_fabric2 = (fi_fabric2_fn)fit_dlnext("fi_fabric2"); + if (!real_fi_fabric2) + return -FI_ENOSYS; + + ret = real_fi_fabric2(info, fabric, flags, context); + if (!ret && fit_armed && fabric && *fabric) + fit_hook_fabric(*fabric); + return ret; +} + +FIT_PUBLIC +int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_fabric2", fit_obj); + fit_ret = fi_fabric2_inner(info, fabric, flags, context); + fit_record_ctrl_exit("fi_fabric2", fit_obj, fit_ret); + return fit_ret; +} diff --git a/backends/libfabric/src/fit_lttng.c b/backends/libfabric/src/fit_lttng.c new file mode 100644 index 000000000..5adf5fbe5 --- /dev/null +++ b/backends/libfabric/src/fit_lttng.c @@ -0,0 +1,295 @@ +/* + * fit_lttng.c -- replaces fit_writer.c. + * + * The standalone tracer wrote sanitised CSV into a per-thread buffer and + * flushed it to one file per thread, then joined the pieces with + * tools/fitrace-merge. Under THAPI, LTTng owns all of that. What goes away: + * + * - the per-thread buffer, its mutex, the flush-on-exit destructor, and + * FI_TRACE_DIR / FI_TRACE_BUFSZ / FI_TRACE_MAX_RECORDS + * - the hostname, pid and tid columns: LTTng adds vpid and vtid as channel + * contexts and hostname as a trace environment entry, so the hot path + * never touches them + * - the cached getpid() and its pthread_atfork re-read, which existed only + * because the pid was a column + * - the timestamp: LTTng stamps every event from the monotonic clock + * - the CSV quoting in fit_sanitize(); CTF strings are length-delimited. + * fit_sanitize() is kept, because an address string with an embedded + * newline would still make the pretty-printed output unreadable, but it + * now only strips control characters. + * + * What is deliberately kept, because fit_hook.c and fit_addr.c call it and + * should not need editing for this: + * + * - the fit_record() / fit_record_cq() signatures + * - fit_cfg, fit_config_init(), fit_log(), fit_armed + * - fit_now_us(), now a stub returning 0 (see below) + * + * Environment variables are renamed FI_TRACE_* -> LTTNG_UST_FI_* to match the + * house convention. The ones that configured the CSV writer are gone. + */ + +#define _GNU_SOURCE + +#include "fi_trace.h" +#include "libfabric_tracepoints.h" + +#include +#include +#include +#include +#include + +#include + +struct fit_config fit_cfg; +int fit_armed; +_Atomic unsigned int fit_av_gen = 0; + +/* ------------------------------------------------------------------ */ +/* logging (stderr, not the trace -- this is for debugging the tracer) */ +/* ------------------------------------------------------------------ */ + +void fit_log(const char *fmt, ...) { + va_list ap; + + if (!fit_cfg.verbose) + return; + + fprintf(stderr, "[fi_trace] "); + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fputc('\n', stderr); +} + +/* ------------------------------------------------------------------ */ +/* configuration */ +/* ------------------------------------------------------------------ */ + +static int env_int(const char *name, int dflt) { + const char *v = getenv(name); + + return v ? atoi(v) : dflt; +} + +/* + * LTTNG_UST_FI_OPS selects which op families get their vtable swapped at all. + * This is not the same thing as enabling or disabling tracepoints: a slot we + * never swap costs the application literally nothing, where a disabled + * tracepoint still costs the branch. For an application that only does tagged + * messaging, "LTTNG_UST_FI_OPS=tagged" is measurably cheaper than filtering on + * the LTTng side. + */ +static void parse_ops(const char *v) { + if (!v || !*v) { + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 1; + return; + } + + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 0; + + if (strstr(v, "msg")) + fit_cfg.trace_msg = 1; + if (strstr(v, "tagged")) + fit_cfg.trace_tagged = 1; + if (strstr(v, "rma")) + fit_cfg.trace_rma = 1; +} + +void fit_config_init(void) { + fit_cfg.enabled = env_int("LTTNG_UST_FI_ENABLE", 1); + fit_cfg.resolve = env_int("LTTNG_UST_FI_RESOLVE", 1); + fit_cfg.trace_rx = env_int("LTTNG_UST_FI_RX", 1); + fit_cfg.trace_srx = env_int("LTTNG_UST_FI_SRX", 0); + fit_cfg.trace_eagain = env_int("LTTNG_UST_FI_EAGAIN", 1); + fit_cfg.trace_cq = env_int("LTTNG_UST_FI_CQ", 1); + fit_cfg.verbose = env_int("LTTNG_UST_FI_VERBOSE", 0); + parse_ops(getenv("LTTNG_UST_FI_OPS")); + + /* + * fit_cfg.host / host_len / dir / bufsz / max_records are no longer + * read. They stay in struct fit_config only so that fi_trace.h does + * not have to change; drop them once you are sure nothing else refers + * to them. + */ +} + +/* ------------------------------------------------------------------ */ +/* string hygiene */ +/* ------------------------------------------------------------------ */ + +/* + * CTF strings are length-delimited, so commas and quotes no longer need + * escaping. Control characters still do: they would corrupt the aligned + * columns of `iprof -t` output and, in the case of a stray NUL, truncate the + * field. Measured once where the string is produced, as before, so the hot + * path only memcpys. + */ +void fit_sanitize(char *s, size_t *len_out) { + size_t n = 0; + + if (!s) { + if (len_out) + *len_out = 0; + return; + } + + for (; s[n]; n++) { + if ((unsigned char)s[n] < 0x20 || (unsigned char)s[n] == 0x7f) + s[n] = '_'; + } + + if (len_out) + *len_out = n; +} + +/* + * Kept so fit_cq_report() in fit_hook.c compiles unchanged. LTTng timestamps + * every event itself from the monotonic clock, with better resolution than + * clock_gettime() in userspace gives you after the syscall, so there is + * nothing for this to do. + * + * Note the consequence for completions: a single fi_cq_read() returning a + * burst of N entries used to stamp all N with one fit_now_us() value, which + * was honest about the batch having no internal resolution. LTTng will now + * stamp them microseconds apart as the loop runs. Treat sub-microsecond + * spacing between completions from one reap as an artifact of the loop, not a + * real arrival pattern. + */ +uint64_t fit_now_us(void) { return 0; } + +/* ------------------------------------------------------------------ */ +/* data-path records */ +/* ------------------------------------------------------------------ */ + +/* + * Entry marker. Cheap on purpose: no AV lookup, no iov summation, no + * sanitisation. Its job is to make a hang or a crash inside the provider + * visible as an entry with no matching exit, per AGEND_SKILLS.md. All the + * payload rides on the exit event, where it was already being computed before + * this port. + */ +void fit_record_entry(struct fit_ep *h, const char *op, char dir, const void *context) { + if (dir == 'R' && !fit_cfg.trace_rx) + return; + + tracepoint(lttng_ust_fi, op_entry, op, h->ep_id, context); +} + +void fit_record(struct fit_ep *h, + const char *op, + char dir, + fi_addr_t peer, + size_t bytes, + size_t iov_count, + ssize_t ret, + const void *context) { + char scratch[FIT_ADDR_MAX]; + const char *peer_str; + size_t peer_len = 0; + + if (dir == 'R' && !fit_cfg.trace_rx) + return; + + /* + * A -FI_EAGAIN return means the provider refused the operation and the + * application will retry; no bytes moved. Applications that spin on a + * busy endpoint can generate thousands of these, which swamps the real + * transfers, so they can be dropped. + * + * Dropping the exit here leaves the matching op_entry unpaired, and the + * analysis side discards an entry that never closes. That is the + * intended behaviour, but it does mean LTTNG_UST_FI_EAGAIN=0 makes the + * entry/exit counts in a raw trace dump disagree -- do not go looking + * for a bug there. + */ + if (ret == -FI_EAGAIN && !fit_cfg.trace_eagain) + return; + + if (!h->src_valid) + fit_ep_resolve_src(h); + + peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), &peer_len); + + tracepoint(lttng_ust_fi, op_exit, op, h->ep_id, h->src, peer_str, (dir == 'R') ? "rx" : "tx", + (uint64_t)iov_count, (uint64_t)bytes, (int64_t)ret, + /* An operation the provider refused never became + * outstanding, so it will produce no completion and must + * not be joined to one. */ + ret ? NULL : context); +} + +void fit_record_cq(struct fit_cq *c, + const char *op, + fi_addr_t peer, + size_t bytes, + ssize_t ret, + const void *context, + uint64_t now) { + char scratch[FIT_ADDR_MAX]; + const char *peer_str = "fi_addr_unspec"; + size_t peer_len = sizeof("fi_addr_unspec") - 1; + const char *src = "unknown"; + struct fit_ep *h = c->ep; + + (void)now; /* LTTng stamps the event; see fit_now_us() */ + + /* The CQ usually serves exactly one endpoint, in which case its local + * address and id describe this record too. */ + if (h) { + if (!h->src_valid) + fit_ep_resolve_src(h); + src = h->src; + peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), &peer_len); + } + + tracepoint(lttng_ust_fi, cq_completion, op, h ? h->ep_id : c->cq_id, src, peer_str, + (uint64_t)bytes, (int64_t)ret, context); +} + +/* ------------------------------------------------------------------ */ +/* control-path records */ +/* ------------------------------------------------------------------ */ + +void fit_record_ctrl_entry(const char *op, unsigned int obj_id) { + tracepoint(lttng_ust_fi, ctrl_entry, op, obj_id); +} + +void fit_record_ctrl_exit(const char *op, unsigned int obj_id, int ret) { + tracepoint(lttng_ust_fi, ctrl_exit, op, obj_id, (int64_t)ret); +} + +/* ------------------------------------------------------------------ */ +/* endpoint description */ +/* ------------------------------------------------------------------ */ + +/* + * Replaces fit_topo_record(), which appended a row to the sidecar CSV that + * fitrace-merge later joined against. Same data, same once-per-endpoint + * cadence (fit_addr.c guards it with h->topo_written); it is now an event and + * the analysis side builds the map in memory. + * + * The signature gained the ep's source address because the sidecar was keyed + * by NIC address while the interval stream is keyed by ep_id, and the mapping + * between them has to come from somewhere. + */ +void fit_record_ep_info(unsigned int ep_id, + const char *src, + size_t src_len, + const struct fit_topo *t) { + (void)src_len; + + if (!fit_armed || !src) + return; + + tracepoint(lttng_ust_fi, ep_info, ep_id, src, (t && t->valid) ? t->name : "", + (t && t->valid) ? 1u : 0u, (t && t->valid) ? t->group : 0u, + (t && t->valid) ? t->sw : 0u, (t && t->valid) ? t->port : 0u); +} + +/* + * No __attribute__((destructor)) flush. LTTng's own atexit handling drains + * the ring buffers, and the blocking channel that iprof configures + * (--blocking-timeout=inf) means nothing is dropped on the way out. + */ diff --git a/backends/libfabric/tracer_libfabric.sh.in b/backends/libfabric/tracer_libfabric.sh.in new file mode 100644 index 000000000..cb9a48f19 --- /dev/null +++ b/backends/libfabric/tracer_libfabric.sh.in @@ -0,0 +1,90 @@ +#!/bin/sh +# +# Standalone driver: set up an LTTng session, preload the tracer, run the +# application. No analysis -- inspect the resulting CTF with +# babeltrace_thapi, or just `babeltrace2 ~/lttng-traces/thapi-fi-session-*`. +# +# This is a developer convenience for checking that tracepoints fire at all. +# For real use go through iprof, which also handles the per-rank session setup +# that this script does not: +# +# iprof --backends fi -- ./app +# mpirun -n 64 -- iprof --backends fi -- ./app +# +exec_prefix=@prefix@ +libdir=@libdir@ +prefix=@prefix@ +datarootdir=@datarootdir@ +datadir=@datadir@ + +display_help() { + echo "Usage:" + echo " $(basename $0) [options] [--] " + echo " --help Show this screen" + echo " --version Print the version string" + echo "" + echo "Environment:" + echo " LTTNG_UST_FI_ENABLE=0 Load but do not hook anything" + echo " LTTNG_UST_FI_OPS=msg,tagged,rma Which op families to swap in" + echo " LTTNG_UST_FI_RX=0 Skip receive postings" + echo " LTTNG_UST_FI_SRX=0 Skip shared receive contexts" + echo " LTTNG_UST_FI_EAGAIN=0 Drop -FI_EAGAIN postings" + echo " LTTNG_UST_FI_CQ=0 Do not hook completion queues" + echo " LTTNG_UST_FI_RESOLVE=0 Do not resolve addresses via the AV" + echo " LTTNG_UST_FI_VERBOSE=1 Tracer diagnostics on stderr" + exit 1 +} + +display_version() { + cat $datadir/thapi_version + exit 1 +} + +while true; do + case "$1" in + --help) + display_help + exit + ;; + --version) + display_version + exit + ;; + --) + shift + break + ;; + *) break ;; + esac +done + +if [ "$#" -eq 0 ]; then + display_help +fi + +lttng-sessiond --daemonize --quiet +lttng create thapi-fi-session +lttng enable-channel --userspace --blocking-timeout=inf blocking-channel +lttng add-context --userspace --channel=blocking-channel -t vpid -t vtid +lttng enable-event --channel=blocking-channel --userspace lttng_ust_fi:* + +# No LD_LIBRARY_PATH manipulation and no soname shadowing: the tracer is +# preloaded alongside the real libfabric.so and reaches it with +# dlvsym(RTLD_NEXT, "fi_fabric", ...). Everything below fi_fabric is reached +# through vtable pointers the provider itself handed us. +export LD_PRELOAD=$libdir/libTracerLibfabric.so +export LTTNG_UST_ALLOW_BLOCKING=1 + +lttng start + +ctrl_c() { + lttng stop + lttng destroy + exit +} + +trap ctrl_c INT + +"$@" +lttng stop +lttng destroy diff --git a/configure.ac b/configure.ac index 623b77b25..75d682fbb 100644 --- a/configure.ac +++ b/configure.ac @@ -178,6 +178,7 @@ AC_CONFIG_FILES([ backends/itt/Makefile backends/cxi/Makefile backends/pytorch/Makefile + backends/libfabric/Makefile ]) AC_CONFIG_FILES([utils/test_wrapper_thapi_text_pretty.sh], [chmod +x utils/test_wrapper_thapi_text_pretty.sh]) AC_CONFIG_FILES([backends/opencl/tracer_opencl.sh], [chmod +x backends/opencl/tracer_opencl.sh]) @@ -189,9 +190,10 @@ AC_CONFIG_FILES([backends/hip/tracer_hip.sh], [chmod +x backends/hip/tracer_hip. AC_CONFIG_FILES([backends/mpi/tracer_mpi.sh], [chmod +x backends/mpi/tracer_mpi.sh]) AC_CONFIG_FILES([backends/itt/tracer_itt.sh], [chmod +x backends/itt/tracer_itt.sh]) AC_CONFIG_FILES([backends/pytorch/tracer_pytorch.sh], [chmod +x backends/pytorch/tracer_pytorch.sh]) +AC_CONFIG_FILES([backends/libfabric/tracer_libfabric.sh], [chmod +x backends/libfabric/tracer_libfabric.sh]) AC_CONFIG_FILES([utils/babeltrace_thapi], [chmod +x utils/babeltrace_thapi]) AC_CONFIG_FILES([xprof/xprof.rb], [chmod +x xprof/xprof.rb]) AC_CONFIG_FILES([xprof/iprof:xprof/xprof_wrapper.rb.in], [chmod +x xprof/iprof]) AC_CONFIG_FILES([thapi.pc]) -AC_OUTPUT +AC_OUTPUT \ No newline at end of file diff --git a/utils/babeltrace_thapi.in b/utils/babeltrace_thapi.in index ac45321b5..2d5853e53 100755 --- a/utils/babeltrace_thapi.in +++ b/utils/babeltrace_thapi.in @@ -170,6 +170,7 @@ def get_components(names) require 'babeltrace_hip_lib' if $options[:'backend-names'].include?('hip') require 'babeltrace_itt_lib' if $options[:'backend-names'].include?('itt') require 'babeltrace_pytorch_lib' if $options[:'backend-names'].include?('pytorch') + require 'babeltrace_libfabric_lib' if $options[:'backend-names'].include?('fi') f = lambda { |iterator, _| iterator.next_messages.each do |m| next unless m.type == :BT_MESSAGE_TYPE_EVENT @@ -414,7 +415,7 @@ class BabeltraceParserThapi < OptionParserWithDefaultAndValidation on('-h', '--help', 'Prints this help') { print_help_and_exit(self, exit_code: 0) } on('-b', '--backends BACKENDS', Array, "Select which and how backends' need to handled.", 'Format: backend_name[:backend_level],...', - default: ['mpi:3', 'omp:2', 'cl:1', 'ze:1', 'cuda:1', 'hip:1', 'cxi:4', 'itt:5', 'pytorch:6']) + default: ['mpi:3', 'omp:2', 'cl:1', 'ze:1', 'cuda:1', 'hip:1', 'cxi:4', 'itt:5', 'pytorch:6', 'fi:7']) on('--debug', default: false) on('--archive SESSION-NAME') on('--archive-session-found-file-path PATH') diff --git a/utils/xprof_utils.hpp b/utils/xprof_utils.hpp index daec319d1..34bfe109c 100644 --- a/utils/xprof_utils.hpp +++ b/utils/xprof_utils.hpp @@ -24,6 +24,7 @@ enum backend_e { BACKEND_CXI = 8, BACKEND_ITT = 9, BACKEND_PYTORCH = 10, + BACKEND_FI = 11, }; typedef enum backend_e backend_t; typedef unsigned backend_level_t; @@ -41,6 +42,7 @@ const std::unordered_map pretty_backend_name_g = { {"cxi", BACKEND_CXI}, {"itt", BACKEND_ITT}, {"pytorch", BACKEND_PYTORCH}, + {"fi", BACKEND_FI}, }; const std::unordered_map backend_levels_g = { @@ -55,6 +57,7 @@ const std::unordered_map backend_levels_g = { {BACKEND_CXI, 4}, {BACKEND_ITT, 5}, {BACKEND_PYTORCH, 6}, + {BACKEND_FI, 7}, }; typedef std::string thapi_metadata_t; @@ -218,4 +221,4 @@ static inline std::string strip_event_class_name_entry(const char *str) { static inline std::string strip_event_class_name_exit(const char *str) { return strip_event_class_name(str, strlen("_exit")); -} +} \ No newline at end of file diff --git a/xprof/xprof.rb.in b/xprof/xprof.rb.in index ebc16acde..3c8909384 100755 --- a/xprof/xprof.rb.in +++ b/xprof/xprof.rb.in @@ -659,6 +659,13 @@ def enable_events_pytorch(channel_name, tracing_mode: 'default', profiling: fals LOGGER.debug("Tracing mode #{tracing_mode} similar to default for pytorch") unless tracing_mode == 'default' end +def enable_events_fi(channel_name, tracing_mode: 'default', profiling: false) + lttng_enable = "lttng enable-event --userspace --session=#{lttng_session_uuid} --channel=#{channel_name}" + exec("#{lttng_enable} lttng_ust_fi:*") + LOGGER.debug('Profiling is ignored for fi') if profiling + LOGGER.debug("Tracing mode #{tracing_mode} similar to default for fi") unless tracing_mode == 'default' +end + def enable_events_metadata(channel_name, tracing_mode: 'default', profiling: true) lttng_enable = "lttng enable-event --userspace --session=#{lttng_session_uuid} --channel=#{channel_name}" LOGGER.debug('Profiling is ignored for metadata') if profiling @@ -741,7 +748,12 @@ module LocalMaster # if LTTNG_HOME doesn't exist FileUtils.mkdir_p(lttng_home_dir) FileUtils.mkdir_p(lttng_trace_dir_tmp) - exec('lttng-sessiond --daemonize') + # if ENV['THAPI_SESSIOND_FOREGROUND'] == '1' + # lttng_start_foreground + # else + # exec('lttng-sessiond --daemonize') + # end + lttng_start_foreground exec("lttng create #{lttng_session_uuid} -o #{lttng_trace_dir_tmp}") File.write(File.join(lttng_trace_dir_tmp, 'thapi_metadata.yaml'), { type: 'lttng' }.to_yaml) @@ -785,7 +797,66 @@ module LocalMaster exec("lttng destroy #{lttng_session_uuid}") end + # Optional workaround for installations where daemonized startup hangs. + # Keep daemon output out of Open3 pipes and wait for a working client socket. + def lttng_start_foreground + log_path = File.join(lttng_home_dir, 'sessiond.log') + pid = Process.spawn('lttng-sessiond', '--no-kernel', + in: File::NULL, out: log_path, err: [:child, :out]) + @sessiond_waiter = Process.detach(pid) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 20 + ready = false + begin + loop do + if @sessiond_waiter.join(0) + raise "lttng-sessiond exited: #{@sessiond_waiter.value}; see #{log_path}" + end + + # Bound each client probe as well as the overall startup wait. + ready = system('timeout', '--kill-after=1s', '2s', + 'lttng', '--no-sessiond', 'list', + out: File::NULL, err: File::NULL) + break if ready + + if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + raise "lttng-sessiond readiness timed out; see #{log_path}" + end + sleep 0.1 + end + ensure + lttng_stop_foreground unless ready + end + end + + def lttng_stop_foreground + waiter = @sessiond_waiter + return unless waiter + + unless waiter.join(0) + begin + Process.kill('TERM', waiter.pid) + rescue Errno::ESRCH + # The daemon exited between the check and the signal. + end + unless waiter.join(5) + begin + Process.kill('KILL', waiter.pid) + rescue Errno::ESRCH + # The daemon exited between the check and the signal. + end + waiter.join + end + end + @sessiond_waiter = nil + end + def lttng_kill_sessiond + if @sessiond_waiter + lttng_stop_foreground + FileUtils.rm_rf(lttng_home_dir) + return + end + # Need to kill the sessiond Daemon. It's safe because each job has their own # # In theory, opening the lttng-sessiond.pid file is racy. @@ -992,6 +1063,14 @@ def all_env_tracers(usr_binary) end end + if OPTIONS[:'backend-names'].include?('fi') + backends << 'fi' + h[%w[LD_PRELOAD prepend]] << File.join(LIBDIR, 'libTracerLibfabric.so') + h['LTTNG_UST_FI_OPS'] = OPTIONS[:'fi-ops'] if OPTIONS[:'fi-ops'] + h['LTTNG_UST_FI_SRX'] = 1 if OPTIONS[:'fi-srx'] + h['LTTNG_UST_FI_VERBOSE'] = 1 if LOGGER.level <= Logger::DEBUG + end + # Sample if SamplingDaemon.active? LOGGER.debug('Sampling Enabled') @@ -1085,7 +1164,7 @@ if $thapi_launch || __FILE__ == $PROGRAM_NAME # General Options parser.on('-b', '--backends BACKENDS', Array, 'Select which backends to use and their grouping level.', 'Format: backend_name[:backend_level],...', - default: ['mpi:3', 'omp:2', 'cl:1', 'ze:1', 'cuda:1', 'hip:1', 'cxi:4', 'itt:5', 'pytorch:6']) + default: ['mpi:3', 'omp:2', 'cl:1', 'ze:1', 'cuda:1', 'hip:1', 'cxi:4', 'itt:5', 'pytorch:6', 'fi:7']) parser.on('--[no-]archive', 'Enable or disable archive support.', default: false) # Analysis