From a8afb505d205b7a007b89a2b4f8620a8917e780d Mon Sep 17 00:00:00 2001 From: rsc18 Date: Thu, 15 Oct 2026 19:26:23 +0000 Subject: [PATCH 1/7] Add initial libfabric fabric entry and exit tracer --- backends/libfabric/libfabric_events.yaml | 21 +++++++++ backends/libfabric/tracer_libfabric.c | 55 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 backends/libfabric/libfabric_events.yaml create mode 100644 backends/libfabric/tracer_libfabric.c diff --git a/backends/libfabric/libfabric_events.yaml b/backends/libfabric/libfabric_events.yaml new file mode 100644 index 000000000..d011508a1 --- /dev/null +++ b/backends/libfabric/libfabric_events.yaml @@ -0,0 +1,21 @@ +lttng_ust_libfabric: + events: + - name: fi_fabric_entry + args: + - ["const void *", attr] + - ["const void *", fabric] + - ["const void *", context] + fields: + - [ctf_integer_hex, uintptr_t, attr, "(uintptr_t)attr"] + - [ctf_integer_hex, uintptr_t, fabric, "(uintptr_t)fabric"] + - [ctf_integer_hex, uintptr_t, context, "(uintptr_t)context"] + + - name: fi_fabric_exit + args: + - [int, result] + - ["const void *", fabric] + fields: + - [ctf_integer, int, result, result] + - [ctf_integer_hex, uintptr_t, fabric, "(uintptr_t)fabric"] + + diff --git a/backends/libfabric/tracer_libfabric.c b/backends/libfabric/tracer_libfabric.c new file mode 100644 index 000000000..b99601e98 --- /dev/null +++ b/backends/libfabric/tracer_libfabric.c @@ -0,0 +1,55 @@ +#define _GNU_SOURCE + +#include +#include +#include + +#include +#include + +#include "libfabric_tracepoints.h" + +typedef int (*fi_fabric_fn)( + struct fi_fabric_attr *, + struct fid_fabric **, + void * +); + +static fi_fabric_fn original_fi_fabric; +static pthread_once_t resolve_once = PTHREAD_ONCE_INIT; + +static void resolve_fi_fabric(void) +{ + original_fi_fabric = + (fi_fabric_fn)dlsym(RTLD_NEXT, "fi_fabric"); +} + +int fi_fabric(struct fi_fabric_attr *attr, + struct fid_fabric **fabric, + void *context) +{ + int incoming_errno = errno; + + pthread_once(&resolve_once, resolve_fi_fabric); + + tracepoint(lttng_ust_libfabric, fi_fabric_entry, + attr, fabric, context); + + errno = incoming_errno; + + int result = original_fi_fabric + ? original_fi_fabric(attr, fabric, context) + : -FI_ENOSYS; + + int provider_errno = errno; + + struct fid_fabric *created = NULL; + if (result == 0 && fabric != NULL) + created = *fabric; + + tracepoint(lttng_ust_libfabric, fi_fabric_exit, + result, created); + + errno = provider_errno; + return result; +} \ No newline at end of file From 102688fe4e112e64df3b783c416d210c9991d8f7 Mon Sep 17 00:00:00 2001 From: rsc18 Date: Thu, 15 Oct 2026 19:39:41 +0000 Subject: [PATCH 2/7] Import fi_trace sources as libfabric tracer baseline --- backends/libfabric/FI_TRACE_REVISION | 1 + backends/libfabric/fi_trace.h | 297 ++++++ backends/libfabric/fit_addr.c | 492 ++++++++++ backends/libfabric/fit_hook.c | 1267 ++++++++++++++++++++++++++ backends/libfabric/fit_writer.c | 531 +++++++++++ 5 files changed, 2588 insertions(+) create mode 100644 backends/libfabric/FI_TRACE_REVISION create mode 100644 backends/libfabric/fi_trace.h create mode 100644 backends/libfabric/fit_addr.c create mode 100644 backends/libfabric/fit_hook.c create mode 100644 backends/libfabric/fit_writer.c diff --git a/backends/libfabric/FI_TRACE_REVISION b/backends/libfabric/FI_TRACE_REVISION new file mode 100644 index 000000000..83eaa20ab --- /dev/null +++ b/backends/libfabric/FI_TRACE_REVISION @@ -0,0 +1 @@ +1efdf1b1159a8e07092672e19e9d369a40640154 diff --git a/backends/libfabric/fi_trace.h b/backends/libfabric/fi_trace.h new file mode 100644 index 000000000..3530e6b80 --- /dev/null +++ b/backends/libfabric/fi_trace.h @@ -0,0 +1,297 @@ +/* + * fi_trace -- LD_PRELOAD per-operation tracer for libfabric + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef FI_TRACE_H +#define FI_TRACE_H + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#define FIT_ADDR_MAX 96 /* "fi_addr_cxi://0x04656600" and friends */ +#define FIT_RAW_ADDR_MAX 64 +#define FIT_REC_MAX 1024 /* worst-case bytes of one CSV line */ + +#ifndef fit_container_of +#define fit_container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) +#endif + +/* ------------------------------------------------------------------ */ +/* configuration (read once from the environment at load time) */ +/* ------------------------------------------------------------------ */ + +struct fit_config { + int enabled; /* FI_TRACE_ENABLE */ + int resolve; /* FI_TRACE_RESOLVE */ + int trace_msg; /* FI_TRACE_OPS */ + int trace_tagged; + int trace_rma; + int trace_rx; /* FI_TRACE_RX */ + int trace_srx; /* FI_TRACE_SRX */ + int trace_eagain; /* FI_TRACE_EAGAIN */ + int trace_cq; /* FI_TRACE_CQ */ + int verbose; /* FI_TRACE_VERBOSE */ + size_t bufsz; /* FI_TRACE_BUFSZ */ + uint64_t max_records; /* FI_TRACE_MAX_RECORDS */ + char dir[4096]; /* FI_TRACE_DIR */ + char host[256]; + size_t host_len; +}; + +/* CSV fields are sanitised and measured once, where they are produced, + * so the hot path only has to memcpy them. */ +void fit_sanitize(char *s, size_t *len_out); + +extern struct fit_config fit_cfg; +extern int fit_armed; + +void fit_config_init(void); +void fit_log(const char *fmt, ...); + +/* ------------------------------------------------------------------ */ +/* peer (fi_addr_t -> NIC address string) cache */ +/* */ +/* Entries are individually allocated and never moved or freed while */ +/* the endpoint lives, so a lookup can hand back a stable pointer and */ +/* the hot path neither copies the string nor takes a lock. Only */ +/* inserts (one per peer, ever) serialise on the endpoint mutex. */ +/* ------------------------------------------------------------------ */ + +struct fit_peer { + fi_addr_t key; + size_t len; /* strlen(str), precomputed */ + char str[FIT_ADDR_MAX]; +}; + +struct fit_peer_tab { + size_t cap; /* power of two */ + struct fit_peer_tab *older; /* retired tables, freed at ep close */ + _Atomic(struct fit_peer *) slot[]; +}; + +/* ------------------------------------------------------------------ */ +/* where a NIC sits in the fabric */ +/* */ +/* A dragonfly address is the part of "which NIC talked to which NIC" */ +/* that the NIC address itself cannot tell you: two endpoints whose */ +/* addresses look equally far apart may be on the same switch or */ +/* three hops away. CXI exposes it through a domain extension, so it */ +/* is resolved once per domain and inherited by every endpoint on it. */ +/* */ +/* This is deliberately not a per-record column. It never changes */ +/* for the life of a domain, so repeating it on millions of rows */ +/* would cost bandwidth and tell you nothing new; it goes in a */ +/* sidecar file keyed by NIC address, and the merge tool joins it. */ +/* ------------------------------------------------------------------ */ + +struct fit_topo { + int valid; /* 0 if the provider would not say */ + unsigned group; /* dragonfly group */ + unsigned sw; /* switch within the group */ + unsigned port; /* port on that switch */ + char name[64]; /* domain name, e.g. "cxi2" */ + size_t name_len; +}; + +/* Ask the provider where this domain's NIC is. Leaves t->valid 0 on + * anything that is not CXI, or a CXI too old to answer. */ +void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, + const struct fi_info *info); + +/* Record one endpoint's location, once, when its address is known. */ +void fit_topo_record(const char *src, size_t src_len, unsigned int ep_id, + const struct fit_topo *t); + +/* ------------------------------------------------------------------ */ +/* per-object hook state */ +/* */ +/* Each wrapped object gets a private *copy* of the provider vtable */ +/* embedded in the hook struct. The object's vtable pointer is then */ +/* retargeted at that copy, so a wrapper can recover its own state */ +/* with container_of() -- no hash lookup on the hot path. */ +/* ------------------------------------------------------------------ */ + +struct fit_ep { + /* our vtables -- must stay first-class members for container_of */ + struct fi_ops_msg msg; + struct fi_ops_tagged tagged; + struct fi_ops_rma rma; + struct fi_ops fid_ops; + struct fi_ops_ep ep_ops; + + /* the provider's originals */ + const struct fi_ops_msg *o_msg; + const struct fi_ops_tagged *o_tagged; + const struct fi_ops_rma *o_rma; + const struct fi_ops *o_fid; + const struct fi_ops_ep *o_ep; + + struct fid_ep *ep; + struct fid_av *av; /* bound AV, used to render addresses */ + unsigned int ep_id; + + /* Points into the owning domain's hook state, which outlives every + * endpoint on it. NULL for an endpoint we met without its domain. */ + const struct fit_topo *topo; + int topo_written; + + char src[FIT_ADDR_MAX]; + size_t src_len; + int src_valid; + + pthread_mutex_t lock; /* serialises inserts + src resolution */ + _Atomic(struct fit_peer_tab *) peers; + size_t peer_cnt; /* written under lock */ + unsigned int peer_gen; /* fit_av_gen when the table was built */ +}; + +/* + * An fi_addr_t is an index into the address vector, and fi_av_remove() + * frees that index for reuse. A cached fi_addr_t -> NIC address mapping + * is therefore only valid until the next removal. Removals are rare + * (most applications never call it), so rather than track which entries + * died, bump a global counter and let each endpoint retire its whole + * table the next time it looks something up. + */ +extern _Atomic unsigned int fit_av_gen; + +void fit_ep_peers_free(struct fit_ep *h); + +struct fit_domain { + struct fi_ops_domain ops; + struct fi_ops fid_ops; + const struct fi_ops_domain *orig; + const struct fi_ops *o_fid; + + /* Resolved once at hook time and handed to every endpoint this + * domain creates. The address of this member is what endpoints + * hold, so the struct must not move. */ + struct fit_topo topo; +}; + +struct fit_fabric { + struct fi_ops_fabric ops; + struct fi_ops fid_ops; + const struct fi_ops_fabric *orig; + const struct fi_ops *o_fid; +}; + +struct fit_av { + struct fi_ops_av ops; + struct fi_ops fid_ops; + const struct fi_ops_av *orig; + const struct fi_ops *o_fid; +}; + +/* + * Completion queue. Hooking this is what turns a log of *submitted* + * operations into a log of transfers with a measurable duration: the + * posting record and the completion record carry the same op_context, so + * subtracting their timestamps gives the latency the application saw. + */ +struct fit_cq { + struct fi_ops_cq ops; + struct fi_ops fid_ops; + const struct fi_ops_cq *orig; + const struct fi_ops *o_fid; + + struct fid_cq *cq; + enum fi_cq_format format; + size_t entry_sz; /* 0 if the format is unknown */ + + /* The endpoint this CQ was bound to, so a completion can be + * attributed to a local NIC address. Left NULL if more than one + * endpoint binds the same CQ, because then it would be a guess. */ + struct fit_ep *ep; + int ep_ambiguous; + unsigned int cq_id; +}; + +/* ------------------------------------------------------------------ */ +/* writer */ +/* ------------------------------------------------------------------ */ + +void fit_writer_init(void); +void fit_writer_flush_all(void); + +/* Emit one CSV record. dir is 'T' (local initiates) or 'R' (local posts + * a receive). peer is the fi_addr_t supplied by the caller; it is + * rendered through the endpoint's AV when possible. context is the + * application's op_context, recorded so the completion can be matched to + * this record; it is dropped when the provider refused the operation, + * since nothing became outstanding. */ +void fit_record(struct fit_ep *h, const char *op, char dir, fi_addr_t peer, + size_t bytes, size_t iov_count, ssize_t ret, + const void *context); + +/* Emit one completion record. now is the time the batch was reaped. */ +void fit_record_cq(struct fit_cq *c, const char *op, fi_addr_t peer, + size_t bytes, ssize_t ret, const void *context, + uint64_t now); + +uint64_t fit_now_us(void); + +/* ------------------------------------------------------------------ */ +/* address rendering */ +/* ------------------------------------------------------------------ */ + +void fit_ep_resolve_src(struct fit_ep *h); + +/* Returns a sanitised, NUL-terminated peer string and its length. The + * result may point into the endpoint's peer cache (stable until the + * endpoint closes) or into the caller's scratch buffer. */ +const char *fit_ep_peer_str(struct fit_ep *h, fi_addr_t peer, char *scratch, + size_t scratch_len, size_t *len_out); + +/* ------------------------------------------------------------------ */ +/* installers */ +/* ------------------------------------------------------------------ */ + +void fit_hook_fabric(struct fid_fabric *fabric); +void fit_hook_domain(struct fid_domain *domain, const struct fi_info *info); +/* topo may be NULL; it is borrowed from the owning domain and outlives + * the endpoint. */ +void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo); +void fit_hook_av(struct fid_av *av); +void fit_hook_cq(struct fid_cq *cq, enum fi_cq_format format); + +static inline size_t fit_iov_len(const struct iovec *iov, size_t count) +{ + size_t i, total = 0; + + if (!iov) + return 0; + for (i = 0; i < count; i++) + total += iov[i].iov_len; + return total; +} + +static inline size_t fit_rma_iov_len(const struct fi_rma_iov *iov, size_t count) +{ + size_t i, total = 0; + + if (!iov) + return 0; + for (i = 0; i < count; i++) + total += iov[i].len; + return total; +} + +#endif /* FI_TRACE_H */ diff --git a/backends/libfabric/fit_addr.c b/backends/libfabric/fit_addr.c new file mode 100644 index 000000000..8a7563617 --- /dev/null +++ b/backends/libfabric/fit_addr.c @@ -0,0 +1,492 @@ +/* + * fi_trace -- NIC address rendering + * + * The local address comes from fi_getname() on the endpoint. A remote + * peer is identified in the API only by an opaque fi_addr_t, so it is + * reverse-mapped through the endpoint's address vector with + * fi_av_lookup() and then formatted with fi_av_straddr() -- which is + * what produces the "fi_addr_cxi://0x04656600" form. + * + * Both are cached: the local address once per endpoint, remote ones in + * a small open-addressing table keyed by fi_addr_t. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "fi_trace.h" + +#include +#include +#include +#include +#include +#include + +#define FIT_PEER_CAP_INIT 64 + +static void raw_to_hex(const void *addr, size_t len, char *out, size_t out_len) +{ + static const char hex[] = "0123456789abcdef"; + const unsigned char *b = addr; + size_t i, n = 0; + + if (out_len < 4) { + if (out_len) + out[0] = '\0'; + return; + } + out[n++] = '0'; + out[n++] = 'x'; + for (i = 0; i < len && n + 2 < out_len; i++) { + out[n++] = hex[b[i] >> 4]; + out[n++] = hex[b[i] & 0xf]; + } + out[n] = '\0'; +} + +/* Format a raw provider address, preferring the provider's own + * stringifier so the output matches fi_info / existing telemetry. */ +static void format_raw(struct fid_av *av, const void *raw, size_t raw_len, + char *out, size_t out_len) +{ + if (av && av->ops && av->ops->straddr) { + size_t len = out_len; + const char *s = fi_av_straddr(av, raw, out, &len); + + if (s) { + out[out_len - 1] = '\0'; + return; + } + } + raw_to_hex(raw, raw_len, out, out_len); +} + +void fit_ep_resolve_src(struct fit_ep *h) +{ + char raw[FIT_RAW_ADDR_MAX]; + size_t raw_len = sizeof(raw); + int ret; + + pthread_mutex_lock(&h->lock); + if (h->src_valid) + goto out; + + /* Mark resolved up front: if the provider cannot answer we do not + * want to retry fi_getname() on every single operation. */ + h->src_valid = 1; + snprintf(h->src, sizeof(h->src), "unknown"); + + if (!fit_cfg.resolve || !h->ep || !h->ep->cm || !h->ep->cm->getname) + goto out; + + ret = fi_getname(&h->ep->fid, raw, &raw_len); + if (ret) { + /* Not enabled yet -- allow a later retry. */ + if (ret == -FI_EOPBADSTATE || ret == -FI_ENOENT) + h->src_valid = 0; + goto out; + } + if (raw_len > sizeof(raw)) + goto out; + + format_raw(h->av, raw, raw_len, h->src, sizeof(h->src)); +out: + fit_sanitize(h->src, &h->src_len); + + /* The two halves of "where is this endpoint" become available at + * different moments: the dragonfly coordinates at domain creation, + * the NIC address only after FI_ENABLE. This is the first point + * where both are known, so it is where the pair gets written -- + * once per endpoint, never on the hot path. */ + if (h->src_valid && !h->topo_written) { + h->topo_written = 1; + fit_topo_record(h->src, h->src_len, h->ep_id, h->topo); + } + + pthread_mutex_unlock(&h->lock); +} + +/* ------------------------------------------------------------------ */ +/* dragonfly location */ +/* */ +/* CXI will say which group, switch and port a NIC is wired to, but */ +/* not through the core API -- it is a provider extension, reached by */ +/* opening a named ops structure on the domain. The name carries a */ +/* version, and ->topology() does not exist in the oldest ones, so try */ +/* newest first and stop at the first that answers. */ +/* ------------------------------------------------------------------ */ + +#ifdef HAVE_FI_CXI_TOPOLOGY +#include +#include +#endif + +void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, + const struct fi_info *info) +{ + memset(t, 0, sizeof(*t)); + + /* The NIC's name ("cxi3") is worth keeping even when the provider + * will not give coordinates: it is how the node's own tooling and + * the counter telemetry refer to the device. */ + if (info && info->domain_attr && info->domain_attr->name) + snprintf(t->name, sizeof(t->name), "%s", + info->domain_attr->name); + else + snprintf(t->name, sizeof(t->name), "unknown"); + fit_sanitize(t->name, &t->name_len); + +#ifdef HAVE_FI_CXI_TOPOLOGY + { + static const char *const versions[] = { + "dom_ops_v6", "dom_ops_v5", "dom_ops_v4", NULL + }; + int i; + + if (!domain || !domain->fid.ops || !domain->fid.ops->ops_open) { + fit_log("topology %s: domain has no ops_open", t->name); + return; + } + + /* Keep the table local to this domain; never reuse another NIC's + * extension pointer. Try the same versions as bidir_intra.cpp. */ + for (i = 0; versions[i]; i++) { + struct fi_cxi_dom_ops *ops = NULL; + unsigned grp = 0, sw = 0, port = 0; + int ret = fi_open_ops(&domain->fid, versions[i], 0, + (void **)&ops, NULL); + + if (ret || !ops || !ops->topology) { + fit_log("topology %s: %s unavailable (ret=%d)", + t->name, versions[i], ret); + continue; + } + + ret = ops->topology(&domain->fid, &grp, &sw, &port); + if (ret) { + fit_log("topology %s: %s query failed (ret=%d)", + t->name, versions[i], ret); + continue; + } + + t->group = grp; + t->sw = sw; + t->port = port; + t->valid = 1; + fit_log("topology %s: %s group=%u switch=%u port=%u", + t->name, versions[i], grp, sw, port); + return; + } + fit_log("topology %s: no supported query succeeded", t->name); + } +#else + (void)domain; + fit_log("topology %s: disabled at build time; run make check-topology " + "with the target libfabric headers, then rebuild", t->name); +#endif +} + +/* ------------------------------------------------------------------ */ +/* the topology sidecar */ +/* */ +/* One row per endpoint, in its own small file next to the traces. */ +/* The alternative -- four more columns on every record -- would repeat*/ +/* a constant tens of millions of times to say something that changes */ +/* never. The merge tool reads the union of these files and joins on */ +/* the NIC address, which is enough to place both ends of every flow: */ +/* a peer this rank only ever sent to is some other rank's local */ +/* endpoint, and that rank wrote the row. */ +/* */ +/* Writing is rare -- once per endpoint for the life of the process -- */ +/* so it takes a lock and writes straight through, rather than paying */ +/* for the buffered per-thread machinery the hot path needs. */ +/* ------------------------------------------------------------------ */ + +static pthread_mutex_t fit_topo_lock = PTHREAD_MUTEX_INITIALIZER; +static int fit_topo_fd = -1; +static int fit_topo_bad; + +static const char fit_topo_header[] = + "host,pid,ep_id,src_addr,nic,group,switch,port\n"; + +static void topo_open_locked(void) +{ + /* dir is 4096, host up to 255, plus the fixed parts and the pid. */ + char path[4480]; + + if (fit_topo_fd >= 0 || fit_topo_bad) + return; + + /* Per process, not per thread: endpoints are a process-wide thing + * and there are only a handful of them. */ + snprintf(path, sizeof(path), "%s/fitrace-topo.%s.%d.csv", + fit_cfg.dir, fit_cfg.host, (int)getpid()); + + fit_topo_fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); + if (fit_topo_fd < 0) { + fit_log("cannot open %s: %s", path, strerror(errno)); + fit_topo_bad = 1; + return; + } + if (write(fit_topo_fd, fit_topo_header, + sizeof(fit_topo_header) - 1) < 0) + fit_topo_bad = 1; +} + +void fit_topo_record(const char *src, size_t src_len, unsigned int ep_id, + const struct fit_topo *t) +{ + char line[FIT_REC_MAX]; + int n; + + if (!fit_armed || !src || !src_len) + return; + + pthread_mutex_lock(&fit_topo_lock); + topo_open_locked(); + if (fit_topo_bad || fit_topo_fd < 0) { + pthread_mutex_unlock(&fit_topo_lock); + return; + } + + /* An endpoint whose provider would not place it still earns a row: + * knowing that this address exists on this host, under this NIC + * name, is most of the value, and an empty coordinate is honest + * about the rest. */ + if (t && t->valid) + n = snprintf(line, sizeof(line), "%s,%d,%u,%s,%s,%u,%u,%u\n", + fit_cfg.host, (int)getpid(), ep_id, src, + t->name, t->group, t->sw, t->port); + else + n = snprintf(line, sizeof(line), "%s,%d,%u,%s,%s,,,\n", + fit_cfg.host, (int)getpid(), ep_id, src, + t ? t->name : ""); + + if (n > 0 && (size_t)n < sizeof(line)) { + if (write(fit_topo_fd, line, (size_t)n) < 0) + fit_topo_bad = 1; + } + pthread_mutex_unlock(&fit_topo_lock); +} + +/* ------------------------------------------------------------------ */ +/* peer cache */ +/* ------------------------------------------------------------------ */ + +static inline size_t peer_hash(fi_addr_t key, size_t cap) +{ + uint64_t h = (uint64_t)key; + + h ^= h >> 33; + h *= 0xff51afd7ed558ccdull; + h ^= h >> 33; + return (size_t)(h & (cap - 1)); +} + +/* Lock-free lookup. Entry pointers are stable for the life of the + * endpoint, so the caller may keep the returned string. */ +static const char *peer_find(struct fit_ep *h, fi_addr_t key, size_t *len_out) +{ + struct fit_peer_tab *tab; + size_t i, slot, cap; + + tab = atomic_load_explicit(&h->peers, memory_order_acquire); + if (!tab) + return NULL; + + cap = tab->cap; + slot = peer_hash(key, cap); + for (i = 0; i < cap; i++) { + struct fit_peer *e = atomic_load_explicit( + &tab->slot[(slot + i) & (cap - 1)], + memory_order_acquire); + + if (!e) + return NULL; + if (e->key == key) { + *len_out = e->len; + return e->str; + } + } + return NULL; +} + +/* Caller holds h->lock. */ +static void tab_place(struct fit_peer_tab *tab, struct fit_peer *e) +{ + size_t slot = peer_hash(e->key, tab->cap); + size_t i; + + for (i = 0; i < tab->cap; i++) { + _Atomic(struct fit_peer *) *s = + &tab->slot[(slot + i) & (tab->cap - 1)]; + + if (!atomic_load_explicit(s, memory_order_relaxed)) { + atomic_store_explicit(s, e, memory_order_release); + return; + } + } +} + +/* Caller holds h->lock. The old table is retired rather than freed so + * that a concurrent reader still walking it stays safe. */ +static int peer_grow(struct fit_ep *h) +{ + struct fit_peer_tab *old = + atomic_load_explicit(&h->peers, memory_order_relaxed); + size_t new_cap = old ? old->cap * 2 : FIT_PEER_CAP_INIT; + struct fit_peer_tab *fresh; + size_t i; + + fresh = calloc(1, sizeof(*fresh) + new_cap * sizeof(fresh->slot[0])); + if (!fresh) + return -1; + + fresh->cap = new_cap; + fresh->older = old; + + if (old) { + for (i = 0; i < old->cap; i++) { + struct fit_peer *e = atomic_load_explicit( + &old->slot[i], memory_order_relaxed); + + if (e) + tab_place(fresh, e); + } + } + + atomic_store_explicit(&h->peers, fresh, memory_order_release); + return 0; +} + +/* Caller holds h->lock. Returns the stable string, or NULL on OOM. */ +static const char *peer_insert(struct fit_ep *h, fi_addr_t key, + const char *str, size_t *len_out) +{ + struct fit_peer_tab *tab = + atomic_load_explicit(&h->peers, memory_order_relaxed); + struct fit_peer *e; + + /* Keep the load factor at or below 1/2 so probing stays short. */ + if (!tab || (h->peer_cnt + 1) * 2 > tab->cap) { + if (peer_grow(h)) + return NULL; + tab = atomic_load_explicit(&h->peers, memory_order_relaxed); + } + + e = calloc(1, sizeof(*e)); + if (!e) + return NULL; + + e->key = key; + snprintf(e->str, sizeof(e->str), "%s", str); + fit_sanitize(e->str, &e->len); + + tab_place(tab, e); + h->peer_cnt++; + *len_out = e->len; + return e->str; +} + +void fit_ep_peers_free(struct fit_ep *h) +{ + struct fit_peer_tab *tab = + atomic_load_explicit(&h->peers, memory_order_relaxed); + size_t i; + + if (tab) { + for (i = 0; i < tab->cap; i++) + free(atomic_load_explicit(&tab->slot[i], + memory_order_relaxed)); + } + while (tab) { + struct fit_peer_tab *older = tab->older; + + free(tab); + tab = older; + } + atomic_store_explicit(&h->peers, NULL, memory_order_relaxed); +} + +/* + * fi_av_remove() frees an fi_addr_t for reuse, so every cached mapping + * becomes suspect the moment one happens. Retire the whole table rather + * than try to work out which entries died: removals are rare, and the + * retired table stays reachable through ->older so a concurrent reader + * walking it is still safe. Caller holds h->lock. + */ +static void peer_retire_stale(struct fit_ep *h, unsigned int gen) +{ + struct fit_peer_tab *old, *fresh; + + if (h->peer_gen == gen) + return; + + old = atomic_load_explicit(&h->peers, memory_order_relaxed); + if (old) { + fresh = calloc(1, sizeof(*fresh) + + FIT_PEER_CAP_INIT * sizeof(fresh->slot[0])); + if (!fresh) + return; /* keep the stale table over none */ + fresh->cap = FIT_PEER_CAP_INIT; + fresh->older = old; + atomic_store_explicit(&h->peers, fresh, memory_order_release); + fit_log("address vector changed; dropped %u cached peers", + (unsigned)h->peer_cnt); + h->peer_cnt = 0; + } + h->peer_gen = gen; +} + +const char *fit_ep_peer_str(struct fit_ep *h, fi_addr_t peer, char *scratch, + size_t scratch_len, size_t *len_out) +{ + char raw[FIT_RAW_ADDR_MAX]; + size_t raw_len = sizeof(raw); + const char *cached; + unsigned int gen; + int resolvable; + + if (peer == FI_ADDR_UNSPEC) { + *len_out = sizeof("fi_addr_unspec") - 1; + return "fi_addr_unspec"; + } + if (peer == FI_ADDR_NOTAVAIL) { + *len_out = sizeof("fi_addr_notavail") - 1; + return "fi_addr_notavail"; + } + + /* The common case: already seen this peer, and no fi_av_remove() + * has invalidated the mapping since. Both resolved and unresolved + * renderings are cached, so neither path formats a string more than + * once per peer. */ + gen = atomic_load_explicit(&fit_av_gen, memory_order_acquire); + if (gen == h->peer_gen) { + cached = peer_find(h, peer, len_out); + if (cached) + return cached; + } + + resolvable = fit_cfg.resolve && h->av && h->av->ops && + h->av->ops->lookup; + + if (resolvable && !fi_av_lookup(h->av, peer, raw, &raw_len) && + raw_len <= sizeof(raw)) + format_raw(h->av, raw, raw_len, scratch, scratch_len); + else + snprintf(scratch, scratch_len, "fi_addr://0x%016llx", + (unsigned long long)peer); + + pthread_mutex_lock(&h->lock); + peer_retire_stale(h, gen); + cached = peer_find(h, peer, len_out); /* lost a race? */ + if (!cached) + cached = peer_insert(h, peer, scratch, len_out); + pthread_mutex_unlock(&h->lock); + + if (cached) + return cached; + + fit_sanitize(scratch, len_out); + return scratch; +} diff --git a/backends/libfabric/fit_hook.c b/backends/libfabric/fit_hook.c new file mode 100644 index 000000000..28aef3b7c --- /dev/null +++ b/backends/libfabric/fit_hook.c @@ -0,0 +1,1267 @@ +/* + * fi_trace -- LD_PRELOAD interposition and vtable wrapping + * + * Why this is not a plain dlsym() shim + * ------------------------------------ + * fi_send(), fi_recv(), fi_tsend(), fi_write() and friends are *static + * inline* functions in that dispatch straight + * through the provider vtables (ep->msg->send, ep->tagged->recv, ...). + * They are not exported symbols, so dlsym(RTLD_NEXT, "fi_send") cannot + * find them and an interposed definition would never be called. + * + * The only data-path-reaching entry points libfabric actually exports + * are fi_fabric() and fi_fabric2(). So we interpose those, and from + * the fabric we walk down the object tree -- fabric -> domain -> + * endpoint -- retargeting each object's vtable pointer at a private + * copy whose data-path slots are our wrappers. The provider's own + * objects are never replaced, only the pointers the application + * dispatches through, so provider-internal container_of() stays valid. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "fi_trace.h" + +#include +#include +#include +#include +#include + +static atomic_uint fit_ep_counter; + +static void fit_cq_attach_ep(struct fid *bfid, struct fit_ep *h); + +/* ------------------------------------------------------------------ */ +/* recovering hook state from a vtable pointer */ +/* ------------------------------------------------------------------ */ + +#define EP_FROM_MSG(ep) fit_container_of((ep)->msg, struct fit_ep, msg) +#define EP_FROM_TAGGED(ep) fit_container_of((ep)->tagged, struct fit_ep, tagged) +#define EP_FROM_RMA(ep) fit_container_of((ep)->rma, struct fit_ep, rma) +#define EP_FROM_FID(fid) fit_container_of((fid)->ops, struct fit_ep, fid_ops) +#define EP_FROM_EPOPS(ep) fit_container_of((ep)->ops, struct fit_ep, ep_ops) + +#define CALL_OR_ENOSYS(fn, args) ((fn) ? (fn) args : -FI_ENOSYS) + +/* ================================================================== */ +/* message operations */ +/* ================================================================== */ + +static ssize_t fit_msg_send(struct fid_ep *ep, const void *buf, size_t len, + void *desc, fi_addr_t dest_addr, void *context) +{ + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_msg->send, + (ep, buf, len, desc, dest_addr, context)); + + fit_record(h, "fi_send", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_msg_sendv(struct fid_ep *ep, const struct iovec *iov, + void **desc, size_t count, fi_addr_t dest_addr, + void *context) +{ + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_msg->sendv, + (ep, iov, desc, count, dest_addr, context)); + + fit_record(h, "fi_sendv", 'T', dest_addr, fit_iov_len(iov, count), + count, ret, context); + return ret; +} + +static ssize_t fit_msg_sendmsg(struct fid_ep *ep, const struct fi_msg *msg, + uint64_t flags) +{ + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_msg->sendmsg, (ep, msg, flags)); + + fit_record(h, "fi_sendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, + msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); + return ret; +} + +static ssize_t fit_msg_inject(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr) +{ + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_msg->inject, (ep, buf, len, dest_addr)); + + fit_record(h, "fi_inject", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_msg_senddata(struct fid_ep *ep, const void *buf, size_t len, + void *desc, uint64_t data, fi_addr_t dest_addr, + void *context) +{ + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_msg->senddata, + (ep, buf, len, desc, data, dest_addr, + context)); + + fit_record(h, "fi_senddata", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_msg_injectdata(struct fid_ep *ep, const void *buf, + size_t len, uint64_t data, + fi_addr_t dest_addr) +{ + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_msg->injectdata, + (ep, buf, len, data, dest_addr)); + + fit_record(h, "fi_injectdata", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_msg_recv(struct fid_ep *ep, void *buf, size_t len, + void *desc, fi_addr_t src_addr, void *context) +{ + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_msg->recv, + (ep, buf, len, desc, src_addr, context)); + + fit_record(h, "fi_recv", 'R', src_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_msg_recvv(struct fid_ep *ep, const struct iovec *iov, + void **desc, size_t count, fi_addr_t src_addr, + void *context) +{ + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_msg->recvv, + (ep, iov, desc, count, src_addr, context)); + + fit_record(h, "fi_recvv", 'R', src_addr, fit_iov_len(iov, count), + count, ret, context); + return ret; +} + +static ssize_t fit_msg_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, + uint64_t flags) +{ + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_msg->recvmsg, (ep, msg, flags)); + + fit_record(h, "fi_recvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, + msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); + return ret; +} + +/* ================================================================== */ +/* tagged operations */ +/* ================================================================== */ + +static ssize_t fit_tag_send(struct fid_ep *ep, const void *buf, size_t len, + void *desc, fi_addr_t dest_addr, uint64_t tag, + void *context) +{ + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->send, + (ep, buf, len, desc, dest_addr, tag, + context)); + + fit_record(h, "fi_tsend", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_tag_sendv(struct fid_ep *ep, const struct iovec *iov, + void **desc, size_t count, fi_addr_t dest_addr, + uint64_t tag, void *context) +{ + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->sendv, + (ep, iov, desc, count, dest_addr, tag, + context)); + + fit_record(h, "fi_tsendv", 'T', dest_addr, fit_iov_len(iov, count), + count, ret, context); + return ret; +} + +static ssize_t fit_tag_sendmsg(struct fid_ep *ep, + const struct fi_msg_tagged *msg, uint64_t flags) +{ + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->sendmsg, (ep, msg, flags)); + + fit_record(h, "fi_tsendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, + msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); + return ret; +} + +static ssize_t fit_tag_inject(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t tag) +{ + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->inject, + (ep, buf, len, dest_addr, tag)); + + fit_record(h, "fi_tinject", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_tag_senddata(struct fid_ep *ep, const void *buf, size_t len, + void *desc, uint64_t data, fi_addr_t dest_addr, + uint64_t tag, void *context) +{ + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->senddata, + (ep, buf, len, desc, data, dest_addr, tag, + context)); + + fit_record(h, "fi_tsenddata", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_tag_injectdata(struct fid_ep *ep, const void *buf, + size_t len, uint64_t data, + fi_addr_t dest_addr, uint64_t tag) +{ + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->injectdata, + (ep, buf, len, data, dest_addr, tag)); + + fit_record(h, "fi_tinjectdata", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_tag_recv(struct fid_ep *ep, void *buf, size_t len, + void *desc, fi_addr_t src_addr, uint64_t tag, + uint64_t ignore, void *context) +{ + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->recv, + (ep, buf, len, desc, src_addr, tag, ignore, + context)); + + fit_record(h, "fi_trecv", 'R', src_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_tag_recvv(struct fid_ep *ep, const struct iovec *iov, + void **desc, size_t count, fi_addr_t src_addr, + uint64_t tag, uint64_t ignore, void *context) +{ + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->recvv, + (ep, iov, desc, count, src_addr, tag, + ignore, context)); + + fit_record(h, "fi_trecvv", 'R', src_addr, fit_iov_len(iov, count), + count, ret, context); + return ret; +} + +static ssize_t fit_tag_recvmsg(struct fid_ep *ep, + const struct fi_msg_tagged *msg, uint64_t flags) +{ + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->recvmsg, (ep, msg, flags)); + + fit_record(h, "fi_trecvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, + msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); + return ret; +} + +/* ================================================================== */ +/* RMA operations */ +/* ================================================================== */ + +static ssize_t fit_rma_write(struct fid_ep *ep, const void *buf, size_t len, + void *desc, fi_addr_t dest_addr, uint64_t addr, + uint64_t key, void *context) +{ + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_rma->write, + (ep, buf, len, desc, dest_addr, addr, key, + context)); + + fit_record(h, "fi_write", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_rma_writev(struct fid_ep *ep, const struct iovec *iov, + void **desc, size_t count, fi_addr_t dest_addr, + uint64_t addr, uint64_t key, void *context) +{ + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_rma->writev, + (ep, iov, desc, count, dest_addr, addr, + key, context)); + + fit_record(h, "fi_writev", 'T', dest_addr, fit_iov_len(iov, count), + count, ret, context); + return ret; +} + +static ssize_t fit_rma_writemsg(struct fid_ep *ep, + const struct fi_msg_rma *msg, uint64_t flags) +{ + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_rma->writemsg, (ep, msg, flags)); + + fit_record(h, "fi_writemsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, + msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); + return ret; +} + +static ssize_t fit_rma_inject(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t addr, uint64_t key) +{ + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_rma->inject, + (ep, buf, len, dest_addr, addr, key)); + + fit_record(h, "fi_inject_write", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_rma_writedata(struct fid_ep *ep, const void *buf, + size_t len, void *desc, uint64_t data, + fi_addr_t dest_addr, uint64_t addr, + uint64_t key, void *context) +{ + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_rma->writedata, + (ep, buf, len, desc, data, dest_addr, addr, + key, context)); + + fit_record(h, "fi_writedata", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_rma_injectdata(struct fid_ep *ep, const void *buf, + size_t len, uint64_t data, + fi_addr_t dest_addr, uint64_t addr, + uint64_t key) +{ + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_rma->injectdata, + (ep, buf, len, data, dest_addr, addr, key)); + + fit_record(h, "fi_inject_writedata", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_rma_read(struct fid_ep *ep, void *buf, size_t len, + void *desc, fi_addr_t src_addr, uint64_t addr, + uint64_t key, void *context) +{ + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_rma->read, + (ep, buf, len, desc, src_addr, addr, key, + context)); + + fit_record(h, "fi_read", 'T', src_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_rma_readv(struct fid_ep *ep, const struct iovec *iov, + void **desc, size_t count, fi_addr_t src_addr, + uint64_t addr, uint64_t key, void *context) +{ + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_rma->readv, + (ep, iov, desc, count, src_addr, addr, key, + context)); + + fit_record(h, "fi_readv", 'T', src_addr, fit_iov_len(iov, count), + count, ret, context); + return ret; +} + +static ssize_t fit_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg, + uint64_t flags) +{ + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret = CALL_OR_ENOSYS(h->o_rma->readmsg, (ep, msg, flags)); + + fit_record(h, "fi_readmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, + msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); + return ret; +} + +/* ================================================================== */ +/* endpoint lifecycle */ +/* ================================================================== */ + +/* + * Install our copies of the endpoint's data-path vtables. + * + * Idempotent, and deliberately re-run after every bind and after + * FI_ENABLE. An endpoint is not finished at the moment fi_endpoint() + * returns: a provider may hand back placeholder ops and swap in the real + * ones later, once it knows which capabilities were negotiated and the + * transmit and receive contexts exist. Patching only at creation then + * loses the race -- the provider overwrites our pointers, every + * subsequent operation goes straight to the provider, and nothing + * reports an error. The symptom is a trace with a full set of "hooked" + * lines and not one data record, which is precisely what CXI produced. + * + * Re-arming is cheap (three pointer compares when nothing changed) and + * happens only on the control path, never per operation. + */ +static void fit_ep_arm_ops(struct fit_ep *h, const char *when) +{ + struct fid_ep *ep = h->ep; + + if (fit_cfg.trace_msg && ep->msg && ep->msg != &h->msg) { + if (h->o_msg) + fit_log("endpoint %u: provider replaced msg ops at %s;" + " re-arming", h->ep_id, when); + h->o_msg = ep->msg; + h->msg = *ep->msg; + h->msg.send = fit_msg_send; + h->msg.sendv = fit_msg_sendv; + h->msg.sendmsg = fit_msg_sendmsg; + h->msg.inject = fit_msg_inject; + h->msg.senddata = fit_msg_senddata; + h->msg.injectdata = fit_msg_injectdata; + h->msg.recv = fit_msg_recv; + h->msg.recvv = fit_msg_recvv; + h->msg.recvmsg = fit_msg_recvmsg; + ep->msg = &h->msg; + } + + if (fit_cfg.trace_tagged && ep->tagged && ep->tagged != &h->tagged) { + if (h->o_tagged) + fit_log("endpoint %u: provider replaced tagged ops at" + " %s; re-arming", h->ep_id, when); + h->o_tagged = ep->tagged; + h->tagged = *ep->tagged; + h->tagged.send = fit_tag_send; + h->tagged.sendv = fit_tag_sendv; + h->tagged.sendmsg = fit_tag_sendmsg; + h->tagged.inject = fit_tag_inject; + h->tagged.senddata = fit_tag_senddata; + h->tagged.injectdata = fit_tag_injectdata; + h->tagged.recv = fit_tag_recv; + h->tagged.recvv = fit_tag_recvv; + h->tagged.recvmsg = fit_tag_recvmsg; + ep->tagged = &h->tagged; + } + + if (fit_cfg.trace_rma && ep->rma && ep->rma != &h->rma) { + if (h->o_rma) + fit_log("endpoint %u: provider replaced rma ops at %s;" + " re-arming", h->ep_id, when); + h->o_rma = ep->rma; + h->rma = *ep->rma; + h->rma.write = fit_rma_write; + h->rma.writev = fit_rma_writev; + h->rma.writemsg = fit_rma_writemsg; + h->rma.inject = fit_rma_inject; + h->rma.writedata = fit_rma_writedata; + h->rma.injectdata = fit_rma_injectdata; + h->rma.read = fit_rma_read; + h->rma.readv = fit_rma_readv; + h->rma.readmsg = fit_rma_readmsg; + ep->rma = &h->rma; + } +} + +static int fit_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags) +{ + struct fit_ep *h = EP_FROM_FID(fid); + int ret = h->o_fid->bind ? h->o_fid->bind(fid, bfid, flags) : -FI_ENOSYS; + + /* Binding a CQ or a counter is exactly the sort of event that tells + * a provider which data path this endpoint will use. */ + if (!ret) + fit_ep_arm_ops(h, "bind"); + + if (ret || !bfid) + return ret; + + /* Remember the AV so remote fi_addr_t values can be reverse-mapped. */ + if (bfid->fclass == FI_CLASS_AV) + h->av = (struct fid_av *)bfid; + + /* And note which endpoint owns the CQ, so its completions can be + * attributed to a local NIC address. */ + if (bfid->fclass == FI_CLASS_CQ) + fit_cq_attach_ep(bfid, h); + + return ret; +} + +static int fit_ep_control(struct fid *fid, int command, void *arg) +{ + struct fit_ep *h = EP_FROM_FID(fid); + int ret = h->o_fid->control ? h->o_fid->control(fid, command, arg) : + -FI_ENOSYS; + + /* The local address only becomes available once enabled -- and so, + * for some providers, does the real data path. */ + if (!ret && command == FI_ENABLE) { + fit_ep_arm_ops(h, "enable"); + fit_ep_resolve_src(h); + } + + return ret; +} + +static int fit_ep_close(struct fid *fid) +{ + struct fit_ep *h = EP_FROM_FID(fid); + int ret; + + fit_writer_flush_all(); + + ret = h->o_fid->close ? h->o_fid->close(fid) : -FI_ENOSYS; + if (ret) + return ret; + + /* The provider object is gone; our side state can go too. */ + fit_ep_peers_free(h); + pthread_mutex_destroy(&h->lock); + free(h); + return 0; +} + +static int fit_ep_tx_ctx(struct fid_ep *sep, int index, + struct fi_tx_attr *attr, struct fid_ep **tx_ep, + void *context) +{ + struct fit_ep *h = EP_FROM_EPOPS(sep); + int ret = h->o_ep->tx_ctx ? + h->o_ep->tx_ctx(sep, index, attr, tx_ep, context) : -FI_ENOSYS; + + if (!ret && tx_ep && *tx_ep) + fit_hook_ep(*tx_ep, h->topo); + return ret; +} + +static int fit_ep_rx_ctx(struct fid_ep *sep, int index, + struct fi_rx_attr *attr, struct fid_ep **rx_ep, + void *context) +{ + struct fit_ep *h = EP_FROM_EPOPS(sep); + int ret = h->o_ep->rx_ctx ? + h->o_ep->rx_ctx(sep, index, attr, rx_ep, context) : -FI_ENOSYS; + + if (!ret && rx_ep && *rx_ep) + fit_hook_ep(*rx_ep, h->topo); + return ret; +} + +void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo) +{ + struct fit_ep *h; + + if (!ep) + return; + + h = calloc(1, sizeof(*h)); + if (!h) { + fit_log("out of memory hooking endpoint"); + return; + } + + pthread_mutex_init(&h->lock, NULL); + h->ep = ep; + h->ep_id = atomic_fetch_add(&fit_ep_counter, 1); + /* Borrowed from the domain, which outlives every endpoint on it. */ + h->topo = topo; + snprintf(h->src, sizeof(h->src), "unknown"); + fit_sanitize(h->src, &h->src_len); + + fit_ep_arm_ops(h, "create"); + + if (ep->fid.ops) { + h->o_fid = ep->fid.ops; + h->fid_ops = *ep->fid.ops; + h->fid_ops.bind = fit_ep_bind; + h->fid_ops.control = fit_ep_control; + h->fid_ops.close = fit_ep_close; + ep->fid.ops = &h->fid_ops; + } + + if (ep->ops) { + h->o_ep = ep->ops; + h->ep_ops = *ep->ops; + h->ep_ops.tx_ctx = fit_ep_tx_ctx; + h->ep_ops.rx_ctx = fit_ep_rx_ctx; + ep->ops = &h->ep_ops; + } + + fit_log("hooked endpoint %u (%p)", h->ep_id, (void *)ep); +} + +/* ================================================================== */ +/* address vector */ +/* */ +/* Wrapped for one reason only: fi_av_remove() frees an fi_addr_t for */ +/* reuse, which silently invalidates every cached fi_addr_t -> NIC */ +/* address mapping. Bump a generation counter so the caches notice. */ +/* ================================================================== */ + +#define AV_FROM_OPS(av) fit_container_of((av)->ops, struct fit_av, ops) + +static int fit_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count, + uint64_t flags) +{ + struct fit_av *a = AV_FROM_OPS(av); + int ret = a->orig->remove ? + a->orig->remove(av, fi_addr, count, flags) : -FI_ENOSYS; + + if (!ret) + atomic_fetch_add_explicit(&fit_av_gen, 1, memory_order_release); + return ret; +} + +static int fit_av_close(struct fid *fid) +{ + struct fit_av *a = fit_container_of(fid->ops, struct fit_av, fid_ops); + int ret = a->o_fid->close ? a->o_fid->close(fid) : -FI_ENOSYS; + + if (!ret) { + atomic_fetch_add_explicit(&fit_av_gen, 1, memory_order_release); + free(a); + } + return ret; +} + +void fit_hook_av(struct fid_av *av) +{ + struct fit_av *a; + + if (!av || !av->ops || av->ops->remove == fit_av_remove) + return; + + a = calloc(1, sizeof(*a)); + if (!a) + return; + + if (av->fid.ops) { + a->o_fid = av->fid.ops; + a->fid_ops = *av->fid.ops; + a->fid_ops.close = fit_av_close; + av->fid.ops = &a->fid_ops; + } + + a->orig = av->ops; + a->ops = *av->ops; + a->ops.remove = fit_av_remove; + av->ops = &a->ops; + + fit_log("hooked address vector %p", (void *)av); +} + +/* ================================================================== */ +/* completion queue */ +/* */ +/* This is where an operation acquires a duration. The posting record */ +/* says when the application handed the transfer to the provider; the */ +/* completion record says when the application learned it was done. */ +/* They are joined on op_context. */ +/* */ +/* The completion also carries two things the posting record cannot: */ +/* the length the provider actually transferred (a receive posts a */ +/* buffer, not a message size), and, via fi_cq_readfrom(), the address */ +/* the data really came from. */ +/* ================================================================== */ + +#define CQ_FROM_OPS(cq) fit_container_of((cq)->ops, struct fit_cq, ops) + +static atomic_uint fit_cq_counter; + +/* Every fi_cq_*_entry layout begins with these fields, in this order, + * so one accessor serves all of them. FI_CQ_FORMAT_CONTEXT stops after + * op_context, which is why entry_sz decides how much may be read. */ +struct fit_cqe_head { + void *op_context; + uint64_t flags; + size_t len; +}; + +static size_t fit_cq_entry_size(enum fi_cq_format format) +{ + switch (format) { + case FI_CQ_FORMAT_CONTEXT: return sizeof(struct fi_cq_entry); + case FI_CQ_FORMAT_MSG: return sizeof(struct fi_cq_msg_entry); + case FI_CQ_FORMAT_DATA: return sizeof(struct fi_cq_data_entry); + case FI_CQ_FORMAT_TAGGED: return sizeof(struct fi_cq_tagged_entry); + default: return 0; + } +} + +/* + * Name the operation the completion refers to. Flags are only present + * from FI_CQ_FORMAT_MSG upwards; with FI_CQ_FORMAT_CONTEXT the provider + * tells us nothing but the context, and the posting record it joins to + * has to supply the operation type. + */ +static const char *fit_cq_opname(uint64_t flags, int have_flags) +{ + if (!have_flags) + return "cq_completion"; + + if (flags & FI_ATOMIC) + return (flags & FI_REMOTE_WRITE) ? "cq_remote_atomic" : + "cq_atomic"; + if (flags & FI_REMOTE_WRITE) + return "cq_remote_write"; + if (flags & FI_REMOTE_READ) + return "cq_remote_read"; + if (flags & FI_WRITE) + return "cq_write"; + if (flags & FI_READ) + return "cq_read"; + if (flags & FI_SEND) + return (flags & FI_TAGGED) ? "cq_tsend" : "cq_send"; + if (flags & FI_RECV) + return (flags & FI_TAGGED) ? "cq_trecv" : "cq_recv"; + return "cq_completion"; +} + +/* Walk a batch the provider has just filled in and emit one record per + * entry. src is the parallel array fi_cq_readfrom() fills, or NULL. */ +/* + * A provider may implement one read entry point in terms of another, and + * it does so through the *public* inline wrapper -- the tcp provider's + * ofi_cq_read() is literally "return fi_cq_readfrom(cq, buf, count, + * NULL)". That inline dispatches through the vtable, so it lands back in + * here and one application poll would be recorded twice. + * + * Count the nesting and report only from the outermost frame: that is the + * call the application actually made, and it is the one carrying the src + * array if there is one, since an internal re-dispatch can only drop + * information, never add it. + */ +static __thread unsigned fit_cq_depth; + +static void fit_cq_report(struct fit_cq *c, const void *buf, ssize_t n, + const fi_addr_t *src) +{ + int have_flags = c->entry_sz >= sizeof(struct fi_cq_msg_entry); + uint64_t now = fit_now_us(); + ssize_t i; + + if (!c->entry_sz) + return; + + for (i = 0; i < n; i++) { + const struct fit_cqe_head *e = + (const struct fit_cqe_head *)((const char *)buf + + (size_t)i * c->entry_sz); + uint64_t flags = have_flags ? e->flags : 0; + size_t len = have_flags ? e->len : 0; + + fit_record_cq(c, fit_cq_opname(flags, have_flags), + src ? src[i] : FI_ADDR_UNSPEC, len, 0, + e->op_context, now); + } +} + +static ssize_t fit_cq_read(struct fid_cq *cq, void *buf, size_t count) +{ + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; + + ret = CALL_OR_ENOSYS(c->orig->read, (cq, buf, count)); + fit_cq_depth = outer; + + /* A polling loop calls this constantly and almost always gets + * -FI_EAGAIN. Nothing happened, so nothing is recorded. */ + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, NULL); + return ret; +} + +static ssize_t fit_cq_readfrom(struct fid_cq *cq, void *buf, size_t count, + fi_addr_t *src_addr) +{ + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; + + ret = CALL_OR_ENOSYS(c->orig->readfrom, (cq, buf, count, src_addr)); + fit_cq_depth = outer; + + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, src_addr); + return ret; +} + +static ssize_t fit_cq_sread(struct fid_cq *cq, void *buf, size_t count, + const void *cond, int timeout) +{ + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; + + ret = CALL_OR_ENOSYS(c->orig->sread, (cq, buf, count, cond, timeout)); + fit_cq_depth = outer; + + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, NULL); + return ret; +} + +static ssize_t fit_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count, + fi_addr_t *src_addr, const void *cond, + int timeout) +{ + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; + + ret = CALL_OR_ENOSYS(c->orig->sreadfrom, (cq, buf, count, src_addr, cond, timeout)); + fit_cq_depth = outer; + + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, src_addr); + return ret; +} + +/* + * A failed operation never appears in the normal completion stream, so + * without this a transfer that errored would show a posting record and + * no completion, indistinguishable from one still in flight. + */ +static ssize_t fit_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf, + uint64_t flags) +{ + struct fit_cq *c = CQ_FROM_OPS(cq); + ssize_t ret = CALL_OR_ENOSYS(c->orig->readerr, (cq, buf, flags)); + + if (ret > 0 && buf) + fit_record_cq(c, fit_cq_opname(buf->flags, 1), +#ifdef HAVE_FI_CQ_ERR_SRC_ADDR + buf->src_addr, +#else + /* Older libfabric has no src_addr here. */ + FI_ADDR_UNSPEC, +#endif + buf->len, -buf->err, buf->op_context, + fit_now_us()); + return ret; +} + +static int fit_cq_close(struct fid *fid) +{ + struct fit_cq *c = fit_container_of(fid->ops, struct fit_cq, fid_ops); + int ret; + + fit_writer_flush_all(); + + ret = c->o_fid->close ? c->o_fid->close(fid) : -FI_ENOSYS; + if (!ret) + free(c); + return ret; +} + +void fit_hook_cq(struct fid_cq *cq, enum fi_cq_format format) +{ + struct fit_cq *c; + + if (!cq || !cq->ops || cq->ops->read == fit_cq_read) + return; + + c = calloc(1, sizeof(*c)); + if (!c) + return; + + c->cq = cq; + c->cq_id = atomic_fetch_add(&fit_cq_counter, 1); + c->format = format; + c->entry_sz = fit_cq_entry_size(format); + + /* + * FI_CQ_FORMAT_UNSPEC leaves the choice to the provider and there + * is no way to ask what it picked. Every layout starts with + * op_context, so fall back to that much: completions are still + * timestamped and still join to their posting record, they just + * carry no length of their own. + */ + if (!c->entry_sz) { + c->entry_sz = sizeof(struct fi_cq_entry); + fit_log("cq %u opened with FI_CQ_FORMAT_UNSPEC; " + "completion lengths unavailable", c->cq_id); + } + + if (cq->fid.ops) { + c->o_fid = cq->fid.ops; + c->fid_ops = *cq->fid.ops; + c->fid_ops.close = fit_cq_close; + cq->fid.ops = &c->fid_ops; + } + + c->orig = cq->ops; + c->ops = *cq->ops; + c->ops.read = fit_cq_read; + c->ops.readfrom = fit_cq_readfrom; + c->ops.readerr = fit_cq_readerr; + c->ops.sread = fit_cq_sread; + c->ops.sreadfrom = fit_cq_sreadfrom; + cq->ops = &c->ops; + + fit_log("hooked cq %u (%p) format %d", c->cq_id, (void *)cq, + (int)format); +} + +/* Called when an endpoint binds a CQ, so a completion can be attributed + * to a local NIC address. Only safe if we hooked that CQ ourselves -- + * the read slot is the tell. */ +static void fit_cq_attach_ep(struct fid *bfid, struct fit_ep *h) +{ + struct fid_cq *cq = (struct fid_cq *)bfid; + struct fit_cq *c; + + if (!cq->ops || cq->ops->read != fit_cq_read) + return; + + c = CQ_FROM_OPS(cq); + if (c->ep && c->ep != h) { + /* Shared across endpoints; guessing one would be wrong. */ + c->ep_ambiguous = 1; + c->ep = NULL; + return; + } + if (!c->ep_ambiguous) + c->ep = h; +} + +/* ================================================================== */ +/* domain */ +/* ================================================================== */ + +#define DOM_FROM_OPS(dom) fit_container_of((dom)->ops, struct fit_domain, ops) + +static int fit_dom_endpoint(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context) +{ + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->endpoint ? + d->orig->endpoint(domain, info, ep, context) : -FI_ENOSYS; + + if (!ret && ep && *ep) + fit_hook_ep(*ep, &d->topo); + return ret; +} + +static int fit_dom_endpoint2(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, uint64_t flags, void *context) +{ + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->endpoint2 ? + d->orig->endpoint2(domain, info, ep, flags, context) : + -FI_ENOSYS; + + if (!ret && ep && *ep) + fit_hook_ep(*ep, &d->topo); + return ret; +} + +static int fit_dom_scalable_ep(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **sep, void *context) +{ + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->scalable_ep ? + d->orig->scalable_ep(domain, info, sep, context) : -FI_ENOSYS; + + if (!ret && sep && *sep) + fit_hook_ep(*sep, &d->topo); + return ret; +} + +static int fit_dom_srx_ctx(struct fid_domain *domain, struct fi_rx_attr *attr, + struct fid_ep **rx_ep, void *context) +{ + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->srx_ctx ? + d->orig->srx_ctx(domain, attr, rx_ep, context) : -FI_ENOSYS; + + /* Off by default. Several providers (tcp among them) route an + * endpoint's receives through an internal shared receive context, so + * hooking both records every fi_recv twice -- once on the endpoint + * the application called, once on the srx underneath it. Set + * FI_TRACE_SRX=1 if the application posts to an srx directly. */ + if (!ret && rx_ep && *rx_ep && fit_cfg.trace_srx) + fit_hook_ep(*rx_ep, &d->topo); + return ret; +} + +static int fit_dom_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context) +{ + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->cq_open ? + d->orig->cq_open(domain, attr, cq, context) : -FI_ENOSYS; + + if (!ret && cq && *cq && fit_cfg.trace_cq) + fit_hook_cq(*cq, attr ? attr->format : FI_CQ_FORMAT_UNSPEC); + return ret; +} + +static int fit_dom_av_open(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context) +{ + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->av_open ? + d->orig->av_open(domain, attr, av, context) : -FI_ENOSYS; + + if (!ret && av && *av) + fit_hook_av(*av); + return ret; +} + +static int fit_dom_close(struct fid *fid) +{ + struct fit_domain *d = fit_container_of(fid->ops, struct fit_domain, + fid_ops); + int ret = d->o_fid->close ? d->o_fid->close(fid) : -FI_ENOSYS; + + if (!ret) + free(d); + return ret; +} + +void fit_hook_domain(struct fid_domain *domain, const struct fi_info *info) +{ + struct fit_domain *d; + + if (!domain || !domain->ops) + return; + + d = calloc(1, sizeof(*d)); + if (!d) + return; + + /* Before patching anything: the extension interface is reached + * through the domain's own ops, and asking while they are still + * entirely the provider's keeps this independent of what we are + * about to change. */ + fit_topo_resolve(&d->topo, domain, info); + + if (domain->fid.ops) { + d->o_fid = domain->fid.ops; + d->fid_ops = *domain->fid.ops; + d->fid_ops.close = fit_dom_close; + domain->fid.ops = &d->fid_ops; + } + + d->orig = domain->ops; + d->ops = *domain->ops; + d->ops.endpoint = fit_dom_endpoint; + d->ops.endpoint2 = fit_dom_endpoint2; + d->ops.scalable_ep = fit_dom_scalable_ep; + d->ops.srx_ctx = fit_dom_srx_ctx; + d->ops.cq_open = fit_dom_cq_open; + d->ops.av_open = fit_dom_av_open; + domain->ops = &d->ops; + + if (d->topo.valid) + fit_log("hooked domain %p (%s: group %u switch %u port %u)", + (void *)domain, d->topo.name, d->topo.group, + d->topo.sw, d->topo.port); + else + fit_log("hooked domain %p", (void *)domain); +} + +/* ================================================================== */ +/* fabric */ +/* ================================================================== */ + +#define FAB_FROM_OPS(fab) fit_container_of((fab)->ops, struct fit_fabric, ops) + +static int fit_fab_domain(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **dom, void *context) +{ + struct fit_fabric *f = FAB_FROM_OPS(fabric); + int ret = f->orig->domain ? + f->orig->domain(fabric, info, dom, context) : -FI_ENOSYS; + + if (!ret && dom && *dom) + fit_hook_domain(*dom, info); + return ret; +} + +static int fit_fab_domain2(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **dom, uint64_t flags, + void *context) +{ + struct fit_fabric *f = FAB_FROM_OPS(fabric); + int ret = f->orig->domain2 ? + f->orig->domain2(fabric, info, dom, flags, context) : + -FI_ENOSYS; + + if (!ret && dom && *dom) + fit_hook_domain(*dom, info); + return ret; +} + +static int fit_fab_close(struct fid *fid) +{ + struct fit_fabric *f = fit_container_of(fid->ops, struct fit_fabric, + fid_ops); + int ret; + + fit_writer_flush_all(); + + ret = f->o_fid->close ? f->o_fid->close(fid) : -FI_ENOSYS; + if (!ret) + free(f); + return ret; +} + +void fit_hook_fabric(struct fid_fabric *fabric) +{ + struct fit_fabric *f; + + if (!fabric || !fabric->ops) + return; + + f = calloc(1, sizeof(*f)); + if (!f) + return; + + if (fabric->fid.ops) { + f->o_fid = fabric->fid.ops; + f->fid_ops = *fabric->fid.ops; + f->fid_ops.close = fit_fab_close; + fabric->fid.ops = &f->fid_ops; + } + + f->orig = fabric->ops; + f->ops = *fabric->ops; + f->ops.domain = fit_fab_domain; + f->ops.domain2 = fit_fab_domain2; + fabric->ops = &f->ops; + + fit_log("hooked fabric %p", (void *)fabric); +} + +/* ================================================================== */ +/* interposed exported symbols */ +/* ================================================================== */ + +typedef int (*fi_fabric_fn)(struct fi_fabric_attr *, struct fid_fabric **, + void *); +typedef int (*fi_fabric2_fn)(struct fi_info *, struct fid_fabric **, uint64_t, + void *); + +static fi_fabric_fn real_fi_fabric; +static fi_fabric2_fn real_fi_fabric2; + +/* + * libfabric exports several versions of the same name: fi_fabric exists as + * both fi_fabric@FABRIC_1.0 and the default fi_fabric@@FABRIC_1.1. A plain + * dlsym() is free to return either, and on glibc it returns the 1.0 compat + * stub -- which immediately calls fi_fabric() again, lands back in this + * library, and recurses until the stack runs out. + * + * So ask for an explicit version, newest first, and take the first hit. + * The newest defined version of a symbol is by convention the default one. + * A libfabric built without symbol versioning has no versions to match, and + * the plain dlsym() fallback is then unambiguous. + */ +#ifndef __GLIBC__ +#error "fi_trace is Linux/glibc only: it needs LD_PRELOAD symbol interposition, \ +dlvsym() to pick the right versioned fi_fabric, and gettid(2). \ +On macOS or elsewhere, run it in a container: ./build/docker-demo.sh" +#endif + +static void *fit_dlnext(const char *name) +{ + static const char *const vers[] = { + "FABRIC_1.9", "FABRIC_1.8", "FABRIC_1.7", "FABRIC_1.6", + "FABRIC_1.5", "FABRIC_1.4", "FABRIC_1.3", "FABRIC_1.2", + "FABRIC_1.1", "FABRIC_1.0", + }; + size_t i; + void *sym; + + for (i = 0; i < sizeof(vers) / sizeof(vers[0]); i++) { + sym = dlvsym(RTLD_NEXT, name, vers[i]); + if (sym) + return sym; + } + + (void) dlerror(); + return dlsym(RTLD_NEXT, name); +} + +__attribute__((constructor)) +static void fit_init(void) +{ + fit_config_init(); + + if (!fit_cfg.enabled) { + fit_log("disabled via FI_TRACE_ENABLE"); + return; + } + + real_fi_fabric = (fi_fabric_fn)fit_dlnext("fi_fabric"); + real_fi_fabric2 = (fi_fabric2_fn)fit_dlnext("fi_fabric2"); + + if (!real_fi_fabric && !real_fi_fabric2) { + fit_log("cannot find libfabric fi_fabric(); tracing off"); + return; + } + + fit_writer_init(); + fit_armed = 1; + fit_log("armed: dir=%s ops=%s%s%s rx=%d resolve=%d", fit_cfg.dir, + fit_cfg.trace_msg ? "msg " : "", + fit_cfg.trace_tagged ? "tagged " : "", + fit_cfg.trace_rma ? "rma" : "", + fit_cfg.trace_rx, fit_cfg.resolve); +} + +/* These must be exported even though the library is built with + * -fvisibility=hidden -- they are the whole point of the preload. */ +#define FIT_PUBLIC __attribute__((visibility("default"))) + +FIT_PUBLIC int fi_fabric(struct fi_fabric_attr *attr, + struct fid_fabric **fabric, void *context); +FIT_PUBLIC int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, + uint64_t flags, void *context); + +FIT_PUBLIC +int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, + void *context) +{ + int ret; + + if (!real_fi_fabric) + real_fi_fabric = (fi_fabric_fn)fit_dlnext("fi_fabric"); + if (!real_fi_fabric) + return -FI_ENOSYS; + + ret = real_fi_fabric(attr, fabric, context); + if (!ret && fit_armed && fabric && *fabric) + fit_hook_fabric(*fabric); + return ret; +} + +FIT_PUBLIC +int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, + void *context) +{ + int ret; + + if (!real_fi_fabric2) + real_fi_fabric2 = (fi_fabric2_fn)fit_dlnext("fi_fabric2"); + if (!real_fi_fabric2) + return -FI_ENOSYS; + + ret = real_fi_fabric2(info, fabric, flags, context); + if (!ret && fit_armed && fabric && *fabric) + fit_hook_fabric(*fabric); + return ret; +} diff --git a/backends/libfabric/fit_writer.c b/backends/libfabric/fit_writer.c new file mode 100644 index 000000000..cec33f9f0 --- /dev/null +++ b/backends/libfabric/fit_writer.c @@ -0,0 +1,531 @@ +/* + * fi_trace -- record writer + * + * One CSV file per thread, so records never interleave and the only + * lock on the hot path is the thread's own -- uncontended except when a + * shutdown path flushes every buffer. Buffers are flushed on overflow, + * at thread exit, and from a library destructor. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "fi_trace.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct fit_config fit_cfg; +int fit_armed; +_Atomic unsigned int fit_av_gen; + +static const char fit_csv_header[] = + "timestamp_us,pid,tid,ep_id,src_hostname,src_addr,dest_addr," + "transfer_func,direction,iov_count,size_bytes,ret,op_context\n"; + +/* ------------------------------------------------------------------ */ +/* logging */ +/* ------------------------------------------------------------------ */ + +void fit_log(const char *fmt, ...) +{ + char buf[1024]; + va_list ap; + int n; + + if (!fit_cfg.verbose) + return; + + n = snprintf(buf, sizeof(buf), "fi_trace[%d]: ", (int)getpid()); + va_start(ap, fmt); + n += vsnprintf(buf + n, sizeof(buf) - n, fmt, ap); + va_end(ap); + if (n < 0) + return; + if ((size_t)n >= sizeof(buf) - 1) + n = sizeof(buf) - 2; + buf[n++] = '\n'; + (void)!write(STDERR_FILENO, buf, n); +} + +/* ------------------------------------------------------------------ */ +/* configuration */ +/* ------------------------------------------------------------------ */ + +static int env_int(const char *name, int dflt) +{ + const char *v = getenv(name); + + if (!v || !*v) + return dflt; + if (!strcasecmp(v, "yes") || !strcasecmp(v, "true") || + !strcasecmp(v, "on")) + return 1; + if (!strcasecmp(v, "no") || !strcasecmp(v, "false") || + !strcasecmp(v, "off")) + return 0; + return (int)strtol(v, NULL, 0); +} + +static uint64_t env_u64(const char *name, uint64_t dflt) +{ + const char *v = getenv(name); + + return (v && *v) ? strtoull(v, NULL, 0) : dflt; +} + +static void parse_ops(const char *spec) +{ + /* default: everything on */ + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 1; + + if (!spec || !*spec || !strcasecmp(spec, "all")) + return; + + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 0; + if (strcasestr(spec, "msg")) + fit_cfg.trace_msg = 1; + if (strcasestr(spec, "tag")) + fit_cfg.trace_tagged = 1; + if (strcasestr(spec, "rma")) + fit_cfg.trace_rma = 1; +} + +void fit_config_init(void) +{ + const char *dir = getenv("FI_TRACE_DIR"); + + fit_cfg.enabled = env_int("FI_TRACE_ENABLE", 1); + fit_cfg.resolve = env_int("FI_TRACE_RESOLVE", 1); + fit_cfg.trace_rx = env_int("FI_TRACE_RX", 1); + fit_cfg.trace_srx = env_int("FI_TRACE_SRX", 0); + fit_cfg.trace_eagain = env_int("FI_TRACE_EAGAIN", 1); + fit_cfg.trace_cq = env_int("FI_TRACE_CQ", 1); + fit_cfg.verbose = env_int("FI_TRACE_VERBOSE", 0); + fit_cfg.bufsz = (size_t)env_u64("FI_TRACE_BUFSZ", 1u << 20); + fit_cfg.max_records = env_u64("FI_TRACE_MAX_RECORDS", 0); + parse_ops(getenv("FI_TRACE_OPS")); + + if (fit_cfg.bufsz < 4 * FIT_REC_MAX) + fit_cfg.bufsz = 4 * FIT_REC_MAX; + + snprintf(fit_cfg.dir, sizeof(fit_cfg.dir), "%s", + (dir && *dir) ? dir : "."); + + if (gethostname(fit_cfg.host, sizeof(fit_cfg.host)) != 0) + snprintf(fit_cfg.host, sizeof(fit_cfg.host), "unknown"); + fit_cfg.host[sizeof(fit_cfg.host) - 1] = '\0'; + fit_sanitize(fit_cfg.host, &fit_cfg.host_len); +} + +/* ------------------------------------------------------------------ */ +/* per-thread buffers */ +/* ------------------------------------------------------------------ */ + +/* + * Each buffer belongs to one thread, but it is not private to it: + * fit_writer_flush_all() is called from fi_close() and from the library + * destructor, and flushes *every* thread's buffer -- possibly while the + * owning thread is in the middle of appending to it. Without + * synchronisation that races on both ->len and the bytes themselves, and + * produces truncated or interleaved records. + * + * So each buffer carries its own mutex. It is uncontended in the normal + * case (the owning thread is the only one touching it), which on glibc is + * a single atomic compare-exchange, and contended only during the rare + * foreign flush. Holding it across the write() is deliberate: a foreign + * flusher blocks the owner for the duration of one write syscall at + * shutdown, which is much cheaper than double-buffering. + */ +struct fit_tls { + pthread_mutex_t lock; + int fd; + char *buf; + size_t cap; + size_t len; + uint64_t nrec; + int broken; + pid_t tid; + struct fit_tls *next; /* global list, for destructor flush */ +}; + +/* glibc no longer caches getpid(), and a syscall per record dominates + * the hot path, so cache it here and refresh it across fork(). */ +static pid_t fit_pid; + +static void fit_pid_refresh(void) +{ + fit_pid = getpid(); +} + +static pthread_key_t fit_key; +static pthread_once_t fit_once = PTHREAD_ONCE_INIT; +static pthread_mutex_t fit_list_lock = PTHREAD_MUTEX_INITIALIZER; +static struct fit_tls *fit_list; + +/* Caller holds t->lock. */ +static void tls_flush_locked(struct fit_tls *t) +{ + size_t off = 0; + + if (t->broken || t->fd < 0 || !t->len) + goto done; + + while (off < t->len) { + ssize_t n = write(t->fd, t->buf + off, t->len - off); + + if (n > 0) { + off += (size_t)n; + continue; + } + if (n < 0 && errno == EINTR) + continue; + fit_log("write failed: %s", strerror(errno)); + t->broken = 1; + break; + } +done: + t->len = 0; +} + +static void tls_flush(struct fit_tls *t) +{ + pthread_mutex_lock(&t->lock); + tls_flush_locked(t); + pthread_mutex_unlock(&t->lock); +} + +static void tls_destroy(void *arg) +{ + struct fit_tls *t = arg, **pp; + + if (!t) + return; + + tls_flush(t); + + pthread_mutex_lock(&fit_list_lock); + for (pp = &fit_list; *pp; pp = &(*pp)->next) { + if (*pp == t) { + *pp = t->next; + break; + } + } + pthread_mutex_unlock(&fit_list_lock); + + /* Safe to tear down now: a concurrent fit_writer_flush_all() holds + * the list lock for its whole iteration, so it either finished + * before the unlink above or can no longer reach this buffer. */ + if (t->fd >= 0) + close(t->fd); + pthread_mutex_destroy(&t->lock); + free(t->buf); + free(t); +} + +static void key_create(void) +{ + pthread_key_create(&fit_key, tls_destroy); +} + +static struct fit_tls *tls_get(void) +{ + struct fit_tls *t; + char path[sizeof(fit_cfg.dir) + sizeof(fit_cfg.host) + 64]; + pid_t tid; + + pthread_once(&fit_once, key_create); + + t = pthread_getspecific(fit_key); + if (t) + return t; + + t = calloc(1, sizeof(*t)); + if (!t) + return NULL; + + pthread_mutex_init(&t->lock, NULL); + tid = (pid_t)syscall(SYS_gettid); + t->tid = tid; + t->fd = -1; + t->cap = fit_cfg.bufsz; + t->buf = malloc(t->cap); + if (!t->buf) { + pthread_mutex_destroy(&t->lock); + free(t); + return NULL; + } + + /* Set this before any work that could recurse back in. */ + pthread_setspecific(fit_key, t); + + snprintf(path, sizeof(path), "%s/fitrace.%s.%d.%d.csv", fit_cfg.dir, + fit_cfg.host, (int)getpid(), (int)tid); + + t->fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); + if (t->fd < 0) { + fit_log("cannot open %s: %s", path, strerror(errno)); + t->broken = 1; + } else { + memcpy(t->buf, fit_csv_header, sizeof(fit_csv_header) - 1); + t->len = sizeof(fit_csv_header) - 1; + fit_log("tracing thread %d to %s", (int)tid, path); + } + + pthread_mutex_lock(&fit_list_lock); + t->next = fit_list; + fit_list = t; + pthread_mutex_unlock(&fit_list_lock); + + return t; +} + +void fit_writer_flush_all(void) +{ + struct fit_tls *t; + + pthread_mutex_lock(&fit_list_lock); + for (t = fit_list; t; t = t->next) + tls_flush(t); + pthread_mutex_unlock(&fit_list_lock); +} + +void fit_writer_init(void) +{ + struct stat st; + + fit_pid_refresh(); + pthread_atfork(NULL, NULL, fit_pid_refresh); + + if (stat(fit_cfg.dir, &st) != 0 && mkdir(fit_cfg.dir, 0755) != 0 && + errno != EEXIST) + fit_log("cannot create %s: %s", fit_cfg.dir, strerror(errno)); +} + +/* ------------------------------------------------------------------ */ +/* fast field appenders */ +/* ------------------------------------------------------------------ */ + +static inline char *put_u64(char *p, uint64_t v) +{ + char tmp[20]; + int n = 0; + + do { + tmp[n++] = (char)('0' + (v % 10)); + v /= 10; + } while (v); + while (n--) + *p++ = tmp[n]; + return p; +} + +static inline char *put_i64(char *p, int64_t v) +{ + if (v < 0) { + *p++ = '-'; + return put_u64(p, (uint64_t)(-(v + 1)) + 1); + } + return put_u64(p, (uint64_t)v); +} + +/* Replace anything that could break CSV framing, in place, and report + * the length. Called once per distinct string, not per record. */ +void fit_sanitize(char *s, size_t *len_out) +{ + size_t i; + + for (i = 0; s[i]; i++) { + char c = s[i]; + + if (c == ',' || c == '\n' || c == '\r' || c == '"') + s[i] = '_'; + } + *len_out = i; +} + +/* Hot path: the string was sanitised when it was produced. */ +static inline char *put_mem(char *p, const char *s, size_t len) +{ + memcpy(p, s, len); + return p + len; +} + +/* Only used for the short, compile-time-constant op names. */ +static inline char *put_lit(char *p, const char *s) +{ + while (*s) + *p++ = *s++; + return p; +} + +/* The op_context column. It is only ever compared for equality -- it + * joins a posting record to its completion -- so the raw pointer value is + * all that is needed, and hex keeps it short. */ +static inline char *put_ptr(char *p, const void *v) +{ + static const char hex[] = "0123456789abcdef"; + uintptr_t u = (uintptr_t)v; + int shift; + + if (!u) { + *p++ = '0'; + return p; + } + + *p++ = '0'; + *p++ = 'x'; + for (shift = (int)(sizeof(u) * 8 - 4); shift >= 0; shift -= 4) { + unsigned d = (unsigned)((u >> shift) & 0xf); + + if (d || p[-1] != 'x' || shift == 0) + *p++ = hex[d]; + } + return p; +} + +uint64_t fit_now_us(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_REALTIME, &ts); + return (uint64_t)ts.tv_sec * 1000000ull + (uint64_t)ts.tv_nsec / 1000ull; +} + +/* ------------------------------------------------------------------ */ +/* the hot path */ +/* ------------------------------------------------------------------ */ + +/* + * Append one record. Everything that needs resolving has already been + * resolved by the caller, so all this does is format and memcpy under the + * buffer lock. + */ +static void fit_emit(unsigned int obj_id, const char *src, size_t src_len, + const char *peer, size_t peer_len, const char *op, + const char *dir, size_t iov_count, size_t bytes, + ssize_t ret, const void *context, uint64_t now) +{ + struct fit_tls *t; + char *p; + + t = tls_get(); + if (!t || t->broken) + return; + + pthread_mutex_lock(&t->lock); + + if (fit_cfg.max_records && t->nrec >= fit_cfg.max_records) + goto out; + + if (t->len + FIT_REC_MAX > t->cap) + tls_flush_locked(t); + if (t->broken) + goto out; + + p = t->buf + t->len; + + p = put_u64(p, now); *p++ = ','; + p = put_u64(p, (uint64_t)fit_pid); *p++ = ','; + p = put_u64(p, (uint64_t)t->tid); *p++ = ','; + p = put_u64(p, obj_id); *p++ = ','; + p = put_mem(p, fit_cfg.host, fit_cfg.host_len); *p++ = ','; + p = put_mem(p, src, src_len); *p++ = ','; + p = put_mem(p, peer, peer_len); *p++ = ','; + p = put_lit(p, op); *p++ = ','; + p = put_lit(p, dir); *p++ = ','; + p = put_u64(p, iov_count); *p++ = ','; + p = put_u64(p, bytes); *p++ = ','; + p = put_i64(p, (int64_t)ret); *p++ = ','; + p = put_ptr(p, context); *p++ = '\n'; + + t->len = (size_t)(p - t->buf); + t->nrec++; +out: + pthread_mutex_unlock(&t->lock); +} + +void fit_record(struct fit_ep *h, const char *op, char dir, fi_addr_t peer, + size_t bytes, size_t iov_count, ssize_t ret, + const void *context) +{ + char scratch[FIT_ADDR_MAX]; + const char *peer_str; + size_t peer_len = 0; + uint64_t now; + + if (dir == 'R' && !fit_cfg.trace_rx) + return; + + /* A -FI_EAGAIN return means the provider refused the operation and + * the application will retry; no bytes moved. Applications that + * spin on a busy endpoint can generate thousands of these, which + * swamps the real transfers, so they can be dropped. */ + if (ret == -FI_EAGAIN && !fit_cfg.trace_eagain) + return; + + now = fit_now_us(); + + if (!h->src_valid) + fit_ep_resolve_src(h); + + peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), + &peer_len); + + /* An operation the provider refused never became outstanding, so it + * will produce no completion and must not be joined to one. */ + fit_emit(h->ep_id, h->src, h->src_len, peer_str, peer_len, op, + (dir == 'R') ? "rx" : "tx", iov_count, bytes, ret, + ret ? NULL : context, now); +} + +/* + * A completion record. Unlike a posting record this is observed *after* + * the transfer, so size_bytes is the length the NIC actually moved and, + * when the application used fi_cq_readfrom(), peer is the real sender + * rather than the wildcard the receive was posted with. + * + * now is passed in because one fi_cq_read() can return a burst of + * completions, and timestamping them individually would imply a + * resolution the batch does not have. + */ +void fit_record_cq(struct fit_cq *c, const char *op, fi_addr_t peer, + size_t bytes, ssize_t ret, const void *context, + uint64_t now) +{ + char scratch[FIT_ADDR_MAX]; + const char *peer_str = "fi_addr_unspec"; + size_t peer_len = sizeof("fi_addr_unspec") - 1; + const char *src = "unknown"; + size_t src_len = sizeof("unknown") - 1; + struct fit_ep *h = c->ep; + + /* The CQ usually serves exactly one endpoint, in which case its + * local address and id describe this record too. */ + if (h) { + if (!h->src_valid) + fit_ep_resolve_src(h); + src = h->src; + src_len = h->src_len; + peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), + &peer_len); + } + + fit_emit(h ? h->ep_id : c->cq_id, src, src_len, peer_str, peer_len, + op, "cq", 1, bytes, ret, context, now); +} + +__attribute__((destructor)) +static void fit_fini(void) +{ + fit_writer_flush_all(); +} From f4a88de248d7eea1707c374ad32fec18db4a4b65 Mon Sep 17 00:00:00 2001 From: rsc18 Date: Thu, 15 Oct 2026 19:48:54 +0000 Subject: [PATCH 3/7] Separate libfabric configuration from CSV writer --- backends/libfabric/fit_config.c | 100 ++++++++++++++++++++++++++++++++ backends/libfabric/fit_writer.c | 97 ------------------------------- 2 files changed, 100 insertions(+), 97 deletions(-) create mode 100644 backends/libfabric/fit_config.c diff --git a/backends/libfabric/fit_config.c b/backends/libfabric/fit_config.c new file mode 100644 index 000000000..6c328a2af --- /dev/null +++ b/backends/libfabric/fit_config.c @@ -0,0 +1,100 @@ +#include "fi_trace.h" + +#include +#include +#include +#include +#include +#include + +struct fit_config fit_cfg; +int fit_armed; +_Atomic unsigned int fit_av_gen; + +void fit_log(const char *fmt, ...) +{ + char buf[1024]; + va_list ap; + int n; + + if (!fit_cfg.verbose) + return; + + n = snprintf(buf, sizeof(buf), "fi_trace[%d]: ", (int)getpid()); + va_start(ap, fmt); + n += vsnprintf(buf + n, sizeof(buf) - n, fmt, ap); + va_end(ap); + if (n < 0) + return; + if ((size_t)n >= sizeof(buf) - 1) + n = sizeof(buf) - 2; + buf[n++] = '\n'; + (void)!write(STDERR_FILENO, buf, n); +} + + +static int env_int(const char *name, int dflt) +{ + const char *v = getenv(name); + + if (!v || !*v) + return dflt; + if (!strcasecmp(v, "yes") || !strcasecmp(v, "true") || + !strcasecmp(v, "on")) + return 1; + if (!strcasecmp(v, "no") || !strcasecmp(v, "false") || + !strcasecmp(v, "off")) + return 0; + return (int)strtol(v, NULL, 0); +} + +static uint64_t env_u64(const char *name, uint64_t dflt) +{ + const char *v = getenv(name); + + return (v && *v) ? strtoull(v, NULL, 0) : dflt; +} + +static void parse_ops(const char *spec) +{ + /* default: everything on */ + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 1; + + if (!spec || !*spec || !strcasecmp(spec, "all")) + return; + + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 0; + if (strcasestr(spec, "msg")) + fit_cfg.trace_msg = 1; + if (strcasestr(spec, "tag")) + fit_cfg.trace_tagged = 1; + if (strcasestr(spec, "rma")) + fit_cfg.trace_rma = 1; +} + +void fit_config_init(void) +{ + const char *dir = getenv("FI_TRACE_DIR"); + + fit_cfg.enabled = env_int("FI_TRACE_ENABLE", 1); + fit_cfg.resolve = env_int("FI_TRACE_RESOLVE", 1); + fit_cfg.trace_rx = env_int("FI_TRACE_RX", 1); + fit_cfg.trace_srx = env_int("FI_TRACE_SRX", 0); + fit_cfg.trace_eagain = env_int("FI_TRACE_EAGAIN", 1); + fit_cfg.trace_cq = env_int("FI_TRACE_CQ", 1); + fit_cfg.verbose = env_int("FI_TRACE_VERBOSE", 0); + fit_cfg.bufsz = (size_t)env_u64("FI_TRACE_BUFSZ", 1u << 20); + fit_cfg.max_records = env_u64("FI_TRACE_MAX_RECORDS", 0); + parse_ops(getenv("FI_TRACE_OPS")); + + if (fit_cfg.bufsz < 4 * FIT_REC_MAX) + fit_cfg.bufsz = 4 * FIT_REC_MAX; + + snprintf(fit_cfg.dir, sizeof(fit_cfg.dir), "%s", + (dir && *dir) ? dir : "."); + + if (gethostname(fit_cfg.host, sizeof(fit_cfg.host)) != 0) + snprintf(fit_cfg.host, sizeof(fit_cfg.host), "unknown"); + fit_cfg.host[sizeof(fit_cfg.host) - 1] = '\0'; + fit_sanitize(fit_cfg.host, &fit_cfg.host_len); +} \ No newline at end of file diff --git a/backends/libfabric/fit_writer.c b/backends/libfabric/fit_writer.c index cec33f9f0..c00a6ce07 100644 --- a/backends/libfabric/fit_writer.c +++ b/backends/libfabric/fit_writer.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -24,108 +23,12 @@ #include #include -struct fit_config fit_cfg; -int fit_armed; -_Atomic unsigned int fit_av_gen; static const char fit_csv_header[] = "timestamp_us,pid,tid,ep_id,src_hostname,src_addr,dest_addr," "transfer_func,direction,iov_count,size_bytes,ret,op_context\n"; -/* ------------------------------------------------------------------ */ -/* logging */ -/* ------------------------------------------------------------------ */ - -void fit_log(const char *fmt, ...) -{ - char buf[1024]; - va_list ap; - int n; - - if (!fit_cfg.verbose) - return; - - n = snprintf(buf, sizeof(buf), "fi_trace[%d]: ", (int)getpid()); - va_start(ap, fmt); - n += vsnprintf(buf + n, sizeof(buf) - n, fmt, ap); - va_end(ap); - if (n < 0) - return; - if ((size_t)n >= sizeof(buf) - 1) - n = sizeof(buf) - 2; - buf[n++] = '\n'; - (void)!write(STDERR_FILENO, buf, n); -} -/* ------------------------------------------------------------------ */ -/* configuration */ -/* ------------------------------------------------------------------ */ - -static int env_int(const char *name, int dflt) -{ - const char *v = getenv(name); - - if (!v || !*v) - return dflt; - if (!strcasecmp(v, "yes") || !strcasecmp(v, "true") || - !strcasecmp(v, "on")) - return 1; - if (!strcasecmp(v, "no") || !strcasecmp(v, "false") || - !strcasecmp(v, "off")) - return 0; - return (int)strtol(v, NULL, 0); -} - -static uint64_t env_u64(const char *name, uint64_t dflt) -{ - const char *v = getenv(name); - - return (v && *v) ? strtoull(v, NULL, 0) : dflt; -} - -static void parse_ops(const char *spec) -{ - /* default: everything on */ - fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 1; - - if (!spec || !*spec || !strcasecmp(spec, "all")) - return; - - fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 0; - if (strcasestr(spec, "msg")) - fit_cfg.trace_msg = 1; - if (strcasestr(spec, "tag")) - fit_cfg.trace_tagged = 1; - if (strcasestr(spec, "rma")) - fit_cfg.trace_rma = 1; -} - -void fit_config_init(void) -{ - const char *dir = getenv("FI_TRACE_DIR"); - - fit_cfg.enabled = env_int("FI_TRACE_ENABLE", 1); - fit_cfg.resolve = env_int("FI_TRACE_RESOLVE", 1); - fit_cfg.trace_rx = env_int("FI_TRACE_RX", 1); - fit_cfg.trace_srx = env_int("FI_TRACE_SRX", 0); - fit_cfg.trace_eagain = env_int("FI_TRACE_EAGAIN", 1); - fit_cfg.trace_cq = env_int("FI_TRACE_CQ", 1); - fit_cfg.verbose = env_int("FI_TRACE_VERBOSE", 0); - fit_cfg.bufsz = (size_t)env_u64("FI_TRACE_BUFSZ", 1u << 20); - fit_cfg.max_records = env_u64("FI_TRACE_MAX_RECORDS", 0); - parse_ops(getenv("FI_TRACE_OPS")); - - if (fit_cfg.bufsz < 4 * FIT_REC_MAX) - fit_cfg.bufsz = 4 * FIT_REC_MAX; - - snprintf(fit_cfg.dir, sizeof(fit_cfg.dir), "%s", - (dir && *dir) ? dir : "."); - - if (gethostname(fit_cfg.host, sizeof(fit_cfg.host)) != 0) - snprintf(fit_cfg.host, sizeof(fit_cfg.host), "unknown"); - fit_cfg.host[sizeof(fit_cfg.host) - 1] = '\0'; - fit_sanitize(fit_cfg.host, &fit_cfg.host_len); -} /* ------------------------------------------------------------------ */ /* per-thread buffers */ From 038e59326be3862493a8f45eeb51538e63a9d5af Mon Sep 17 00:00:00 2001 From: rsc18 Date: Fri, 16 Oct 2026 16:23:33 +0000 Subject: [PATCH 4/7] libfabric backend added --- Makefile.am | 5 +- ...rofiling_cuda_driver_api.thapi_text_pretty | 6 - ...al_profiling_cuda_memcpy.thapi_text_pretty | 6 - ..._profiling_cuda_multictx.thapi_text_pretty | 14 - ...ofiling_cuda_multistream.thapi_text_pretty | 10 - ...al_profiling_kernel_name.thapi_text_pretty | 8 - ...al_profiling_multithread.thapi_text_pretty | 14 - ...nterval_profiling_normal.thapi_text_pretty | 8 - ...erval_profiling_event-AA.thapi_text_pretty | 3 - ...rval_profiling_event-AAB.thapi_text_pretty | 5 - ...val_profiling_event-AABB.thapi_text_pretty | 6 - ...rval_profiling_event-ABA.thapi_text_pretty | 5 - ...val_profiling_event-ABAB.thapi_text_pretty | 6 - ...val_profiling_event-ABBA.thapi_text_pretty | 6 - ...l_profiling_event-AsAeAs.thapi_text_pretty | 4 - ...profiling_event-AsAeAsAe.thapi_text_pretty | 5 - ...l_profiling_event-AsAsAe.thapi_text_pretty | 4 - ...profiling_event-AsAsAeAe.thapi_text_pretty | 5 - ...rval_profiling_event-BAA.thapi_text_pretty | 5 - ...nterval_profiling_normal.thapi_text_pretty | 2 - ...erval_profiling_task-AAB.thapi_text_pretty | 3 - ...rval_profiling_task-AABB.thapi_text_pretty | 4 - ...rval_profiling_task-ABBA.thapi_text_pretty | 4 - ...erval_profiling_task-BAA.thapi_text_pretty | 3 - backends/libfabric/FI_TRACE_REVISION | 1 - backends/libfabric/Makefile.am | 235 +++++ .../libfabric/btx_fiinterval_callbacks.cpp | 306 +++++++ backends/libfabric/btx_libfabric_model.yaml | 199 +++++ backends/libfabric/fit_config.c | 100 --- backends/libfabric/fit_writer.c | 434 --------- .../libfabric/gen_babeltrace_libfabric_lib.rb | 8 + backends/libfabric/include/rdma/fabric.h | 832 ++++++++++++++++++ backends/libfabric/include/rdma/fi_atomic.h | 300 +++++++ backends/libfabric/include/rdma/fi_cm.h | 137 +++ .../libfabric/include/rdma/fi_collective.h | 316 +++++++ backends/libfabric/include/rdma/fi_cxi_ext.h | 539 ++++++++++++ backends/libfabric/include/rdma/fi_domain.h | 598 +++++++++++++ backends/libfabric/include/rdma/fi_endpoint.h | 369 ++++++++ backends/libfabric/include/rdma/fi_eq.h | 488 ++++++++++ backends/libfabric/include/rdma/fi_errno.h | 208 +++++ backends/libfabric/include/rdma/fi_ext.h | 192 ++++ backends/libfabric/include/rdma/fi_profile.h | 179 ++++ backends/libfabric/include/rdma/fi_rma.h | 168 ++++ backends/libfabric/include/rdma/fi_tagged.h | 168 ++++ backends/libfabric/include/rdma/fi_trigger.h | 191 ++++ .../libfabric/include/rdma/providers/fi_log.h | 134 +++ .../include/rdma/providers/fi_peer.h | 243 +++++ .../include/rdma/providers/fi_prov.h | 121 +++ backends/libfabric/libfabric_events.yaml | 92 +- backends/libfabric/{ => src}/fi_trace.h | 15 +- backends/libfabric/{ => src}/fit_addr.c | 69 +- backends/libfabric/{ => src}/fit_hook.c | 477 ++++++++-- backends/libfabric/src/fit_lttng.c | 305 +++++++ backends/libfabric/tracer_libfabric.c | 55 -- backends/libfabric/tracer_libfabric.sh.in | 90 ++ .../omp/tests/data_op_emi.thapi_text_pretty | 6 - .../omp/tests/submit_emi.thapi_text_pretty | 2 - backends/omp/tests/target.thapi_text_pretty | 2 - .../omp/tests/target_emi.thapi_text_pretty | 2 - .../opencl/tests/api_call.thapi_text_pretty | 2 - .../tests/command_queue.thapi_text_pretty | 6 - ...mmand_queue_other_thread.thapi_text_pretty | 6 - ...enqueue_read_buffer_fast.thapi_text_pretty | 4 - .../interleave_process.thapi_text_pretty | 8 - .../tests/interleave_thread.thapi_text_pretty | 8 - .../tests/nd_range_kernel.thapi_text_pretty | 5 - .../tests/results_first.thapi_text_pretty | 4 - .../opencl/tests/with_error.thapi_text_pretty | 4 - ...erval_profiling_API_call.thapi_text_pretty | 2 - ...nterval_profiling_ignore.thapi_text_pretty | 4 - ...iling_interleave_process.thapi_text_pretty | 8 - ...al_profiling_multithread.thapi_text_pretty | 8 - ...nterval_profiling_normal.thapi_text_pretty | 4 - ...profiling_resubmit_event.thapi_text_pretty | 8 - ...l_profiling_shared_event.thapi_text_pretty | 18 - ...ng_shared_event_resubmit.thapi_text_pretty | 13 - ...ling_shared_event_xphase.thapi_text_pretty | 21 - configure.ac | 4 +- utils/babeltrace_thapi.in | 3 +- utils/xprof_utils.hpp | 5 +- xprof/xprof.rb.in | 83 +- 81 files changed, 6902 insertions(+), 1048 deletions(-) delete mode 100644 backends/cuda/tests/interval_profiling_cuda_driver_api.thapi_text_pretty delete mode 100644 backends/cuda/tests/interval_profiling_cuda_memcpy.thapi_text_pretty delete mode 100644 backends/cuda/tests/interval_profiling_cuda_multictx.thapi_text_pretty delete mode 100644 backends/cuda/tests/interval_profiling_cuda_multistream.thapi_text_pretty delete mode 100644 backends/cuda/tests/interval_profiling_kernel_name.thapi_text_pretty delete mode 100644 backends/cuda/tests/interval_profiling_multithread.thapi_text_pretty delete mode 100644 backends/cuda/tests/interval_profiling_normal.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-AA.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-AAB.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-AABB.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-ABA.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-ABAB.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-ABBA.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-AsAeAs.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-AsAeAsAe.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-AsAsAe.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-AsAsAeAe.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_event-BAA.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_normal.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_task-AAB.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_task-AABB.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_task-ABBA.thapi_text_pretty delete mode 100644 backends/itt/tests/interval_profiling_task-BAA.thapi_text_pretty delete mode 100644 backends/libfabric/FI_TRACE_REVISION create mode 100644 backends/libfabric/Makefile.am create mode 100644 backends/libfabric/btx_fiinterval_callbacks.cpp create mode 100644 backends/libfabric/btx_libfabric_model.yaml delete mode 100644 backends/libfabric/fit_config.c delete mode 100644 backends/libfabric/fit_writer.c create mode 100644 backends/libfabric/gen_babeltrace_libfabric_lib.rb create mode 100644 backends/libfabric/include/rdma/fabric.h create mode 100644 backends/libfabric/include/rdma/fi_atomic.h create mode 100644 backends/libfabric/include/rdma/fi_cm.h create mode 100644 backends/libfabric/include/rdma/fi_collective.h create mode 100644 backends/libfabric/include/rdma/fi_cxi_ext.h create mode 100644 backends/libfabric/include/rdma/fi_domain.h create mode 100644 backends/libfabric/include/rdma/fi_endpoint.h create mode 100644 backends/libfabric/include/rdma/fi_eq.h create mode 100644 backends/libfabric/include/rdma/fi_errno.h create mode 100644 backends/libfabric/include/rdma/fi_ext.h create mode 100644 backends/libfabric/include/rdma/fi_profile.h create mode 100644 backends/libfabric/include/rdma/fi_rma.h create mode 100644 backends/libfabric/include/rdma/fi_tagged.h create mode 100644 backends/libfabric/include/rdma/fi_trigger.h create mode 100644 backends/libfabric/include/rdma/providers/fi_log.h create mode 100644 backends/libfabric/include/rdma/providers/fi_peer.h create mode 100644 backends/libfabric/include/rdma/providers/fi_prov.h rename backends/libfabric/{ => src}/fi_trace.h (96%) rename backends/libfabric/{ => src}/fit_addr.c (85%) rename backends/libfabric/{ => src}/fit_hook.c (74%) create mode 100644 backends/libfabric/src/fit_lttng.c delete mode 100644 backends/libfabric/tracer_libfabric.c create mode 100644 backends/libfabric/tracer_libfabric.sh.in delete mode 100644 backends/omp/tests/data_op_emi.thapi_text_pretty delete mode 100644 backends/omp/tests/submit_emi.thapi_text_pretty delete mode 100644 backends/omp/tests/target.thapi_text_pretty delete mode 100644 backends/omp/tests/target_emi.thapi_text_pretty delete mode 100644 backends/opencl/tests/api_call.thapi_text_pretty delete mode 100644 backends/opencl/tests/command_queue.thapi_text_pretty delete mode 100644 backends/opencl/tests/command_queue_other_thread.thapi_text_pretty delete mode 100644 backends/opencl/tests/enqueue_read_buffer_fast.thapi_text_pretty delete mode 100644 backends/opencl/tests/interleave_process.thapi_text_pretty delete mode 100644 backends/opencl/tests/interleave_thread.thapi_text_pretty delete mode 100644 backends/opencl/tests/nd_range_kernel.thapi_text_pretty delete mode 100644 backends/opencl/tests/results_first.thapi_text_pretty delete mode 100644 backends/opencl/tests/with_error.thapi_text_pretty delete mode 100644 backends/ze/tests/interval_profiling_API_call.thapi_text_pretty delete mode 100644 backends/ze/tests/interval_profiling_ignore.thapi_text_pretty delete mode 100644 backends/ze/tests/interval_profiling_interleave_process.thapi_text_pretty delete mode 100644 backends/ze/tests/interval_profiling_multithread.thapi_text_pretty delete mode 100644 backends/ze/tests/interval_profiling_normal.thapi_text_pretty delete mode 100644 backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty delete mode 100644 backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty delete mode 100644 backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty delete mode 100644 backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty diff --git a/Makefile.am b/Makefile.am index 79a737042..62ae5d592 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,8 +3,9 @@ pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = thapi.pc ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = utils xprof sampling backends/opencl backends/ze backends/cuda backends/omp backends/hip backends/mpi backends/cxi backends/itt backends/pytorch - +SUBDIRS = utils xprof sampling backends/opencl backends/ze backends/cuda backends/omp backends/hip backends/mpi backends/cxi backends/itt backends/pytorch backends/libfabric EXTRA_DIST = autogen.sh README.md .valgrind/dlopen.supp thapi.pc.in CLEANFILES = thapi.pc + +DIST_SUBDIRS = utils xprof sampling backends/opencl backends/ze backends/cuda backends/omp backends/hip backends/mpi backends/cxi backends/itt backends/pytorch backends/libfabric diff --git a/backends/cuda/tests/interval_profiling_cuda_driver_api.thapi_text_pretty b/backends/cuda/tests/interval_profiling_cuda_driver_api.thapi_text_pretty deleted file mode 100644 index bf8f1e839..000000000 --- a/backends/cuda/tests/interval_profiling_cuda_driver_api.thapi_text_pretty +++ /dev/null @@ -1,6 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 3 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } -12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } -12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_cuda_memcpy.thapi_text_pretty b/backends/cuda/tests/interval_profiling_cuda_memcpy.thapi_text_pretty deleted file mode 100644 index 0cc2268e4..000000000 --- a/backends/cuda/tests/interval_profiling_cuda_memcpy.thapi_text_pretty +++ /dev/null @@ -1,6 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 3 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuMemcpyHtoD_v2_entry: { dstDevice: 0x00007f0000000000, srcHost: 0x0000500000000000, ByteCount: 400 } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } -12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuMemcpyHtoD_v2_exit: { cuResult: CUDA_SUCCESS } -12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_cuda_multictx.thapi_text_pretty b/backends/cuda/tests/interval_profiling_cuda_multictx.thapi_text_pretty deleted file mode 100644 index 5a059e3ef..000000000 --- a/backends/cuda/tests/interval_profiling_cuda_multictx.thapi_text_pretty +++ /dev/null @@ -1,14 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 123 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1230000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } -12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } -12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } -12:00:00.060000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxDestroy_v2_entry: { ctx: 0x1230000000000000 } -12:00:00.070000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxDestroy_v2_exit: { cuResult: CUDA_SUCCESS } -12:00:00.080000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x1000000000000000, flags: 0, dev: 456 } -12:00:00.090000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x4560000000000000 } -12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } -12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x3000000000000000, hStop: 0x4000000000000000 } -12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } -12:00:00.130000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x3000000000000000, hStop: 0x4000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_cuda_multistream.thapi_text_pretty b/backends/cuda/tests/interval_profiling_cuda_multistream.thapi_text_pretty deleted file mode 100644 index 18fce0964..000000000 --- a/backends/cuda/tests/interval_profiling_cuda_multistream.thapi_text_pretty +++ /dev/null @@ -1,10 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 3 } -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuStreamCreate_entry: { phStream: 0x0000000000000000, Flags: 0 } -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuStreamCreate_exit: { cuResult: CUDA_SUCCESS, phStream_val: 0x1000000000000000 } -12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 100 } -12:00:00.005000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x2000000000000000 } -12:00:00.006000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } -12:00:00.007000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } -12:00:00.008000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } -12:00:00.009000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_kernel_name.thapi_text_pretty b/backends/cuda/tests/interval_profiling_kernel_name.thapi_text_pretty deleted file mode 100644 index 218e51a32..000000000 --- a/backends/cuda/tests/interval_profiling_kernel_name.thapi_text_pretty +++ /dev/null @@ -1,8 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 3 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuModuleGetFunction_entry: { hfunc: 0x0000000000000000, hmod: 0x0000000000000000, name: 0x0000000000000000, name_val: "testkernel" } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuModuleGetFunction_exit: { cuResult: CUDA_SUCCESS, hfunc_val: 0x1000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchKernel_entry: { f: 0x1000000000000000, gridDimX: 1, gridDimY: 1, gridDimZ: 1, blockDimX: 1, blockDimY: 1, blockDimZ: 1, sharedMemBytes: 0, hStream: 0x0000000000000000, kernelParams: 0x0000000000000000, extra: 0x0000000000000000, extra_vals: [ ] } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } -12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchKernel_exit: { cuResult: CUDA_SUCCESS } -12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_multithread.thapi_text_pretty b/backends/cuda/tests/interval_profiling_multithread.thapi_text_pretty deleted file mode 100644 index 42356344f..000000000 --- a/backends/cuda/tests/interval_profiling_multithread.thapi_text_pretty +++ /dev/null @@ -1,14 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuDevicePrimaryCtxRetain_entry: { pctx: 0x0000000000000000, dev: 3 } -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuDevicePrimaryCtxRetain_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxSetCurrent_entry: { ctx: 0x1000000000000000 } -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxSetCurrent_exit: { cuResult: CUDA_SUCCESS } -12:00:00.004000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 7 } -12:00:00.005000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x2000000000000000 } -12:00:00.006000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } -12:00:00.007000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } -12:00:00.008000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } -12:00:00.009000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x3000000000000000, hStop: 0x4000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } -12:00:00.011000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } -12:00:00.012000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x3000000000000000, hStop: 0x4000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_normal.thapi_text_pretty b/backends/cuda/tests/interval_profiling_normal.thapi_text_pretty deleted file mode 100644 index f5748b895..000000000 --- a/backends/cuda/tests/interval_profiling_normal.thapi_text_pretty +++ /dev/null @@ -1,8 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuDevicePrimaryCtxRetain_entry: { pctx: 0x0000000000000000, dev: 3 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuDevicePrimaryCtxRetain_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxSetCurrent_entry: { ctx: 0x1000000000000000 } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxSetCurrent_exit: { cuResult: CUDA_SUCCESS } -12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } -12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } -12:00:00.070000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } -12:00:00.070000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/itt/tests/interval_profiling_event-AA.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AA.thapi_text_pretty deleted file mode 100644 index d0552fb3c..000000000 --- a/backends/itt/tests/interval_profiling_event-AA.thapi_text_pretty +++ /dev/null @@ -1,3 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AAB.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AAB.thapi_text_pretty deleted file mode 100644 index 73cae7134..000000000 --- a/backends/itt/tests/interval_profiling_event-AAB.thapi_text_pretty +++ /dev/null @@ -1,5 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} -12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} diff --git a/backends/itt/tests/interval_profiling_event-AABB.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AABB.thapi_text_pretty deleted file mode 100644 index 58cd66cfd..000000000 --- a/backends/itt/tests/interval_profiling_event-AABB.thapi_text_pretty +++ /dev/null @@ -1,6 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} -12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} -12:00:00.043000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 1} diff --git a/backends/itt/tests/interval_profiling_event-ABA.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-ABA.thapi_text_pretty deleted file mode 100644 index f6eedaf20..000000000 --- a/backends/itt/tests/interval_profiling_event-ABA.thapi_text_pretty +++ /dev/null @@ -1,5 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} -12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-ABAB.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-ABAB.thapi_text_pretty deleted file mode 100644 index 37b8b263e..000000000 --- a/backends/itt/tests/interval_profiling_event-ABAB.thapi_text_pretty +++ /dev/null @@ -1,6 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} -12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} -12:00:00.043000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 1} diff --git a/backends/itt/tests/interval_profiling_event-ABBA.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-ABBA.thapi_text_pretty deleted file mode 100644 index 33f5f2e38..000000000 --- a/backends/itt/tests/interval_profiling_event-ABBA.thapi_text_pretty +++ /dev/null @@ -1,6 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} -12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 1} -12:00:00.043000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AsAeAs.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AsAeAs.thapi_text_pretty deleted file mode 100644 index 757ebd6bd..000000000 --- a/backends/itt/tests/interval_profiling_event-AsAeAs.thapi_text_pretty +++ /dev/null @@ -1,4 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AsAeAsAe.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AsAeAsAe.thapi_text_pretty deleted file mode 100644 index 0f4f4f9f8..000000000 --- a/backends/itt/tests/interval_profiling_event-AsAeAsAe.thapi_text_pretty +++ /dev/null @@ -1,5 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AsAsAe.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AsAsAe.thapi_text_pretty deleted file mode 100644 index 6c582c7a7..000000000 --- a/backends/itt/tests/interval_profiling_event-AsAsAe.thapi_text_pretty +++ /dev/null @@ -1,4 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AsAsAeAe.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AsAsAeAe.thapi_text_pretty deleted file mode 100644 index 217f318fe..000000000 --- a/backends/itt/tests/interval_profiling_event-AsAsAeAe.thapi_text_pretty +++ /dev/null @@ -1,5 +0,0 @@ -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} -12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-BAA.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-BAA.thapi_text_pretty deleted file mode 100644 index 63aaee275..000000000 --- a/backends/itt/tests/interval_profiling_event-BAA.thapi_text_pretty +++ /dev/null @@ -1,5 +0,0 @@ -12:00:00.001000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} -12:00:00.002000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000010, namelen: 7, name_val: "Event B"} -12:00:00.003000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} -12:00:00.013000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} -12:00:00.023000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_normal.thapi_text_pretty b/backends/itt/tests/interval_profiling_normal.thapi_text_pretty deleted file mode 100644 index 925596dc8..000000000 --- a/backends/itt/tests/interval_profiling_normal.thapi_text_pretty +++ /dev/null @@ -1,2 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain", name__strA_val: "test task"} -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} diff --git a/backends/itt/tests/interval_profiling_task-AAB.thapi_text_pretty b/backends/itt/tests/interval_profiling_task-AAB.thapi_text_pretty deleted file mode 100644 index 788dcd456..000000000 --- a/backends/itt/tests/interval_profiling_task-AAB.thapi_text_pretty +++ /dev/null @@ -1,3 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain.AAB", name__strA_val: "test task A"} -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000100, domain__nameA_val: "test.domain.AAB", name__strA_val: "test task B"} diff --git a/backends/itt/tests/interval_profiling_task-AABB.thapi_text_pretty b/backends/itt/tests/interval_profiling_task-AABB.thapi_text_pretty deleted file mode 100644 index bb41cd7f2..000000000 --- a/backends/itt/tests/interval_profiling_task-AABB.thapi_text_pretty +++ /dev/null @@ -1,4 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain.AABB", name__strA_val: "test task A"} -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000100, domain__nameA_val: "test.domain.AABB", name__strA_val: "test task B"} -12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} diff --git a/backends/itt/tests/interval_profiling_task-ABBA.thapi_text_pretty b/backends/itt/tests/interval_profiling_task-ABBA.thapi_text_pretty deleted file mode 100644 index 209f65e89..000000000 --- a/backends/itt/tests/interval_profiling_task-ABBA.thapi_text_pretty +++ /dev/null @@ -1,4 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain.ABBA", name__strA_val: "test task A"} -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000100, domain__nameA_val: "test.domain.ABBA", name__strA_val: "test task B"} -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} diff --git a/backends/itt/tests/interval_profiling_task-BAA.thapi_text_pretty b/backends/itt/tests/interval_profiling_task-BAA.thapi_text_pretty deleted file mode 100644 index 42535b680..000000000 --- a/backends/itt/tests/interval_profiling_task-BAA.thapi_text_pretty +++ /dev/null @@ -1,3 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain.BAA", name__strA_val: "test task B"} -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000100, domain__nameA_val: "test.domain.BAA", name__strA_val: "test task A"} -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} diff --git a/backends/libfabric/FI_TRACE_REVISION b/backends/libfabric/FI_TRACE_REVISION deleted file mode 100644 index 83eaa20ab..000000000 --- a/backends/libfabric/FI_TRACE_REVISION +++ /dev/null @@ -1 +0,0 @@ -1efdf1b1159a8e07092672e19e9d369a40640154 diff --git a/backends/libfabric/Makefile.am b/backends/libfabric/Makefile.am new file mode 100644 index 000000000..01f34e267 --- /dev/null +++ b/backends/libfabric/Makefile.am @@ -0,0 +1,235 @@ +if STRICT + WERROR = -Werror +else + WERROR = +endif + +# +# ---------------------------------------------------------------------- +# Tracepoint provider +# ---------------------------------------------------------------------- +# +# libfabric_events.yaml -> (gen_custom_probes.rb) -> .tp -> (lttng-gen-tp) +# -> libfabric_tracepoints.{c,h} +# +# No h2yaml and no _model.rb anywhere in this backend: the tracer is +# hand-written, so there is no header to parse. backends/pytorch and +# backends/cxi are the two other backends built this way. +# + +FI_STATIC_PROBES = libfabric_tracepoints + +FI_STATIC_PROBES_TP = $(FI_STATIC_PROBES:=.tp) + +FI_STATIC_PROBES_INCL = $(FI_STATIC_PROBES:=.h) + +FI_STATIC_PROBES_SRC = $(FI_STATIC_PROBES:=.c) + +$(FI_STATIC_PROBES_TP): %.tp: $(top_srcdir)/utils/gen_custom_probes.rb $(srcdir)/libfabric_events.yaml $(top_srcdir)/utils/gen_probe_base.rb + $(RUBY) $< $(srcdir)/libfabric_events.yaml lttng_ust_fi > $@ + +%.h %.c: %.tp + $(LTTNG_GEN_TP) $< -o $*.c -o $*.h + +noinst_LTLIBRARIES = libfitracepoints.la + +nodist_libfitracepoints_la_SOURCES = \ + $(FI_STATIC_PROBES_INCL) \ + $(FI_STATIC_PROBES_SRC) + +libfitracepoints_la_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I./ +libfitracepoints_la_CFLAGS = -fPIC -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Wno-sign-compare $(WERROR) $(LTTNG_UST_CFLAGS) +libfitracepoints_la_LDFLAGS = $(LTTNG_UST_LIBS) + +EXTRA_DIST = \ + libfabric_events.yaml \ + $(top_srcdir)/utils/gen_custom_probes.rb \ + include + +CLEANFILES = \ + $(FI_STATIC_PROBES_TP) \ + $(FI_STATIC_PROBES_INCL) \ + $(FI_STATIC_PROBES_SRC) + +BUILT_SOURCES = \ + $(FI_STATIC_PROBES_INCL) + +# +# ---------------------------------------------------------------------- +# Tracer +# ---------------------------------------------------------------------- +# +# Note what is absent compared to backends/pytorch: there is no dummy_libs +# stanza, because we never link against libfabric. Every provider entry point +# is reached either through dlvsym(RTLD_NEXT, "fi_fabric", ...) or through a +# function pointer in a vtable the provider handed us, so the only thing +# libfabric contributes at build time is its headers. +# +# The headers are vendored under include/rdma/ so the backend builds on a +# machine with no libfabric installed -- same reason backends/mpi vendors +# mpi.h. Update them with: +# cp -r /include/rdma backends/libfabric/include/ +# + +bin_SCRIPTS = \ + tracer_libfabric.sh + +lib_LTLIBRARIES = libTracerLibfabric.la + +nodist_libTracerLibfabric_la_SOURCES = \ + $(FI_STATIC_PROBES_INCL) + +libTracerLibfabric_la_SOURCES = \ + src/fit_hook.c \ + src/fit_addr.c \ + src/fit_lttng.c + +libTracerLibfabric_la_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I$(srcdir)/include -I$(srcdir)/src -I./ +libTracerLibfabric_la_CFLAGS = -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers $(WERROR) $(LTTNG_UST_CFLAGS) +libTracerLibfabric_la_LDFLAGS = $(LTTNG_UST_LIBS) -ldl -avoid-version -module +libTracerLibfabric_la_LIBADD = libfitracepoints.la + +EXTRA_DIST += \ + src/fi_trace.h + +# +# ---------------------------------------------------------------------- +# Analysis: raw lttng_ust_fi trace -> interval stream +# ---------------------------------------------------------------------- +# +# Three names here are load-bearing and fail confusingly rather than loudly if +# you change them: +# +# -p fiinterval -c interval utils/babeltrace_thapi.in:161 looks the +# component up as "filter.#{b}interval.interval" +# where b is the short backend name, "fi". +# libFIInterval.la utils/test_wrapper_thapi_text_pretty.sh.in:69 +# builds lib"${be^^}"Interval.so. +# $(pkglibdir)/bt2 babeltrace_thapi.in:151 scans exactly this one +# directory for plugins. +# + +BTX_FI_GENERATED = \ + btx_filter_fi/metababel/metababel.h \ + btx_filter_fi/metababel/btx_component.h \ + btx_filter_fi/metababel/btx_component.c \ + btx_filter_fi/metababel/btx_upstream.h \ + btx_filter_fi/metababel/btx_upstream.c \ + btx_filter_fi/metababel/btx_downstream.h \ + btx_filter_fi/metababel/btx_downstream.c \ + btx_filter_fi/btx_main.c + +EXTRA_DIST += \ + $(top_srcdir)/xprof/btx_interval_model.yaml \ + $(srcdir)/btx_libfabric_model.yaml + +$(BTX_FI_GENERATED) &: $(top_srcdir)/xprof/btx_interval_model.yaml $(srcdir)/btx_libfabric_model.yaml + $(METABABEL) -u $(srcdir)/btx_libfabric_model.yaml -d $(top_srcdir)/xprof/btx_interval_model.yaml -t FILTER -o btx_filter_fi -p fiinterval -c interval + +CLEANFILES += \ + $(BTX_FI_GENERATED) + +BUILT_SOURCES += \ + $(BTX_FI_GENERATED) + +nodist_libFIInterval_la_SOURCES = \ + $(BTX_FI_GENERATED) + +libFIInterval_la_SOURCES = \ + btx_fiinterval_callbacks.cpp + +libFIInterval_la_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I./ -I./btx_filter_fi +libFIInterval_la_CFLAGS = -Wall -Wextra -Wno-unused-parameter $(WERROR) $(BABELTRACE2_CFLAGS) +libFIInterval_la_CXXFLAGS = -std=c++17 -Wall -Wextra -Wno-unused-parameter $(WERROR) $(BABELTRACE2_CFLAGS) +libFIInterval_la_LDFLAGS = $(BABELTRACE2_LIBS) -avoid-version -module + +bt2dir = $(pkglibdir)/bt2 +bt2_LTLIBRARIES = libFIInterval.la + +# +# ---------------------------------------------------------------------- +# Enable pretty.print in babeltrace_thapi +# ---------------------------------------------------------------------- +# +# The library name uses the *directory* name (libfabric), not the short +# backend name (fi) -- babeltrace_thapi.in:167 does the same for OpenCL, +# requiring babeltrace_opencl_lib for backend key "cl". +# + +babeltrace_libfabric_lib.rb: $(srcdir)/gen_babeltrace_libfabric_lib.rb $(top_srcdir)/utils/gen_babeltrace_lib_helper.rb $(top_srcdir)/utils/gen_probe_base.rb $(srcdir)/btx_libfabric_model.yaml + SRC_DIR=$(srcdir) $(RUBY) $< > $@ + +EXTRA_DIST += \ + gen_babeltrace_libfabric_lib.rb + +CLEANFILES += \ + babeltrace_libfabric_lib.rb + +data_DATA = \ + babeltrace_libfabric_lib.rb + +# +# ---------------------------------------------------------------------- +# Tests +# ---------------------------------------------------------------------- +# +# Offline unit tests: a text dump of a raw trace is compiled into a synthetic +# babeltrace source plugin, piped through the real libFIInterval.so, and the +# output diffed against the committed golden. No NIC, no LTTng, no +# application required -- which is the whole point, since none of those exist +# on CI. +# +# The five cases below are not arbitrary; each one encodes a caveat the +# standalone README documents in prose, which is exactly the kind of thing +# that rots without a test. +# + +TRACE_COMMON = \ + tests/send_matched-post_complete.thapi_text_pretty \ + tests/send_null_context-no_join.thapi_text_pretty \ + tests/send_eagain-no_transfer.thapi_text_pretty \ + tests/completion_orphan-no_posting.thapi_text_pretty \ + tests/recv_wildcard-posted_vs_actual.thapi_text_pretty \ + tests/progress_thread-cross_thread_join.thapi_text_pretty \ + tests/ctrl_nested-domain_ep_cq.thapi_text_pretty + +BTX_FI_GENERATED_SOURCE_TEST = \ + btx_source_fi_test/metababel/metababel.h \ + btx_source_fi_test/metababel/btx_component.h \ + btx_source_fi_test/metababel/btx_component.c \ + btx_source_fi_test/metababel/btx_downstream.h \ + btx_source_fi_test/metababel/btx_downstream.c \ + btx_source_fi_test/btx_main.c + +$(abs_builddir)/btx_libfabric_model.yaml: $(srcdir)/btx_libfabric_model.yaml + cp $< $@ + +$(BTX_FI_GENERATED_SOURCE_TEST) &: $(abs_builddir)/btx_libfabric_model.yaml + $(METABABEL) -d $(abs_builddir)/btx_libfabric_model.yaml -t SOURCE -o btx_source_fi_test -p fitest -c test + +noinst_LTLIBRARIES += libtestfisource.la +nodist_libtestfisource_la_SOURCES = $(BTX_FI_GENERATED_SOURCE_TEST) +libtestfisource_la_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I./btx_source_fi_test/ +libtestfisource_la_CFLAGS = -fPIC -Wall -Wextra -Wno-unused-parameter $(WERROR) $(BABELTRACE2_CFLAGS) + +TRACE_OUT = $(TRACE_COMMON:.thapi_text_pretty=.bt_text_pretty) + +TESTS = $(TRACE_COMMON) + +TEST_EXTENSIONS = .thapi_text_pretty +THAPI_TEXT_PRETTY_LOG_COMPILER = $(top_builddir)/utils/test_wrapper_thapi_text_pretty.sh +AM_THAPI_TEXT_PRETTY_LOG_FLAGS = backends/libfabric fi ./btx_source_fi_test/ .libs/libtestfisource.a + +check_DATA = \ + $(TRACE_COMMON) \ + $(TRACE_OUT) + +EXTRA_DIST += \ + $(TRACE_COMMON) \ + $(TRACE_OUT) \ + tests/mock_provider.c \ + tests/testapp.c + +CLEANFILES += \ + $(BTX_FI_GENERATED_SOURCE_TEST) \ + $(abs_builddir)/btx_libfabric_model.yaml diff --git a/backends/libfabric/btx_fiinterval_callbacks.cpp b/backends/libfabric/btx_fiinterval_callbacks.cpp new file mode 100644 index 000000000..018b759a4 --- /dev/null +++ b/backends/libfabric/btx_fiinterval_callbacks.cpp @@ -0,0 +1,306 @@ +/* + * Analysis side of the libfabric backend: turn the raw lttng_ust_fi trace into + * THAPI's "interval" stream. + * + * Three jobs, in increasing order of difficulty: + * + * 1. Pair op_entry with op_exit for a per-call duration. This is the usual + * THAPI entry/exit pairing and measures how long the libfabric call + * itself took -- i.e. how long the provider spent accepting the request. + * + * 2. Join a posting to its completion for a transfer duration. This is the + * part no other THAPI backend has to do. A libfabric transfer is not a + * function call with a return; it is a posting that the application later + * reaps from a completion queue, possibly from a different thread. The + * two are joined on op_context, which is the application's own pointer. + * + * 3. Attribute bytes to a peer, for the traffic stream. Only the completion + * knows how many bytes actually moved. + * + * ep_info events build a side table so the pretty-printer and the traffic + * metadata can name the NIC and its dragonfly coordinates; this replaces the + * sidecar CSV + fitrace-merge join of the standalone tracer. + */ + +#include "xprof_utils.hpp" +#include + +#include +#include +#include +#include +#include + +/* ------------------------------------------------------------------ */ +/* state */ +/* ------------------------------------------------------------------ */ + +struct posting_s { + int64_t ts; /* when the application posted */ + std::string op; /* "fi_send", "fi_trecv", ... */ + std::string peer; /* as known at posting time; a wildcard + receive will be corrected by the + completion, which knows the real sender */ + uint64_t posted_bytes; /* buffer length, not transferred length */ +}; + +struct ep_info_s { + std::string src_addr; + std::string domain_name; + bool topo_valid; + uint32_t group, switch_id, port; +}; + +struct data_s { + /* + * Per-thread LIFO of open entry timestamps. A wrapper cannot re-enter + * itself, but a layered provider (rxm over verbs, or the tcp provider + * implementing fi_cq_read in terms of fi_cq_readfrom) dispatches back + * through the public inline and lands in another wrapper, so nesting is + * real and a stack is required rather than a single slot. + */ + std::unordered_map>> op_stack; + std::unordered_map>> ctrl_stack; + + /* + * Outstanding postings keyed by {hostname, vpid, op_context}. + * + * Note what is NOT in the key: vtid. A progress thread reaping another + * thread's completions is a normal libfabric design, so keying on the + * posting thread would silently drop every join in such an application. + * The cost is that two threads in one process posting with the same context + * pointer at the same time become ambiguous -- but libfabric already + * requires op_context to be unique among outstanding operations on an + * endpoint, so a correct application cannot hit this. + */ + std::unordered_map outstanding; + + std::unordered_map endpoints; /* key: host/pid/ep_id */ +}; +typedef struct data_s data_t; + +static void btx_initialize_component(void **usr_data) { *usr_data = new data_t; } + +static void btx_finalize_component(void *usr_data) { + delete static_cast(usr_data); +} + +/* ------------------------------------------------------------------ */ +/* helpers */ +/* ------------------------------------------------------------------ */ + +static std::string ctx_key(const char *hostname, int64_t vpid, + uintptr_t op_context) { + return std::string(hostname) + "/" + std::to_string(vpid) + "/" + + std::to_string(op_context); +} + +static std::string ep_key(const char *hostname, int64_t vpid, uint32_t ep_id) { + return std::string(hostname) + "/" + std::to_string(vpid) + "/" + + std::to_string(ep_id); +} + +/* + * Traffic metadata: who the bytes went to or came from, qualified by the local + * NIC when we know it. This is the string that shows up in iprof's traffic + * table, so it wants to be stable and greppable rather than pretty. + */ +static std::string traffic_metadata(data_t *s, const char *hostname, + int64_t vpid, uint32_t ep_id, + const std::string &peer) { + auto it = s->endpoints.find(ep_key(hostname, vpid, ep_id)); + if (it == s->endpoints.end()) + return peer; + + std::string m = it->second.src_addr + "->" + peer; + if (it->second.topo_valid) + m += " [" + it->second.domain_name + " g" + + std::to_string(it->second.group) + "s" + + std::to_string(it->second.switch_id) + "p" + + std::to_string(it->second.port) + "]"; + return m; +} + +/* ------------------------------------------------------------------ */ +/* data path */ +/* ------------------------------------------------------------------ */ + +static void op_entry_callback(void *btx_handle, void *usr_data, int64_t ts, + const char *hostname, int64_t vpid, uint64_t vtid, + char *op, uint32_t ep_id, uintptr_t op_context) { + auto *s = static_cast(usr_data); + s->op_stack[{hostname, vpid, vtid}].push_back({ts, op}); +} + +static void op_exit_callback(void *btx_handle, void *usr_data, int64_t ts, + const char *hostname, int64_t vpid, uint64_t vtid, + char *op, uint32_t ep_id, char *src_addr, + char *peer_addr, char *direction, + uint64_t iov_count, uint64_t size_bytes, + int64_t ret, uintptr_t op_context) { + auto *s = static_cast(usr_data); + auto &stack = s->op_stack[{hostname, vpid, vtid}]; + + /* + * An exit with no entry happens at the head of a trace that started while + * the application was already inside libfabric. Drop it rather than + * inventing a start timestamp. + */ + if (stack.empty()) + return; + + const int64_t entry_ts = stack.back().first; + const std::string entry_op = stack.back().second; + stack.pop_back(); + + /* + * Duration of the libfabric call itself. err_flag is set on any negative + * return, which includes -FI_EAGAIN; that is a legitimate "try later", not a + * failure, but it is also not work the application wanted, and flagging it + * makes it visible in the tally instead of inflating the success counts. + */ + btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, entry_ts, + BACKEND_FI, entry_op.c_str(), (ts - entry_ts), + (ret < 0)); + + if (ret < 0) + return; /* nothing became outstanding; the tracer already nulled context */ + + /* + * A NULL op_context is legal and common -- fi_inject takes no context at + * all, and plenty of applications pass NULL to the others. Such a posting + * can never be joined to a completion, because nothing distinguishes it from + * any other contextless operation in flight. Record it as a call (done + * above) but not as an outstanding transfer. + */ + if (!op_context) + return; + + /* + * Insert overwrites by design. op_context is the application's pointer and + * is recycled the instant an operation completes, so at most one live entry + * per key is correct; an overwrite means we missed a completion. + */ + s->outstanding[ctx_key(hostname, vpid, op_context)] = {entry_ts, entry_op, + peer_addr, size_bytes}; +} + +static void cq_completion_callback(void *btx_handle, void *usr_data, int64_t ts, + const char *hostname, int64_t vpid, + uint64_t vtid, char *op, uint32_t obj_id, + char *src_addr, char *peer_addr, + uint64_t size_bytes, int64_t ret, + uintptr_t op_context) { + auto *s = static_cast(usr_data); + + if (!op_context) + return; + + auto it = s->outstanding.find(ctx_key(hostname, vpid, op_context)); + if (it == s->outstanding.end()) + return; /* completion for a posting we never saw -- normal at trace start */ + + const posting_s post = it->second; + s->outstanding.erase(it); + + /* + * The duration the application actually experienced: post to reap. + * + * This is an upper bound on the transfer time, and deliberately so. An + * application that posts and then does ten milliseconds of compute before + * polling will show ten milliseconds here. That is the right number to + * report -- it is the latency the application lived with -- but it is not + * the NIC's transfer time, and nothing in a libfabric trace can give you + * that. Compare against the CXI counters (iprof --backends fi,cxi) when you + * need the hardware view. + */ + btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, post.ts, + BACKEND_FI, (post.op + ".transfer").c_str(), + (ts - post.ts), (ret < 0)); + + if (ret < 0) + return; + + /* + * Bytes are attributed from the completion, not the posting. A receive + * posts a buffer and completes with the message length; using the posted + * length would overstate received traffic by whatever slack the application + * left. Likewise peer_addr: a wildcard receive posts FI_ADDR_UNSPEC and + * only fi_cq_readfrom() reveals who actually sent. + */ + const std::string peer = + (peer_addr && peer_addr[0] && std::string(peer_addr) != "fi_addr_unspec") + ? peer_addr + : post.peer; + + btx_push_message_lttng_traffic( + btx_handle, hostname, vpid, vtid, post.ts, BACKEND_FI, post.op.c_str(), + size_bytes, traffic_metadata(s, hostname, vpid, obj_id, peer).c_str()); +} + +/* ------------------------------------------------------------------ */ +/* control path */ +/* ------------------------------------------------------------------ */ + +static void ctrl_entry_callback(void *btx_handle, void *usr_data, int64_t ts, + const char *hostname, int64_t vpid, + uint64_t vtid, char *op, uint32_t obj_id) { + auto *s = static_cast(usr_data); + s->ctrl_stack[{hostname, vpid, vtid}].push_back({ts, op}); +} + +static void ctrl_exit_callback(void *btx_handle, void *usr_data, int64_t ts, + const char *hostname, int64_t vpid, + uint64_t vtid, char *op, uint32_t obj_id, + int64_t ret) { + auto *s = static_cast(usr_data); + auto &stack = s->ctrl_stack[{hostname, vpid, vtid}]; + if (stack.empty()) + return; + + /* + * The control path nests for real: fi_domain() calls into the provider, + * which opens an endpoint, which binds a CQ. A separate stack from the data + * path keeps that nesting from interleaving with postings. + */ + const int64_t entry_ts = stack.back().first; + const std::string entry_op = stack.back().second; + stack.pop_back(); + + btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, entry_ts, + BACKEND_FI, entry_op.c_str(), (ts - entry_ts), + (ret < 0)); +} + +/* ------------------------------------------------------------------ */ +/* endpoint description */ +/* ------------------------------------------------------------------ */ + +static void ep_info_callback(void *btx_handle, void *usr_data, int64_t ts, + const char *hostname, int64_t vpid, uint64_t vtid, + uint32_t ep_id, char *src_addr, char *domain_name, + uint32_t topo_valid, uint32_t group, + uint32_t switch_id, uint32_t port) { + auto *s = static_cast(usr_data); + s->endpoints[ep_key(hostname, vpid, ep_id)] = { + src_addr, domain_name, topo_valid != 0, group, switch_id, port}; +} + +/* ------------------------------------------------------------------ */ + +void btx_register_usr_callbacks(void *btx_handle) { + btx_register_callbacks_initialize_component(btx_handle, + &btx_initialize_component); + btx_register_callbacks_finalize_component(btx_handle, + &btx_finalize_component); + + btx_register_callbacks_lttng_ust_fi_op_entry(btx_handle, &op_entry_callback); + btx_register_callbacks_lttng_ust_fi_op_exit(btx_handle, &op_exit_callback); + btx_register_callbacks_lttng_ust_fi_cq_completion(btx_handle, + &cq_completion_callback); + btx_register_callbacks_lttng_ust_fi_ctrl_entry(btx_handle, + &ctrl_entry_callback); + btx_register_callbacks_lttng_ust_fi_ctrl_exit(btx_handle, + &ctrl_exit_callback); + btx_register_callbacks_lttng_ust_fi_ep_info(btx_handle, &ep_info_callback); +} diff --git a/backends/libfabric/btx_libfabric_model.yaml b/backends/libfabric/btx_libfabric_model.yaml new file mode 100644 index 000000000..a8369b694 --- /dev/null +++ b/backends/libfabric/btx_libfabric_model.yaml @@ -0,0 +1,199 @@ +:environment: + :entries: + - :name: hostname + :type: string +:stream_classes: +- :name: thapi_libfabric + :default_clock_class: {} + :packet_context_field_class: + :type: structure + :members: + - :name: cpu_id + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 32 + :event_common_context_field_class: + :type: structure + :members: + - :name: vpid + :field_class: + :type: integer_signed + :cast_type: int64_t + :field_value_range: 64 + - :name: vtid + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 64 + :event_classes: + - :name: lttng_ust_fi:op_entry + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: ep_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: op_context + :field_class: + :type: integer_unsigned + :cast_type: uintptr_t + :field_value_range: 64 + :preferred_display_base: 16 + - :name: lttng_ust_fi:op_exit + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: ep_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: src_addr + :field_class: + :cast_type: char * + :type: string + - :name: peer_addr + :field_class: + :cast_type: char * + :type: string + - :name: direction + :field_class: + :cast_type: char * + :type: string + - :name: iov_count + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 64 + - :name: size_bytes + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 64 + - :name: ret + :field_class: + :type: integer_signed + :cast_type: int64_t + :field_value_range: 64 + - :name: op_context + :field_class: + :type: integer_unsigned + :cast_type: uintptr_t + :field_value_range: 64 + :preferred_display_base: 16 + - :name: lttng_ust_fi:cq_completion + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: obj_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: src_addr + :field_class: + :cast_type: char * + :type: string + - :name: peer_addr + :field_class: + :cast_type: char * + :type: string + - :name: size_bytes + :field_class: + :type: integer_unsigned + :cast_type: uint64_t + :field_value_range: 64 + - :name: ret + :field_class: + :type: integer_signed + :cast_type: int64_t + :field_value_range: 64 + - :name: op_context + :field_class: + :type: integer_unsigned + :cast_type: uintptr_t + :field_value_range: 64 + :preferred_display_base: 16 + - :name: lttng_ust_fi:ctrl_entry + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: obj_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: lttng_ust_fi:ctrl_exit + :payload_field_class: + :type: structure + :members: + - :name: op + :field_class: + :cast_type: char * + :type: string + - :name: obj_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: ret + :field_class: + :type: integer_signed + :cast_type: int64_t + :field_value_range: 64 + - :name: lttng_ust_fi:ep_info + :payload_field_class: + :type: structure + :members: + - :name: ep_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: src_addr + :field_class: + :cast_type: char * + :type: string + - :name: domain_name + :field_class: + :cast_type: char * + :type: string + - :name: topo_valid + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: group + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: switch_id + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 + - :name: port + :field_class: + :type: integer_unsigned + :cast_type: uint32_t + :field_value_range: 32 \ No newline at end of file diff --git a/backends/libfabric/fit_config.c b/backends/libfabric/fit_config.c deleted file mode 100644 index 6c328a2af..000000000 --- a/backends/libfabric/fit_config.c +++ /dev/null @@ -1,100 +0,0 @@ -#include "fi_trace.h" - -#include -#include -#include -#include -#include -#include - -struct fit_config fit_cfg; -int fit_armed; -_Atomic unsigned int fit_av_gen; - -void fit_log(const char *fmt, ...) -{ - char buf[1024]; - va_list ap; - int n; - - if (!fit_cfg.verbose) - return; - - n = snprintf(buf, sizeof(buf), "fi_trace[%d]: ", (int)getpid()); - va_start(ap, fmt); - n += vsnprintf(buf + n, sizeof(buf) - n, fmt, ap); - va_end(ap); - if (n < 0) - return; - if ((size_t)n >= sizeof(buf) - 1) - n = sizeof(buf) - 2; - buf[n++] = '\n'; - (void)!write(STDERR_FILENO, buf, n); -} - - -static int env_int(const char *name, int dflt) -{ - const char *v = getenv(name); - - if (!v || !*v) - return dflt; - if (!strcasecmp(v, "yes") || !strcasecmp(v, "true") || - !strcasecmp(v, "on")) - return 1; - if (!strcasecmp(v, "no") || !strcasecmp(v, "false") || - !strcasecmp(v, "off")) - return 0; - return (int)strtol(v, NULL, 0); -} - -static uint64_t env_u64(const char *name, uint64_t dflt) -{ - const char *v = getenv(name); - - return (v && *v) ? strtoull(v, NULL, 0) : dflt; -} - -static void parse_ops(const char *spec) -{ - /* default: everything on */ - fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 1; - - if (!spec || !*spec || !strcasecmp(spec, "all")) - return; - - fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 0; - if (strcasestr(spec, "msg")) - fit_cfg.trace_msg = 1; - if (strcasestr(spec, "tag")) - fit_cfg.trace_tagged = 1; - if (strcasestr(spec, "rma")) - fit_cfg.trace_rma = 1; -} - -void fit_config_init(void) -{ - const char *dir = getenv("FI_TRACE_DIR"); - - fit_cfg.enabled = env_int("FI_TRACE_ENABLE", 1); - fit_cfg.resolve = env_int("FI_TRACE_RESOLVE", 1); - fit_cfg.trace_rx = env_int("FI_TRACE_RX", 1); - fit_cfg.trace_srx = env_int("FI_TRACE_SRX", 0); - fit_cfg.trace_eagain = env_int("FI_TRACE_EAGAIN", 1); - fit_cfg.trace_cq = env_int("FI_TRACE_CQ", 1); - fit_cfg.verbose = env_int("FI_TRACE_VERBOSE", 0); - fit_cfg.bufsz = (size_t)env_u64("FI_TRACE_BUFSZ", 1u << 20); - fit_cfg.max_records = env_u64("FI_TRACE_MAX_RECORDS", 0); - parse_ops(getenv("FI_TRACE_OPS")); - - if (fit_cfg.bufsz < 4 * FIT_REC_MAX) - fit_cfg.bufsz = 4 * FIT_REC_MAX; - - snprintf(fit_cfg.dir, sizeof(fit_cfg.dir), "%s", - (dir && *dir) ? dir : "."); - - if (gethostname(fit_cfg.host, sizeof(fit_cfg.host)) != 0) - snprintf(fit_cfg.host, sizeof(fit_cfg.host), "unknown"); - fit_cfg.host[sizeof(fit_cfg.host) - 1] = '\0'; - fit_sanitize(fit_cfg.host, &fit_cfg.host_len); -} \ No newline at end of file diff --git a/backends/libfabric/fit_writer.c b/backends/libfabric/fit_writer.c deleted file mode 100644 index c00a6ce07..000000000 --- a/backends/libfabric/fit_writer.c +++ /dev/null @@ -1,434 +0,0 @@ -/* - * fi_trace -- record writer - * - * One CSV file per thread, so records never interleave and the only - * lock on the hot path is the thread's own -- uncontended except when a - * shutdown path flushes every buffer. Buffers are flushed on overflow, - * at thread exit, and from a library destructor. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include "fi_trace.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -static const char fit_csv_header[] = - "timestamp_us,pid,tid,ep_id,src_hostname,src_addr,dest_addr," - "transfer_func,direction,iov_count,size_bytes,ret,op_context\n"; - - - -/* ------------------------------------------------------------------ */ -/* per-thread buffers */ -/* ------------------------------------------------------------------ */ - -/* - * Each buffer belongs to one thread, but it is not private to it: - * fit_writer_flush_all() is called from fi_close() and from the library - * destructor, and flushes *every* thread's buffer -- possibly while the - * owning thread is in the middle of appending to it. Without - * synchronisation that races on both ->len and the bytes themselves, and - * produces truncated or interleaved records. - * - * So each buffer carries its own mutex. It is uncontended in the normal - * case (the owning thread is the only one touching it), which on glibc is - * a single atomic compare-exchange, and contended only during the rare - * foreign flush. Holding it across the write() is deliberate: a foreign - * flusher blocks the owner for the duration of one write syscall at - * shutdown, which is much cheaper than double-buffering. - */ -struct fit_tls { - pthread_mutex_t lock; - int fd; - char *buf; - size_t cap; - size_t len; - uint64_t nrec; - int broken; - pid_t tid; - struct fit_tls *next; /* global list, for destructor flush */ -}; - -/* glibc no longer caches getpid(), and a syscall per record dominates - * the hot path, so cache it here and refresh it across fork(). */ -static pid_t fit_pid; - -static void fit_pid_refresh(void) -{ - fit_pid = getpid(); -} - -static pthread_key_t fit_key; -static pthread_once_t fit_once = PTHREAD_ONCE_INIT; -static pthread_mutex_t fit_list_lock = PTHREAD_MUTEX_INITIALIZER; -static struct fit_tls *fit_list; - -/* Caller holds t->lock. */ -static void tls_flush_locked(struct fit_tls *t) -{ - size_t off = 0; - - if (t->broken || t->fd < 0 || !t->len) - goto done; - - while (off < t->len) { - ssize_t n = write(t->fd, t->buf + off, t->len - off); - - if (n > 0) { - off += (size_t)n; - continue; - } - if (n < 0 && errno == EINTR) - continue; - fit_log("write failed: %s", strerror(errno)); - t->broken = 1; - break; - } -done: - t->len = 0; -} - -static void tls_flush(struct fit_tls *t) -{ - pthread_mutex_lock(&t->lock); - tls_flush_locked(t); - pthread_mutex_unlock(&t->lock); -} - -static void tls_destroy(void *arg) -{ - struct fit_tls *t = arg, **pp; - - if (!t) - return; - - tls_flush(t); - - pthread_mutex_lock(&fit_list_lock); - for (pp = &fit_list; *pp; pp = &(*pp)->next) { - if (*pp == t) { - *pp = t->next; - break; - } - } - pthread_mutex_unlock(&fit_list_lock); - - /* Safe to tear down now: a concurrent fit_writer_flush_all() holds - * the list lock for its whole iteration, so it either finished - * before the unlink above or can no longer reach this buffer. */ - if (t->fd >= 0) - close(t->fd); - pthread_mutex_destroy(&t->lock); - free(t->buf); - free(t); -} - -static void key_create(void) -{ - pthread_key_create(&fit_key, tls_destroy); -} - -static struct fit_tls *tls_get(void) -{ - struct fit_tls *t; - char path[sizeof(fit_cfg.dir) + sizeof(fit_cfg.host) + 64]; - pid_t tid; - - pthread_once(&fit_once, key_create); - - t = pthread_getspecific(fit_key); - if (t) - return t; - - t = calloc(1, sizeof(*t)); - if (!t) - return NULL; - - pthread_mutex_init(&t->lock, NULL); - tid = (pid_t)syscall(SYS_gettid); - t->tid = tid; - t->fd = -1; - t->cap = fit_cfg.bufsz; - t->buf = malloc(t->cap); - if (!t->buf) { - pthread_mutex_destroy(&t->lock); - free(t); - return NULL; - } - - /* Set this before any work that could recurse back in. */ - pthread_setspecific(fit_key, t); - - snprintf(path, sizeof(path), "%s/fitrace.%s.%d.%d.csv", fit_cfg.dir, - fit_cfg.host, (int)getpid(), (int)tid); - - t->fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); - if (t->fd < 0) { - fit_log("cannot open %s: %s", path, strerror(errno)); - t->broken = 1; - } else { - memcpy(t->buf, fit_csv_header, sizeof(fit_csv_header) - 1); - t->len = sizeof(fit_csv_header) - 1; - fit_log("tracing thread %d to %s", (int)tid, path); - } - - pthread_mutex_lock(&fit_list_lock); - t->next = fit_list; - fit_list = t; - pthread_mutex_unlock(&fit_list_lock); - - return t; -} - -void fit_writer_flush_all(void) -{ - struct fit_tls *t; - - pthread_mutex_lock(&fit_list_lock); - for (t = fit_list; t; t = t->next) - tls_flush(t); - pthread_mutex_unlock(&fit_list_lock); -} - -void fit_writer_init(void) -{ - struct stat st; - - fit_pid_refresh(); - pthread_atfork(NULL, NULL, fit_pid_refresh); - - if (stat(fit_cfg.dir, &st) != 0 && mkdir(fit_cfg.dir, 0755) != 0 && - errno != EEXIST) - fit_log("cannot create %s: %s", fit_cfg.dir, strerror(errno)); -} - -/* ------------------------------------------------------------------ */ -/* fast field appenders */ -/* ------------------------------------------------------------------ */ - -static inline char *put_u64(char *p, uint64_t v) -{ - char tmp[20]; - int n = 0; - - do { - tmp[n++] = (char)('0' + (v % 10)); - v /= 10; - } while (v); - while (n--) - *p++ = tmp[n]; - return p; -} - -static inline char *put_i64(char *p, int64_t v) -{ - if (v < 0) { - *p++ = '-'; - return put_u64(p, (uint64_t)(-(v + 1)) + 1); - } - return put_u64(p, (uint64_t)v); -} - -/* Replace anything that could break CSV framing, in place, and report - * the length. Called once per distinct string, not per record. */ -void fit_sanitize(char *s, size_t *len_out) -{ - size_t i; - - for (i = 0; s[i]; i++) { - char c = s[i]; - - if (c == ',' || c == '\n' || c == '\r' || c == '"') - s[i] = '_'; - } - *len_out = i; -} - -/* Hot path: the string was sanitised when it was produced. */ -static inline char *put_mem(char *p, const char *s, size_t len) -{ - memcpy(p, s, len); - return p + len; -} - -/* Only used for the short, compile-time-constant op names. */ -static inline char *put_lit(char *p, const char *s) -{ - while (*s) - *p++ = *s++; - return p; -} - -/* The op_context column. It is only ever compared for equality -- it - * joins a posting record to its completion -- so the raw pointer value is - * all that is needed, and hex keeps it short. */ -static inline char *put_ptr(char *p, const void *v) -{ - static const char hex[] = "0123456789abcdef"; - uintptr_t u = (uintptr_t)v; - int shift; - - if (!u) { - *p++ = '0'; - return p; - } - - *p++ = '0'; - *p++ = 'x'; - for (shift = (int)(sizeof(u) * 8 - 4); shift >= 0; shift -= 4) { - unsigned d = (unsigned)((u >> shift) & 0xf); - - if (d || p[-1] != 'x' || shift == 0) - *p++ = hex[d]; - } - return p; -} - -uint64_t fit_now_us(void) -{ - struct timespec ts; - - clock_gettime(CLOCK_REALTIME, &ts); - return (uint64_t)ts.tv_sec * 1000000ull + (uint64_t)ts.tv_nsec / 1000ull; -} - -/* ------------------------------------------------------------------ */ -/* the hot path */ -/* ------------------------------------------------------------------ */ - -/* - * Append one record. Everything that needs resolving has already been - * resolved by the caller, so all this does is format and memcpy under the - * buffer lock. - */ -static void fit_emit(unsigned int obj_id, const char *src, size_t src_len, - const char *peer, size_t peer_len, const char *op, - const char *dir, size_t iov_count, size_t bytes, - ssize_t ret, const void *context, uint64_t now) -{ - struct fit_tls *t; - char *p; - - t = tls_get(); - if (!t || t->broken) - return; - - pthread_mutex_lock(&t->lock); - - if (fit_cfg.max_records && t->nrec >= fit_cfg.max_records) - goto out; - - if (t->len + FIT_REC_MAX > t->cap) - tls_flush_locked(t); - if (t->broken) - goto out; - - p = t->buf + t->len; - - p = put_u64(p, now); *p++ = ','; - p = put_u64(p, (uint64_t)fit_pid); *p++ = ','; - p = put_u64(p, (uint64_t)t->tid); *p++ = ','; - p = put_u64(p, obj_id); *p++ = ','; - p = put_mem(p, fit_cfg.host, fit_cfg.host_len); *p++ = ','; - p = put_mem(p, src, src_len); *p++ = ','; - p = put_mem(p, peer, peer_len); *p++ = ','; - p = put_lit(p, op); *p++ = ','; - p = put_lit(p, dir); *p++ = ','; - p = put_u64(p, iov_count); *p++ = ','; - p = put_u64(p, bytes); *p++ = ','; - p = put_i64(p, (int64_t)ret); *p++ = ','; - p = put_ptr(p, context); *p++ = '\n'; - - t->len = (size_t)(p - t->buf); - t->nrec++; -out: - pthread_mutex_unlock(&t->lock); -} - -void fit_record(struct fit_ep *h, const char *op, char dir, fi_addr_t peer, - size_t bytes, size_t iov_count, ssize_t ret, - const void *context) -{ - char scratch[FIT_ADDR_MAX]; - const char *peer_str; - size_t peer_len = 0; - uint64_t now; - - if (dir == 'R' && !fit_cfg.trace_rx) - return; - - /* A -FI_EAGAIN return means the provider refused the operation and - * the application will retry; no bytes moved. Applications that - * spin on a busy endpoint can generate thousands of these, which - * swamps the real transfers, so they can be dropped. */ - if (ret == -FI_EAGAIN && !fit_cfg.trace_eagain) - return; - - now = fit_now_us(); - - if (!h->src_valid) - fit_ep_resolve_src(h); - - peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), - &peer_len); - - /* An operation the provider refused never became outstanding, so it - * will produce no completion and must not be joined to one. */ - fit_emit(h->ep_id, h->src, h->src_len, peer_str, peer_len, op, - (dir == 'R') ? "rx" : "tx", iov_count, bytes, ret, - ret ? NULL : context, now); -} - -/* - * A completion record. Unlike a posting record this is observed *after* - * the transfer, so size_bytes is the length the NIC actually moved and, - * when the application used fi_cq_readfrom(), peer is the real sender - * rather than the wildcard the receive was posted with. - * - * now is passed in because one fi_cq_read() can return a burst of - * completions, and timestamping them individually would imply a - * resolution the batch does not have. - */ -void fit_record_cq(struct fit_cq *c, const char *op, fi_addr_t peer, - size_t bytes, ssize_t ret, const void *context, - uint64_t now) -{ - char scratch[FIT_ADDR_MAX]; - const char *peer_str = "fi_addr_unspec"; - size_t peer_len = sizeof("fi_addr_unspec") - 1; - const char *src = "unknown"; - size_t src_len = sizeof("unknown") - 1; - struct fit_ep *h = c->ep; - - /* The CQ usually serves exactly one endpoint, in which case its - * local address and id describe this record too. */ - if (h) { - if (!h->src_valid) - fit_ep_resolve_src(h); - src = h->src; - src_len = h->src_len; - peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), - &peer_len); - } - - fit_emit(h ? h->ep_id : c->cq_id, src, src_len, peer_str, peer_len, - op, "cq", 1, bytes, ret, context, now); -} - -__attribute__((destructor)) -static void fit_fini(void) -{ - fit_writer_flush_all(); -} diff --git a/backends/libfabric/gen_babeltrace_libfabric_lib.rb b/backends/libfabric/gen_babeltrace_libfabric_lib.rb new file mode 100644 index 000000000..3e6ba30ec --- /dev/null +++ b/backends/libfabric/gen_babeltrace_libfabric_lib.rb @@ -0,0 +1,8 @@ +require_relative '../../utils/gen_babeltrace_lib_helper' + +# Passing nil for the naming argument is legal only because no field in +# btx_libfabric_model.yaml carries ":metadata: :be_class:". If you later add +# an enum or bitmask field -- rendering fi_cq flags symbolically, say -- this +# backend will also need a libfabric_library.rb defining the value tables, and +# nil has to become that naming object. backends/mpi is the example to copy. +add_babeltrace_event_callbacks(nil, File.join(SRC_DIR, 'btx_libfabric_model.yaml')) diff --git a/backends/libfabric/include/rdma/fabric.h b/backends/libfabric/include/rdma/fabric.h new file mode 100644 index 000000000..319d09780 --- /dev/null +++ b/backends/libfabric/include/rdma/fabric.h @@ -0,0 +1,832 @@ +/* + * Copyright (c) 2013-2017 Intel Corporation. All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. + * (C) Copyright 2020 Hewlett Packard Enterprise Development LP + * Copyright (c) 2022 DataDirect Networks, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FABRIC_H +#define FABRIC_H + +#include +#include +#include +#include +#include + +#ifdef __GNUC__ +#define FI_DEPRECATED_FUNC __attribute__((deprecated)) +#define FI_DEPRECATED_FIELD __attribute__((deprecated)) +#define FI_FORMAT_PRINTF(string, first) \ + __attribute__ ((__format__ (__printf__, (string), (first)))) +#elif defined(_MSC_VER) +#define FI_DEPRECATED_FUNC __declspec(deprecated) +#define FI_DEPRECATED_FIELD +#define FI_FORMAT_PRINTF(string, first) +#else +#define FI_DEPRECATED_FUNC +#define FI_DEPRECATED_FIELD +#define FI_FORMAT_PRINTF(string, first) +#endif + +#if defined(__GNUC__) && !defined(__clang__) +#define EXTERNALLY_VISIBLE externally_visible +#else +#define EXTERNALLY_VISIBLE +#endif + +#if defined(_WIN32) +#include +typedef SSIZE_T ssize_t; +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define FI_MAJOR_VERSION 2 +#define FI_MINOR_VERSION 3 +#define FI_REVISION_VERSION 1 + +/* Removing these breaks the build for some apps. + * The use of FI_NAME_MAX is undefined. + * FI_ATOMIC_OP_LAST and FI_DATATYPE_LAST values cannot change + * (such as inserting new enum values that they are intended to be the + * last of) without breaking apps that recompile. So, they are hard-coded + * here. + */ +enum { + FI_NAME_MAX = 64, + FI_ATOMIC_OP_LAST = 19, + FI_DATATYPE_LAST = 14, /* not actual last datatype */ +}; + +#define FI_VERSION(major, minor) (((major) << 16) | (minor)) +#define FI_MAJOR(version) (version >> 16) +#define FI_MINOR(version) (version & 0xFFFF) +#define FI_VERSION_GE(v1, v2) (v1 >= v2) +#define FI_VERSION_LT(v1, v2) (v1 < v2) + +uint32_t fi_version(void); + +struct fid; +struct fid_fabric; +struct fid_domain; +struct fid_av; +struct fid_wait; +struct fid_poll; +struct fid_eq; +struct fid_cq; +struct fid_cntr; +struct fid_ep; +struct fid_pep; +struct fid_stx; +struct fid_mr; +struct fid_nic; + +typedef struct fid *fid_t; + +/* + * Flags + * The 64-bit flag field is used as follows: + * 1-grow up common (usable with multiple operations) + * 59-grow down operation specific (used for single call/class) + * 60 - 63 provider specific + */ + +#define FI_MSG (1ULL << 1) +#define FI_RMA (1ULL << 2) +#define FI_TAGGED (1ULL << 3) +#define FI_ATOMIC (1ULL << 4) +#define FI_ATOMICS FI_ATOMIC +#define FI_MULTICAST (1ULL << 5) +#define FI_COLLECTIVE (1ULL << 6) + +#define FI_READ (1ULL << 8) +#define FI_WRITE (1ULL << 9) +#define FI_RECV (1ULL << 10) +#define FI_SEND (1ULL << 11) +#define FI_TRANSMIT FI_SEND +#define FI_REMOTE_READ (1ULL << 12) +#define FI_REMOTE_WRITE (1ULL << 13) + +#define FI_MULTI_RECV (1ULL << 16) +#define FI_REMOTE_CQ_DATA (1ULL << 17) +#define FI_MORE (1ULL << 18) +#define FI_PEEK (1ULL << 19) +#define FI_TRIGGER (1ULL << 20) +#define FI_FENCE (1ULL << 21) +/* #define FI_PRIORITY (1ULL << 22) */ + +#define FI_COMPLETION (1ULL << 24) +#define FI_EVENT FI_COMPLETION +#define FI_INJECT (1ULL << 25) +#define FI_INJECT_COMPLETE (1ULL << 26) +#define FI_TRANSMIT_COMPLETE (1ULL << 27) +#define FI_DELIVERY_COMPLETE (1ULL << 28) +#define FI_AFFINITY (1ULL << 29) +#define FI_COMMIT_COMPLETE (1ULL << 30) +#define FI_MATCH_COMPLETE (1ULL << 31) + +#define FI_RESCAN (1ULL << 35) +#define FI_PEER_TRANSFER (1ULL << 36) +/* #define FI_MR_DMABUF (1ULL << 40) */ +#define FI_AV_USER_ID (1ULL << 41) +#define FI_FIREWALL_ADDR (1ULL << 42) +#define FI_PEER (1ULL << 43) +/* #define FI_XPU_TRIGGER (1ULL << 44) */ + +#define FI_TAGGED_DIRECTED_RECV (1ULL << 45) +#define FI_TAGGED_MULTI_RECV (1ULL << 46) +#define FI_HMEM (1ULL << 47) +#define FI_EXACT_DIRECTED_RECV (1ULL << 48) +#define FI_RMA_PMEM (1ULL << 49) +#define FI_SOURCE_ERR (1ULL << 50) +#define FI_LOCAL_COMM (1ULL << 51) +#define FI_REMOTE_COMM (1ULL << 52) +#define FI_SHARED_AV (1ULL << 53) +#define FI_PROV_ATTR_ONLY (1ULL << 54) +#define FI_NUMERICHOST (1ULL << 55) +#define FI_RMA_EVENT (1ULL << 56) +#define FI_SOURCE (1ULL << 57) +#define FI_NAMED_RX_CTX (1ULL << 58) +#define FI_DIRECTED_RECV (1ULL << 59) + + +/* Tagged messages, buffered receives, CQ flags */ +#define FI_CLAIM (1ULL << 59) +#define FI_DISCARD (1ULL << 58) +#define FI_AUTH_KEY (1ULL << 57) + +struct fi_ioc { + void *addr; + size_t count; +}; + +/* + * Format for transport addresses to insert into address vectors + */ +enum { + FI_FORMAT_UNSPEC, /* void * */ + FI_SOCKADDR, /* struct sockaddr */ + FI_SOCKADDR_IN, /* struct sockaddr_in */ + FI_SOCKADDR_IN6, /* struct sockaddr_in6 */ + FI_SOCKADDR_IB, /* struct sockaddr_ib */ + /* PSMX provider is deprecated. + * We will keep this value in order to save binary compatibility. + */ + FI_ADDR_PSMX, /* uint64_t */ + FI_ADDR_GNI, + FI_ADDR_BGQ, + FI_ADDR_MLX, + FI_ADDR_STR, /* formatted char * */ + FI_ADDR_PSMX2, /* uint64_t[2] */ + FI_ADDR_IB_UD, /* uint64_t[4] */ + FI_ADDR_EFA, + FI_ADDR_PSMX3, /* uint64_t[4] */ + FI_ADDR_OPX, + FI_ADDR_CXI, + FI_ADDR_UCX, + + FI_SOCKADDR_IP, /* FI_SOCKADDR_IN and FI_SOCKADDR_IN6 */ +}; + +#define FI_ADDR_UNSPEC ((uint64_t) -1) +#define FI_ADDR_NOTAVAIL ((uint64_t) -1) +#define FI_KEY_NOTAVAIL ((uint64_t) -1) +#define FI_SHARED_CONTEXT SIZE_MAX +#define FI_AV_AUTH_KEY SIZE_MAX +typedef uint64_t fi_addr_t; + +enum fi_av_type { + FI_AV_UNSPEC, + FI_AV_MAP, + FI_AV_TABLE +}; + +#define FI_MR_UNSPEC _Pragma("GCC warning \"'FI_MR_UNSPEC' is deprecated\"") (0) +#define FI_MR_BASIC _Pragma("GCC warning \"'FI_MR_BASIC' is deprecated\"") (1 << 0) +#define FI_MR_SCALABLE _Pragma("GCC warning \"'FI_MR_SCALABLE' is deprecated\"") (1 << 1) + +#define FI_MR_LOCAL (1 << 2) +#define FI_MR_RAW (1 << 3) +#define FI_MR_VIRT_ADDR (1 << 4) +#define FI_MR_ALLOCATED (1 << 5) +#define FI_MR_PROV_KEY (1 << 6) +#define FI_MR_MMU_NOTIFY (1 << 7) +#define FI_MR_RMA_EVENT (1 << 8) +#define FI_MR_ENDPOINT (1 << 9) +#define FI_MR_HMEM (1 << 10) +#define FI_MR_COLLECTIVE (1 << 11) + +enum fi_progress { + FI_PROGRESS_UNSPEC, + FI_PROGRESS_AUTO, + FI_PROGRESS_MANUAL, + FI_PROGRESS_CONTROL_UNIFIED, +}; + +enum fi_threading { + FI_THREAD_UNSPEC, + FI_THREAD_SAFE, + FI_THREAD_FID, + FI_THREAD_DOMAIN, + FI_THREAD_COMPLETION, + FI_THREAD_ENDPOINT, +}; + +enum fi_resource_mgmt { + FI_RM_UNSPEC, + FI_RM_DISABLED, + FI_RM_ENABLED +}; + +#define FI_ORDER_NONE _Pragma("GCC warning \"'FI_ORDER_NONE' is deprecated\"") 0ULL +#define FI_ORDER_RAR (1ULL << 0) +#define FI_ORDER_RAW (1ULL << 1) +#define FI_ORDER_RAS (1ULL << 2) +#define FI_ORDER_WAR (1ULL << 3) +#define FI_ORDER_WAW (1ULL << 4) +#define FI_ORDER_WAS (1ULL << 5) +#define FI_ORDER_SAR (1ULL << 6) +#define FI_ORDER_SAW (1ULL << 7) +#define FI_ORDER_SAS (1ULL << 8) +#define FI_ORDER_STRICT _Pragma("GCC warning \"'FI_ORDER_STRICT' is deprecated\"") 0x1FF + +#define FI_ORDER_RMA_RAR (1ULL << 32) +#define FI_ORDER_RMA_RAW (1ULL << 33) +#define FI_ORDER_RMA_WAR (1ULL << 34) +#define FI_ORDER_RMA_WAW (1ULL << 35) +#define FI_ORDER_ATOMIC_RAR (1ULL << 36) +#define FI_ORDER_ATOMIC_RAW (1ULL << 37) +#define FI_ORDER_ATOMIC_WAR (1ULL << 38) +#define FI_ORDER_ATOMIC_WAW (1ULL << 39) + +#define FI_ORDER_DATA _Pragma("GCC warning \"'FI_ORDER_DATA' is deprecated\"") (1ULL << 16) + +enum fi_ep_type { + FI_EP_UNSPEC, + FI_EP_MSG, + FI_EP_DGRAM, + FI_EP_RDM, + /* FI_EP_SOCK_STREAM, */ + /* FI_EP_SOCK_DGRAM, */ +}; + +/* Endpoint protocol + * If two providers support the same protocol, then they shall interoperate + * when the protocol capabilities match. + */ +enum { + FI_PROTO_UNSPEC, + FI_PROTO_RDMA_CM_IB_RC, + FI_PROTO_IWARP, + FI_PROTO_IB_UD, + /* PSMX provider is deprecated. + * We will keep this value in order to save binary compatibility. + */ + FI_PROTO_PSMX, + FI_PROTO_UDP, + FI_PROTO_SOCK_TCP, + /* MXM provider is deprecated. + * We will keep this value in order to save binary compatibility. + */ + FI_PROTO_MXM, + FI_PROTO_IWARP_RDM, + FI_PROTO_IB_RDM, + FI_PROTO_GNI, + FI_PROTO_RXM, + FI_PROTO_RXD, + FI_PROTO_MLX, + FI_PROTO_NETWORKDIRECT, + FI_PROTO_PSMX2, + FI_PROTO_SHM, + FI_PROTO_MRAIL, + FI_PROTO_RSTREAM, + FI_PROTO_RDMA_CM_IB_XRC, + FI_PROTO_EFA, + FI_PROTO_PSMX3, + FI_PROTO_RXM_TCP, + FI_PROTO_OPX, + FI_PROTO_CXI, + FI_PROTO_XNET, + FI_PROTO_COLL, + FI_PROTO_UCX, + FI_PROTO_SM2, + FI_PROTO_CXI_RNR, + FI_PROTO_LPP, + FI_PROTO_LNX, +}; + +enum { + FI_TAG_BITS, + FI_TAG_MPI, + FI_TAG_CCL, + FI_TAG_MAX_FORMAT = (1ULL << 16), +}; + +enum { + FI_TC_UNSPEC = 0, + FI_TC_DSCP = 0x100, + FI_TC_LABEL = 0x200, + FI_TC_BEST_EFFORT = FI_TC_LABEL, + FI_TC_LOW_LATENCY, + FI_TC_DEDICATED_ACCESS, + FI_TC_BULK_DATA, + FI_TC_SCAVENGER, + FI_TC_NETWORK_CTRL, +}; + +static inline uint32_t fi_tc_dscp_set(uint8_t dscp) +{ + return ((uint32_t) dscp) | FI_TC_DSCP; +} + +static inline uint8_t fi_tc_dscp_get(uint32_t tclass) +{ + return tclass & FI_TC_DSCP ? (uint8_t) tclass : 0; +} + +/* Mode bits */ +#define FI_CONTEXT (1ULL << 59) +#define FI_MSG_PREFIX (1ULL << 58) +#define FI_ASYNC_IOV (1ULL << 57) +#define FI_RX_CQ_DATA (1ULL << 56) +#define FI_LOCAL_MR _Pragma("GCC warning \"'FI_LOCAL_MR' is deprecated\"") (1ULL << 55) +/* #define FI_NOTIFY_FLAGS_ONLY (1ULL << 54) */ +/* #define FI_RESTRICTED_COMP (1ULL << 53) */ +#define FI_CONTEXT2 (1ULL << 52) +/* #define FI_BUFFERED_RECV (1ULL << 51) */ +/* #define FI_PEER_TRANSFER (1ULL << 36) */ + +struct fi_tx_attr { + uint64_t caps; + uint64_t mode; + uint64_t op_flags; + uint64_t msg_order; + uint64_t comp_order; + size_t inject_size; + size_t size; + size_t iov_limit; + size_t rma_iov_limit; + uint32_t tclass; +}; + +struct fi_rx_attr { + uint64_t caps; + uint64_t mode; + uint64_t op_flags; + uint64_t msg_order; + uint64_t comp_order; + size_t total_buffered_recv; + size_t size; + size_t iov_limit; +}; + +struct fi_ep_attr { + enum fi_ep_type type; + uint32_t protocol; + uint32_t protocol_version; + size_t max_msg_size; + size_t msg_prefix_size; + size_t max_order_raw_size; + size_t max_order_war_size; + size_t max_order_waw_size; + uint64_t mem_tag_format; + size_t tx_ctx_cnt; + size_t rx_ctx_cnt; + size_t auth_key_size; + uint8_t *auth_key; +}; + +struct fi_domain_attr { + struct fid_domain *domain; + char *name; + enum fi_threading threading; + enum fi_progress control_progress; + union { + enum fi_progress data_progress; + enum fi_progress progress; + }; + enum fi_resource_mgmt resource_mgmt; + enum fi_av_type av_type; + int mr_mode; + size_t mr_key_size; + size_t cq_data_size; + size_t cq_cnt; + size_t ep_cnt; + size_t tx_ctx_cnt; + size_t rx_ctx_cnt; + size_t max_ep_tx_ctx; + size_t max_ep_rx_ctx; + size_t max_ep_stx_ctx; + size_t max_ep_srx_ctx; + size_t cntr_cnt; + size_t mr_iov_limit; + uint64_t caps; + uint64_t mode; + uint8_t *auth_key; + size_t auth_key_size; + size_t max_err_data; + size_t mr_cnt; + uint32_t tclass; + size_t max_ep_auth_key; + uint32_t max_group_id; +}; + +struct fi_fabric_attr { + struct fid_fabric *fabric; + char *name; + char *prov_name; + uint32_t prov_version; + uint32_t api_version; +}; + +struct fi_info { + struct fi_info *next; + uint64_t caps; + uint64_t mode; + uint32_t addr_format; + size_t src_addrlen; + size_t dest_addrlen; + void *src_addr; + void *dest_addr; + fid_t handle; + struct fi_tx_attr *tx_attr; + struct fi_rx_attr *rx_attr; + struct fi_ep_attr *ep_attr; + struct fi_domain_attr *domain_attr; + struct fi_fabric_attr *fabric_attr; + struct fid_nic *nic; +}; + +struct fi_device_attr { + char *name; + char *device_id; + char *device_version; + char *vendor_id; + char *driver; + char *firmware; +}; + +enum fi_bus_type { + FI_BUS_UNSPEC, + FI_BUS_UNKNOWN = FI_BUS_UNSPEC, + FI_BUS_PCI, +}; + +struct fi_pci_attr { + uint16_t domain_id; + uint8_t bus_id; + uint8_t device_id; + uint8_t function_id; +}; + +struct fi_bus_attr { + enum fi_bus_type bus_type; + union { + struct fi_pci_attr pci; + } attr; +}; + +enum fi_link_state { + FI_LINK_UNKNOWN, + FI_LINK_DOWN, + FI_LINK_UP, +}; + +struct fi_link_attr { + char *address; + size_t mtu; + size_t speed; + enum fi_link_state state; + char *network_type; +}; + +enum { + FI_CLASS_UNSPEC, + FI_CLASS_FABRIC, + FI_CLASS_DOMAIN, + FI_CLASS_EP, + FI_CLASS_SEP, + FI_CLASS_RX_CTX, + FI_CLASS_SRX_CTX, + FI_CLASS_TX_CTX, + FI_CLASS_STX_CTX, + FI_CLASS_PEP, + FI_CLASS_INTERFACE, + FI_CLASS_AV, + FI_CLASS_MR, + FI_CLASS_EQ, + FI_CLASS_CQ, + FI_CLASS_CNTR, + FI_CLASS_WAIT, + FI_CLASS_POLL, + FI_CLASS_CONNREQ, + FI_CLASS_MC, + FI_CLASS_NIC, + FI_CLASS_AV_SET, + FI_CLASS_MR_CACHE, + FI_CLASS_MEM_MONITOR, + FI_CLASS_PEER_CQ, + FI_CLASS_PEER_SRX, + FI_CLASS_LOG, + FI_CLASS_PEER_AV, + FI_CLASS_PEER_AV_SET, + FI_CLASS_PEER_CNTR, + FI_CLASS_PROFILE, +}; + +struct fi_eq_attr; +struct fi_wait_attr; + +/* fi_bind()-specific flags */ +#define FI_SELECTIVE_COMPLETION (1ULL << 59) + +struct fi_ops { + size_t size; + int (*close)(struct fid *fid); + int (*bind)(struct fid *fid, struct fid *bfid, uint64_t flags); + int (*control)(struct fid *fid, int command, void *arg); + int (*ops_open)(struct fid *fid, const char *name, + uint64_t flags, void **ops, void *context); + int (*tostr)(const struct fid *fid, char *buf, size_t len); + int (*ops_set)(struct fid *fid, const char *name, uint64_t flags, + void *ops, void *context); +}; + +/* All fabric interface descriptors must start with this structure */ +struct fid { + size_t fclass; + void *context; + struct fi_ops *ops; +}; + +int fi_getinfo(uint32_t version, const char *node, const char *service, + uint64_t flags, const struct fi_info *hints, + struct fi_info **info); +void fi_freeinfo(struct fi_info *info); +struct fi_info *fi_dupinfo(const struct fi_info *info); + +static inline struct fi_info *fi_allocinfo(void) +{ + return fi_dupinfo(NULL); +} + +struct fi_ops_fabric { + size_t size; + int (*domain)(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **dom, void *context); + int (*passive_ep)(struct fid_fabric *fabric, struct fi_info *info, + struct fid_pep **pep, void *context); + int (*eq_open)(struct fid_fabric *fabric, struct fi_eq_attr *attr, + struct fid_eq **eq, void *context); + int (*wait_open)(struct fid_fabric *fabric, struct fi_wait_attr *attr, + struct fid_wait **waitset); + int (*trywait)(struct fid_fabric *fabric, struct fid **fids, + int count); + int (*domain2)(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **dom, uint64_t flags, void *context); +}; + +struct fid_fabric { + struct fid fid; + struct fi_ops_fabric *ops; + uint32_t api_version; +}; + +int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, + uint64_t flags, void *context); +int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, + void *context); +int fi_open(uint32_t version, const char *name, void *attr, size_t attr_len, + uint64_t flags, struct fid **fid, void *context); + +struct fid_nic { + struct fid fid; + struct fi_device_attr *device_attr; + struct fi_bus_attr *bus_attr; + struct fi_link_attr *link_attr; + void *prov_attr; +}; + +#define FI_CHECK_OP(ops, opstype, op) \ + (ops && (ops->size > offsetof(opstype, op)) && ops->op) + +static inline int fi_close(struct fid *fid) +{ + return fid->ops->close(fid); +} + +struct fi_alias { + struct fid **fid; + uint64_t flags; +}; + +struct fi_fid_var { + int name; + void *val; +}; + +struct fi_mr_raw_attr { + uint64_t flags; + uint64_t *base_addr; + uint8_t *raw_key; + size_t *key_size; +}; + +struct fi_mr_map_raw { + uint64_t flags; + uint64_t base_addr; + uint8_t *raw_key; + size_t key_size; + uint64_t *key; +}; + +/* control commands */ +enum { + FI_GETFIDFLAG, /* uint64_t flags */ + FI_SETFIDFLAG, /* uint64_t flags */ + FI_GETOPSFLAG, /* uint64_t flags */ + FI_SETOPSFLAG, /* uint64_t flags */ + FI_ALIAS, /* struct fi_alias * */ + FI_GETWAIT, /* void * wait object */ + FI_ENABLE, /* NULL */ + FI_BACKLOG, /* integer * */ + FI_GET_RAW_MR, /* fi_mr_raw_attr */ + FI_MAP_RAW_MR, /* fi_mr_map_raw */ + FI_UNMAP_KEY, /* uint64_t key */ + FI_QUEUE_WORK, /* struct fi_deferred_work */ + FI_CANCEL_WORK, /* struct fi_deferred_work */ + FI_FLUSH_WORK, /* NULL */ + FI_REFRESH, /* mr: fi_mr_modify */ + FI_DUP, /* struct fid ** */ + FI_GETWAITOBJ, /*enum fi_wait_obj * */ + FI_GET_VAL, /* struct fi_fid_var */ + FI_SET_VAL, /* struct fi_fid_var */ + FI_EXPORT_FID, /* struct fi_fid_export */ + FI_GET_FD, /* int */ +}; + +static inline int fi_control(struct fid *fid, int command, void *arg) +{ + return fid->ops->control(fid, command, arg); +} + +static inline int fi_alias(struct fid *fid, struct fid **alias_fid, uint64_t flags) +{ + struct fi_alias alias; + alias.fid = alias_fid; + alias.flags = flags; + return fi_control(fid, FI_ALIAS, &alias); +} + +/* Provider specific names should set the uppermost bit. */ + +static inline int fi_get_val(struct fid *fid, int name, void *val) +{ + struct fi_fid_var var; + var.name = name; + var.val = val; + return fi_control(fid, FI_GET_VAL, &var); +} + +static inline int fi_set_val(struct fid *fid, int name, void *val) +{ + struct fi_fid_var var; + var.name = name; + var.val = val; + return fi_control(fid, FI_SET_VAL, &var); +} + +static inline int +fi_open_ops(struct fid *fid, const char *name, uint64_t flags, + void **ops, void *context) +{ + return fid->ops->ops_open(fid, name, flags, ops, context); +} + +static inline int +fi_set_ops(struct fid *fid, const char *name, uint64_t flags, + void *ops, void *context) +{ + return FI_CHECK_OP(fid->ops, struct fi_ops, ops_set) ? + fid->ops->ops_set(fid, name, flags, ops, context) : -FI_ENOSYS; +} + +enum fi_type { + FI_TYPE_INFO, + FI_TYPE_EP_TYPE, + FI_TYPE_CAPS, + FI_TYPE_OP_FLAGS, + FI_TYPE_ADDR_FORMAT, + FI_TYPE_TX_ATTR, + FI_TYPE_RX_ATTR, + FI_TYPE_EP_ATTR, + FI_TYPE_DOMAIN_ATTR, + FI_TYPE_FABRIC_ATTR, + FI_TYPE_THREADING, + FI_TYPE_PROGRESS, + FI_TYPE_PROTOCOL, + FI_TYPE_MSG_ORDER, + FI_TYPE_MODE, + FI_TYPE_AV_TYPE, + FI_TYPE_ATOMIC_TYPE, + FI_TYPE_ATOMIC_OP, + FI_TYPE_VERSION, + FI_TYPE_EQ_EVENT, + FI_TYPE_CQ_EVENT_FLAGS, + FI_TYPE_MR_MODE, + FI_TYPE_OP_TYPE, + FI_TYPE_FID, + FI_TYPE_COLLECTIVE_OP, + FI_TYPE_HMEM_IFACE, + FI_TYPE_CQ_FORMAT, + FI_TYPE_LOG_LEVEL, + FI_TYPE_LOG_SUBSYS, + FI_TYPE_AV_ATTR, + FI_TYPE_CQ_ATTR, + FI_TYPE_MR_ATTR, + FI_TYPE_CNTR_ATTR, + FI_TYPE_CQ_ERR_ENTRY, +}; + +char *fi_tostr(const void *data, enum fi_type datatype); +char *fi_tostr_r(char *buf, size_t len, const void *data, + enum fi_type datatype); + +enum fi_param_type { + FI_PARAM_STRING, + FI_PARAM_INT, + FI_PARAM_BOOL, + FI_PARAM_SIZE_T, +}; + +struct fi_param { + const char *name; + enum fi_param_type type; + const char *help_string; + const char *value; +}; + +int fi_getparams(struct fi_param **params, int *count); +void fi_freeparams(struct fi_param *params); + +/* Dummy definitions for removed flags/caps/types. For compiling old fabtests */ +#define FI_VARIABLE_MSG 0ULL +#define FI_NOTIFY_FLAGS_ONLY 0ULL +#define FI_RESTRICTED_COMP 0ULL +#define FI_EP_SOCK_STREAM FI_EP_UNSPEC + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_ +struct fi_context { + void *internal[4]; +}; + +struct fi_context2 { + void *internal[8]; +}; +#endif + +struct fi_recv_context { + struct fid_ep *ep; + void *context; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* FABRIC_H */ diff --git a/backends/libfabric/include/rdma/fi_atomic.h b/backends/libfabric/include/rdma/fi_atomic.h new file mode 100644 index 000000000..cc8b1e520 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_atomic.h @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_ATOMIC_H +#define FI_ATOMIC_H + +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Atomic flags */ +#define FI_FETCH_ATOMIC (1ULL << 58) +#define FI_COMPARE_ATOMIC (1ULL << 59) + +struct fi_atomic_attr { + size_t count; + size_t size; +}; + +struct fi_msg_atomic { + const struct fi_ioc *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t addr; + const struct fi_rma_ioc *rma_iov; + size_t rma_iov_count; + enum fi_datatype datatype; + enum fi_op op; + void *context; + uint64_t data; +}; + +struct fi_msg_fetch { + struct fi_ioc *msg_iov; + void **desc; + size_t iov_count; +}; + +struct fi_msg_compare { + const struct fi_ioc *msg_iov; + void **desc; + size_t iov_count; +}; + +struct fi_ops_atomic { + size_t size; + ssize_t (*write)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*writev)(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*writemsg)(struct fid_ep *ep, + const struct fi_msg_atomic *msg, uint64_t flags); + ssize_t (*inject)(struct fid_ep *ep, const void *buf, size_t count, + fi_addr_t dest_addr, uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op); + + ssize_t (*readwrite)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*readwritev)(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*readwritemsg)(struct fid_ep *ep, + const struct fi_msg_atomic *msg, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + uint64_t flags); + + ssize_t (*compwrite)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + const void *compare, void *compare_desc, + void *result, void *result_desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*compwritev)(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + const struct fi_ioc *comparev, void **compare_desc, size_t compare_count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context); + ssize_t (*compwritemsg)(struct fid_ep *ep, + const struct fi_msg_atomic *msg, + const struct fi_ioc *comparev, void **compare_desc, size_t compare_count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + uint64_t flags); + + int (*writevalid)(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count); + int (*readwritevalid)(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count); + int (*compwritevalid)(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count); +}; + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_ATOMIC + +static inline ssize_t +fi_atomic(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->write(ep, buf, count, desc, dest_addr, addr, key, + datatype, op, context); +} + +static inline ssize_t +fi_atomicv(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->writev(ep, iov, desc, count, dest_addr, addr, key, + datatype, op, context); +} + +static inline ssize_t +fi_atomicmsg(struct fid_ep *ep, + const struct fi_msg_atomic *msg, uint64_t flags) +{ + return ep->atomic->writemsg(ep, msg, flags); +} + +static inline ssize_t +fi_inject_atomic(struct fid_ep *ep, const void *buf, size_t count, + fi_addr_t dest_addr, uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op) +{ + return ep->atomic->inject(ep, buf, count, dest_addr, addr, + key, datatype, op); +} + +static inline ssize_t +fi_fetch_atomic(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->readwrite(ep, buf, count, desc, result, result_desc, + dest_addr, addr, key, datatype, op, context); +} + +static inline ssize_t +fi_fetch_atomicv(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->readwritev(ep, iov, desc, count, + resultv, result_desc, result_count, + dest_addr, addr, key, datatype, op, context); +} + +static inline ssize_t +fi_fetch_atomicmsg(struct fid_ep *ep, + const struct fi_msg_atomic *msg, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + uint64_t flags) +{ + return ep->atomic->readwritemsg(ep, msg, resultv, result_desc, + result_count, flags); +} + +static inline ssize_t +fi_compare_atomic(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + const void *compare, void *compare_desc, + void *result, void *result_desc, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->compwrite(ep, buf, count, desc, + compare, compare_desc, result, result_desc, + dest_addr, addr, key, datatype, op, context); +} + +static inline ssize_t +fi_compare_atomicv(struct fid_ep *ep, + const struct fi_ioc *iov, void **desc, size_t count, + const struct fi_ioc *comparev, void **compare_desc, size_t compare_count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + fi_addr_t dest_addr, + uint64_t addr, uint64_t key, + enum fi_datatype datatype, enum fi_op op, void *context) +{ + return ep->atomic->compwritev(ep, iov, desc, count, + comparev, compare_desc, compare_count, + resultv, result_desc, result_count, + dest_addr, addr, key, datatype, op, context); +} + +static inline ssize_t +fi_compare_atomicmsg(struct fid_ep *ep, + const struct fi_msg_atomic *msg, + const struct fi_ioc *comparev, void **compare_desc, size_t compare_count, + struct fi_ioc *resultv, void **result_desc, size_t result_count, + uint64_t flags) +{ + return ep->atomic->compwritemsg(ep, msg, + comparev, compare_desc, compare_count, + resultv, result_desc, result_count, flags); +} + +static inline int +fi_atomicvalid(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count) +{ + return ep->atomic->writevalid(ep, datatype, op, count); +} + +static inline int +fi_fetch_atomicvalid(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count) +{ + return ep->atomic->readwritevalid(ep, datatype, op, count); +} + +static inline int +fi_compare_atomicvalid(struct fid_ep *ep, + enum fi_datatype datatype, enum fi_op op, size_t *count) +{ + return ep->atomic->compwritevalid(ep, datatype, op, count); +} + +static inline int +fi_query_atomic(struct fid_domain *domain, + enum fi_datatype datatype, enum fi_op op, + struct fi_atomic_attr *attr, uint64_t flags) +{ + return FI_CHECK_OP(domain->ops, struct fi_ops_domain, query_atomic) ? + domain->ops->query_atomic(domain, datatype, op, attr, flags) : + -FI_ENOSYS; +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_ATOMIC_H */ diff --git a/backends/libfabric/include/rdma/fi_cm.h b/backends/libfabric/include/rdma/fi_cm.h new file mode 100644 index 000000000..c0faf1f37 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_cm.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2013-2016 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_CM_H +#define FI_CM_H + +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +struct fid_mc { + struct fid fid; + fi_addr_t fi_addr; +}; + +struct fi_ops_cm { + size_t size; + int (*setname)(fid_t fid, void *addr, size_t addrlen); + int (*getname)(fid_t fid, void *addr, size_t *addrlen); + int (*getpeer)(struct fid_ep *ep, void *addr, size_t *addrlen); + int (*connect)(struct fid_ep *ep, const void *addr, + const void *param, size_t paramlen); + int (*listen)(struct fid_pep *pep); + int (*accept)(struct fid_ep *ep, const void *param, size_t paramlen); + int (*reject)(struct fid_pep *pep, fid_t handle, + const void *param, size_t paramlen); + int (*shutdown)(struct fid_ep *ep, uint64_t flags); + int (*join)(struct fid_ep *ep, const void *addr, uint64_t flags, + struct fid_mc **mc, void *context); +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_CM + +static inline int fi_setname(fid_t fid, void *addr, size_t addrlen) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->cm->setname(fid, addr, addrlen); +} + +static inline int fi_getname(fid_t fid, void *addr, size_t *addrlen) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->cm->getname(fid, addr, addrlen); +} + +static inline int fi_getpeer(struct fid_ep *ep, void *addr, size_t *addrlen) +{ + return ep->cm->getpeer(ep, addr, addrlen); +} + +static inline int fi_listen(struct fid_pep *pep) +{ + return pep->cm->listen(pep); +} + +static inline int +fi_connect(struct fid_ep *ep, const void *addr, + const void *param, size_t paramlen) +{ + return ep->cm->connect(ep, addr, param, paramlen); +} + +static inline int +fi_accept(struct fid_ep *ep, const void *param, size_t paramlen) +{ + return ep->cm->accept(ep, param, paramlen); +} + +static inline int +fi_reject(struct fid_pep *pep, fid_t handle, + const void *param, size_t paramlen) +{ + return pep->cm->reject(pep, handle, param, paramlen); +} + +static inline int fi_shutdown(struct fid_ep *ep, uint64_t flags) +{ + return ep->cm->shutdown(ep, flags); +} + +static inline int fi_join(struct fid_ep *ep, const void *addr, uint64_t flags, + struct fid_mc **mc, void *context) +{ + return FI_CHECK_OP(ep->cm, struct fi_ops_cm, join) ? + ep->cm->join(ep, addr, flags, mc, context) : -FI_ENOSYS; +} + +static inline fi_addr_t fi_mc_addr(struct fid_mc *mc) +{ + return mc->fi_addr; +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_CM_H */ diff --git a/backends/libfabric/include/rdma/fi_collective.h b/backends/libfabric/include/rdma/fi_collective.h new file mode 100644 index 000000000..9e7ac6297 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_collective.h @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2019 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_COLLECTIVE_H +#define FI_COLLECTIVE_H + +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + + +struct fi_ops_av_set { + size_t size; + int (*set_union)(struct fid_av_set *dst, + const struct fid_av_set *src); + int (*intersect)(struct fid_av_set *dst, + const struct fid_av_set *src); + int (*diff)(struct fid_av_set *dst, const struct fid_av_set *src); + int (*insert)(struct fid_av_set *set, fi_addr_t addr); + int (*remove)(struct fid_av_set *set, fi_addr_t addr); + int (*addr)(struct fid_av_set *set, fi_addr_t *coll_addr); +}; + +struct fid_av_set { + struct fid fid; + struct fi_ops_av_set *ops; +}; + +struct fi_collective_attr { + enum fi_op op; + enum fi_datatype datatype; + struct fi_atomic_attr datatype_attr; + size_t max_members; + uint64_t mode; +}; + +struct fi_collective_addr { + const struct fid_av_set *set; + fi_addr_t coll_addr; +}; + +struct fi_msg_collective { + const struct fi_ioc *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t coll_addr; + fi_addr_t root_addr; + enum fi_collective_op coll; + enum fi_datatype datatype; + enum fi_op op; + void *context; +}; + +struct fi_ops_collective { + size_t size; + + ssize_t (*barrier)(struct fid_ep *ep, fi_addr_t coll_addr, void *context); + ssize_t (*broadcast)(struct fid_ep *ep, + void *buf, size_t count, void *desc, + fi_addr_t coll_addr, fi_addr_t root_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*alltoall)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*allreduce)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context); + ssize_t (*allgather)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*reduce_scatter)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context); + ssize_t (*reduce)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + fi_addr_t root_addr, enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context); + ssize_t (*scatter)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t coll_addr, fi_addr_t root_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*gather)(struct fid_ep *ep, + const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t coll_addr, fi_addr_t root_addr, + enum fi_datatype datatype, uint64_t flags, void *context); + ssize_t (*msg)(struct fid_ep *ep, + const struct fi_msg_collective *msg, + struct fi_ioc *resultv, void **result_desc, + size_t result_count, uint64_t flags); + ssize_t (*barrier2)(struct fid_ep *ep, fi_addr_t coll_addr, uint64_t flags, + void *context); +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_COLLECTIVE + +static inline int +fi_av_set(struct fid_av *av, struct fi_av_set_attr *attr, + struct fid_av_set **set, void * context) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, av_set) ? + av->ops->av_set(av, attr, set, context) : -FI_ENOSYS; +} + +static inline int +fi_av_set_union(struct fid_av_set *dst, const struct fid_av_set *src) +{ + return dst->ops->set_union(dst, src); +} + +static inline int +fi_av_set_intersect(struct fid_av_set *dst, const struct fid_av_set *src) +{ + return dst->ops->intersect(dst, src); +} + +static inline int +fi_av_set_diff(struct fid_av_set *dst, const struct fid_av_set *src) +{ + return dst->ops->diff(dst, src); +} + +static inline int +fi_av_set_insert(struct fid_av_set *set, fi_addr_t addr) +{ + return set->ops->insert(set, addr); +} + +static inline int +fi_av_set_remove(struct fid_av_set *set, fi_addr_t addr) +{ + return set->ops->remove(set, addr); +} + +static inline int +fi_av_set_addr(struct fid_av_set *set, fi_addr_t *coll_addr) +{ + return set->ops->addr(set, coll_addr); +} + +static inline int +fi_join_collective(struct fid_ep *ep, fi_addr_t coll_addr, + const struct fid_av_set *set, + uint64_t flags, struct fid_mc **mc, void *context) +{ + struct fi_collective_addr addr; + + addr.set = set; + addr.coll_addr = coll_addr; + return fi_join(ep, &addr, flags | FI_COLLECTIVE, mc, context); +} + +static inline ssize_t +fi_barrier(struct fid_ep *ep, fi_addr_t coll_addr, void *context) +{ + return ep->collective->barrier(ep, coll_addr, context); +} + +static inline ssize_t +fi_barrier2(struct fid_ep *ep, fi_addr_t coll_addr, uint64_t flags, void *context) +{ + if (!flags) + return fi_barrier(ep, coll_addr, context); + + return FI_CHECK_OP(ep->collective, struct fi_ops_collective, barrier2) ? + ep->collective->barrier2(ep, coll_addr, flags, context) : + -FI_ENOSYS; +} + +static inline ssize_t +fi_broadcast(struct fid_ep *ep, void *buf, size_t count, void *desc, + fi_addr_t coll_addr, fi_addr_t root_addr, + enum fi_datatype datatype, uint64_t flags, void *context) +{ + return ep->collective->broadcast(ep, buf, count, desc, + coll_addr, root_addr, datatype, flags, context); +} + +static inline ssize_t +fi_alltoall(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, + fi_addr_t coll_addr, enum fi_datatype datatype, + uint64_t flags, void *context) +{ + return ep->collective->alltoall(ep, buf, count, desc, + result, result_desc, coll_addr, datatype, flags, context); +} + +static inline ssize_t +fi_allreduce(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context) +{ + return ep->collective->allreduce(ep, buf, count, desc, + result, result_desc, coll_addr, datatype, op, flags, context); +} + +static inline ssize_t +fi_allgather(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, uint64_t flags, void *context) +{ + return ep->collective->allgather(ep, buf, count, desc, + result, result_desc, coll_addr, datatype, flags, context); +} + +static inline ssize_t +fi_reduce_scatter(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context) +{ + return ep->collective->reduce_scatter(ep, buf, count, desc, + result, result_desc, coll_addr, datatype, op, flags, context); +} + +static inline ssize_t +fi_reduce(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + fi_addr_t root_addr, enum fi_datatype datatype, enum fi_op op, + uint64_t flags, void *context) +{ + return ep->collective->reduce(ep, buf, count, desc, result, result_desc, + coll_addr, root_addr, datatype, op, flags, context); +} + + +static inline ssize_t +fi_scatter(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + fi_addr_t root_addr, enum fi_datatype datatype, + uint64_t flags, void *context) +{ + return ep->collective->scatter(ep, buf, count, desc, result, result_desc, + coll_addr, root_addr, datatype, flags, context); +} + + +static inline ssize_t +fi_gather(struct fid_ep *ep, const void *buf, size_t count, void *desc, + void *result, void *result_desc, fi_addr_t coll_addr, + fi_addr_t root_addr, enum fi_datatype datatype, + uint64_t flags, void *context) +{ + return ep->collective->gather(ep, buf, count, desc, result, result_desc, + coll_addr, root_addr, datatype, flags, context); +} + +static inline +int fi_query_collective(struct fid_domain *domain, enum fi_collective_op coll, + struct fi_collective_attr *attr, uint64_t flags) +{ + return FI_CHECK_OP(domain->ops, struct fi_ops_domain, query_collective) ? + domain->ops->query_collective(domain, coll, attr, flags) : + -FI_ENOSYS; +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_COLLECTIVE_H */ diff --git a/backends/libfabric/include/rdma/fi_cxi_ext.h b/backends/libfabric/include/rdma/fi_cxi_ext.h new file mode 100644 index 000000000..b69bb0a20 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_cxi_ext.h @@ -0,0 +1,539 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only + * + * Copyright (c) 2020-2024 Hewlett Packard Enterprise Development LP + */ + +#ifndef _FI_CXI_EXT_H_ +#define _FI_CXI_EXT_H_ + +/* CXI provider specific NIC attributes. This information is returned in + * fi_info::nid::prov_attr. + + * Users can optionally modify some fields. Depending on the field adjusted, + * this can impact fi_domain() or other endpoint allocation behavior. + */ + +#define FI_CXI_NIC_ATTR_VER 1U + +struct cxip_nic_attr { + /* Version of NIC attr. Must remain at the top of this struct. */ + uint32_t version; + + /* NIC address. Should never be modified. */ + const unsigned int addr; + + /* On output from fi_getinfo(), rgroup_id will be set in the following + * order: + * 1. Resource group ID returned from SLINGSHOT_SVC_ID environment + * variable + * 2. First resource group ID with matching UID + * 3. First resource group ID with matching GID + * 4. First resource group ID with open permissions + */ + const unsigned int default_rgroup_id; + + /* Default VNI used with the rgroup ID. */ + const unsigned int default_vni; +}; + +/* + * TODO: The following should be integrated into the include/rdma/fi_ext.h + * and are use for provider specific fi_control() operations. + */ +#define FI_PROV_SPECIFIC_CXI (0xccc << 16) + +enum { + FI_OPT_CXI_SET_TCLASS = -FI_PROV_SPECIFIC_CXI, /* uint32_t */ + FI_OPT_CXI_SET_MSG_ORDER, /* uint64_t */ + + /* fid_nic control operation to refresh NIC attributes. */ + FI_OPT_CXI_NIC_REFRESH_ATTR, + + FI_OPT_CXI_SET_MR_MATCH_EVENTS, /* bool */ + FI_OPT_CXI_GET_MR_MATCH_EVENTS, /* bool */ + FI_OPT_CXI_SET_OPTIMIZED_MRS, /* bool */ + FI_OPT_CXI_GET_OPTIMIZED_MRS, /* bool */ + FI_OPT_CXI_SET_PROV_KEY_CACHE, /* bool */ + FI_OPT_CXI_GET_PROV_KEY_CACHE, /* bool */ + FI_OPT_CXI_SET_RNR_MAX_RETRY_TIME, /* uint64_t */ +}; + +/* + * Defines for compatibility between main branch features and release + * branches back porting of features required for use. Any value + * included here should map exactly to the value established in the + * main branch (enum or define) and this CXI equivalent will exist forever. + */ +#define FI_CXI_CNTR_EVENTS_BYTES 1 /* FI_CNTR_EVENTS_BYTES */ + +/* + * CXI provider specific counter flag to return current/cached counter value + * in host memory. A request to update the count is requested, but the routine + * does not wait for the update to complete. Subsequent reads will pick up + * the updated counter value. The normal behavior is to wait for a memory update + * to complete (or to use the domain ops counter routines). + */ +#define FI_CXI_CNTR_CACHED (1ULL << 32) + +/* + * TODO: Set this to the upstream value prior to releasing software. + * This flag returned in a completion and indicates that the message was + * truncated and that the length indicates the truncated message length. + */ +#define FI_CXI_TRUNC (1ULL << 56) + +/* + * Execute a given libfabric atomic memory operation as a PCIe operation as + * compared to a NIC operation. + * + * Note: Ordering between PCIe atomic operations and NIC atomic/RMA operations + * is undefined. + * + * Note: This flag overloads the bit used for FI_SOURCE. But, since FI_SOURCE + * is invalid for AMO operations, overloading this bit is not an issue. + */ +#define FI_CXI_PCIE_AMO (1ULL << 57) + +/* + * Flag an accelerated collective as pre-reduced. + * + * This can be passed to the accelerated collectives operations to indicate + * that the supplied data is a pre-reduced cxip_coll_accumulator structure. + * + * Note: This flag overloads FI_CXI_PCIE_AMO. Accelerated collectives do not + * use FI_CXI_PCIE_AMO or FI_SOURCE. + */ +#define FI_CXI_PRE_REDUCED (1ULL << 57) + +/* + * Use CXI High Rate Puts (HRP). Increases message rate performance. Applies to + * RMA and unreliable, non-fetching AMO operations. + */ +#define FI_CXI_HRP (1ULL << 60) + +/* + * Disable AMO reliability. Increases message rate performance. Applies to + * non-fetching AMOs. Required for HRP AMOs. + */ +#define FI_CXI_UNRELIABLE (1ULL << 61) + +/* Depreciated. */ +#define FI_CXI_WEAK_FENCE \ + _Pragma ("GCC warning \"'FI_CXI_WEAK_FENCE' macro is deprecated\"") \ + (1ULL << 63) + +/* + * Used in conjunction with the deferred work queue API. If a deferred work + * queue operation has this flag set, the CXI provider will ensure a counter + * writeback occurs once the deferred work queue operation completes. + * Note: Addition hardware resources will be used to ensure a counter writeback + * occurs at the completion of the deferred work queue operation. + */ +#define FI_CXI_CNTR_WB (1ULL << 62) +#define FI_CXI_COUNTER_OPS "cxi_counter_ops" + +struct fi_cxi_cntr_ops { + /* Set the counter writeback address to a client provided address. */ + int (*set_wb_buffer)(struct fid *fid, void *buf, size_t len); + + /* Get the counter MMIO region. */ + int (*get_mmio_addr)(struct fid *fid, void **addr, size_t *len); +}; + +/* Success values cannot exceed FI_CXI_CNTR_SUCCESS_MAX */ +#define FI_CXI_CNTR_SUCCESS_MAX ((1ULL << 48) - 1) + +/* Failure values cannot exceed FI_CXI_CNTR_FAILURE_MAX */ +#define FI_CXI_CNTR_FAILURE_MAX ((1ULL << 7) - 1) + +/* fi_cntr_read() equivalent but for the writeback buffer. */ +static inline uint64_t fi_cxi_cntr_wb_read(const void *wb_buf) +{ + return (*(uint64_t *)wb_buf) & FI_CXI_CNTR_SUCCESS_MAX; +}; + +/* fi_cntr_reader() equivalent but for the writeback buffer. */ +static inline uint64_t fi_cxi_cntr_wb_readerr(const void *wb_buf) +{ + return ((*(uint64_t *)wb_buf) >> 48) & FI_CXI_CNTR_FAILURE_MAX; +}; + +/* Generate a counter success value which can be polled on. */ +static inline int fi_cxi_gen_cntr_success(uint64_t value, uint64_t *cxi_value) +{ + if (value > FI_CXI_CNTR_SUCCESS_MAX) + return -FI_EINVAL; + + *cxi_value = (1ULL << 63) | value; + return FI_SUCCESS; +}; + +/* fi_cntr_add() equivalent but for the MMIO region. */ +static inline int fi_cxi_cntr_add(void *cntr_mmio, uint64_t value) +{ + /* Success counter is only 48 bits wide. */ + if (value > FI_CXI_CNTR_SUCCESS_MAX) + return -FI_EINVAL; + + *((uint64_t *)cntr_mmio) = value; + return FI_SUCCESS; +} + +/* fi_cntr_adderr() equivalent but for the MMIO region. */ +static inline int fi_cxi_cntr_adderr(void *cntr_mmio, uint64_t value) +{ + /* Error counter is only 7 bits wide. */ + if (value > FI_CXI_CNTR_FAILURE_MAX) + return -FI_EINVAL; + + *((uint64_t *)cntr_mmio + 8) = value; + return FI_SUCCESS; +} + +/* fi_cntr_set() equivalent but for the MMIO region. */ +static inline int fi_cxi_cntr_set(void *cntr_mmio, uint64_t value) +{ + /* Only set of zero is supported through MMIO region. */ + if (value > 0) + return -FI_EINVAL; + + *((uint64_t *)cntr_mmio + 16) = 0; + return FI_SUCCESS; +} + +/* fi_cntr_seterr() equivalent but for MMIO region. */ +static inline int fi_cxi_cntr_seterr(void *cntr_mmio, uint64_t value) +{ + /* Only set of zero is supported through MMIO region. */ + if (value > 0) + return -FI_EINVAL; + + *((uint64_t *)cntr_mmio + 24) = 0; + return FI_SUCCESS; +} + +/* fi_cntr_add() equivalent but for the MMIO region. */ +static inline void *fi_cxi_get_cntr_add_addr(void *cntr_mmio) +{ + return cntr_mmio; +} + +/* fi_cntr_adderr() equivalent but for the MMIO region. */ +static inline void *fi_cxi_get_cntr_adderr_addr(void *cntr_mmio) +{ + return (void *)((uint64_t *)cntr_mmio + 8); +} + +/* fi_cntr_set() equivalent but for the MMIO region reset. + * NOTE: CXI does not support set to counter MMIO region. Only reset. + */ +static inline void *fi_cxi_get_cntr_reset_addr(void *cntr_mmio) +{ + return (void *)((uint64_t *)cntr_mmio + 16); +} + +/* fi_cntr_seterr() equivalent but for MMIO region reset. + * NOTE: CXI does not support set to counter MMIO region. Only reset. + */ +static inline void *fi_cxi_get_cntr_reseterr_addr(void *cntr_mmio) +{ + return (void *)((uint64_t *)cntr_mmio + 24); +} + +#define FI_CXI_DOM_OPS_1 "dom_ops_v1" +#define FI_CXI_DOM_OPS_2 "dom_ops_v2" +#define FI_CXI_DOM_OPS_3 "dom_ops_v3" +#define FI_CXI_DOM_OPS_4 "dom_ops_v4" +#define FI_CXI_DOM_OPS_5 "dom_ops_v5" +#define FI_CXI_DOM_OPS_6 "dom_ops_v6" + +/* v1 to v6 can use the same struct since they only appended a routine */ +struct fi_cxi_dom_ops { + int (*cntr_read)(struct fid *fid, unsigned int cntr, uint64_t *value, + struct timespec *ts); + int (*topology)(struct fid *fid, unsigned int *group_id, + unsigned int *switch_id, unsigned int *port_id); + + /* Enable hybrid MR desc mode. Hybrid MR desc allows for libfabric users + * to optionally pass in a valid MR desc for local communication + * operations. + * + * When enabled, if the MR desc is NULL, the provider will + * perform internal memory registration. Else, the provider will assume + * the MR desc field is valid and skip internal memory registration. + * + * When disabled, the provider will ignore the MR desc field and always + * perform internal memory registration. This is the default behavior. + * + * All child endpoints will inherit the current domain status of hybrid + * MR desc only during endpoint creation. Dynamically changing the + * domain hybrid MR desc status with endpoint allocate may not propagate + * to child endpoints. Thus, it is recommended to set hybrid MR desc + * status prior to allocating endpoints. + */ + int (*enable_hybrid_mr_desc)(struct fid *fid, bool enable); + + /* Get unexpected message information. + * + * Obtain a list of unexpected messages associated with the endpoint. + * The list is returned as an array of CQ tagged entries. The following + * is how the fields in fi_cq_tagged_entry are used. + * + * op_context: NULL since this message has not matched a posted receive + * flags: A combination of FI_MSG, FI_TAGGED, FI_RECV, + * and/or FI_REMOTE_CQ_DATA + * len: Unexpected message request length + * data: Completion queue data (only valid if FI_REMOTE_CQ_DATA + * is set) + * tag: Unexpected message tag (only valid if FI_TAGGED is set) + * + * @ep: Endpoint FID to have unexpected messages returned to user. + * @entry: Tagged entry array to be filled in by the provider. If the + * entry is NULL, only ux_count will be set. + * @count: Number of entries in entry and src_addr array. If count is + * zero,then only the ux_count will be set on return. + * @src_addr: Source address array to be filled in by the provider. If + * the entry is NULL, only ux_count will be set. + * @ux_count: Output variable used to return the number of unexpected + * messages queued on the given endpoint. + * + * Return: On success, number of entries copied into the users entry + * and src_addr arrays. On error, -FI_ERRNO. + */ + size_t (*ep_get_unexp_msgs)(struct fid_ep *fid_ep, + struct fi_cq_tagged_entry *entry, + size_t count, fi_addr_t *src_addr, + size_t *ux_count); + + /* Get the depth of the deferred work queue. The depth is the number of + * of triggered operation commands which can be queued to hardware. The + * depth is not per fi_domain. The depth is across all processes using + * the same CXI service which usually maps to a job-step. + */ + int (*get_dwq_depth)(struct fid *fid, size_t *depth); + + /* The following two functions have been deprecated in favor of + * using the fi_control() standardized interface. They will be + * removed in a future software release, but are left here initially + * to allow early users to adjust their usage. + */ + int (*enable_mr_match_events)(struct fid *fid, bool enable); + int (*enable_optimized_mrs)(struct fid *fid, bool enable); +}; + +/* + * CXI Authorization Key + */ +struct cxi_auth_key { + /* The CXI service assigned to the Domain and Endpoints. A CXI service + * is associated with a set of local resource limits, VNIs, and Traffic + * Classes. + * + * The svc_id used by an OFI Domain must match all Endpoints belonging + * to the Domain. + */ + uint32_t svc_id; + + /* The Virtual Network ID (VNI) assigned to the Endpoint. Two Endpoints + * must use the same VNI in order to communicate. + * + * Note that while the CXI service may define one or more VNIs which a + * process can access, an Endpoint is assigned to only one. + */ + uint16_t vni; +}; + +/* + * CXI Collectives + */ + +/* + * AV Set communication key. + * + * For production: + * - Set cxip_comm_key.keytype = COMM_KEY_NONE. + * - Initialize cxip_comm_key structure to zeros. + * - Create one av_set on each node. + * - Initialize each av_set to contain the NIC addresses of all endpoints. + * - Call fi_join_collective() once on each endpoint. + * - dest_addr is a multicast address created by the join. + * - hwroot_nic is assigned by the join. + * - The PTE will receive at the multicast ID value, index extension of zero. + * - Sending to the multicast ID will cause delivery to nodes according to the + * tree topology. + * + * For testing with externally established multicast address: + * - NOT IMPLEMENTED. + * + * For testing on a multinode system without multicast: + * - Set cxip_comm_key.keytype = COMM_KEY_UNICAST. + * - Set cxip_comm_key.ucast.hwroot_idx to the desired hw_root index. + * - Create one av_set on each node. + * - Initialize each av_set to contain the NIC addresses of all endpoints. + * - Call fi_join_collective() once one each endpoint. + * - hwroot_nic is the NIC address of the node that serves as the emulated + * hardware root of the tree. + * - The PTE will use the EP source NIC address and process PID, with a + * PID_IDX of CXIP_PTL_IDX_COLL. + * - Sending to any (valid) node address with CXIP_PTL_IDX_COLL will target the + * collectives PTE on that node. + * - The root/leaf send routines will distribute one or more packets to all + * fi_addr_t in the av_set as appropriate. + * + * For testing under NETSIM on a single node: + * - Set cxip_comm_key.keytype = COMM_KEY_RANK. + * - Set cxip_comm_key.rank.hwroot_idx to the desired hw_root index. + * - Set cxip_comm_key.rank.rank to the simulated rank. + * - Create N av_set objects, one for each simulated rank. + * - Call fi_join_collective() once for each simulated endpoint. + * - dest_addr is the MC object index. + * - hwroot_nic is the MC object index for the MC object to serve as the + * simulated hardware root. + * - The PTE will use the EP source NIC address and process PID, with a PID_IDX + * of 16 + dest_addr (MC object index). + * - Sending to the node's own address with a PID_IDX of 16 + MC index will + * target the appropriate MC object. + * - Simulation is limited to 32 simulated endpoints. + */ +enum cxip_comm_key_type { + COMM_KEY_NONE = 0, + COMM_KEY_MULTICAST, + COMM_KEY_UNICAST, + COMM_KEY_RANK, + COMM_KEY_MAX +}; + +/* Extends C_RC_* driver errors for libfabric */ +/* Translated to strings by cxip_strerror() -- keep synchronized */ +enum cxip_coll_prov_errno { + /* C_RC_* from cxi-driver overlaps first 6 bits of space [0,63] */ + + /* collectives CQ reduction error codes + * highest number error predominates + */ + FI_CXI_ERRNO_RED_FIRST = 1024, + FI_CXI_ERRNO_RED_FLT_OVERFLOW = 1024, + /* double precision value overflow */ + FI_CXI_ERRNO_RED_FLT_INVALID = 1025, + /* double precision sNAN/inf value */ + FI_CXI_ERRNO_RED_INT_OVERFLOW = 1026, + /* reproducible sum overflow */ + FI_CXI_ERRNO_RED_CONTR_OVERFLOW = 1027, + /* reduction contribution overflow */ + FI_CXI_ERRNO_RED_OP_MISMATCH = 1028, + /* reduction opcode mismatch */ + FI_CXI_ERRNO_RED_MC_FAILURE = 1029, + /* unused */ + FI_CXI_COLL_RC_RDMA_FAILURE = 1030, + /* leaf rdma read error */ + FI_CXI_COLL_RC_RDMA_DATA_FAILURE = 1031, + /* leaf rdma read data miscompare, unexpected packet data */ + FI_CXI_ERRNO_RED_OTHER = 1032, + /* non-specific reduction error, fatal */ + FI_CXI_ERRNO_RED_LAST = 1033, + + /* collectives EQ join error codes + * highest number error predominates + */ + FI_CXI_ERRNO_JOIN_FIRST = 2048, + FI_CXI_ERRNO_JOIN_MCAST_INUSE = 2048, + /* endpoint already using mcast address */ + FI_CXI_ERRNO_JOIN_HWROOT_INUSE = 2049, + /* endpoint already serving as HWRoot */ + FI_CXI_ERRNO_JOIN_MCAST_INVALID = 2050, + /* mcast address from FM is invalid */ + FI_CXI_ERRNO_JOIN_HWROOT_INVALID = 2051, + /* HWRoot address from FM is invalid */ + FI_CXI_ERRNO_JOIN_CURL_FAILED = 2052, + /* libcurl initiation failed */ + FI_CXI_ERRNO_JOIN_CURL_TIMEOUT = 2053, + /* libcurl timed out */ + FI_CXI_ERRNO_JOIN_SERVER_ERR = 2054, + /* unhandled CURL response code */ + FI_CXI_ERRNO_JOIN_FAIL_PTE = 2055, + /* libfabric PTE allocation failed */ + FI_CXI_ERRNO_JOIN_OTHER = 2056, + /* non-specific JOIN error, fatal */ + FI_CXI_ERRNO_JOIN_FAIL_RDMA = 2057, + /* root or leaf rdma init failure */ + FI_CXI_ERRNO_JOIN_LAST = FI_CXI_ERRNO_JOIN_FIRST + 43, + /* LAST is determined by the 43-bit error mask . + * Result is the OR of all bits set by different endpoints. + * This reserves space for all 43 bits for new errors. + */ +}; + +typedef unsigned int cxip_coll_op_t; // CXI collective opcode + +struct cxip_coll_mcast_key { + uint32_t hwroot_idx; // index of hwroot in av_set list + uint32_t mcast_addr; // 13-bit multicast address id +}; + +struct cxip_coll_unicast_key { + uint32_t hwroot_idx; // index of hwroot in av_set list + uint32_t mcast_addr; // 13-bit simulated multcast address +}; + +struct cxip_coll_rank_key { + uint32_t hwroot_idx; // index of hwroot in av_set list + uint32_t rank; // rank of this object + bool rx_discard; // clear to report RX events +}; + +struct cxip_comm_key { + enum cxip_comm_key_type keytype; + union { + struct cxip_coll_mcast_key mcast; + struct cxip_coll_unicast_key ucast; + struct cxip_coll_rank_key rank; + }; +}; + +/* Extended reduction opcodes. + * + * Only the following standard FI_ATOMIC operations are supported: + * - FI_MIN : INT or FLT + * - FI_MAX : INT or FLT + * - FI_SUM : INT or FLT + * - FI_BOR : INT + * - FI_BAND : INT + * - FI_BXOR : INT + * + * The codes below extend this standard FI_ATOMIC set to explicitly take + * advantage of extended hardware operations. These can be used as opcodes for + * any of the collective operations, just like FI_MIN or FI_SUM. + * + * Note that the current FI_ATOMIC set ends at opcode == 19. We start this one + * at 32, to accommodate possible expansion of the FI_ATOMIC set, and check for + * overlap during initialization. + */ +enum cxip_coll_op { + FI_CXI_MINMAXLOC = 32, // FLT or INT + FI_CXI_REPSUM, // FLT only + FI_CXI_OP_LAST +}; + +/* Extended accelerated reduction structures. + */ +struct cxip_coll_intminmax { + int64_t minval; + uint64_t minidx; + int64_t maxval; + uint64_t maxidx; +}; + +struct cxip_coll_fltminmax { + double minval; + uint64_t minidx; + double maxval; + uint64_t maxidx; +}; + +/* opaque export of struct cxip_coll_data */ +struct cxip_coll_accumulator { + uint8_t accum[64]; +}; + +#endif /* _FI_CXI_EXT_H_ */ diff --git a/backends/libfabric/include/rdma/fi_domain.h b/backends/libfabric/include/rdma/fi_domain.h new file mode 100644 index 000000000..0e6d0acb6 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_domain.h @@ -0,0 +1,598 @@ +/* + * Copyright (c) 2013-2017 Intel Corporation. All rights reserved. + * (C) Copyright 2020 Hewlett Packard Enterprise Development LP + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_DOMAIN_H +#define FI_DOMAIN_H + +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* + * AV = Address Vector + * Maps and stores transport/network addresses. + */ + +#define FI_SYMMETRIC (1ULL << 59) +#define FI_SYNC_ERR (1ULL << 58) +#define FI_UNIVERSE (1ULL << 57) +#define FI_BARRIER_SET (1ULL << 40) +#define FI_BROADCAST_SET (1ULL << 41) +#define FI_ALLTOALL_SET (1ULL << 42) +#define FI_ALLREDUCE_SET (1ULL << 43) +#define FI_ALLGATHER_SET (1ULL << 44) +#define FI_REDUCE_SCATTER_SET (1ULL << 45) +#define FI_REDUCE_SET (1ULL << 46) +#define FI_SCATTER_SET (1ULL << 47) +#define FI_GATHER_SET (1ULL << 48) + +struct fi_av_attr { + enum fi_av_type type; + int rx_ctx_bits; + size_t count; + size_t ep_per_node; + const char *name; + void *map_addr; + uint64_t flags; +}; + +struct fi_av_set_attr { + size_t count; + fi_addr_t start_addr; + fi_addr_t end_addr; + uint64_t stride; + size_t comm_key_size; + uint8_t *comm_key; + uint64_t flags; +}; + +struct fid_av_set; + +struct fi_ops_av { + size_t size; + int (*insert)(struct fid_av *av, const void *addr, size_t count, + fi_addr_t *fi_addr, uint64_t flags, void *context); + int (*insertsvc)(struct fid_av *av, const char *node, + const char *service, fi_addr_t *fi_addr, + uint64_t flags, void *context); + int (*insertsym)(struct fid_av *av, const char *node, size_t nodecnt, + const char *service, size_t svccnt, fi_addr_t *fi_addr, + uint64_t flags, void *context); + int (*remove)(struct fid_av *av, fi_addr_t *fi_addr, size_t count, + uint64_t flags); + int (*lookup)(struct fid_av *av, fi_addr_t fi_addr, void *addr, + size_t *addrlen); + const char * (*straddr)(struct fid_av *av, const void *addr, + char *buf, size_t *len); + int (*av_set)(struct fid_av *av, struct fi_av_set_attr *attr, + struct fid_av_set **av_set, void *context); + int (*insert_auth_key)(struct fid_av *av, const void *auth_key, + size_t auth_key_size, fi_addr_t *fi_addr, + uint64_t flags); + int (*lookup_auth_key)(struct fid_av *av, fi_addr_t fi_addr, + void *auth_key, size_t *auth_key_size); + int (*set_user_id)(struct fid_av *av, fi_addr_t fi_addr, + fi_addr_t user_id, uint64_t flags); +}; + +struct fid_av { + struct fid fid; + struct fi_ops_av *ops; +}; + + +/* + * MR = Memory Region + * Tracks registered memory regions, primarily for remote access, + * but also for local access until we can remove that need. + */ + +#define FI_MR_DMABUF (1ULL << 40) +#define FI_MR_SINGLE_USE (1ULL << 41) +#define FI_HMEM_HOST_ALLOC (1ULL << 45) +#define FI_HMEM_DEVICE_ONLY (1ULL << 46) + +struct fid_mr { + struct fid fid; + void *mem_desc; + uint64_t key; +}; + +enum fi_hmem_iface { + FI_HMEM_SYSTEM = 0, + FI_HMEM_CUDA, + FI_HMEM_ROCR, + FI_HMEM_ZE, + FI_HMEM_NEURON, + FI_HMEM_SYNAPSEAI, +}; + +static inline int fi_hmem_ze_device(int driver_index, int device_index) +{ + return driver_index << 16 | device_index; +} + +struct fi_mr_dmabuf { + int fd; + uint64_t offset; + size_t len; + void *base_addr; +}; + +struct fi_mr_auth_key { + struct fid_av *av; + fi_addr_t src_addr; +}; + +struct fi_mr_attr { + union { + const struct iovec *mr_iov; + const struct fi_mr_dmabuf *dmabuf; + }; + size_t iov_count; + uint64_t access; + uint64_t offset; + uint64_t requested_key; + void *context; + size_t auth_key_size; + uint8_t *auth_key; + enum fi_hmem_iface iface; + union { + uint64_t reserved; + int cuda; + int ze; + int neuron; + int synapseai; + } device; + void *hmem_data; + size_t page_size; + const struct fid_mr *base_mr; + size_t sub_mr_cnt; +}; + +struct fi_mr_modify { + uint64_t flags; + struct fi_mr_attr attr; +}; + +#define FI_SET_OPS_HMEM_OVERRIDE "hmem_override_ops" + +struct fi_hmem_override_ops { + size_t size; + + ssize_t (*copy_from_hmem_iov)(void *dest, size_t size, + enum fi_hmem_iface iface, uint64_t device, + const struct iovec *hmem_iov, + size_t hmem_iov_count, + uint64_t hmem_iov_offset); + + ssize_t (*copy_to_hmem_iov)(enum fi_hmem_iface iface, uint64_t device, + const struct iovec *hmem_iov, + size_t hmem_iov_count, + uint64_t hmem_iov_offset, const void *src, + size_t size); +}; + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_ATOMIC_DEF + +#define FI_COLLECTIVE_OFFSET 256 + +enum fi_datatype { + FI_INT8, + FI_UINT8, + FI_INT16, + FI_UINT16, + FI_INT32, + FI_UINT32, + FI_INT64, + FI_UINT64, + FI_FLOAT, + FI_DOUBLE, + FI_FLOAT_COMPLEX, + FI_DOUBLE_COMPLEX, + FI_LONG_DOUBLE, + FI_LONG_DOUBLE_COMPLEX, + FI_INT128, + FI_UINT128, + FI_FLOAT16, + FI_BFLOAT16, + FI_FLOAT8_E4M3, + FI_FLOAT8_E5M2, + + /* Collective datatypes */ + FI_VOID = FI_COLLECTIVE_OFFSET, +}; + +enum fi_op { + FI_MIN, + FI_MAX, + FI_SUM, + FI_PROD, + FI_LOR, + FI_LAND, + FI_BOR, + FI_BAND, + FI_LXOR, + FI_BXOR, + FI_ATOMIC_READ, + FI_ATOMIC_WRITE, + FI_CSWAP, + FI_CSWAP_NE, + FI_CSWAP_LE, + FI_CSWAP_LT, + FI_CSWAP_GE, + FI_CSWAP_GT, + FI_MSWAP, + FI_DIFF, + + /* Collective datatypes */ + FI_NOOP = FI_COLLECTIVE_OFFSET, +}; + +#endif + +#ifndef FABRIC_DIRECT_COLLECTIVE_DEF + +enum fi_collective_op { + FI_BARRIER, + FI_BROADCAST, + FI_ALLTOALL, + FI_ALLREDUCE, + FI_ALLGATHER, + FI_REDUCE_SCATTER, + FI_REDUCE, + FI_SCATTER, + FI_GATHER, +}; + +#endif + + +struct fi_atomic_attr; +struct fi_cq_attr; +struct fi_cntr_attr; +struct fi_collective_attr; + +struct fi_ops_domain { + size_t size; + int (*av_open)(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context); + int (*cq_open)(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context); + int (*endpoint)(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context); + int (*scalable_ep)(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **sep, void *context); + int (*cntr_open)(struct fid_domain *domain, struct fi_cntr_attr *attr, + struct fid_cntr **cntr, void *context); + int (*poll_open)(struct fid_domain *domain, struct fi_poll_attr *attr, + struct fid_poll **pollset); + int (*stx_ctx)(struct fid_domain *domain, + struct fi_tx_attr *attr, struct fid_stx **stx, + void *context); + int (*srx_ctx)(struct fid_domain *domain, + struct fi_rx_attr *attr, struct fid_ep **rx_ep, + void *context); + int (*query_atomic)(struct fid_domain *domain, + enum fi_datatype datatype, enum fi_op op, + struct fi_atomic_attr *attr, uint64_t flags); + int (*query_collective)(struct fid_domain *domain, + enum fi_collective_op coll, + struct fi_collective_attr *attr, uint64_t flags); + int (*endpoint2)(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, uint64_t flags, void *context); +}; + +/* Memory registration flags */ +/* #define FI_RMA_EVENT (1ULL << 56) */ + +struct fi_ops_mr { + size_t size; + int (*reg)(struct fid *fid, const void *buf, size_t len, + uint64_t access, uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context); + int (*regv)(struct fid *fid, const struct iovec *iov, + size_t count, uint64_t access, + uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context); + int (*regattr)(struct fid *fid, const struct fi_mr_attr *attr, + uint64_t flags, struct fid_mr **mr); +}; + +/* Domain bind flags */ +#define FI_REG_MR _Pragma("GCC warning \"'FI_REG_MR' is deprecated\"") (1ULL << 59) + +struct fid_domain { + struct fid fid; + struct fi_ops_domain *ops; + struct fi_ops_mr *mr; +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_DOMAIN + +static inline int +fi_domain(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **domain, void *context) +{ + return fabric->ops->domain(fabric, info, domain, context); +} + +static inline int +fi_domain2(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **domain, uint64_t flags, void *context) +{ + if (!flags) + return fi_domain(fabric, info, domain, context); + + return FI_CHECK_OP(fabric->ops, struct fi_ops_fabric, domain2) ? + fabric->ops->domain2(fabric, info, domain, flags, context) : + -FI_ENOSYS; +} + +static inline int +fi_domain_bind(struct fid_domain *domain, struct fid *fid, uint64_t flags) +{ + return domain->fid.ops->bind(&domain->fid, fid, flags); +} + +static inline int +fi_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context) +{ + return domain->ops->cq_open(domain, attr, cq, context); +} + +static inline int +fi_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr, + struct fid_cntr **cntr, void *context) +{ + return domain->ops->cntr_open(domain, attr, cntr, context); +} + +static inline FI_DEPRECATED_FUNC int +fi_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr, + struct fid_wait **waitset) +{ + return fabric->ops->wait_open(fabric, attr, waitset); +} + +static inline FI_DEPRECATED_FUNC int +fi_poll_open(struct fid_domain *domain, struct fi_poll_attr *attr, + struct fid_poll **pollset) +{ + return domain->ops->poll_open(domain, attr, pollset); +} + +static inline int +fi_mr_reg(struct fid_domain *domain, const void *buf, size_t len, + uint64_t acs, uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context) +{ + return domain->mr->reg(&domain->fid, buf, len, acs, offset, + requested_key, flags, mr, context); +} + +static inline int +fi_mr_regv(struct fid_domain *domain, const struct iovec *iov, + size_t count, uint64_t acs, + uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context) +{ + return domain->mr->regv(&domain->fid, iov, count, acs, + offset, requested_key, flags, mr, context); +} + +static inline int +fi_mr_regattr(struct fid_domain *domain, const struct fi_mr_attr *attr, + uint64_t flags, struct fid_mr **mr) +{ + return domain->mr->regattr(&domain->fid, attr, flags, mr); +} + +static inline void *fi_mr_desc(struct fid_mr *mr) +{ + return mr->mem_desc; +} + +static inline uint64_t fi_mr_key(struct fid_mr *mr) +{ + return mr->key; +} + +static inline int +fi_mr_raw_attr(struct fid_mr *mr, uint64_t *base_addr, + uint8_t *raw_key, size_t *key_size, uint64_t flags) +{ + struct fi_mr_raw_attr attr; + attr.flags = flags; + attr.base_addr = base_addr; + attr.raw_key = raw_key; + attr.key_size = key_size; + return mr->fid.ops->control(&mr->fid, FI_GET_RAW_MR, &attr); +} + +static inline int +fi_mr_map_raw(struct fid_domain *domain, uint64_t base_addr, + uint8_t *raw_key, size_t key_size, uint64_t *key, uint64_t flags) +{ + struct fi_mr_map_raw map; + map.flags = flags; + map.base_addr = base_addr; + map.raw_key = raw_key; + map.key_size = key_size; + map.key = key; + return domain->fid.ops->control(&domain->fid, FI_MAP_RAW_MR, &map); +} + +static inline int +fi_mr_unmap_key(struct fid_domain *domain, uint64_t key) +{ + return domain->fid.ops->control(&domain->fid, FI_UNMAP_KEY, &key); +} + +static inline int fi_mr_bind(struct fid_mr *mr, struct fid *bfid, uint64_t flags) +{ + return mr->fid.ops->bind(&mr->fid, bfid, flags); +} + +static inline int +fi_mr_refresh(struct fid_mr *mr, const struct iovec *iov, size_t count, + uint64_t flags) +{ + struct fi_mr_modify modify; + memset(&modify, 0, sizeof(modify)); + modify.flags = flags; + modify.attr.mr_iov = iov; + modify.attr.iov_count = count; + return mr->fid.ops->control(&mr->fid, FI_REFRESH, &modify); +} + +static inline int fi_mr_enable(struct fid_mr *mr) +{ + return mr->fid.ops->control(&mr->fid, FI_ENABLE, NULL); +} + +static inline int +fi_av_open(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context) +{ + return domain->ops->av_open(domain, attr, av, context); +} + +static inline FI_DEPRECATED_FUNC int +fi_av_bind(struct fid_av *av, struct fid *fid, uint64_t flags) +{ + return av->fid.ops->bind(&av->fid, fid, flags); +} + +static inline int +fi_av_insert(struct fid_av *av, const void *addr, size_t count, + fi_addr_t *fi_addr, uint64_t flags, void *context) +{ + return av->ops->insert(av, addr, count, fi_addr, flags, context); +} + +static inline int +fi_av_insertsvc(struct fid_av *av, const char *node, const char *service, + fi_addr_t *fi_addr, uint64_t flags, void *context) +{ + return av->ops->insertsvc(av, node, service, fi_addr, flags, context); +} + +static inline int +fi_av_insertsym(struct fid_av *av, const char *node, size_t nodecnt, + const char *service, size_t svccnt, + fi_addr_t *fi_addr, uint64_t flags, void *context) +{ + return av->ops->insertsym(av, node, nodecnt, service, svccnt, + fi_addr, flags, context); +} + +static inline int +fi_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count, uint64_t flags) +{ + return av->ops->remove(av, fi_addr, count, flags); +} + +static inline int +fi_av_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr, size_t *addrlen) +{ + return av->ops->lookup(av, fi_addr, addr, addrlen); +} + +static inline const char * +fi_av_straddr(struct fid_av *av, const void *addr, char *buf, size_t *len) +{ + return av->ops->straddr(av, addr, buf, len); +} + +static inline int +fi_av_insert_auth_key(struct fid_av *av, const void *auth_key, + size_t auth_key_size, fi_addr_t *fi_addr, uint64_t flags) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, insert_auth_key) ? + av->ops->insert_auth_key(av, auth_key, auth_key_size, fi_addr, + flags) : -FI_ENOSYS; +} + +static inline int +fi_av_lookup_auth_key(struct fid_av *av, fi_addr_t addr, void *auth_key, + size_t *auth_key_size) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, lookup_auth_key) ? + av->ops->lookup_auth_key(av, addr, auth_key, auth_key_size) : + -FI_ENOSYS; +} + +static inline int +fi_av_set_user_id(struct fid_av *av, fi_addr_t fi_addr, fi_addr_t user_id, + uint64_t flags) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, set_user_id) ? + av->ops->set_user_id(av, fi_addr, user_id, flags) : -FI_ENOSYS; +} + +static inline fi_addr_t +fi_rx_addr(fi_addr_t fi_addr, int rx_index, int rx_ctx_bits) +{ + return (fi_addr_t) (((uint64_t) rx_index << (64 - rx_ctx_bits)) | fi_addr); +} + +static inline fi_addr_t +fi_group_addr(fi_addr_t fi_addr, uint32_t group_id) +{ + return (fi_addr_t) (((uint64_t) group_id << 32) | fi_addr); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_DOMAIN_H */ diff --git a/backends/libfabric/include/rdma/fi_endpoint.h b/backends/libfabric/include/rdma/fi_endpoint.h new file mode 100644 index 000000000..918972958 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_endpoint.h @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_ENDPOINT_H +#define FI_ENDPOINT_H + +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +struct fi_msg { + const struct iovec *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t addr; + void *context; + uint64_t data; +}; + +/* Endpoint option levels */ +enum { + FI_OPT_ENDPOINT +}; + +/* FI_OPT_ENDPOINT option names */ +enum { + FI_OPT_MIN_MULTI_RECV, /* size_t */ + FI_OPT_CM_DATA_SIZE, /* size_t */ + FI_OPT_BUFFERED_MIN, /* size_t */ + FI_OPT_BUFFERED_LIMIT, /* size_t */ + FI_OPT_SEND_BUF_SIZE, + FI_OPT_RECV_BUF_SIZE, + FI_OPT_TX_SIZE, + FI_OPT_RX_SIZE, + FI_OPT_FI_HMEM_P2P, /* int */ + FI_OPT_XPU_TRIGGER, /* reserved for compatibility */ + FI_OPT_CUDA_API_PERMITTED, /* bool */ + FI_OPT_SHARED_MEMORY_PERMITTED, /* bool */ + FI_OPT_MAX_MSG_SIZE, /* size_t */ + FI_OPT_MAX_TAGGED_SIZE, /* size_t */ + FI_OPT_MAX_RMA_SIZE, /* size_t */ + FI_OPT_MAX_ATOMIC_SIZE, /* size_t */ + FI_OPT_INJECT_MSG_SIZE, /* size_t */ + FI_OPT_INJECT_TAGGED_SIZE, /* size_t */ + FI_OPT_INJECT_RMA_SIZE, /* size_t */ + FI_OPT_INJECT_ATOMIC_SIZE, /* size_t */ + FI_OPT_FIREWALL_ADDR, /* bool */ +}; + +/* + * Parameters for FI_OPT_HMEM_P2P to allow endpoint control over peer to peer + * support and FI_HMEM. + */ +enum { + FI_HMEM_P2P_ENABLED, /* Provider decides when to use P2P, default. */ + FI_HMEM_P2P_REQUIRED, /* Must use P2P for all transfers */ + FI_HMEM_P2P_PREFERRED, /* Should use P2P for all transfers if available */ + FI_HMEM_P2P_DISABLED /* Do not use P2P */ +}; + +struct fi_ops_ep { + size_t size; + ssize_t (*cancel)(fid_t fid, void *context); + int (*getopt)(fid_t fid, int level, int optname, + void *optval, size_t *optlen); + int (*setopt)(fid_t fid, int level, int optname, + const void *optval, size_t optlen); + int (*tx_ctx)(struct fid_ep *sep, int index, + struct fi_tx_attr *attr, struct fid_ep **tx_ep, + void *context); + int (*rx_ctx)(struct fid_ep *sep, int index, + struct fi_rx_attr *attr, struct fid_ep **rx_ep, + void *context); + ssize_t (*rx_size_left)(struct fid_ep *ep); + ssize_t (*tx_size_left)(struct fid_ep *ep); +}; + +struct fi_ops_msg { + size_t size; + ssize_t (*recv)(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, void *context); + ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, void *context); + ssize_t (*recvmsg)(struct fid_ep *ep, const struct fi_msg *msg, + uint64_t flags); + ssize_t (*send)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, void *context); + ssize_t (*sendv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, void *context); + ssize_t (*sendmsg)(struct fid_ep *ep, const struct fi_msg *msg, + uint64_t flags); + ssize_t (*inject)(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr); + ssize_t (*senddata)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, void *context); + ssize_t (*injectdata)(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr); +}; + +struct fi_ops_cm; +struct fi_ops_rma; +struct fi_ops_tagged; +struct fi_ops_atomic; +struct fi_ops_collective; + +/* + * Calls which modify the properties of a endpoint (control, setopt, bind, ...) + * must be serialized against all other operations. Those calls may modify the + * operations referenced by a endpoint in order to optimize the data transfer code + * paths. + * + * A provider may allocate the minimal size structure needed to support the + * ops requested by the user. + */ +struct fid_ep { + struct fid fid; + struct fi_ops_ep *ops; + struct fi_ops_cm *cm; + struct fi_ops_msg *msg; + struct fi_ops_rma *rma; + struct fi_ops_tagged *tagged; + struct fi_ops_atomic *atomic; + struct fi_ops_collective *collective; +}; + +struct fid_pep { + struct fid fid; + struct fi_ops_ep *ops; + struct fi_ops_cm *cm; +}; + +struct fid_stx { + struct fid fid; + struct fi_ops_ep *ops; +}; + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_ENDPOINT + +static inline int +fi_passive_ep(struct fid_fabric *fabric, struct fi_info *info, + struct fid_pep **pep, void *context) +{ + return fabric->ops->passive_ep(fabric, info, pep, context); +} + +static inline int +fi_endpoint(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context) +{ + return domain->ops->endpoint(domain, info, ep, context); +} + +static inline int +fi_endpoint2(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, uint64_t flags, void *context) +{ + if (!flags) + return fi_endpoint(domain, info, ep, context); + + return FI_CHECK_OP(domain->ops, struct fi_ops_domain, endpoint2) ? + domain->ops->endpoint2(domain, info, ep, flags, context) : + -FI_ENOSYS; +} + +static inline int +fi_scalable_ep(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **sep, void *context) +{ + return domain->ops->scalable_ep(domain, info, sep, context); +} + +static inline int fi_ep_bind(struct fid_ep *ep, struct fid *bfid, uint64_t flags) +{ + return ep->fid.ops->bind(&ep->fid, bfid, flags); +} + +static inline int fi_pep_bind(struct fid_pep *pep, struct fid *bfid, uint64_t flags) +{ + return pep->fid.ops->bind(&pep->fid, bfid, flags); +} + +static inline int fi_scalable_ep_bind(struct fid_ep *sep, struct fid *bfid, uint64_t flags) +{ + return sep->fid.ops->bind(&sep->fid, bfid, flags); +} + +static inline int fi_enable(struct fid_ep *ep) +{ + return ep->fid.ops->control(&ep->fid, FI_ENABLE, NULL); +} + +static inline ssize_t fi_cancel(fid_t fid, void *context) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->ops->cancel(fid, context); +} + +static inline int +fi_setopt(fid_t fid, int level, int optname, + const void *optval, size_t optlen) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->ops->setopt(fid, level, optname, optval, optlen); +} + +static inline int +fi_getopt(fid_t fid, int level, int optname, + void *optval, size_t *optlen) +{ + struct fid_ep *ep = (struct fid_ep *) fid; + return ep->ops->getopt(fid, level, optname, optval, optlen); +} + +static inline int fi_ep_alias(struct fid_ep *ep, struct fid_ep **alias_ep, + uint64_t flags) +{ + int ret; + struct fid *fid; + ret = fi_alias(&ep->fid, &fid, flags); + if (!ret) + *alias_ep = (struct fid_ep *) fid; + return ret; +} + +static inline int +fi_tx_context(struct fid_ep *ep, int idx, struct fi_tx_attr *attr, + struct fid_ep **tx_ep, void *context) +{ + return ep->ops->tx_ctx(ep, idx, attr, tx_ep, context); +} + +static inline int +fi_rx_context(struct fid_ep *ep, int idx, struct fi_rx_attr *attr, + struct fid_ep **rx_ep, void *context) +{ + return ep->ops->rx_ctx(ep, idx, attr, rx_ep, context); +} + +static inline FI_DEPRECATED_FUNC ssize_t +fi_rx_size_left(struct fid_ep *ep) +{ + return ep->ops->rx_size_left(ep); +} + +static inline FI_DEPRECATED_FUNC ssize_t +fi_tx_size_left(struct fid_ep *ep) +{ + return ep->ops->tx_size_left(ep); +} + +static inline int +fi_stx_context(struct fid_domain *domain, struct fi_tx_attr *attr, + struct fid_stx **stx, void *context) +{ + return domain->ops->stx_ctx(domain, attr, stx, context); +} + +static inline int +fi_srx_context(struct fid_domain *domain, struct fi_rx_attr *attr, + struct fid_ep **rx_ep, void *context) +{ + return domain->ops->srx_ctx(domain, attr, rx_ep, context); +} + +static inline ssize_t +fi_recv(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, + void *context) +{ + return ep->msg->recv(ep, buf, len, desc, src_addr, context); +} + +static inline ssize_t +fi_recvv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, void *context) +{ + return ep->msg->recvv(ep, iov, desc, count, src_addr, context); +} + +static inline ssize_t +fi_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) +{ + return ep->msg->recvmsg(ep, msg, flags); +} + +static inline ssize_t +fi_send(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, void *context) +{ + return ep->msg->send(ep, buf, len, desc, dest_addr, context); +} + +static inline ssize_t +fi_sendv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, void *context) +{ + return ep->msg->sendv(ep, iov, desc, count, dest_addr, context); +} + +static inline ssize_t +fi_sendmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) +{ + return ep->msg->sendmsg(ep, msg, flags); +} + +static inline ssize_t +fi_inject(struct fid_ep *ep, const void *buf, size_t len, fi_addr_t dest_addr) +{ + return ep->msg->inject(ep, buf, len, dest_addr); +} + +static inline ssize_t +fi_senddata(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, void *context) +{ + return ep->msg->senddata(ep, buf, len, desc, data, dest_addr, context); +} + +static inline ssize_t +fi_injectdata(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr) +{ + return ep->msg->injectdata(ep, buf, len, data, dest_addr); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_ENDPOINT_H */ diff --git a/backends/libfabric/include/rdma/fi_eq.h b/backends/libfabric/include/rdma/fi_eq.h new file mode 100644 index 000000000..ee213986b --- /dev/null +++ b/backends/libfabric/include/rdma/fi_eq.h @@ -0,0 +1,488 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_EQ_H +#define FI_EQ_H + +#include + +#ifndef _WIN32 +#include +#endif /* _WIN32 */ + +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + + +/* + * Wait Set + * Allows associating multiple EQs and counters with a single wait object. + */ + +/* Use fi_control GETWAIT to get underlying wait object(s) */ +enum fi_wait_obj { + FI_WAIT_NONE, + FI_WAIT_UNSPEC, + FI_WAIT_SET, + FI_WAIT_FD, + FI_WAIT_MUTEX_COND, /* pthread mutex & cond, deprecated */ + FI_WAIT_YIELD, + FI_WAIT_POLLFD, +}; + +struct fi_wait_attr { + enum fi_wait_obj wait_obj; + uint64_t flags; +}; + +/* deprecated */ +struct fi_ops_wait { + size_t size; + int (*wait)(struct fid_wait *waitset, int timeout); +}; + +/* deprecated */ +struct fid_wait { + struct fid fid; + struct fi_ops_wait *ops; +}; + +#ifndef _WIN32 +struct fi_mutex_cond { + pthread_mutex_t *mutex; + pthread_cond_t *cond; +}; +#endif /* _WIN32 */ + +struct fi_wait_pollfd { + uint64_t change_index; + size_t nfds; + struct pollfd *fd; +}; + +/* + * Poll Set + * Allows polling multiple event queues and counters for progress + */ + +struct fi_poll_attr { + uint64_t flags; +}; + +/* deprecated */ +struct fi_ops_poll { + size_t size; + int (*poll)(struct fid_poll *pollset, void **context, int count); + int (*poll_add)(struct fid_poll *pollset, struct fid *event_fid, + uint64_t flags); + int (*poll_del)(struct fid_poll *pollset, struct fid *event_fid, + uint64_t flags); +}; + +/* deprecated */ +struct fid_poll { + struct fid fid; + struct fi_ops_poll *ops; +}; + +/* + * EQ = Event Queue + * Used to report various control (not data transfer) events and operations. + */ + +struct fi_eq_attr { + size_t size; + uint64_t flags; + enum fi_wait_obj wait_obj; + int signaling_vector; + struct fid_wait *wait_set; /* deprecated */ +}; + +/* Standard EQ events */ +enum { + FI_NOTIFY, + FI_CONNREQ, + FI_CONNECTED, + FI_SHUTDOWN, + FI_MR_COMPLETE, + FI_AV_COMPLETE, + FI_JOIN_COMPLETE, +}; + +struct fi_eq_entry { + fid_t fid; + void *context; + uint64_t data; +}; + +struct fi_eq_err_entry { + fid_t fid; + void *context; + uint64_t data; + int err; + int prov_errno; + /* err_data is available until the next time the EQ is read */ + void *err_data; + size_t err_data_size; +}; + +struct fi_eq_cm_entry { + fid_t fid; + /* user must call fi_freeinfo to release info */ + struct fi_info *info; + /* connection data placed here, up to space provided */ + uint8_t data[]; +}; + +struct fi_ops_eq { + size_t size; + ssize_t (*read)(struct fid_eq *eq, uint32_t *event, + void *buf, size_t len, uint64_t flags); + ssize_t (*readerr)(struct fid_eq *eq, struct fi_eq_err_entry *buf, + uint64_t flags); + ssize_t (*write)(struct fid_eq *eq, uint32_t event, + const void *buf, size_t len, uint64_t flags); + ssize_t (*sread)(struct fid_eq *eq, uint32_t *event, + void *buf, size_t len, int timeout, uint64_t flags); + const char * (*strerror)(struct fid_eq *eq, int prov_errno, + const void *err_data, char *buf, size_t len); +}; + +struct fid_eq { + struct fid fid; + struct fi_ops_eq *ops; +}; + + +/* + * CQ = Complete Queue + * Used to report the completion of data transfer operations. + */ + +enum fi_cq_format { + FI_CQ_FORMAT_UNSPEC, + FI_CQ_FORMAT_CONTEXT, + FI_CQ_FORMAT_MSG, + FI_CQ_FORMAT_DATA, + FI_CQ_FORMAT_TAGGED, +}; + +struct fi_cq_entry { + void *op_context; +}; + +struct fi_cq_msg_entry { + void *op_context; + uint64_t flags; + size_t len; +}; + +struct fi_cq_data_entry { + void *op_context; + uint64_t flags; + size_t len; + void *buf; + /* data depends on operation and/or flags - e.g. remote EQ data */ + uint64_t data; +}; + +struct fi_cq_tagged_entry { + void *op_context; + uint64_t flags; + size_t len; + void *buf; + uint64_t data; + uint64_t tag; +}; + +struct fi_cq_err_entry { + void *op_context; + uint64_t flags; + size_t len; + void *buf; + uint64_t data; + uint64_t tag; + size_t olen; + int err; + int prov_errno; + /* err_data is available until the next time the CQ is read */ + void *err_data; + size_t err_data_size; + fi_addr_t src_addr; +}; + +enum fi_cq_wait_cond { + FI_CQ_COND_NONE, + FI_CQ_COND_THRESHOLD /* size_t threshold */ +}; + +struct fi_cq_attr { + size_t size; + uint64_t flags; + enum fi_cq_format format; + enum fi_wait_obj wait_obj; + int signaling_vector; + enum fi_cq_wait_cond wait_cond; + struct fid_wait *wait_set; /* deprecated */ +}; + +struct fi_ops_cq { + size_t size; + ssize_t (*read)(struct fid_cq *cq, void *buf, size_t count); + ssize_t (*readfrom)(struct fid_cq *cq, void *buf, size_t count, + fi_addr_t *src_addr); + ssize_t (*readerr)(struct fid_cq *cq, struct fi_cq_err_entry *buf, + uint64_t flags); + ssize_t (*sread)(struct fid_cq *cq, void *buf, size_t count, + const void *cond, int timeout); + ssize_t (*sreadfrom)(struct fid_cq *cq, void *buf, size_t count, + fi_addr_t *src_addr, const void *cond, int timeout); + int (*signal)(struct fid_cq *cq); + const char * (*strerror)(struct fid_cq *cq, int prov_errno, + const void *err_data, char *buf, size_t len); +}; + +struct fid_cq { + struct fid fid; + struct fi_ops_cq *ops; +}; + + +/* + * CNTR = Counter + * Used to report the number of completed of asynchronous operations. + */ + +enum fi_cntr_events { + FI_CNTR_EVENTS_COMP, + FI_CNTR_EVENTS_BYTES /* count bytes not completeion events */ +}; + +struct fi_cntr_attr { + enum fi_cntr_events events; + enum fi_wait_obj wait_obj; + struct fid_wait *wait_set; /* deprecated */ + uint64_t flags; +}; + +struct fi_ops_cntr { + size_t size; + uint64_t (*read)(struct fid_cntr *cntr); + uint64_t (*readerr)(struct fid_cntr *cntr); + int (*add)(struct fid_cntr *cntr, uint64_t value); + int (*set)(struct fid_cntr *cntr, uint64_t value); + int (*wait)(struct fid_cntr *cntr, uint64_t threshold, int timeout); + int (*adderr)(struct fid_cntr *cntr, uint64_t value); + int (*seterr)(struct fid_cntr *cntr, uint64_t value); +}; + +struct fid_cntr { + struct fid fid; + struct fi_ops_cntr *ops; +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_EQ + +static inline int +fi_trywait(struct fid_fabric *fabric, struct fid **fids, int count) +{ + return fabric->ops->trywait(fabric, fids, count); +} + +static inline FI_DEPRECATED_FUNC int +fi_wait(struct fid_wait *waitset, int timeout) +{ + return waitset->ops->wait(waitset, timeout); +} + +static inline FI_DEPRECATED_FUNC int +fi_poll(struct fid_poll *pollset, void **context, int count) +{ + return pollset->ops->poll(pollset, context, count); +} + +static inline FI_DEPRECATED_FUNC int +fi_poll_add(struct fid_poll *pollset, struct fid *event_fid, uint64_t flags) +{ + return pollset->ops->poll_add(pollset, event_fid, flags); +} + +static inline FI_DEPRECATED_FUNC int +fi_poll_del(struct fid_poll *pollset, struct fid *event_fid, uint64_t flags) +{ + return pollset->ops->poll_del(pollset, event_fid, flags); +} + +static inline int +fi_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr, + struct fid_eq **eq, void *context) +{ + return fabric->ops->eq_open(fabric, attr, eq, context); +} + +static inline ssize_t +fi_eq_read(struct fid_eq *eq, uint32_t *event, void *buf, + size_t len, uint64_t flags) +{ + return eq->ops->read(eq, event, buf, len, flags); +} + +static inline ssize_t +fi_eq_readerr(struct fid_eq *eq, struct fi_eq_err_entry *buf, uint64_t flags) +{ + return eq->ops->readerr(eq, buf, flags); +} + +static inline ssize_t +fi_eq_write(struct fid_eq *eq, uint32_t event, const void *buf, + size_t len, uint64_t flags) +{ + return eq->ops->write(eq, event, buf, len, flags); +} + +static inline ssize_t +fi_eq_sread(struct fid_eq *eq, uint32_t *event, void *buf, size_t len, + int timeout, uint64_t flags) +{ + return eq->ops->sread(eq, event, buf, len, timeout, flags); +} + +static inline const char * +fi_eq_strerror(struct fid_eq *eq, int prov_errno, const void *err_data, + char *buf, size_t len) +{ + return eq->ops->strerror(eq, prov_errno, err_data, buf, len); +} + + +static inline ssize_t fi_cq_read(struct fid_cq *cq, void *buf, size_t count) +{ + return cq->ops->read(cq, buf, count); +} + +static inline ssize_t +fi_cq_readfrom(struct fid_cq *cq, void *buf, size_t count, fi_addr_t *src_addr) +{ + return cq->ops->readfrom(cq, buf, count, src_addr); +} + +static inline ssize_t +fi_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf, uint64_t flags) +{ + /* For compatibility with older providers. */ + if (buf) + buf->src_addr = FI_ADDR_NOTAVAIL; + return cq->ops->readerr(cq, buf, flags); +} + +static inline ssize_t +fi_cq_sread(struct fid_cq *cq, void *buf, size_t count, const void *cond, int timeout) +{ + return cq->ops->sread(cq, buf, count, cond, timeout); +} + +static inline ssize_t +fi_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count, + fi_addr_t *src_addr, const void *cond, int timeout) +{ + return cq->ops->sreadfrom(cq, buf, count, src_addr, cond, timeout); +} + +static inline int fi_cq_signal(struct fid_cq *cq) +{ + return cq->ops->signal(cq); +} + +static inline const char * +fi_cq_strerror(struct fid_cq *cq, int prov_errno, const void *err_data, + char *buf, size_t len) +{ + return cq->ops->strerror(cq, prov_errno, err_data, buf, len); +} + + +static inline uint64_t fi_cntr_read(struct fid_cntr *cntr) +{ + return cntr->ops->read(cntr); +} + +static inline uint64_t fi_cntr_readerr(struct fid_cntr *cntr) +{ + return cntr->ops->readerr(cntr); +} + +static inline int fi_cntr_add(struct fid_cntr *cntr, uint64_t value) +{ + return cntr->ops->add(cntr, value); +} + +static inline int fi_cntr_adderr(struct fid_cntr *cntr, uint64_t value) +{ + return FI_CHECK_OP(cntr->ops, struct fi_ops_cntr, adderr) ? + cntr->ops->adderr(cntr, value) : -FI_ENOSYS; +} + +static inline int fi_cntr_set(struct fid_cntr *cntr, uint64_t value) +{ + return cntr->ops->set(cntr, value); +} + +static inline int fi_cntr_seterr(struct fid_cntr *cntr, uint64_t value) +{ + return FI_CHECK_OP(cntr->ops, struct fi_ops_cntr, seterr) ? + cntr->ops->seterr(cntr, value) : -FI_ENOSYS; +} + +static inline int +fi_cntr_wait(struct fid_cntr *cntr, uint64_t threshold, int timeout) +{ + return cntr->ops->wait(cntr, threshold, timeout); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_EQ_H */ diff --git a/backends/libfabric/include/rdma/fi_errno.h b/backends/libfabric/include/rdma/fi_errno.h new file mode 100644 index 000000000..11286ab5c --- /dev/null +++ b/backends/libfabric/include/rdma/fi_errno.h @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_ERRNO_H +#define FI_ERRNO_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* FI directly mapped errno values */ + +#define FI_SUCCESS 0 + +#define FI_EPERM EPERM /* Operation not permitted */ +#define FI_ENOENT ENOENT /* No such file or directory */ +//#define FI_ESRCH ESRCH /* No such process */ +#define FI_EINTR EINTR /* Interrupted system call */ +#define FI_EIO EIO /* I/O error */ +//#define FI_ENXIO ENXIO /* No such device or address */ +#define FI_E2BIG E2BIG /* Argument list too long */ +//#define FI_ENOEXEC ENOEXEC /* Exec format error */ +#define FI_EBADF EBADF /* Bad file number */ +//#define FI_ECHILD ECHILD /* No child processes */ +#define FI_EAGAIN EAGAIN /* Try again */ +#define FI_ENOMEM ENOMEM /* Out of memory */ +#define FI_EACCES EACCES /* Permission denied */ +#define FI_EFAULT EFAULT /* Bad address */ +//#define FI_ENOTBLK ENOTBLK /* Block device required */ +#define FI_EBUSY EBUSY /* Device or resource busy */ +//#define FI_EEXIST EEXIST /* File exists */ +//#define FI_EXDEV EXDEV /* Cross-device link */ +#define FI_ENODEV ENODEV /* No such device */ +//#define FI_ENOTDIR ENOTDIR /* Not a directory */ +//#define FI_EISDIR EISDIR /* Is a directory */ +#define FI_EINVAL EINVAL /* Invalid argument */ +//#define FI_ENFILE ENFILE /* File table overflow */ +#define FI_EMFILE EMFILE /* Too many open files */ +//#define FI_ENOTTY ENOTTY /* Not a typewriter */ +//#define FI_ETXTBSY ETXTBSY /* Text file busy */ +//#define FI_EFBIG EFBIG /* File too large */ +#define FI_ENOSPC ENOSPC /* No space left on device */ +//#define FI_ESPIPE ESPIPE /* Illegal seek */ +//#define FI_EROFS EROFS /* Read-only file system */ +//#define FI_EMLINK EMLINK /* Too many links */ +//#define FI_EPIPE EPIPE /* Broken pipe */ +//#define FI_EDOM EDOM /* Math argument out of domain of func */ +//#define FI_ERANGE ERANGE /* Math result not representable */ +//#define FI_EDEADLK EDEADLK /* Resource deadlock would occur */ +//#define FI_ENAMETOOLONG ENAMETOLONG /* File name too long */ +//#define FI_ENOLCK ENOLCK /* No record locks available */ +#define FI_ENOSYS ENOSYS /* Function not implemented */ +//#define FI_ENOTEMPTY ENOTEMPTY /* Directory not empty */ +//#define FI_ELOOP ELOOP /* Too many symbolic links encountered */ +#define FI_EWOULDBLOCK EWOULDBLOCK /* Operation would block */ +#define FI_ENOMSG ENOMSG /* No message of desired type */ +//#define FI_EIDRM EIDRM /* Identifier removed */ +//#define FI_ECHRNG ECHRNG /* Channel number out of range */ +//#define FI_EL2NSYNC EL2NSYCN /* Level 2 not synchronized */ +//#define FI_EL3HLT EL3HLT /* Level 3 halted */ +//#define FI_EL3RST EL3RST /* Level 3 reset */ +//#define FI_ELNRNG ELNRNG /* Link number out of range */ +//#define FI_EUNATCH EUNATCH /* Protocol driver not attached */ +//#define FI_ENOCSI ENOCSI /* No CSI structure available */ +//#define FI_EL2HLT EL2HLT /* Level 2 halted */ +//#define FI_EBADE EBADE /* Invalid exchange */ +//#define FI_EBADR EBADDR /* Invalid request descriptor */ +//#define FI_EXFULL EXFULL /* Exchange full */ +//#define FI_ENOANO ENOANO /* No anode */ +//#define FI_EBADRQC EBADRQC /* Invalid request code */ +//#define FI_EBADSLT EBADSLT /* Invalid slot */ +//#define FI_EDEADLOCK EDEADLOCK /* Resource deadlock would occur */ +//#define FI_EBFONT EBFONT /* Bad font file format */ +//#define FI_ENOSTR ENOSTR /* Device not a stream */ +#define FI_ENODATA ENODATA /* No data available */ +//#define FI_ETIME ETIME /* Timer expired */ +//#define FI_ENOSR ENOSR /* Out of streams resources */ +//#define FI_ENONET ENONET /* Machine is not on the network */ +//#define FI_ENOPKG ENOPKG /* Package not installed */ +//#define FI_EREMOTE EREMOTE /* Object is remote */ +//#define FI_ENOLINK ENOLINK /* Link has been severed */ +//#define FI_EADV EADV /* Advertise error */ +//#define FI_ESRMNT ESRMNT /* Srmount error */ +//#define FI_ECOMM ECOMM /* Communication error on send */ +#define FI_EPROTO EPROTO /* Protocol error */ +//#define FI_EMULTIHOP EMULTIHOP /* Multihop attempted */ +//#define FI_EDOTDOT EDOTDOT /* RFS specific error */ +//#define FI_EBADMSG EBADMSG /* Not a data message */ +#define FI_EOVERFLOW EOVERFLOW /* Value too large for defined data type */ +//#define FI_ENOTUNIQ ENOTUNIQ /* Name not unique on network */ +//#define FI_EBADFD EBADFD /* File descriptor in bad state */ +//#define FI_EREMCHG EREMCHG /* Remote address changed */ +//#define FI_ELIBACC ELIBACC /* Can not access a needed shared library */ +//#define FI_ELIBBAD ELIBBAD /* Accessing a corrupted shared library */ +//#define FI_ELIBSCN ELIBSCN /* .lib section in a.out corrupted */ +//#define FI_ELIBMAX ELIBMAX /* Attempting to link in too many shared libraries */ +//#define FI_ELIBEXEC ELIBEXEC /* Cannot exec a shared library directly */ +//#define FI_EILSEQ EILSEQ /* Illegal byte sequence */ +//#define FI_ERESTART ERESTART /* Interrupted system call should be restarted */ +//#define FI_ESTRPIPE ESTRPIPE /* Streams pipe error */ +//#define FI_EUSERS EUSERS /* Too many users */ +//#define FI_ENOTSOCK ENOTSOCK /* Socket operation on non-socket */ +//#define FI_EDESTADDRREQ EDESTADDRREQ /* Destination address required */ +#define FI_EMSGSIZE EMSGSIZE /* Message too long */ +//#define FI_EPROTOTYPE EPROTOTYPE /* Protocol wrong type for endpoint */ +#define FI_ENOPROTOOPT ENOPROTOOPT /* Protocol not available */ +//#define FI_EPROTONOSUPPORT EPROTONOSUPPORT /* Protocol not supported */ +//#define FI_ESOCKTNOSUPPORT ESOCKTNOSUPPORT /* Socket type not supported */ +#define FI_EOPNOTSUPP EOPNOTSUPP /* Operation not supported on transport endpoint */ +//#define FI_EPFNOSUPPORT EPFNOSUPPORT /* Protocol family not supported */ +//#define FI_EAFNOSUPPORT EAFNOSUPPORT /* Address family not supported by protocol */ +#define FI_EADDRINUSE EADDRINUSE /* Address already in use */ +#define FI_EADDRNOTAVAIL EADDRNOTAVAIL /* Cannot assign requested address */ +#define FI_ENETDOWN ENETDOWN /* Network is down */ +#define FI_ENETUNREACH ENETUNREACH /* Network is unreachable */ +//#define FI_ENETRESET ENETRESET /* Network dropped connection because of reset */ +#define FI_ECONNABORTED ECONNABORTED /* Software caused connection abort */ +#define FI_ECONNRESET ECONNRESET /* Connection reset by peer */ +#define FI_ENOBUFS ENOBUFS /* No buffer space available */ +#define FI_EISCONN EISCONN /* Transport endpoint is already connected */ +#define FI_ENOTCONN ENOTCONN /* Transport endpoint is not connected */ +#define FI_ESHUTDOWN ESHUTDOWN /* Cannot send after transport endpoint shutdown */ +//#define FI_ETOOMANYREFS ETOOMANYREFS /* Too many references: cannot splice */ +#define FI_ETIMEDOUT ETIMEDOUT /* Connection timed out */ +#define FI_ECONNREFUSED ECONNREFUSED /* Connection refused */ +#define FI_EHOSTDOWN EHOSTDOWN /* Host is down */ +#define FI_EHOSTUNREACH EHOSTUNREACH /* No route to host */ +#define FI_EALREADY EALREADY /* Operation already in progress */ +#define FI_EINPROGRESS EINPROGRESS /* Operation now in progress */ +//#define FI_ESTALE ESTALE /* Stale NFS file handle */ +//#define FI_EUCLEAN EUNCLEAN /* Structure needs cleaning */ +//#define FI_ENOTNAM ENOTNAM /* Not a XENIX named type file */ +//#define FI_ENAVAIL ENAVAIL /* No XENIX semaphores available */ +//#define FI_EISNAM EISNAM /* Is a named type file */ +#define FI_EREMOTEIO EREMOTEIO /* Remote I/O error */ +//#define FI_EDQUOT EDQUOT /* Quota exceeded */ +//#define FI_ENOMEDIUM ENOMEDIUM /* No medium found */ +//#define FI_EMEDIUMTYPE EMEDIUMTYPE /* Wrong medium type */ +#define FI_ECANCELED ECANCELED /* Operation Canceled */ + +//#define FI_EKEYEXPIRED EKEYEXPIRED /* Key has expired */ +//#define FI_EKEYREVOKED EKEYREVOKED /* Key has been revoked */ +#define FI_EKEYREJECTED EKEYREJECTED /* Key was rejected by service */ +//#define FI_EOWNERDEAD EOWNERDEAD /* Owner died */ +//#define FI_ENOTRECOVERABLE ENOTRECOVERABLE /* State not recoverable */ + +/* FI specific return values: >= 256 */ +#define FI_ERRNO_OFFSET 256 + +enum { + FI_EOTHER = FI_ERRNO_OFFSET, /* Unspecified error */ + FI_ETOOSMALL = 257, /* Provided buffer is too small */ + FI_EOPBADSTATE = 258, /* Operation not permitted in current state */ + FI_EAVAIL = 259, /* Error available */ + FI_EBADFLAGS = 260, /* Flags not supported */ + FI_ENOEQ = 261, /* Missing or unavailable event queue */ + FI_EDOMAIN = 262, /* Invalid resource domain */ + FI_ENOCQ = 263, /* Missing or unavailable completion queue */ + FI_ECRC = 264, /* CRC error */ + FI_ETRUNC = 265, /* Truncation error */ + FI_ENOKEY = 266, /* Required key not available */ + FI_ENOAV = 267, /* Missing or unavailable address vector */ + FI_EOVERRUN = 268, /* Queue has been overrun */ + FI_ENORX = 269, /* Receiver not ready, no receive buffers available */ + FI_ENOMR = 270, /* No more memory registrations available */ + FI_EFIREWALLADDR = 271, /* Host unreachable because it is behind firewall */ + FI_ERRNO_MAX +}; + +const char *fi_strerror(int errnum); + +#ifdef __cplusplus +} +#endif + +#endif /* FI_ERRNO_H */ diff --git a/backends/libfabric/include/rdma/fi_ext.h b/backends/libfabric/include/rdma/fi_ext.h new file mode 100644 index 000000000..9932ba0cb --- /dev/null +++ b/backends/libfabric/include/rdma/fi_ext.h @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2021-2023 Intel Corporation. All rights reserved. + * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright (c) 2022 DataDirect Networks, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_EXT_H +#define FI_EXT_H + +#include +#include +#include +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Each provider needs to define an unique 12-bit provider + * specific code to avoid overlapping with other providers, + * then bit left shift the code 16 bits. Note that the + * highest 4 bits are not touched, so they are still left + * to 0. The lowest 16 bits can be used to define provider + * specific values. E.g., + * + * define FI_PROV_SPECIFIC_XXX (0xabc << 16) + * + * enum { + * FI_PROV_XXX_FOO = -(FI_PROV_SPECIFIC_XXX), + * FI_PROV_XXX_BAR, + * } + */ + +#define FI_PROV_SPECIFIC_EFA (0xefa << 16) +#define FI_PROV_SPECIFIC_TCP (0x7cb << 16) + + +/* negative options are provider specific */ +enum { + FI_OPT_EFA_RNR_RETRY = -FI_PROV_SPECIFIC_EFA, + FI_OPT_EFA_EMULATED_READ, /* bool */ + FI_OPT_EFA_EMULATED_WRITE, /* bool */ + FI_OPT_EFA_EMULATED_ATOMICS, /* bool */ + FI_OPT_EFA_USE_DEVICE_RDMA, /* bool */ + FI_OPT_EFA_SENDRECV_IN_ORDER_ALIGNED_128_BYTES, /* bool */ + FI_OPT_EFA_WRITE_IN_ORDER_ALIGNED_128_BYTES, /* bool */ + FI_OPT_EFA_HOMOGENEOUS_PEERS, /* bool */ +}; + +struct fi_fid_export { + struct fid **fid; + uint64_t flags; + void *context; +}; + +static inline int +fi_export_fid(struct fid *fid, uint64_t flags, + struct fid **expfid, void *context) +{ + struct fi_fid_export exp; + + exp.fid = expfid; + exp.flags = flags; + exp.context = context; + return fi_control(fid, FI_EXPORT_FID, &exp); +} + +static inline int +fi_import_fid(struct fid *fid, struct fid *expfid, uint64_t flags) +{ + return fid->ops->bind(fid, expfid, flags); +} + + +/* + * System memory monitor import extension: + * To use, open mr_cache fid and import. + */ + +struct fid_mem_monitor; + +struct fi_ops_mem_monitor { + size_t size; + int (*start)(struct fid_mem_monitor *monitor); + void (*stop)(struct fid_mem_monitor *monitor); + int (*subscribe)(struct fid_mem_monitor *monitor, + const void *addr, size_t len); + void (*unsubscribe)(struct fid_mem_monitor *monitor, + const void *addr, size_t len); + bool (*valid)(struct fid_mem_monitor *monitor, + const void *addr, size_t len); +}; + +struct fi_ops_mem_notify { + size_t size; + void (*notify)(struct fid_mem_monitor *monitor, const void *addr, + size_t len); +}; + +struct fid_mem_monitor { + struct fid fid; + struct fi_ops_mem_monitor *export_ops; + struct fi_ops_mem_notify *import_ops; +}; + + +/* + * System logging import extension: + * To use, open logging fid and import. + */ + +#define FI_LOG_PROV_FILTERED (1ULL << 0) /* Filter provider */ + +struct fi_ops_log { + size_t size; + int (*enabled)(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, uint64_t flags); + int (*ready)(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, uint64_t flags, uint64_t *showtime); + void (*log)(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, const char *func, int line, + const char *msg); +}; + +struct fid_logging { + struct fid fid; + struct fi_ops_log *ops; +}; + +static inline int fi_import(uint32_t version, const char *name, void *attr, + size_t attr_len, uint64_t flags, struct fid *fid, + void *context) +{ + struct fid *open_fid; + int ret; + + ret = fi_open(version, name, attr, attr_len, flags, &open_fid, context); + if (ret != FI_SUCCESS) + return ret; + + ret = fi_import_fid(open_fid, fid, flags); + fi_close(open_fid); + return ret; +} + +static inline int fi_import_log(uint32_t version, uint64_t flags, + struct fid_logging *log_fid) +{ + log_fid->fid.fclass = FI_CLASS_LOG; + log_fid->ops->size = sizeof(struct fi_ops_log); + + return fi_import(version, "logging", NULL, 0, flags, &log_fid->fid, + log_fid); +} + +#ifdef __cplusplus +} +#endif + +#endif /* FI_EXT_H */ diff --git a/backends/libfabric/include/rdma/fi_profile.h b/backends/libfabric/include/rdma/fi_profile.h new file mode 100644 index 000000000..cc8d8918b --- /dev/null +++ b/backends/libfabric/include/rdma/fi_profile.h @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2023 Intel Corporation, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_PROFILE_H +#define FI_PROFILE_H + +#include +#include +#include + +#include + +/* + * pre-defined profiling variables + */ +enum { + FI_VAR_UNEXP_MSG_CNT, // datatype: FI_UINT64 + FI_VAR_UNEXP_MSG_QUEUE, // datatype: FI_TYPE_CQ_ERR_ENTRY + FI_VAR_MSG_QUEUE_CNT, // datatype: FI_UNIT64 + FI_VAR_CONNECTION_CNT, // datatype: FI_UNIT64 + FI_VAR_CONN_REQUEST, // datatype: FI_UNIT64 + FI_VAR_CONN_ACCEPT, // datatype: FI_UNIT64 + FI_VAR_CONN_REJECT, // datatype: FI_UNIT64 + FI_VAR_OFI_MEM, // datatype: FI_UINT64 +}; + +/* + * pre-defined events which can change the profiling variables + */ +enum { + FI_EVENT_UNEXP_MSG_RECVD, // var = FI_VAR_UNEXP_MSG_CNT + FI_EVENT_UNEXP_MSG_MATCHED, // var = FI_VAR_UNEXP_MSG_CNT +}; + +enum fi_profile_type { + fi_primitive_type, + fi_defined_type, +}; + +struct fi_profile_desc { + uint32_t id; + enum fi_profile_type datatype_sel; + union { + enum fi_datatype primitive; + enum fi_type defined; + } datatype; + uint64_t flags; + size_t size; + const char *name; + const char *desc; +}; + +struct fid_profile; + +struct fi_profile_ops { + size_t size; + void (*reset)(struct fid_profile *prof_fid, uint64_t flags); + ssize_t (*query_vars)(struct fid_profile *prof_fid, + struct fi_profile_desc *varlist, size_t *count); + ssize_t (*query_events)(struct fid_profile *prof_fid, + struct fi_profile_desc *eventlist, size_t *count); + ssize_t (*read_var)(struct fid_profile *prof_fid, uint32_t var_id, + void *data, size_t *size); + int (*reg_callback)(struct fid_profile *prof_fid, uint32_t event_id, + int (*callback)(struct fid_profile *prof_fid, + struct fi_profile_desc *event, void *param, size_t size, + void *context), + void *context); + void (*start_reads)(struct fid_profile *prof_fid, uint64_t flags); + void (*end_reads)(struct fid_profile *prof_fid, uint64_t flags); +}; + +struct fid_profile { + struct fid fid; + struct fi_profile_ops *ops; +}; + +static inline void +fi_profile_reset(struct fid_profile *prof_fid, uint64_t flags) +{ + return prof_fid->ops->reset(prof_fid, flags); +} + + +static inline ssize_t +fi_profile_query_vars(struct fid_profile *prof_fid, + struct fi_profile_desc *varlist, size_t *count) +{ + return prof_fid->ops->query_vars(prof_fid, varlist, count); +} + +static inline ssize_t +fi_profile_query_events(struct fid_profile *prof_fid, + struct fi_profile_desc *eventlist, size_t *count) +{ + return prof_fid->ops->query_events(prof_fid, eventlist, count); +} + +static inline ssize_t +fi_profile_read_u64(struct fid_profile *prof_fid, uint32_t var_id, + uint64_t *data) +{ + size_t size = sizeof(uint64_t); + ssize_t ret = prof_fid->ops->read_var(prof_fid, var_id, + (void *)data, &size); + return (ret > 0) ? 0 : ret; +} + +static inline int +fi_profile_register_callback(struct fid_profile *prof_fid, uint32_t event_id, + int (*callback)(struct fid_profile *prof_fid, + struct fi_profile_desc *event, void *param, + size_t size, void *context), + void *context) +{ + return prof_fid->ops->reg_callback(prof_fid, event_id, callback, context); +} + +static inline void +fi_profile_start_reads(struct fid_profile *prof_fid, uint64_t flags) +{ + return prof_fid->ops->start_reads(prof_fid, flags); +} + +static inline void +fi_profile_end_reads(struct fid_profile *prof_fid, uint64_t flags) +{ + return prof_fid->ops->end_reads(prof_fid, flags); +} + +static inline int +fi_profile_open(struct fid *fid, uint64_t flags, + struct fid_profile **prof_fid, void *context) +{ + struct fi_profile_ops *ops; + int ret = fi_open_ops(fid, "fi_profile_ops", flags, + (void **)&ops, context); + if (!ret) + *prof_fid = container_of(ops, struct fid_profile, ops); + + return ret; +} + +static inline int +fi_profile_close(struct fid_profile *prof_fid) +{ + return prof_fid->fid.ops->close(&(prof_fid->fid)); +} + +#endif /* FI_PROFILE_H */ diff --git a/backends/libfabric/include/rdma/fi_rma.h b/backends/libfabric/include/rdma/fi_rma.h new file mode 100644 index 000000000..003c10fd9 --- /dev/null +++ b/backends/libfabric/include/rdma/fi_rma.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_RMA_H +#define FI_RMA_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct fi_rma_iov { + uint64_t addr; + size_t len; + uint64_t key; +}; + +struct fi_rma_ioc { + uint64_t addr; + size_t count; + uint64_t key; +}; + +struct fi_msg_rma { + const struct iovec *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t addr; + const struct fi_rma_iov *rma_iov; + size_t rma_iov_count; + void *context; + uint64_t data; +}; + +struct fi_ops_rma { + size_t size; + ssize_t (*read)(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context); + ssize_t (*readv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key, + void *context); + ssize_t (*readmsg)(struct fid_ep *ep, const struct fi_msg_rma *msg, + uint64_t flags); + ssize_t (*write)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context); + ssize_t (*writev)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key, + void *context); + ssize_t (*writemsg)(struct fid_ep *ep, const struct fi_msg_rma *msg, + uint64_t flags); + ssize_t (*inject)(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t addr, uint64_t key); + ssize_t (*writedata)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, + void *context); + ssize_t (*injectdata)(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key); +}; + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_RMA + +static inline ssize_t +fi_read(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context) +{ + return ep->rma->read(ep, buf, len, desc, src_addr, addr, key, context); +} + +static inline ssize_t +fi_readv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key, + void *context) +{ + return ep->rma->readv(ep, iov, desc, count, src_addr, addr, key, context); +} + +static inline ssize_t +fi_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) +{ + return ep->rma->readmsg(ep, msg, flags); +} + +static inline ssize_t +fi_write(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) +{ + return ep->rma->write(ep, buf, len, desc, dest_addr, addr, key, context); +} + +static inline ssize_t +fi_writev(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key, + void *context) +{ + return ep->rma->writev(ep, iov, desc, count, dest_addr, addr, key, context); +} + +static inline ssize_t +fi_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) +{ + return ep->rma->writemsg(ep, msg, flags); +} + +static inline ssize_t +fi_inject_write(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t addr, uint64_t key) +{ + return ep->rma->inject(ep, buf, len, dest_addr, addr, key); +} + +static inline ssize_t +fi_writedata(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, + void *context) +{ + return ep->rma->writedata(ep, buf, len, desc,data, dest_addr, + addr, key, context); +} + +static inline ssize_t +fi_inject_writedata(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key) +{ + return ep->rma->injectdata(ep, buf, len, data, dest_addr, addr, key); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_RMA_H */ diff --git a/backends/libfabric/include/rdma/fi_tagged.h b/backends/libfabric/include/rdma/fi_tagged.h new file mode 100644 index 000000000..9230c34df --- /dev/null +++ b/backends/libfabric/include/rdma/fi_tagged.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_TAGGED_H +#define FI_TAGGED_H + +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + + +#define FI_MPI_IGNORE_TAG ((uint64_t) UINT32_MAX) +#define FI_MPI_IGNORE_PAYLOAD (((uint64_t) UINT8_MAX) << 32) + + +static inline uint64_t +fi_tag_mpi(int tag, uint8_t payload_id) +{ + return (((uint64_t) payload_id) << 32) | ((uint64_t) (uint32_t) tag); +} + +struct fi_msg_tagged { + const struct iovec *msg_iov; + void **desc; + size_t iov_count; + fi_addr_t addr; + uint64_t tag; + uint64_t ignore; + void *context; + uint64_t data; +}; + +struct fi_ops_tagged { + size_t size; + ssize_t (*recv)(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, + uint64_t tag, uint64_t ignore, void *context); + ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, + uint64_t tag, uint64_t ignore, void *context); + ssize_t (*recvmsg)(struct fid_ep *ep, const struct fi_msg_tagged *msg, + uint64_t flags); + ssize_t (*send)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, uint64_t tag, void *context); + ssize_t (*sendv)(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, uint64_t tag, void *context); + ssize_t (*sendmsg)(struct fid_ep *ep, const struct fi_msg_tagged *msg, + uint64_t flags); + ssize_t (*inject)(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t tag); + ssize_t (*senddata)(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, uint64_t tag, void *context); + ssize_t (*injectdata)(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr, uint64_t tag); +}; + + +#ifdef FABRIC_DIRECT +#include +#endif /* FABRIC_DIRECT */ + +#ifndef FABRIC_DIRECT_TAGGED + +static inline ssize_t +fi_trecv(struct fid_ep *ep, void *buf, size_t len, void *desc, + fi_addr_t src_addr, uint64_t tag, uint64_t ignore, void *context) +{ + return ep->tagged->recv(ep, buf, len, desc, src_addr, tag, ignore, + context); +} + +static inline ssize_t +fi_trecvv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t src_addr, uint64_t tag, uint64_t ignore, + void *context) +{ + return ep->tagged->recvv(ep, iov, desc, count, src_addr, tag, ignore, + context); +} + +static inline ssize_t +fi_trecvmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) +{ + return ep->tagged->recvmsg(ep, msg, flags); +} + +static inline ssize_t +fi_tsend(struct fid_ep *ep, const void *buf, size_t len, void *desc, + fi_addr_t dest_addr, uint64_t tag, void *context) +{ + return ep->tagged->send(ep, buf, len, desc, dest_addr, tag, context); +} + +static inline ssize_t +fi_tsendv(struct fid_ep *ep, const struct iovec *iov, void **desc, + size_t count, fi_addr_t dest_addr, uint64_t tag, void *context) +{ + return ep->tagged->sendv(ep, iov, desc, count, dest_addr,tag, context); +} + +static inline ssize_t +fi_tsendmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) +{ + return ep->tagged->sendmsg(ep, msg, flags); +} + +static inline ssize_t +fi_tinject(struct fid_ep *ep, const void *buf, size_t len, + fi_addr_t dest_addr, uint64_t tag) +{ + return ep->tagged->inject(ep, buf, len, dest_addr, tag); +} + +static inline ssize_t +fi_tsenddata(struct fid_ep *ep, const void *buf, size_t len, void *desc, + uint64_t data, fi_addr_t dest_addr, uint64_t tag, void *context) +{ + return ep->tagged->senddata(ep, buf, len, desc, data, + dest_addr, tag, context); +} + +static inline ssize_t +fi_tinjectdata(struct fid_ep *ep, const void *buf, size_t len, + uint64_t data, fi_addr_t dest_addr, uint64_t tag) +{ + return ep->tagged->injectdata(ep, buf, len, data, dest_addr, tag); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* FI_TAGGED_H */ diff --git a/backends/libfabric/include/rdma/fi_trigger.h b/backends/libfabric/include/rdma/fi_trigger.h new file mode 100644 index 000000000..80f7d648b --- /dev/null +++ b/backends/libfabric/include/rdma/fi_trigger.h @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_TRIGGER_H +#define FI_TRIGGER_H + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum fi_trigger_event { + FI_TRIGGER_THRESHOLD, + FI_TRIGGER_XPU, +}; + +enum fi_op_type { + FI_OP_RECV, + FI_OP_SEND, + FI_OP_TRECV, + FI_OP_TSEND, + FI_OP_READ, + FI_OP_WRITE, + FI_OP_ATOMIC, + FI_OP_FETCH_ATOMIC, + FI_OP_COMPARE_ATOMIC, + FI_OP_CNTR_SET, + FI_OP_CNTR_ADD +}; + +struct fi_trigger_threshold { + struct fid_cntr *cntr; + size_t threshold; +}; + +struct fi_trigger_var { + enum fi_datatype datatype; + int count; + void *addr; + union { + uint8_t val8; + uint16_t val16; + uint32_t val32; + uint64_t val64; + uint8_t *data; + } value; +}; + +struct fi_trigger_xpu { + int count; + enum fi_hmem_iface iface; + union { + uint64_t reserved; + int cuda; + int ze; + } device; + struct fi_trigger_var *var; +}; + +struct fi_op_msg { + struct fid_ep *ep; + struct fi_msg msg; + uint64_t flags; +}; + +struct fi_op_tagged { + struct fid_ep *ep; + struct fi_msg_tagged msg; + uint64_t flags; +}; + +struct fi_op_rma { + struct fid_ep *ep; + struct fi_msg_rma msg; + uint64_t flags; +}; + +struct fi_op_atomic { + struct fid_ep *ep; + struct fi_msg_atomic msg; + uint64_t flags; +}; + +struct fi_op_fetch_atomic { + struct fid_ep *ep; + struct fi_msg_atomic msg; + struct fi_msg_fetch fetch; + uint64_t flags; +}; + +struct fi_op_compare_atomic { + struct fid_ep *ep; + struct fi_msg_atomic msg; + struct fi_msg_fetch fetch; + struct fi_msg_compare compare; + uint64_t flags; +}; + +struct fi_op_cntr { + struct fid_cntr *cntr; + uint64_t value; +}; + +#ifdef FABRIC_DIRECT +#include +#endif + +#ifndef FABRIC_DIRECT_TRIGGER + +/* Size must match struct fi_context */ +struct fi_triggered_context { + enum fi_trigger_event event_type; + union { + struct fi_trigger_threshold threshold; + struct fi_trigger_xpu xpu; + void *internal[3]; + } trigger; +}; + +/* Size must match struct fi_context2 */ +struct fi_triggered_context2 { + enum fi_trigger_event event_type; + union { + struct fi_trigger_threshold threshold; + struct fi_trigger_xpu xpu; + void *internal[7]; + } trigger; +}; + +struct fi_deferred_work { + struct fi_context2 context; + + uint64_t threshold; + struct fid_cntr *triggering_cntr; + struct fid_cntr *completion_cntr; + + enum fi_op_type op_type; + + union { + struct fi_op_msg *msg; + struct fi_op_tagged *tagged; + struct fi_op_rma *rma; + struct fi_op_atomic *atomic; + struct fi_op_fetch_atomic *fetch_atomic; + struct fi_op_compare_atomic *compare_atomic; + struct fi_op_cntr *cntr; + } op; +}; + +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* FI_TRIGGER_H */ diff --git a/backends/libfabric/include/rdma/providers/fi_log.h b/backends/libfabric/include/rdma/providers/fi_log.h new file mode 100644 index 000000000..9268a2ee8 --- /dev/null +++ b/backends/libfabric/include/rdma/providers/fi_log.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2015-2016, Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015, Intel Corp., Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#ifndef FI_LOG_H +#define FI_LOG_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum fi_log_subsys { + FI_LOG_CORE, + FI_LOG_FABRIC, + FI_LOG_DOMAIN, + FI_LOG_EP_CTRL, + FI_LOG_EP_DATA, + FI_LOG_AV, + FI_LOG_CQ, + FI_LOG_EQ, + FI_LOG_MR, + FI_LOG_CNTR, +}; + +enum fi_log_level { + FI_LOG_WARN, + FI_LOG_TRACE, + FI_LOG_INFO, + FI_LOG_DEBUG, +}; + +int fi_log_enabled(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys); +int fi_log_ready(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, uint64_t *showtime); +void fi_log(const struct fi_provider *prov, enum fi_log_level level, + enum fi_log_subsys subsys, const char *func, int line, + const char *fmt, ...) FI_FORMAT_PRINTF(6, 7); + +#define FI_LOG(prov, level, subsystem, ...) \ + do { \ + if (fi_log_enabled(prov, level, subsystem)) { \ + int saved_errno = errno; \ + fi_log(prov, level, subsystem, \ + __func__, __LINE__, __VA_ARGS__); \ + errno = saved_errno; \ + } \ + } while (0) + +#define FI_LOG_SPARSE(prov, level, subsystem, ...) \ + do { \ + static uint64_t showtime; \ + if (fi_log_ready(prov, level, subsystem, &showtime)) { \ + int saved_errno = errno; \ + fi_log(prov, level, subsystem, \ + __func__, __LINE__, __VA_ARGS__); \ + errno = saved_errno; \ + } \ + } while (0) + +#define FI_WARN(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_WARN, subsystem, __VA_ARGS__) +#define FI_WARN_SPARSE(prov, subsystem, ...) \ + FI_LOG_SPARSE(prov, FI_LOG_WARN, subsystem, __VA_ARGS__) + +#define FI_TRACE(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_TRACE, subsystem, __VA_ARGS__) + +#define FI_INFO(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_INFO, subsystem, __VA_ARGS__) + +#if defined(ENABLE_DEBUG) && ENABLE_DEBUG +#define FI_DBG(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_DEBUG, subsystem, __VA_ARGS__) +#define FI_DBG_TRACE(prov, subsystem, ...) \ + FI_LOG(prov, FI_LOG_TRACE, subsystem, __VA_ARGS__) +#else +#define FI_DBG(prov_name, subsystem, ...) \ + do {} while (0) +#define FI_DBG_TRACE(prov, subsystem, ...) \ + do {} while (0) +#endif + +#define FI_WARN_ONCE(prov, subsystem, ...) \ + do { \ + static int warned = 0; \ + if (!warned && \ + fi_log_enabled(prov, FI_LOG_WARN, subsystem)) { \ + int saved_errno = errno; \ + fi_log(prov, FI_LOG_WARN, subsystem, \ + __func__, __LINE__, __VA_ARGS__); \ + warned = 1; \ + errno = saved_errno; \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* FI_LOG_H */ diff --git a/backends/libfabric/include/rdma/providers/fi_peer.h b/backends/libfabric/include/rdma/providers/fi_peer.h new file mode 100644 index 000000000..63ebce01f --- /dev/null +++ b/backends/libfabric/include/rdma/providers/fi_peer.h @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2021-2023 Intel Corporation. All rights reserved. + * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright (c) 2022 DataDirect Networks, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_PEER_H +#define FI_PEER_H + +#include +#include +#include +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Peer provider AV support. + */ +struct fid_peer_av; + +struct fi_ops_av_owner { + size_t size; + int (*query)(struct fid_peer_av *av, struct fi_av_attr *attr); + fi_addr_t (*ep_addr)(struct fid_peer_av *av, struct fid_ep *ep); +}; + +struct fid_peer_av { + struct fid fid; + struct fi_ops_av_owner *owner_ops; +}; + +struct fi_peer_av_context { + size_t size; + struct fid_peer_av *av; +}; + + +/* + * Peer provider AV set support. + */ +struct fid_peer_av_set; + +struct fi_ops_av_set_owner { + size_t size; + int (*members)(struct fid_peer_av_set *av, fi_addr_t *addr, + size_t *count); +}; + +struct fid_peer_av_set { + struct fid fid; + struct fi_ops_av_set_owner *owner_ops; +}; + +struct fi_peer_av_set_context { + size_t size; + struct fi_peer_av_set *av_set; +}; + + +/* + * Peer provider CQ support. + */ +struct fid_peer_cq; + +struct fi_ops_cq_owner { + size_t size; + ssize_t (*write)(struct fid_peer_cq *cq, void *context, uint64_t flags, + size_t len, void *buf, uint64_t data, uint64_t tag, + fi_addr_t src); + ssize_t (*writeerr)(struct fid_peer_cq *cq, + const struct fi_cq_err_entry *err_entry); +}; + +struct fid_peer_cq { + struct fid fid; + struct fi_ops_cq_owner *owner_ops; +}; + +struct fi_peer_cq_context { + size_t size; + struct fid_peer_cq *cq; +}; + +/* + * Peer provider counter support. + */ +struct fid_peer_cntr; + +struct fi_ops_cntr_owner { + size_t size; + void (*inc)(struct fid_peer_cntr *cntr); + void (*incerr)(struct fid_peer_cntr *cntr); +}; + +struct fid_peer_cntr { + struct fid fid; + struct fi_ops_cntr_owner *owner_ops; +}; + +struct fi_peer_cntr_context { + size_t size; + struct fid_peer_cntr *cntr; +}; + +/* + * Peer provider domain support. + */ +struct fi_peer_domain_context { + size_t size; + struct fid_domain *domain; +}; + + +/* + * Peer provider EQ support. + */ +struct fi_peer_eq_context { + size_t size; + struct fid_eq *eq; +}; + + +/* + * Peer shared rx context + */ +struct fid_peer_srx; + +struct fi_peer_rx_entry { + struct fid_peer_srx *srx; + fi_addr_t addr; + size_t msg_size; + uint64_t tag; + uint64_t cq_data; + uint64_t flags; + void *context; + size_t count; + void **desc; + void *peer_context; + void *owner_context; + struct iovec *iov; +}; + +struct fi_peer_match_attr { + fi_addr_t addr; + size_t msg_size; + uint64_t tag; +}; + +struct fi_ops_srx_owner { + size_t size; + int (*get_msg)(struct fid_peer_srx *srx, + struct fi_peer_match_attr *attr, + struct fi_peer_rx_entry **entry); + int (*get_tag)(struct fid_peer_srx *srx, + struct fi_peer_match_attr *attr, + struct fi_peer_rx_entry **entry); + int (*queue_msg)(struct fi_peer_rx_entry *entry); + int (*queue_tag)(struct fi_peer_rx_entry *entry); + void (*foreach_unspec_addr)(struct fid_peer_srx *srx, + fi_addr_t (*get_addr)(struct fi_peer_rx_entry *)); + + void (*free_entry)(struct fi_peer_rx_entry *entry); +}; + +struct fi_ops_srx_peer { + size_t size; + int (*start_msg)(struct fi_peer_rx_entry *entry); + int (*start_tag)(struct fi_peer_rx_entry *entry); + int (*discard_msg)(struct fi_peer_rx_entry *entry); + int (*discard_tag)(struct fi_peer_rx_entry *entry); +}; + +struct fid_peer_srx { + struct fid_ep ep_fid; + struct fi_ops_srx_owner *owner_ops; + struct fi_ops_srx_peer *peer_ops; +}; + +struct fi_peer_srx_context { + size_t size; + struct fid_peer_srx *srx; +}; + + +/* + * Peer transfers + */ +struct fi_peer_transfer_context; + +struct fi_ops_transfer_peer { + size_t size; + ssize_t (*complete)(struct fid_ep *ep, struct fi_cq_tagged_entry *buf, + fi_addr_t src_addr); + ssize_t (*comperr)(struct fid_ep *ep, struct fi_cq_err_entry *buf); +}; + +struct fi_peer_transfer_context { + size_t size; + struct fi_info *info; + struct fid_ep *ep; + struct fi_ops_transfer_peer *peer_ops; +}; + + +#ifdef __cplusplus +} +#endif + +#endif /* FI_PEER_H */ diff --git a/backends/libfabric/include/rdma/providers/fi_prov.h b/backends/libfabric/include/rdma/providers/fi_prov.h new file mode 100644 index 000000000..ab8858d8f --- /dev/null +++ b/backends/libfabric/include/rdma/providers/fi_prov.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006, 2016 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * Copyright (c) 2013-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef FI_PROV_H +#define FI_PROV_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Extension that dl-loaded providers should add to their .so filename + * (probably via libtool "-release" option). For example a provider + * driver named "foo" should build a plug-in named "libfoo-fi.so", and + * place it in $prefix/$libdir/libfabric/ + */ +#define FI_LIB_EXTENSION "fi" +#define FI_LIB_SUFFIX FI_LIB_EXTENSION ".so" + +/* + * Dynamically loaded providers must export the following entry point. + * This is invoked by the libfabric framework when the provider library + * is loaded. + */ +#define FI_EXT_INI \ + __attribute__((visibility ("default"),EXTERNALLY_VISIBLE)) \ + struct fi_provider* fi_prov_ini(void) + +struct fi_provider { + uint32_t version; + uint32_t fi_version; + struct fi_context context; + const char *name; + int (*getinfo)(uint32_t version, const char *node, const char *service, + uint64_t flags, const struct fi_info *hints, + struct fi_info **info); + int (*fabric)(struct fi_fabric_attr *attr, struct fid_fabric **fabric, + void *context); + void (*cleanup)(void); +}; + + +/* + * Defines a configuration parameter for use with libfabric. + */ +int fi_param_define(const struct fi_provider *provider, const char *param_name, + enum fi_param_type type, const char *help_string_fmt, ...); + +/* + * Get the value of a configuration variable. + * + * Currently, configuration parameter will only be read from the + * environment. Someday this call could be expanded to also check + * config files. + */ +int fi_param_get(struct fi_provider *provider, const char *param_name, + void *value); + +static inline int +fi_param_get_str(struct fi_provider *provider, const char *param_name, char **value) +{ + return fi_param_get(provider, param_name, value); +} + +static inline int +fi_param_get_int(struct fi_provider *provider, const char *param_name, int *value) +{ + return fi_param_get(provider, param_name, value); +} + +static inline int +fi_param_get_bool(struct fi_provider *provider, const char *param_name, int *value) +{ + return fi_param_get(provider, param_name, value); +} + +static inline int +fi_param_get_size_t(struct fi_provider *provider, const char *param_name, size_t *value) +{ + return fi_param_get(provider, param_name, value); +} + +#ifdef __cplusplus +} +#endif + +#endif /* FI_PROV_H */ diff --git a/backends/libfabric/libfabric_events.yaml b/backends/libfabric/libfabric_events.yaml index d011508a1..fe9e13a12 100644 --- a/backends/libfabric/libfabric_events.yaml +++ b/backends/libfabric/libfabric_events.yaml @@ -1,21 +1,87 @@ -lttng_ust_libfabric: +lttng_ust_fi: events: - - name: fi_fabric_entry + - name: op_entry args: - - ["const void *", attr] - - ["const void *", fabric] - - ["const void *", context] + - ["const char *", op] + - [uint32_t, ep_id] + - ["const void *", op_context] fields: - - [ctf_integer_hex, uintptr_t, attr, "(uintptr_t)attr"] - - [ctf_integer_hex, uintptr_t, fabric, "(uintptr_t)fabric"] - - [ctf_integer_hex, uintptr_t, context, "(uintptr_t)context"] + - [ctf_string, op, op] + - [ctf_integer, uint32_t, ep_id, ep_id] + - [ctf_integer_hex, uintptr_t, op_context, "(uintptr_t)op_context"] - - name: fi_fabric_exit + - name: op_exit args: - - [int, result] - - ["const void *", fabric] + - ["const char *", op] + - [uint32_t, ep_id] + - ["const char *", src_addr] + - ["const char *", peer_addr] + - ["const char *", direction] + - [uint64_t, iov_count] + - [uint64_t, size_bytes] + - [int64_t, ret] + - ["const void *", op_context] fields: - - [ctf_integer, int, result, result] - - [ctf_integer_hex, uintptr_t, fabric, "(uintptr_t)fabric"] + - [ctf_string, op, op] + - [ctf_integer, uint32_t, ep_id, ep_id] + - [ctf_string, src_addr, src_addr] + - [ctf_string, peer_addr, peer_addr] + - [ctf_string, direction, direction] + - [ctf_integer, uint64_t, iov_count, iov_count] + - [ctf_integer, uint64_t, size_bytes, size_bytes] + - [ctf_integer, int64_t, ret, ret] + - [ctf_integer_hex, uintptr_t, op_context, "(uintptr_t)op_context"] + - name: cq_completion + args: + - ["const char *", op] + - [uint32_t, obj_id] + - ["const char *", src_addr] + - ["const char *", peer_addr] + - [uint64_t, size_bytes] + - [int64_t, ret] + - ["const void *", op_context] + fields: + - [ctf_string, op, op] + - [ctf_integer, uint32_t, obj_id, obj_id] + - [ctf_string, src_addr, src_addr] + - [ctf_string, peer_addr, peer_addr] + - [ctf_integer, uint64_t, size_bytes, size_bytes] + - [ctf_integer, int64_t, ret, ret] + - [ctf_integer_hex, uintptr_t, op_context, "(uintptr_t)op_context"] + + - name: ctrl_entry + args: + - ["const char *", op] + - [uint32_t, obj_id] + fields: + - [ctf_string, op, op] + - [ctf_integer, uint32_t, obj_id, obj_id] + - name: ctrl_exit + args: + - ["const char *", op] + - [uint32_t, obj_id] + - [int64_t, ret] + fields: + - [ctf_string, op, op] + - [ctf_integer, uint32_t, obj_id, obj_id] + - [ctf_integer, int64_t, ret, ret] + + - name: ep_info + args: + - [uint32_t, ep_id] + - ["const char *", src_addr] + - ["const char *", domain_name] + - [uint32_t, topo_valid] + - [uint32_t, group] + - [uint32_t, switch_id] + - [uint32_t, port] + fields: + - [ctf_integer, uint32_t, ep_id, ep_id] + - [ctf_string, src_addr, src_addr] + - [ctf_string, domain_name, domain_name] + - [ctf_integer, uint32_t, topo_valid, topo_valid] + - [ctf_integer, uint32_t, group, group] + - [ctf_integer, uint32_t, switch_id, switch_id] + - [ctf_integer, uint32_t, port, port] diff --git a/backends/libfabric/fi_trace.h b/backends/libfabric/src/fi_trace.h similarity index 96% rename from backends/libfabric/fi_trace.h rename to backends/libfabric/src/fi_trace.h index 3530e6b80..7976e6518 100644 --- a/backends/libfabric/fi_trace.h +++ b/backends/libfabric/src/fi_trace.h @@ -33,6 +33,17 @@ ((type *)((char *)(ptr) - offsetof(type, member))) #endif +struct fit_ep; +struct fit_cq; +struct fit_topo; + +void fit_record_entry(struct fit_ep *h, const char *op, char dir, + const void *context); +void fit_record_ep_info(unsigned int ep_id, const char *src, + size_t src_len, const struct fit_topo *t); +void fit_record_ctrl_entry(const char *op, unsigned int obj_id); +void fit_record_ctrl_exit(const char *op, unsigned int obj_id, int ret); + /* ------------------------------------------------------------------ */ /* configuration (read once from the environment at load time) */ /* ------------------------------------------------------------------ */ @@ -116,8 +127,6 @@ void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, const struct fi_info *info); /* Record one endpoint's location, once, when its address is known. */ -void fit_topo_record(const char *src, size_t src_len, unsigned int ep_id, - const struct fit_topo *t); /* ------------------------------------------------------------------ */ /* per-object hook state */ @@ -228,7 +237,7 @@ struct fit_cq { /* writer */ /* ------------------------------------------------------------------ */ -void fit_writer_init(void); +// void fit_writer_init(void); void fit_writer_flush_all(void); /* Emit one CSV record. dir is 'T' (local initiates) or 'R' (local posts diff --git a/backends/libfabric/fit_addr.c b/backends/libfabric/src/fit_addr.c similarity index 85% rename from backends/libfabric/fit_addr.c rename to backends/libfabric/src/fit_addr.c index 8a7563617..a7ba1b259 100644 --- a/backends/libfabric/fit_addr.c +++ b/backends/libfabric/src/fit_addr.c @@ -100,7 +100,7 @@ void fit_ep_resolve_src(struct fit_ep *h) * once per endpoint, never on the hot path. */ if (h->src_valid && !h->topo_written) { h->topo_written = 1; - fit_topo_record(h->src, h->src_len, h->ep_id, h->topo); + fit_record_ep_info(h->ep_id, h->src, h->src_len, h->topo); } pthread_mutex_unlock(&h->lock); @@ -202,73 +202,6 @@ void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, /* for the buffered per-thread machinery the hot path needs. */ /* ------------------------------------------------------------------ */ -static pthread_mutex_t fit_topo_lock = PTHREAD_MUTEX_INITIALIZER; -static int fit_topo_fd = -1; -static int fit_topo_bad; - -static const char fit_topo_header[] = - "host,pid,ep_id,src_addr,nic,group,switch,port\n"; - -static void topo_open_locked(void) -{ - /* dir is 4096, host up to 255, plus the fixed parts and the pid. */ - char path[4480]; - - if (fit_topo_fd >= 0 || fit_topo_bad) - return; - - /* Per process, not per thread: endpoints are a process-wide thing - * and there are only a handful of them. */ - snprintf(path, sizeof(path), "%s/fitrace-topo.%s.%d.csv", - fit_cfg.dir, fit_cfg.host, (int)getpid()); - - fit_topo_fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); - if (fit_topo_fd < 0) { - fit_log("cannot open %s: %s", path, strerror(errno)); - fit_topo_bad = 1; - return; - } - if (write(fit_topo_fd, fit_topo_header, - sizeof(fit_topo_header) - 1) < 0) - fit_topo_bad = 1; -} - -void fit_topo_record(const char *src, size_t src_len, unsigned int ep_id, - const struct fit_topo *t) -{ - char line[FIT_REC_MAX]; - int n; - - if (!fit_armed || !src || !src_len) - return; - - pthread_mutex_lock(&fit_topo_lock); - topo_open_locked(); - if (fit_topo_bad || fit_topo_fd < 0) { - pthread_mutex_unlock(&fit_topo_lock); - return; - } - - /* An endpoint whose provider would not place it still earns a row: - * knowing that this address exists on this host, under this NIC - * name, is most of the value, and an empty coordinate is honest - * about the rest. */ - if (t && t->valid) - n = snprintf(line, sizeof(line), "%s,%d,%u,%s,%s,%u,%u,%u\n", - fit_cfg.host, (int)getpid(), ep_id, src, - t->name, t->group, t->sw, t->port); - else - n = snprintf(line, sizeof(line), "%s,%d,%u,%s,%s,,,\n", - fit_cfg.host, (int)getpid(), ep_id, src, - t ? t->name : ""); - - if (n > 0 && (size_t)n < sizeof(line)) { - if (write(fit_topo_fd, line, (size_t)n) < 0) - fit_topo_bad = 1; - } - pthread_mutex_unlock(&fit_topo_lock); -} - /* ------------------------------------------------------------------ */ /* peer cache */ /* ------------------------------------------------------------------ */ diff --git a/backends/libfabric/fit_hook.c b/backends/libfabric/src/fit_hook.c similarity index 74% rename from backends/libfabric/fit_hook.c rename to backends/libfabric/src/fit_hook.c index 28aef3b7c..96749adda 100644 --- a/backends/libfabric/fit_hook.c +++ b/backends/libfabric/src/fit_hook.c @@ -52,9 +52,11 @@ static ssize_t fit_msg_send(struct fid_ep *ep, const void *buf, size_t len, void *desc, fi_addr_t dest_addr, void *context) { struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_msg->send, - (ep, buf, len, desc, dest_addr, context)); + ssize_t ret; + fit_record_entry(h, "fi_send", 'T', context); + ret = CALL_OR_ENOSYS(h->o_msg->send, + (ep, buf, len, desc, dest_addr, context)); fit_record(h, "fi_send", 'T', dest_addr, len, 1, ret, context); return ret; } @@ -64,9 +66,11 @@ static ssize_t fit_msg_sendv(struct fid_ep *ep, const struct iovec *iov, void *context) { struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_msg->sendv, - (ep, iov, desc, count, dest_addr, context)); + ssize_t ret; + fit_record_entry(h, "fi_sendv", 'T', context); + ret = CALL_OR_ENOSYS(h->o_msg->sendv, + (ep, iov, desc, count, dest_addr, context)); fit_record(h, "fi_sendv", 'T', dest_addr, fit_iov_len(iov, count), count, ret, context); return ret; @@ -76,8 +80,10 @@ static ssize_t fit_msg_sendmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) { struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_msg->sendmsg, (ep, msg, flags)); + ssize_t ret; + fit_record_entry(h, "fi_sendmsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_msg->sendmsg, (ep, msg, flags)); fit_record(h, "fi_sendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); @@ -88,8 +94,10 @@ static ssize_t fit_msg_inject(struct fid_ep *ep, const void *buf, size_t len, fi_addr_t dest_addr) { struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_msg->inject, (ep, buf, len, dest_addr)); + ssize_t ret; + fit_record_entry(h, "fi_inject", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_msg->inject, (ep, buf, len, dest_addr)); fit_record(h, "fi_inject", 'T', dest_addr, len, 1, ret, NULL); return ret; } @@ -99,10 +107,12 @@ static ssize_t fit_msg_senddata(struct fid_ep *ep, const void *buf, size_t len, void *context) { struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_msg->senddata, - (ep, buf, len, desc, data, dest_addr, - context)); + ssize_t ret; + fit_record_entry(h, "fi_senddata", 'T', context); + ret = CALL_OR_ENOSYS(h->o_msg->senddata, + (ep, buf, len, desc, data, dest_addr, + context)); fit_record(h, "fi_senddata", 'T', dest_addr, len, 1, ret, context); return ret; } @@ -112,9 +122,11 @@ static ssize_t fit_msg_injectdata(struct fid_ep *ep, const void *buf, fi_addr_t dest_addr) { struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_msg->injectdata, - (ep, buf, len, data, dest_addr)); + ssize_t ret; + fit_record_entry(h, "fi_injectdata", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_msg->injectdata, + (ep, buf, len, data, dest_addr)); fit_record(h, "fi_injectdata", 'T', dest_addr, len, 1, ret, NULL); return ret; } @@ -123,9 +135,11 @@ static ssize_t fit_msg_recv(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, void *context) { struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_msg->recv, - (ep, buf, len, desc, src_addr, context)); + ssize_t ret; + fit_record_entry(h, "fi_recv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_msg->recv, + (ep, buf, len, desc, src_addr, context)); fit_record(h, "fi_recv", 'R', src_addr, len, 1, ret, context); return ret; } @@ -135,9 +149,11 @@ static ssize_t fit_msg_recvv(struct fid_ep *ep, const struct iovec *iov, void *context) { struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_msg->recvv, - (ep, iov, desc, count, src_addr, context)); + ssize_t ret; + fit_record_entry(h, "fi_recvv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_msg->recvv, + (ep, iov, desc, count, src_addr, context)); fit_record(h, "fi_recvv", 'R', src_addr, fit_iov_len(iov, count), count, ret, context); return ret; @@ -147,8 +163,10 @@ static ssize_t fit_msg_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) { struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_msg->recvmsg, (ep, msg, flags)); + ssize_t ret; + fit_record_entry(h, "fi_recvmsg", 'R', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_msg->recvmsg, (ep, msg, flags)); fit_record(h, "fi_recvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); @@ -164,10 +182,12 @@ static ssize_t fit_tag_send(struct fid_ep *ep, const void *buf, size_t len, void *context) { struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->send, - (ep, buf, len, desc, dest_addr, tag, - context)); + ssize_t ret; + fit_record_entry(h, "fi_tsend", 'T', context); + ret = CALL_OR_ENOSYS(h->o_tagged->send, + (ep, buf, len, desc, dest_addr, tag, + context)); fit_record(h, "fi_tsend", 'T', dest_addr, len, 1, ret, context); return ret; } @@ -177,10 +197,12 @@ static ssize_t fit_tag_sendv(struct fid_ep *ep, const struct iovec *iov, uint64_t tag, void *context) { struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->sendv, - (ep, iov, desc, count, dest_addr, tag, - context)); + ssize_t ret; + fit_record_entry(h, "fi_tsendv", 'T', context); + ret = CALL_OR_ENOSYS(h->o_tagged->sendv, + (ep, iov, desc, count, dest_addr, tag, + context)); fit_record(h, "fi_tsendv", 'T', dest_addr, fit_iov_len(iov, count), count, ret, context); return ret; @@ -190,8 +212,10 @@ static ssize_t fit_tag_sendmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) { struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->sendmsg, (ep, msg, flags)); + ssize_t ret; + fit_record_entry(h, "fi_tsendmsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->sendmsg, (ep, msg, flags)); fit_record(h, "fi_tsendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); @@ -202,9 +226,11 @@ static ssize_t fit_tag_inject(struct fid_ep *ep, const void *buf, size_t len, fi_addr_t dest_addr, uint64_t tag) { struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->inject, - (ep, buf, len, dest_addr, tag)); + ssize_t ret; + fit_record_entry(h, "fi_tinject", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->inject, + (ep, buf, len, dest_addr, tag)); fit_record(h, "fi_tinject", 'T', dest_addr, len, 1, ret, NULL); return ret; } @@ -214,10 +240,12 @@ static ssize_t fit_tag_senddata(struct fid_ep *ep, const void *buf, size_t len, uint64_t tag, void *context) { struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->senddata, - (ep, buf, len, desc, data, dest_addr, tag, - context)); + ssize_t ret; + fit_record_entry(h, "fi_tsenddata", 'T', context); + ret = CALL_OR_ENOSYS(h->o_tagged->senddata, + (ep, buf, len, desc, data, dest_addr, tag, + context)); fit_record(h, "fi_tsenddata", 'T', dest_addr, len, 1, ret, context); return ret; } @@ -227,9 +255,11 @@ static ssize_t fit_tag_injectdata(struct fid_ep *ep, const void *buf, fi_addr_t dest_addr, uint64_t tag) { struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->injectdata, - (ep, buf, len, data, dest_addr, tag)); + ssize_t ret; + fit_record_entry(h, "fi_tinjectdata", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->injectdata, + (ep, buf, len, data, dest_addr, tag)); fit_record(h, "fi_tinjectdata", 'T', dest_addr, len, 1, ret, NULL); return ret; } @@ -239,10 +269,12 @@ static ssize_t fit_tag_recv(struct fid_ep *ep, void *buf, size_t len, uint64_t ignore, void *context) { struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->recv, - (ep, buf, len, desc, src_addr, tag, ignore, - context)); + ssize_t ret; + fit_record_entry(h, "fi_trecv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_tagged->recv, + (ep, buf, len, desc, src_addr, tag, ignore, + context)); fit_record(h, "fi_trecv", 'R', src_addr, len, 1, ret, context); return ret; } @@ -252,10 +284,12 @@ static ssize_t fit_tag_recvv(struct fid_ep *ep, const struct iovec *iov, uint64_t tag, uint64_t ignore, void *context) { struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->recvv, - (ep, iov, desc, count, src_addr, tag, - ignore, context)); + ssize_t ret; + fit_record_entry(h, "fi_trecvv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_tagged->recvv, + (ep, iov, desc, count, src_addr, tag, + ignore, context)); fit_record(h, "fi_trecvv", 'R', src_addr, fit_iov_len(iov, count), count, ret, context); return ret; @@ -265,8 +299,10 @@ static ssize_t fit_tag_recvmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) { struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_tagged->recvmsg, (ep, msg, flags)); + ssize_t ret; + fit_record_entry(h, "fi_trecvmsg", 'R', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->recvmsg, (ep, msg, flags)); fit_record(h, "fi_trecvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); @@ -282,10 +318,12 @@ static ssize_t fit_rma_write(struct fid_ep *ep, const void *buf, size_t len, uint64_t key, void *context) { struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_rma->write, - (ep, buf, len, desc, dest_addr, addr, key, - context)); + ssize_t ret; + fit_record_entry(h, "fi_write", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->write, + (ep, buf, len, desc, dest_addr, addr, key, + context)); fit_record(h, "fi_write", 'T', dest_addr, len, 1, ret, context); return ret; } @@ -295,10 +333,12 @@ static ssize_t fit_rma_writev(struct fid_ep *ep, const struct iovec *iov, uint64_t addr, uint64_t key, void *context) { struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_rma->writev, - (ep, iov, desc, count, dest_addr, addr, - key, context)); + ssize_t ret; + fit_record_entry(h, "fi_writev", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->writev, + (ep, iov, desc, count, dest_addr, addr, + key, context)); fit_record(h, "fi_writev", 'T', dest_addr, fit_iov_len(iov, count), count, ret, context); return ret; @@ -308,8 +348,10 @@ static ssize_t fit_rma_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) { struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_rma->writemsg, (ep, msg, flags)); + ssize_t ret; + fit_record_entry(h, "fi_writemsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_rma->writemsg, (ep, msg, flags)); fit_record(h, "fi_writemsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); @@ -320,9 +362,11 @@ static ssize_t fit_rma_inject(struct fid_ep *ep, const void *buf, size_t len, fi_addr_t dest_addr, uint64_t addr, uint64_t key) { struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_rma->inject, - (ep, buf, len, dest_addr, addr, key)); + ssize_t ret; + fit_record_entry(h, "fi_inject_write", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_rma->inject, + (ep, buf, len, dest_addr, addr, key)); fit_record(h, "fi_inject_write", 'T', dest_addr, len, 1, ret, NULL); return ret; } @@ -333,10 +377,12 @@ static ssize_t fit_rma_writedata(struct fid_ep *ep, const void *buf, uint64_t key, void *context) { struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_rma->writedata, - (ep, buf, len, desc, data, dest_addr, addr, - key, context)); + ssize_t ret; + fit_record_entry(h, "fi_writedata", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->writedata, + (ep, buf, len, desc, data, dest_addr, addr, + key, context)); fit_record(h, "fi_writedata", 'T', dest_addr, len, 1, ret, context); return ret; } @@ -347,9 +393,11 @@ static ssize_t fit_rma_injectdata(struct fid_ep *ep, const void *buf, uint64_t key) { struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_rma->injectdata, - (ep, buf, len, data, dest_addr, addr, key)); + ssize_t ret; + fit_record_entry(h, "fi_inject_writedata", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_rma->injectdata, + (ep, buf, len, data, dest_addr, addr, key)); fit_record(h, "fi_inject_writedata", 'T', dest_addr, len, 1, ret, NULL); return ret; } @@ -359,10 +407,12 @@ static ssize_t fit_rma_read(struct fid_ep *ep, void *buf, size_t len, uint64_t key, void *context) { struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_rma->read, - (ep, buf, len, desc, src_addr, addr, key, - context)); + ssize_t ret; + fit_record_entry(h, "fi_read", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->read, + (ep, buf, len, desc, src_addr, addr, key, + context)); fit_record(h, "fi_read", 'T', src_addr, len, 1, ret, context); return ret; } @@ -372,10 +422,12 @@ static ssize_t fit_rma_readv(struct fid_ep *ep, const struct iovec *iov, uint64_t addr, uint64_t key, void *context) { struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_rma->readv, - (ep, iov, desc, count, src_addr, addr, key, - context)); + ssize_t ret; + fit_record_entry(h, "fi_readv", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->readv, + (ep, iov, desc, count, src_addr, addr, key, + context)); fit_record(h, "fi_readv", 'T', src_addr, fit_iov_len(iov, count), count, ret, context); return ret; @@ -385,8 +437,10 @@ static ssize_t fit_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) { struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret = CALL_OR_ENOSYS(h->o_rma->readmsg, (ep, msg, flags)); + ssize_t ret; + fit_record_entry(h, "fi_readmsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_rma->readmsg, (ep, msg, flags)); fit_record(h, "fi_readmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); @@ -472,7 +526,7 @@ static void fit_ep_arm_ops(struct fit_ep *h, const char *when) } } -static int fit_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags) +static int fit_ep_bind_inner(struct fid *fid, struct fid *bfid, uint64_t flags) { struct fit_ep *h = EP_FROM_FID(fid); int ret = h->o_fid->bind ? h->o_fid->bind(fid, bfid, flags) : -FI_ENOSYS; @@ -497,7 +551,19 @@ static int fit_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags) return ret; } -static int fit_ep_control(struct fid *fid, int command, void *arg) +static int fit_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_ep_bind", fit_obj); + fit_ret = fit_ep_bind_inner(fid, bfid, flags); + fit_record_ctrl_exit("fi_ep_bind", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_control_inner(struct fid *fid, int command, void *arg) { struct fit_ep *h = EP_FROM_FID(fid); int ret = h->o_fid->control ? h->o_fid->control(fid, command, arg) : @@ -513,13 +579,23 @@ static int fit_ep_control(struct fid *fid, int command, void *arg) return ret; } -static int fit_ep_close(struct fid *fid) +static int fit_ep_control(struct fid *fid, int command, void *arg) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_ep_control", fit_obj); + fit_ret = fit_ep_control_inner(fid, command, arg); + fit_record_ctrl_exit("fi_ep_control", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_close_inner(struct fid *fid) { struct fit_ep *h = EP_FROM_FID(fid); int ret; - fit_writer_flush_all(); - ret = h->o_fid->close ? h->o_fid->close(fid) : -FI_ENOSYS; if (ret) return ret; @@ -531,7 +607,19 @@ static int fit_ep_close(struct fid *fid) return 0; } -static int fit_ep_tx_ctx(struct fid_ep *sep, int index, +static int fit_ep_close(struct fid *fid) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_close_ep", fit_obj); + fit_ret = fit_ep_close_inner(fid); + fit_record_ctrl_exit("fi_close_ep", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_tx_ctx_inner(struct fid_ep *sep, int index, struct fi_tx_attr *attr, struct fid_ep **tx_ep, void *context) { @@ -544,7 +632,21 @@ static int fit_ep_tx_ctx(struct fid_ep *sep, int index, return ret; } -static int fit_ep_rx_ctx(struct fid_ep *sep, int index, +static int fit_ep_tx_ctx(struct fid_ep *sep, int index, + struct fi_tx_attr *attr, struct fid_ep **tx_ep, + void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_EPOPS(sep)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_tx_context", fit_obj); + fit_ret = fit_ep_tx_ctx_inner(sep, index, attr, tx_ep, context); + fit_record_ctrl_exit("fi_tx_context", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_rx_ctx_inner(struct fid_ep *sep, int index, struct fi_rx_attr *attr, struct fid_ep **rx_ep, void *context) { @@ -557,6 +659,20 @@ static int fit_ep_rx_ctx(struct fid_ep *sep, int index, return ret; } +static int fit_ep_rx_ctx(struct fid_ep *sep, int index, + struct fi_rx_attr *attr, struct fid_ep **rx_ep, + void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_EPOPS(sep)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_rx_context", fit_obj); + fit_ret = fit_ep_rx_ctx_inner(sep, index, attr, rx_ep, context); + fit_record_ctrl_exit("fi_rx_context", fit_obj, fit_ret); + return fit_ret; +} + void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo) { struct fit_ep *h; @@ -610,7 +726,7 @@ void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo) #define AV_FROM_OPS(av) fit_container_of((av)->ops, struct fit_av, ops) -static int fit_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count, +static int fit_av_remove_inner(struct fid_av *av, fi_addr_t *fi_addr, size_t count, uint64_t flags) { struct fit_av *a = AV_FROM_OPS(av); @@ -622,7 +738,20 @@ static int fit_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count, return ret; } -static int fit_av_close(struct fid *fid) +static int fit_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count, + uint64_t flags) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_av_remove", fit_obj); + fit_ret = fit_av_remove_inner(av, fi_addr, count, flags); + fit_record_ctrl_exit("fi_av_remove", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_av_close_inner(struct fid *fid) { struct fit_av *a = fit_container_of(fid->ops, struct fit_av, fid_ops); int ret = a->o_fid->close ? a->o_fid->close(fid) : -FI_ENOSYS; @@ -634,6 +763,18 @@ static int fit_av_close(struct fid *fid) return ret; } +static int fit_av_close(struct fid *fid) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_close_av", fit_obj); + fit_ret = fit_av_close_inner(fid); + fit_record_ctrl_exit("fi_close_av", fit_obj, fit_ret); + return fit_ret; +} + void fit_hook_av(struct fid_av *av) { struct fit_av *a; @@ -852,19 +993,29 @@ static ssize_t fit_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf, return ret; } -static int fit_cq_close(struct fid *fid) +static int fit_cq_close_inner(struct fid *fid) { struct fit_cq *c = fit_container_of(fid->ops, struct fit_cq, fid_ops); int ret; - fit_writer_flush_all(); - ret = c->o_fid->close ? c->o_fid->close(fid) : -FI_ENOSYS; if (!ret) free(c); return ret; } +static int fit_cq_close(struct fid *fid) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(fit_container_of(fid->ops, struct fit_cq, fid_ops)->cq_id); + int fit_ret; + + fit_record_ctrl_entry("fi_close_cq", fit_obj); + fit_ret = fit_cq_close_inner(fid); + fit_record_ctrl_exit("fi_close_cq", fit_obj, fit_ret); + return fit_ret; +} + void fit_hook_cq(struct fid_cq *cq, enum fi_cq_format format) { struct fit_cq *c; @@ -942,7 +1093,7 @@ static void fit_cq_attach_ep(struct fid *bfid, struct fit_ep *h) #define DOM_FROM_OPS(dom) fit_container_of((dom)->ops, struct fit_domain, ops) -static int fit_dom_endpoint(struct fid_domain *domain, struct fi_info *info, +static int fit_dom_endpoint_inner(struct fid_domain *domain, struct fi_info *info, struct fid_ep **ep, void *context) { struct fit_domain *d = DOM_FROM_OPS(domain); @@ -954,7 +1105,20 @@ static int fit_dom_endpoint(struct fid_domain *domain, struct fi_info *info, return ret; } -static int fit_dom_endpoint2(struct fid_domain *domain, struct fi_info *info, +static int fit_dom_endpoint(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_endpoint", fit_obj); + fit_ret = fit_dom_endpoint_inner(domain, info, ep, context); + fit_record_ctrl_exit("fi_endpoint", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_endpoint2_inner(struct fid_domain *domain, struct fi_info *info, struct fid_ep **ep, uint64_t flags, void *context) { struct fit_domain *d = DOM_FROM_OPS(domain); @@ -967,7 +1131,20 @@ static int fit_dom_endpoint2(struct fid_domain *domain, struct fi_info *info, return ret; } -static int fit_dom_scalable_ep(struct fid_domain *domain, struct fi_info *info, +static int fit_dom_endpoint2(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, uint64_t flags, void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_endpoint2", fit_obj); + fit_ret = fit_dom_endpoint2_inner(domain, info, ep, flags, context); + fit_record_ctrl_exit("fi_endpoint2", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_scalable_ep_inner(struct fid_domain *domain, struct fi_info *info, struct fid_ep **sep, void *context) { struct fit_domain *d = DOM_FROM_OPS(domain); @@ -979,7 +1156,20 @@ static int fit_dom_scalable_ep(struct fid_domain *domain, struct fi_info *info, return ret; } -static int fit_dom_srx_ctx(struct fid_domain *domain, struct fi_rx_attr *attr, +static int fit_dom_scalable_ep(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **sep, void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_scalable_ep", fit_obj); + fit_ret = fit_dom_scalable_ep_inner(domain, info, sep, context); + fit_record_ctrl_exit("fi_scalable_ep", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_srx_ctx_inner(struct fid_domain *domain, struct fi_rx_attr *attr, struct fid_ep **rx_ep, void *context) { struct fit_domain *d = DOM_FROM_OPS(domain); @@ -996,7 +1186,20 @@ static int fit_dom_srx_ctx(struct fid_domain *domain, struct fi_rx_attr *attr, return ret; } -static int fit_dom_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, +static int fit_dom_srx_ctx(struct fid_domain *domain, struct fi_rx_attr *attr, + struct fid_ep **rx_ep, void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_srx_context", fit_obj); + fit_ret = fit_dom_srx_ctx_inner(domain, attr, rx_ep, context); + fit_record_ctrl_exit("fi_srx_context", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_cq_open_inner(struct fid_domain *domain, struct fi_cq_attr *attr, struct fid_cq **cq, void *context) { struct fit_domain *d = DOM_FROM_OPS(domain); @@ -1008,7 +1211,20 @@ static int fit_dom_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, return ret; } -static int fit_dom_av_open(struct fid_domain *domain, struct fi_av_attr *attr, +static int fit_dom_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_cq_open", fit_obj); + fit_ret = fit_dom_cq_open_inner(domain, attr, cq, context); + fit_record_ctrl_exit("fi_cq_open", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_av_open_inner(struct fid_domain *domain, struct fi_av_attr *attr, struct fid_av **av, void *context) { struct fit_domain *d = DOM_FROM_OPS(domain); @@ -1020,7 +1236,20 @@ static int fit_dom_av_open(struct fid_domain *domain, struct fi_av_attr *attr, return ret; } -static int fit_dom_close(struct fid *fid) +static int fit_dom_av_open(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_av_open", fit_obj); + fit_ret = fit_dom_av_open_inner(domain, attr, av, context); + fit_record_ctrl_exit("fi_av_open", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_close_inner(struct fid *fid) { struct fit_domain *d = fit_container_of(fid->ops, struct fit_domain, fid_ops); @@ -1031,6 +1260,18 @@ static int fit_dom_close(struct fid *fid) return ret; } +static int fit_dom_close(struct fid *fid) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_close_domain", fit_obj); + fit_ret = fit_dom_close_inner(fid); + fit_record_ctrl_exit("fi_close_domain", fit_obj, fit_ret); + return fit_ret; +} + void fit_hook_domain(struct fid_domain *domain, const struct fi_info *info) { struct fit_domain *d; @@ -1079,7 +1320,7 @@ void fit_hook_domain(struct fid_domain *domain, const struct fi_info *info) #define FAB_FROM_OPS(fab) fit_container_of((fab)->ops, struct fit_fabric, ops) -static int fit_fab_domain(struct fid_fabric *fabric, struct fi_info *info, +static int fit_fab_domain_inner(struct fid_fabric *fabric, struct fi_info *info, struct fid_domain **dom, void *context) { struct fit_fabric *f = FAB_FROM_OPS(fabric); @@ -1091,7 +1332,20 @@ static int fit_fab_domain(struct fid_fabric *fabric, struct fi_info *info, return ret; } -static int fit_fab_domain2(struct fid_fabric *fabric, struct fi_info *info, +static int fit_fab_domain(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **dom, void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_domain", fit_obj); + fit_ret = fit_fab_domain_inner(fabric, info, dom, context); + fit_record_ctrl_exit("fi_domain", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_fab_domain2_inner(struct fid_fabric *fabric, struct fi_info *info, struct fid_domain **dom, uint64_t flags, void *context) { @@ -1105,20 +1359,44 @@ static int fit_fab_domain2(struct fid_fabric *fabric, struct fi_info *info, return ret; } -static int fit_fab_close(struct fid *fid) +static int fit_fab_domain2(struct fid_fabric *fabric, struct fi_info *info, + struct fid_domain **dom, uint64_t flags, + void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_domain2", fit_obj); + fit_ret = fit_fab_domain2_inner(fabric, info, dom, flags, context); + fit_record_ctrl_exit("fi_domain2", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_fab_close_inner(struct fid *fid) { struct fit_fabric *f = fit_container_of(fid->ops, struct fit_fabric, fid_ops); int ret; - fit_writer_flush_all(); - ret = f->o_fid->close ? f->o_fid->close(fid) : -FI_ENOSYS; if (!ret) free(f); return ret; } +static int fit_fab_close(struct fid *fid) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_close_fabric", fit_obj); + fit_ret = fit_fab_close_inner(fid); + fit_record_ctrl_exit("fi_close_fabric", fit_obj, fit_ret); + return fit_ret; +} + void fit_hook_fabric(struct fid_fabric *fabric) { struct fit_fabric *f; @@ -1214,7 +1492,7 @@ static void fit_init(void) return; } - fit_writer_init(); + // fit_writer_init(); fit_armed = 1; fit_log("armed: dir=%s ops=%s%s%s rx=%d resolve=%d", fit_cfg.dir, fit_cfg.trace_msg ? "msg " : "", @@ -1232,8 +1510,7 @@ FIT_PUBLIC int fi_fabric(struct fi_fabric_attr *attr, FIT_PUBLIC int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, void *context); -FIT_PUBLIC -int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, +static int fi_fabric_inner(struct fi_fabric_attr *attr, struct fid_fabric **fabric, void *context) { int ret; @@ -1250,7 +1527,20 @@ int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, } FIT_PUBLIC -int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, +int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, + void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_fabric", fit_obj); + fit_ret = fi_fabric_inner(attr, fabric, context); + fit_record_ctrl_exit("fi_fabric", fit_obj, fit_ret); + return fit_ret; +} + +static int fi_fabric2_inner(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, void *context) { int ret; @@ -1265,3 +1555,18 @@ int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, fit_hook_fabric(*fabric); return ret; } + +FIT_PUBLIC +int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, + void *context) +{ + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_fabric2", fit_obj); + fit_ret = fi_fabric2_inner(info, fabric, flags, context); + fit_record_ctrl_exit("fi_fabric2", fit_obj, fit_ret); + return fit_ret; +} + diff --git a/backends/libfabric/src/fit_lttng.c b/backends/libfabric/src/fit_lttng.c new file mode 100644 index 000000000..358d58d42 --- /dev/null +++ b/backends/libfabric/src/fit_lttng.c @@ -0,0 +1,305 @@ +/* + * fit_lttng.c -- replaces fit_writer.c. + * + * The standalone tracer wrote sanitised CSV into a per-thread buffer and + * flushed it to one file per thread, then joined the pieces with + * tools/fitrace-merge. Under THAPI, LTTng owns all of that. What goes away: + * + * - the per-thread buffer, its mutex, the flush-on-exit destructor, and + * FI_TRACE_DIR / FI_TRACE_BUFSZ / FI_TRACE_MAX_RECORDS + * - the hostname, pid and tid columns: LTTng adds vpid and vtid as channel + * contexts and hostname as a trace environment entry, so the hot path + * never touches them + * - the cached getpid() and its pthread_atfork re-read, which existed only + * because the pid was a column + * - the timestamp: LTTng stamps every event from the monotonic clock + * - the CSV quoting in fit_sanitize(); CTF strings are length-delimited. + * fit_sanitize() is kept, because an address string with an embedded + * newline would still make the pretty-printed output unreadable, but it + * now only strips control characters. + * + * What is deliberately kept, because fit_hook.c and fit_addr.c call it and + * should not need editing for this: + * + * - the fit_record() / fit_record_cq() signatures + * - fit_cfg, fit_config_init(), fit_log(), fit_armed + * - fit_now_us(), now a stub returning 0 (see below) + * + * Environment variables are renamed FI_TRACE_* -> LTTNG_UST_FI_* to match the + * house convention. The ones that configured the CSV writer are gone. + */ + +#define _GNU_SOURCE + +#include "fi_trace.h" +#include "libfabric_tracepoints.h" + +#include +#include +#include +#include +#include + +#include + +struct fit_config fit_cfg; +int fit_armed; +_Atomic unsigned int fit_av_gen = 0; + +/* ------------------------------------------------------------------ */ +/* logging (stderr, not the trace -- this is for debugging the tracer) */ +/* ------------------------------------------------------------------ */ + +void fit_log(const char *fmt, ...) +{ + va_list ap; + + if (!fit_cfg.verbose) + return; + + fprintf(stderr, "[fi_trace] "); + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fputc('\n', stderr); +} + +/* ------------------------------------------------------------------ */ +/* configuration */ +/* ------------------------------------------------------------------ */ + +static int env_int(const char *name, int dflt) +{ + const char *v = getenv(name); + + return v ? atoi(v) : dflt; +} + +/* + * LTTNG_UST_FI_OPS selects which op families get their vtable swapped at all. + * This is not the same thing as enabling or disabling tracepoints: a slot we + * never swap costs the application literally nothing, where a disabled + * tracepoint still costs the branch. For an application that only does tagged + * messaging, "LTTNG_UST_FI_OPS=tagged" is measurably cheaper than filtering on + * the LTTng side. + */ +static void parse_ops(const char *v) +{ + if (!v || !*v) { + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 1; + return; + } + + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 0; + + if (strstr(v, "msg")) + fit_cfg.trace_msg = 1; + if (strstr(v, "tagged")) + fit_cfg.trace_tagged = 1; + if (strstr(v, "rma")) + fit_cfg.trace_rma = 1; +} + +void fit_config_init(void) +{ + fit_cfg.enabled = env_int("LTTNG_UST_FI_ENABLE", 1); + fit_cfg.resolve = env_int("LTTNG_UST_FI_RESOLVE", 1); + fit_cfg.trace_rx = env_int("LTTNG_UST_FI_RX", 1); + fit_cfg.trace_srx = env_int("LTTNG_UST_FI_SRX", 0); + fit_cfg.trace_eagain = env_int("LTTNG_UST_FI_EAGAIN", 1); + fit_cfg.trace_cq = env_int("LTTNG_UST_FI_CQ", 1); + fit_cfg.verbose = env_int("LTTNG_UST_FI_VERBOSE", 0); + parse_ops(getenv("LTTNG_UST_FI_OPS")); + + /* + * fit_cfg.host / host_len / dir / bufsz / max_records are no longer + * read. They stay in struct fit_config only so that fi_trace.h does + * not have to change; drop them once you are sure nothing else refers + * to them. + */ +} + +/* ------------------------------------------------------------------ */ +/* string hygiene */ +/* ------------------------------------------------------------------ */ + +/* + * CTF strings are length-delimited, so commas and quotes no longer need + * escaping. Control characters still do: they would corrupt the aligned + * columns of `iprof -t` output and, in the case of a stray NUL, truncate the + * field. Measured once where the string is produced, as before, so the hot + * path only memcpys. + */ +void fit_sanitize(char *s, size_t *len_out) +{ + size_t n = 0; + + if (!s) { + if (len_out) + *len_out = 0; + return; + } + + for (; s[n]; n++) { + if ((unsigned char)s[n] < 0x20 || (unsigned char)s[n] == 0x7f) + s[n] = '_'; + } + + if (len_out) + *len_out = n; +} + +/* + * Kept so fit_cq_report() in fit_hook.c compiles unchanged. LTTng timestamps + * every event itself from the monotonic clock, with better resolution than + * clock_gettime() in userspace gives you after the syscall, so there is + * nothing for this to do. + * + * Note the consequence for completions: a single fi_cq_read() returning a + * burst of N entries used to stamp all N with one fit_now_us() value, which + * was honest about the batch having no internal resolution. LTTng will now + * stamp them microseconds apart as the loop runs. Treat sub-microsecond + * spacing between completions from one reap as an artifact of the loop, not a + * real arrival pattern. + */ +uint64_t fit_now_us(void) +{ + return 0; +} + +/* ------------------------------------------------------------------ */ +/* data-path records */ +/* ------------------------------------------------------------------ */ + +/* + * Entry marker. Cheap on purpose: no AV lookup, no iov summation, no + * sanitisation. Its job is to make a hang or a crash inside the provider + * visible as an entry with no matching exit, per AGEND_SKILLS.md. All the + * payload rides on the exit event, where it was already being computed before + * this port. + */ +void fit_record_entry(struct fit_ep *h, const char *op, char dir, + const void *context) +{ + if (dir == 'R' && !fit_cfg.trace_rx) + return; + + tracepoint(lttng_ust_fi, op_entry, op, h->ep_id, context); +} + +void fit_record(struct fit_ep *h, const char *op, char dir, fi_addr_t peer, + size_t bytes, size_t iov_count, ssize_t ret, + const void *context) +{ + char scratch[FIT_ADDR_MAX]; + const char *peer_str; + size_t peer_len = 0; + + if (dir == 'R' && !fit_cfg.trace_rx) + return; + + /* + * A -FI_EAGAIN return means the provider refused the operation and the + * application will retry; no bytes moved. Applications that spin on a + * busy endpoint can generate thousands of these, which swamps the real + * transfers, so they can be dropped. + * + * Dropping the exit here leaves the matching op_entry unpaired, and the + * analysis side discards an entry that never closes. That is the + * intended behaviour, but it does mean LTTNG_UST_FI_EAGAIN=0 makes the + * entry/exit counts in a raw trace dump disagree -- do not go looking + * for a bug there. + */ + if (ret == -FI_EAGAIN && !fit_cfg.trace_eagain) + return; + + if (!h->src_valid) + fit_ep_resolve_src(h); + + peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), &peer_len); + + tracepoint(lttng_ust_fi, op_exit, op, h->ep_id, h->src, peer_str, + (dir == 'R') ? "rx" : "tx", (uint64_t)iov_count, + (uint64_t)bytes, (int64_t)ret, + /* An operation the provider refused never became + * outstanding, so it will produce no completion and must + * not be joined to one. */ + ret ? NULL : context); +} + +void fit_record_cq(struct fit_cq *c, const char *op, fi_addr_t peer, + size_t bytes, ssize_t ret, const void *context, + uint64_t now) +{ + char scratch[FIT_ADDR_MAX]; + const char *peer_str = "fi_addr_unspec"; + size_t peer_len = sizeof("fi_addr_unspec") - 1; + const char *src = "unknown"; + struct fit_ep *h = c->ep; + + (void)now; /* LTTng stamps the event; see fit_now_us() */ + + /* The CQ usually serves exactly one endpoint, in which case its local + * address and id describe this record too. */ + if (h) { + if (!h->src_valid) + fit_ep_resolve_src(h); + src = h->src; + peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), + &peer_len); + } + + tracepoint(lttng_ust_fi, cq_completion, op, + h ? h->ep_id : c->cq_id, src, peer_str, + (uint64_t)bytes, (int64_t)ret, context); +} + +/* ------------------------------------------------------------------ */ +/* control-path records */ +/* ------------------------------------------------------------------ */ + +void fit_record_ctrl_entry(const char *op, unsigned int obj_id) +{ + tracepoint(lttng_ust_fi, ctrl_entry, op, obj_id); +} + +void fit_record_ctrl_exit(const char *op, unsigned int obj_id, int ret) +{ + tracepoint(lttng_ust_fi, ctrl_exit, op, obj_id, (int64_t)ret); +} + +/* ------------------------------------------------------------------ */ +/* endpoint description */ +/* ------------------------------------------------------------------ */ + +/* + * Replaces fit_topo_record(), which appended a row to the sidecar CSV that + * fitrace-merge later joined against. Same data, same once-per-endpoint + * cadence (fit_addr.c guards it with h->topo_written); it is now an event and + * the analysis side builds the map in memory. + * + * The signature gained the ep's source address because the sidecar was keyed + * by NIC address while the interval stream is keyed by ep_id, and the mapping + * between them has to come from somewhere. + */ +void fit_record_ep_info(unsigned int ep_id, const char *src, size_t src_len, + const struct fit_topo *t) +{ + (void)src_len; + + if (!fit_armed || !src) + return; + + tracepoint(lttng_ust_fi, ep_info, ep_id, src, + (t && t->valid) ? t->name : "", + (t && t->valid) ? 1u : 0u, + (t && t->valid) ? t->group : 0u, + (t && t->valid) ? t->sw : 0u, + (t && t->valid) ? t->port : 0u); +} + +/* + * No __attribute__((destructor)) flush. LTTng's own atexit handling drains + * the ring buffers, and the blocking channel that iprof configures + * (--blocking-timeout=inf) means nothing is dropped on the way out. + */ diff --git a/backends/libfabric/tracer_libfabric.c b/backends/libfabric/tracer_libfabric.c deleted file mode 100644 index b99601e98..000000000 --- a/backends/libfabric/tracer_libfabric.c +++ /dev/null @@ -1,55 +0,0 @@ -#define _GNU_SOURCE - -#include -#include -#include - -#include -#include - -#include "libfabric_tracepoints.h" - -typedef int (*fi_fabric_fn)( - struct fi_fabric_attr *, - struct fid_fabric **, - void * -); - -static fi_fabric_fn original_fi_fabric; -static pthread_once_t resolve_once = PTHREAD_ONCE_INIT; - -static void resolve_fi_fabric(void) -{ - original_fi_fabric = - (fi_fabric_fn)dlsym(RTLD_NEXT, "fi_fabric"); -} - -int fi_fabric(struct fi_fabric_attr *attr, - struct fid_fabric **fabric, - void *context) -{ - int incoming_errno = errno; - - pthread_once(&resolve_once, resolve_fi_fabric); - - tracepoint(lttng_ust_libfabric, fi_fabric_entry, - attr, fabric, context); - - errno = incoming_errno; - - int result = original_fi_fabric - ? original_fi_fabric(attr, fabric, context) - : -FI_ENOSYS; - - int provider_errno = errno; - - struct fid_fabric *created = NULL; - if (result == 0 && fabric != NULL) - created = *fabric; - - tracepoint(lttng_ust_libfabric, fi_fabric_exit, - result, created); - - errno = provider_errno; - return result; -} \ No newline at end of file diff --git a/backends/libfabric/tracer_libfabric.sh.in b/backends/libfabric/tracer_libfabric.sh.in new file mode 100644 index 000000000..cb9a48f19 --- /dev/null +++ b/backends/libfabric/tracer_libfabric.sh.in @@ -0,0 +1,90 @@ +#!/bin/sh +# +# Standalone driver: set up an LTTng session, preload the tracer, run the +# application. No analysis -- inspect the resulting CTF with +# babeltrace_thapi, or just `babeltrace2 ~/lttng-traces/thapi-fi-session-*`. +# +# This is a developer convenience for checking that tracepoints fire at all. +# For real use go through iprof, which also handles the per-rank session setup +# that this script does not: +# +# iprof --backends fi -- ./app +# mpirun -n 64 -- iprof --backends fi -- ./app +# +exec_prefix=@prefix@ +libdir=@libdir@ +prefix=@prefix@ +datarootdir=@datarootdir@ +datadir=@datadir@ + +display_help() { + echo "Usage:" + echo " $(basename $0) [options] [--] " + echo " --help Show this screen" + echo " --version Print the version string" + echo "" + echo "Environment:" + echo " LTTNG_UST_FI_ENABLE=0 Load but do not hook anything" + echo " LTTNG_UST_FI_OPS=msg,tagged,rma Which op families to swap in" + echo " LTTNG_UST_FI_RX=0 Skip receive postings" + echo " LTTNG_UST_FI_SRX=0 Skip shared receive contexts" + echo " LTTNG_UST_FI_EAGAIN=0 Drop -FI_EAGAIN postings" + echo " LTTNG_UST_FI_CQ=0 Do not hook completion queues" + echo " LTTNG_UST_FI_RESOLVE=0 Do not resolve addresses via the AV" + echo " LTTNG_UST_FI_VERBOSE=1 Tracer diagnostics on stderr" + exit 1 +} + +display_version() { + cat $datadir/thapi_version + exit 1 +} + +while true; do + case "$1" in + --help) + display_help + exit + ;; + --version) + display_version + exit + ;; + --) + shift + break + ;; + *) break ;; + esac +done + +if [ "$#" -eq 0 ]; then + display_help +fi + +lttng-sessiond --daemonize --quiet +lttng create thapi-fi-session +lttng enable-channel --userspace --blocking-timeout=inf blocking-channel +lttng add-context --userspace --channel=blocking-channel -t vpid -t vtid +lttng enable-event --channel=blocking-channel --userspace lttng_ust_fi:* + +# No LD_LIBRARY_PATH manipulation and no soname shadowing: the tracer is +# preloaded alongside the real libfabric.so and reaches it with +# dlvsym(RTLD_NEXT, "fi_fabric", ...). Everything below fi_fabric is reached +# through vtable pointers the provider itself handed us. +export LD_PRELOAD=$libdir/libTracerLibfabric.so +export LTTNG_UST_ALLOW_BLOCKING=1 + +lttng start + +ctrl_c() { + lttng stop + lttng destroy + exit +} + +trap ctrl_c INT + +"$@" +lttng stop +lttng destroy diff --git a/backends/omp/tests/data_op_emi.thapi_text_pretty b/backends/omp/tests/data_op_emi.thapi_text_pretty deleted file mode 100644 index 89ddff7aa..000000000 --- a/backends/omp/tests/data_op_emi.thapi_text_pretty +++ /dev/null @@ -1,6 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_begin, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_alloc, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x0000000000000000, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } -12:00:00.001000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_end, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_alloc, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x00007ff074080040, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } -12:00:00.002000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_begin, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_delete, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x0000000000000000, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } -12:00:00.003000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_end, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_delete, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x00007ff074080040, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } -12:00:00.004000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_begin, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_associate, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x0000000000000000, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } -12:00:00.005000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_end, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_associate, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x00007ff074080040, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } diff --git a/backends/omp/tests/submit_emi.thapi_text_pretty b/backends/omp/tests/submit_emi.thapi_text_pretty deleted file mode 100644 index 91440d5f2..000000000 --- a/backends/omp/tests/submit_emi.thapi_text_pretty +++ /dev/null @@ -1,2 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_submit_emi: { endpoint: ompt_scope_begin, target_data: 0x0000000000000008, host_op_id: 0x0000000000000009, requested_num_teams: 3823427656 } -12:00:00.001000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_submit_emi: { endpoint: ompt_scope_end, target_data: 0x0000000000000008, host_op_id: 0x0000000000000009, requested_num_teams: 3823427656 } diff --git a/backends/omp/tests/target.thapi_text_pretty b/backends/omp/tests/target.thapi_text_pretty deleted file mode 100644 index f5f2c48e4..000000000 --- a/backends/omp/tests/target.thapi_text_pretty +++ /dev/null @@ -1,2 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target: { kind: ompt_target, endpoint: ompt_scope_begin, device_num: 0, task_data: 0x00007ff0755cfe80, task_data: 0x00007ff0755eac98, target_id: 1, codeptr_ra: 0x00007ff07af1643a } -12:00:00.001000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target: { kind: ompt_target, endpoint: ompt_scope_end, device_num: 0, task_data: 0x00007ff0755cfe80, task_data: 0x00007ff0755eac98, target_id: 1, codeptr_ra: 0x00007ff07af1643a } diff --git a/backends/omp/tests/target_emi.thapi_text_pretty b/backends/omp/tests/target_emi.thapi_text_pretty deleted file mode 100644 index 36b35c18b..000000000 --- a/backends/omp/tests/target_emi.thapi_text_pretty +++ /dev/null @@ -1,2 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_emi: { kind: ompt_target, endpoint: ompt_scope_begin, device_num: 0, task_data: 0x00007ff0755cfe80, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, codeptr_ra: 0x00007ff07af1643a } -12:00:00.001000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_emi: { kind: ompt_target, endpoint: ompt_scope_end, device_num: 0, task_data: 0x00007ff0755cfe80, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, codeptr_ra: 0x00007ff07af1643a } diff --git a/backends/opencl/tests/api_call.thapi_text_pretty b/backends/opencl/tests/api_call.thapi_text_pretty deleted file mode 100644 index f1af96107..000000000 --- a/backends/opencl/tests/api_call.thapi_text_pretty +++ /dev/null @@ -1,2 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clLinkProgram_entry: { context: 1, num_devices: 1, device_list: 0, options: 0, num_input_programs: 0, input_programs: 0, pfn_notify: 0, user_data: 0, errcode_ret: 0, _device_list_vals_length: 0, device_list_vals: 0, options_val: "", _input_programs_vals_length: 0, input_programs_vals: 0 } -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clLinkProgram_exit: { program: 1, errcode_ret_val: 0 } diff --git a/backends/opencl/tests/command_queue.thapi_text_pretty b/backends/opencl/tests/command_queue.thapi_text_pretty deleted file mode 100644 index e0947f8ce..000000000 --- a/backends/opencl/tests/command_queue.thapi_text_pretty +++ /dev/null @@ -1,6 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clCreateCommandQueue_entry: { context: 0, device: 10, properties: 0, errcode_ret: 0 } -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clCreateCommandQueue_exit: { command_queue: 23, errcode_ret_val: 0 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 23, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } -12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } -12:00:00.005000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 65 } diff --git a/backends/opencl/tests/command_queue_other_thread.thapi_text_pretty b/backends/opencl/tests/command_queue_other_thread.thapi_text_pretty deleted file mode 100644 index 66d09928e..000000000 --- a/backends/opencl/tests/command_queue_other_thread.thapi_text_pretty +++ /dev/null @@ -1,6 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl:clCreateCommandQueue_entry: { context: 0, device: 10, properties: 0, errcode_ret: 0 } -12:00:00.001000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl:clCreateCommandQueue_exit: { command_queue: 23, errcode_ret_val: 0 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 23, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } -12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } -12:00:00.005000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 65 } diff --git a/backends/opencl/tests/enqueue_read_buffer_fast.thapi_text_pretty b/backends/opencl/tests/enqueue_read_buffer_fast.thapi_text_pretty deleted file mode 100644 index af2e61f07..000000000 --- a/backends/opencl/tests/enqueue_read_buffer_fast.thapi_text_pretty +++ /dev/null @@ -1,4 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 10 } -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } diff --git a/backends/opencl/tests/interleave_process.thapi_text_pretty b/backends/opencl/tests/interleave_process.thapi_text_pretty deleted file mode 100644 index 799317e71..000000000 --- a/backends/opencl/tests/interleave_process.thapi_text_pretty +++ /dev/null @@ -1,8 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 11, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } -12:00:00.001000000 - testhost - vpid: 20, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 12, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 11 } -12:00:00.003000000 - testhost - vpid: 20, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 12 } -12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 11 } -12:00:00.005000000 - testhost - vpid: 20, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 12 } -12:00:00.006000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 11, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 10 } -12:00:00.007000000 - testhost - vpid: 20, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 12, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 20 } diff --git a/backends/opencl/tests/interleave_thread.thapi_text_pretty b/backends/opencl/tests/interleave_thread.thapi_text_pretty deleted file mode 100644 index 1cdcefeeb..000000000 --- a/backends/opencl/tests/interleave_thread.thapi_text_pretty +++ /dev/null @@ -1,8 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 11, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } -12:00:00.001000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 12, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 11 } -12:00:00.003000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 12 } -12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 11 } -12:00:00.005000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 12 } -12:00:00.006000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 11, event_command_exec_status: 0, queued_status: 0, queued: 10, submit_status: 0, submit: 0, start_status: 0, start: 15, end_status: 0, end: 25 } -12:00:00.007000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl_profiling:event_profiling_results: { event: 12, event_command_exec_status: 0, queued_status: 0, queued: 100, submit_status: 0, submit: 0, start_status: 0, start: 115, end_status: 0, end: 135 } diff --git a/backends/opencl/tests/nd_range_kernel.thapi_text_pretty b/backends/opencl/tests/nd_range_kernel.thapi_text_pretty deleted file mode 100644 index 1820d0550..000000000 --- a/backends/opencl/tests/nd_range_kernel.thapi_text_pretty +++ /dev/null @@ -1,5 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_arguments:kernel_info: { kernel: 12, function_name: "foo", num_args: 0, context: 1, program: 1, attributes: "" } -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueNDRangeKernel_entry: { command_queue: 10, kernel: 12, work_dim: 1, global_work_offset: 0, global_work_size: 1024, local_work_size: 256, num_events_in_wait_list: 0, event_wait_list: 0, event: 0, _event_wait_list_vals_length: 0, event_wait_list_vals: 0, _global_work_offset_vals_length: 0, global_work_offset_vals: 0, _global_work_size_vals_length: 0, global_work_size_vals: 0, _local_work_size_vals_length: 0, local_work_size_vals: 0 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueNDRangeKernel_exit: { errcode_ret_val: 0, event_val: 1 } -12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 10 } diff --git a/backends/opencl/tests/results_first.thapi_text_pretty b/backends/opencl/tests/results_first.thapi_text_pretty deleted file mode 100644 index 05b255e3e..000000000 --- a/backends/opencl/tests/results_first.thapi_text_pretty +++ /dev/null @@ -1,4 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 10, submit_status: 0, submit: 0, start_status: 0, start: 20, end_status: 0, end: 30 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } diff --git a/backends/opencl/tests/with_error.thapi_text_pretty b/backends/opencl/tests/with_error.thapi_text_pretty deleted file mode 100644 index cd0557fd3..000000000 --- a/backends/opencl/tests/with_error.thapi_text_pretty +++ /dev/null @@ -1,4 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } -12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: -58, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 10 } -12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } -12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } diff --git a/backends/ze/tests/interval_profiling_API_call.thapi_text_pretty b/backends/ze/tests/interval_profiling_API_call.thapi_text_pretty deleted file mode 100644 index d5dba5b84..000000000 --- a/backends/ze/tests/interval_profiling_API_call.thapi_text_pretty +++ /dev/null @@ -1,2 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeDriverGet_entry: { pCount: 1, phDrivers: 0, pCount_val: 1 } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeDriverGet_exit: { zeResult: ZE_RESULT_SUCCESS, pCount_val: 1, _phDrivers_vals_length: 0, phDrivers_vals: 0 } diff --git a/backends/ze/tests/interval_profiling_ignore.thapi_text_pretty b/backends/ze/tests/interval_profiling_ignore.thapi_text_pretty deleted file mode 100644 index 816a893b4..000000000 --- a/backends/ze/tests/interval_profiling_ignore.thapi_text_pretty +++ /dev/null @@ -1,4 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeEventHostSignal_entry: { hEvent: 0x1000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x2000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeEventHostSignal_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x2000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } diff --git a/backends/ze/tests/interval_profiling_interleave_process.thapi_text_pretty b/backends/ze/tests/interval_profiling_interleave_process.thapi_text_pretty deleted file mode 100644 index aaa1cda2c..000000000 --- a/backends/ze/tests/interval_profiling_interleave_process.thapi_text_pretty +++ /dev/null @@ -1,8 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.010000000 - testhost - vpid: 2, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.020000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x1000000000000000 } -12:00:00.030000000 - testhost - vpid: 2, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x2000000000000000 } -12:00:00.040000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.050000000 - testhost - vpid: 2, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.060000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x1000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } -12:00:00.070000000 - testhost - vpid: 2, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x2000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 20, contextStart: 0, contextEnd: 10 } diff --git a/backends/ze/tests/interval_profiling_multithread.thapi_text_pretty b/backends/ze/tests/interval_profiling_multithread.thapi_text_pretty deleted file mode 100644 index aa0ec4468..000000000 --- a/backends/ze/tests/interval_profiling_multithread.thapi_text_pretty +++ /dev/null @@ -1,8 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.000000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x2000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x1000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x2000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.030000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x2000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 20, contextStart: 0, contextEnd: 20 } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x1000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } diff --git a/backends/ze/tests/interval_profiling_normal.thapi_text_pretty b/backends/ze/tests/interval_profiling_normal.thapi_text_pretty deleted file mode 100644 index f6db269ab..000000000 --- a/backends/ze/tests/interval_profiling_normal.thapi_text_pretty +++ /dev/null @@ -1,4 +0,0 @@ -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x1000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x1000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } diff --git a/backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty b/backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty deleted file mode 100644 index b4c3ca9b4..000000000 --- a/backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty +++ /dev/null @@ -1,8 +0,0 @@ -# 1 Append, but the underlying cl is Executed twice in a real run, so -# 2 results arrive for the same hEvent. Both are attributed to that one -# Append. -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } -12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } diff --git a/backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty b/backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty deleted file mode 100644 index 64199d25b..000000000 --- a/backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty +++ /dev/null @@ -1,18 +0,0 @@ -# 4 Appends share one hEvent. Each Append's result is attributed back to -# its own Append, in submission order. -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.210000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.220000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.300000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.310000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.320000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.400000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } -12:00:00.410000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } -12:00:00.420000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 200, globalEnd: 240, contextStart: 200, contextEnd: 240 } -12:00:00.430000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 300, globalEnd: 350, contextStart: 300, contextEnd: 350 } diff --git a/backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty b/backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty deleted file mode 100644 index fb64b5d7a..000000000 --- a/backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty +++ /dev/null @@ -1,13 +0,0 @@ -# 2 Appends share one hEvent, then the underlying cl is Executed twice, -# so 4 results arrive. Each submission's pair of results is attributed to -# the two Appends in submission order. -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } -12:00:00.210000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } -12:00:00.300000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 200, globalEnd: 240, contextStart: 200, contextEnd: 240 } -12:00:00.310000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 300, globalEnd: 350, contextStart: 300, contextEnd: 350 } diff --git a/backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty b/backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty deleted file mode 100644 index e9d336d89..000000000 --- a/backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty +++ /dev/null @@ -1,21 +0,0 @@ -# Two build phases on the same cl, both reusing the same hEvent. -# Phase 1: 2 Appends, cl Executed twice -> 4 results. -# Phase 2: cl is Reset, then 1 Append, cl Executed three times -> 3 results. -# Results from a phase are attributed only to that phase's Appends; the -# phase-1 Appends do not bleed into the phase-2 results. -12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } -12:00:00.210000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } -12:00:00.300000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 200, globalEnd: 240, contextStart: 200, contextEnd: 240 } -12:00:00.310000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 300, globalEnd: 350, contextStart: 300, contextEnd: 350 } -12:00:00.400000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.410000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } -12:00:00.420000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } -12:00:00.500000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 400, globalEnd: 410, contextStart: 400, contextEnd: 410 } -12:00:00.510000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 500, globalEnd: 520, contextStart: 500, contextEnd: 520 } -12:00:00.520000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 600, globalEnd: 640, contextStart: 600, contextEnd: 640 } diff --git a/configure.ac b/configure.ac index 623b77b25..75d682fbb 100644 --- a/configure.ac +++ b/configure.ac @@ -178,6 +178,7 @@ AC_CONFIG_FILES([ backends/itt/Makefile backends/cxi/Makefile backends/pytorch/Makefile + backends/libfabric/Makefile ]) AC_CONFIG_FILES([utils/test_wrapper_thapi_text_pretty.sh], [chmod +x utils/test_wrapper_thapi_text_pretty.sh]) AC_CONFIG_FILES([backends/opencl/tracer_opencl.sh], [chmod +x backends/opencl/tracer_opencl.sh]) @@ -189,9 +190,10 @@ AC_CONFIG_FILES([backends/hip/tracer_hip.sh], [chmod +x backends/hip/tracer_hip. AC_CONFIG_FILES([backends/mpi/tracer_mpi.sh], [chmod +x backends/mpi/tracer_mpi.sh]) AC_CONFIG_FILES([backends/itt/tracer_itt.sh], [chmod +x backends/itt/tracer_itt.sh]) AC_CONFIG_FILES([backends/pytorch/tracer_pytorch.sh], [chmod +x backends/pytorch/tracer_pytorch.sh]) +AC_CONFIG_FILES([backends/libfabric/tracer_libfabric.sh], [chmod +x backends/libfabric/tracer_libfabric.sh]) AC_CONFIG_FILES([utils/babeltrace_thapi], [chmod +x utils/babeltrace_thapi]) AC_CONFIG_FILES([xprof/xprof.rb], [chmod +x xprof/xprof.rb]) AC_CONFIG_FILES([xprof/iprof:xprof/xprof_wrapper.rb.in], [chmod +x xprof/iprof]) AC_CONFIG_FILES([thapi.pc]) -AC_OUTPUT +AC_OUTPUT \ No newline at end of file diff --git a/utils/babeltrace_thapi.in b/utils/babeltrace_thapi.in index ac45321b5..2d5853e53 100755 --- a/utils/babeltrace_thapi.in +++ b/utils/babeltrace_thapi.in @@ -170,6 +170,7 @@ def get_components(names) require 'babeltrace_hip_lib' if $options[:'backend-names'].include?('hip') require 'babeltrace_itt_lib' if $options[:'backend-names'].include?('itt') require 'babeltrace_pytorch_lib' if $options[:'backend-names'].include?('pytorch') + require 'babeltrace_libfabric_lib' if $options[:'backend-names'].include?('fi') f = lambda { |iterator, _| iterator.next_messages.each do |m| next unless m.type == :BT_MESSAGE_TYPE_EVENT @@ -414,7 +415,7 @@ class BabeltraceParserThapi < OptionParserWithDefaultAndValidation on('-h', '--help', 'Prints this help') { print_help_and_exit(self, exit_code: 0) } on('-b', '--backends BACKENDS', Array, "Select which and how backends' need to handled.", 'Format: backend_name[:backend_level],...', - default: ['mpi:3', 'omp:2', 'cl:1', 'ze:1', 'cuda:1', 'hip:1', 'cxi:4', 'itt:5', 'pytorch:6']) + default: ['mpi:3', 'omp:2', 'cl:1', 'ze:1', 'cuda:1', 'hip:1', 'cxi:4', 'itt:5', 'pytorch:6', 'fi:7']) on('--debug', default: false) on('--archive SESSION-NAME') on('--archive-session-found-file-path PATH') diff --git a/utils/xprof_utils.hpp b/utils/xprof_utils.hpp index daec319d1..34bfe109c 100644 --- a/utils/xprof_utils.hpp +++ b/utils/xprof_utils.hpp @@ -24,6 +24,7 @@ enum backend_e { BACKEND_CXI = 8, BACKEND_ITT = 9, BACKEND_PYTORCH = 10, + BACKEND_FI = 11, }; typedef enum backend_e backend_t; typedef unsigned backend_level_t; @@ -41,6 +42,7 @@ const std::unordered_map pretty_backend_name_g = { {"cxi", BACKEND_CXI}, {"itt", BACKEND_ITT}, {"pytorch", BACKEND_PYTORCH}, + {"fi", BACKEND_FI}, }; const std::unordered_map backend_levels_g = { @@ -55,6 +57,7 @@ const std::unordered_map backend_levels_g = { {BACKEND_CXI, 4}, {BACKEND_ITT, 5}, {BACKEND_PYTORCH, 6}, + {BACKEND_FI, 7}, }; typedef std::string thapi_metadata_t; @@ -218,4 +221,4 @@ static inline std::string strip_event_class_name_entry(const char *str) { static inline std::string strip_event_class_name_exit(const char *str) { return strip_event_class_name(str, strlen("_exit")); -} +} \ No newline at end of file diff --git a/xprof/xprof.rb.in b/xprof/xprof.rb.in index ebc16acde..b8c1ccd94 100755 --- a/xprof/xprof.rb.in +++ b/xprof/xprof.rb.in @@ -659,6 +659,13 @@ def enable_events_pytorch(channel_name, tracing_mode: 'default', profiling: fals LOGGER.debug("Tracing mode #{tracing_mode} similar to default for pytorch") unless tracing_mode == 'default' end +def enable_events_fi(channel_name, tracing_mode: 'default', profiling: false) + lttng_enable = "lttng enable-event --userspace --session=#{lttng_session_uuid} --channel=#{channel_name}" + exec("#{lttng_enable} lttng_ust_fi:*") + LOGGER.debug('Profiling is ignored for fi') if profiling + LOGGER.debug("Tracing mode #{tracing_mode} similar to default for fi") unless tracing_mode == 'default' +end + def enable_events_metadata(channel_name, tracing_mode: 'default', profiling: true) lttng_enable = "lttng enable-event --userspace --session=#{lttng_session_uuid} --channel=#{channel_name}" LOGGER.debug('Profiling is ignored for metadata') if profiling @@ -741,7 +748,12 @@ module LocalMaster # if LTTNG_HOME doesn't exist FileUtils.mkdir_p(lttng_home_dir) FileUtils.mkdir_p(lttng_trace_dir_tmp) - exec('lttng-sessiond --daemonize') + # if ENV['THAPI_SESSIOND_FOREGROUND'] == '1' + # lttng_start_foreground + # else + # exec('lttng-sessiond --daemonize') + # end + lttng_start_foreground exec("lttng create #{lttng_session_uuid} -o #{lttng_trace_dir_tmp}") File.write(File.join(lttng_trace_dir_tmp, 'thapi_metadata.yaml'), { type: 'lttng' }.to_yaml) @@ -785,7 +797,66 @@ module LocalMaster exec("lttng destroy #{lttng_session_uuid}") end + # Optional workaround for installations where daemonized startup hangs. + # Keep daemon output out of Open3 pipes and wait for a working client socket. + def lttng_start_foreground + log_path = File.join(lttng_home_dir, 'sessiond.log') + pid = Process.spawn('lttng-sessiond', '--no-kernel', + in: File::NULL, out: log_path, err: [:child, :out]) + @sessiond_waiter = Process.detach(pid) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 20 + + begin + loop do + if @sessiond_waiter.join(0) + raise "lttng-sessiond exited: #{@sessiond_waiter.value}; see #{log_path}" + end + + # Bound each client probe as well as the overall startup wait. + ready = system('timeout', '--kill-after=1s', '2s', + 'lttng', '--no-sessiond', 'list', + out: File::NULL, err: File::NULL) + break if ready + + if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + raise "lttng-sessiond readiness timed out; see #{log_path}" + end + sleep 0.1 + end + rescue Exception + lttng_stop_foreground + raise + end + end + + def lttng_stop_foreground + waiter = @sessiond_waiter + return unless waiter + + unless waiter.join(0) + begin + Process.kill('TERM', waiter.pid) + rescue Errno::ESRCH + # The daemon exited between the check and the signal. + end + unless waiter.join(5) + begin + Process.kill('KILL', waiter.pid) + rescue Errno::ESRCH + end + waiter.join + end + end + @sessiond_waiter = nil + end + def lttng_kill_sessiond + if @sessiond_waiter + lttng_stop_foreground + FileUtils.rm_rf(lttng_home_dir) + return + end + # Need to kill the sessiond Daemon. It's safe because each job has their own # # In theory, opening the lttng-sessiond.pid file is racy. @@ -992,6 +1063,14 @@ def all_env_tracers(usr_binary) end end + if OPTIONS[:'backend-names'].include?('fi') + backends << 'fi' + h[%w[LD_PRELOAD prepend]] << File.join(LIBDIR, 'libTracerLibfabric.so') + h['LTTNG_UST_FI_OPS'] = OPTIONS[:'fi-ops'] if OPTIONS[:'fi-ops'] + h['LTTNG_UST_FI_SRX'] = 1 if OPTIONS[:'fi-srx'] + h['LTTNG_UST_FI_VERBOSE'] = 1 if LOGGER.level <= Logger::DEBUG + end + # Sample if SamplingDaemon.active? LOGGER.debug('Sampling Enabled') @@ -1085,7 +1164,7 @@ if $thapi_launch || __FILE__ == $PROGRAM_NAME # General Options parser.on('-b', '--backends BACKENDS', Array, 'Select which backends to use and their grouping level.', 'Format: backend_name[:backend_level],...', - default: ['mpi:3', 'omp:2', 'cl:1', 'ze:1', 'cuda:1', 'hip:1', 'cxi:4', 'itt:5', 'pytorch:6']) + default: ['mpi:3', 'omp:2', 'cl:1', 'ze:1', 'cuda:1', 'hip:1', 'cxi:4', 'itt:5', 'pytorch:6', 'fi:7']) parser.on('--[no-]archive', 'Enable or disable archive support.', default: false) # Analysis From 09e3bb0e114ab223bdc4d5b3bffa689c6aa04473 Mon Sep 17 00:00:00 2001 From: rsc18 Date: Fri, 16 Oct 2026 16:47:21 +0000 Subject: [PATCH 5/7] add thapi_text_prettry that was removed --- ...rofiling_cuda_driver_api.thapi_text_pretty | 6 ++++++ ...al_profiling_cuda_memcpy.thapi_text_pretty | 6 ++++++ ..._profiling_cuda_multictx.thapi_text_pretty | 14 +++++++++++++ ...ofiling_cuda_multistream.thapi_text_pretty | 10 +++++++++ ...al_profiling_kernel_name.thapi_text_pretty | 8 +++++++ ...al_profiling_multithread.thapi_text_pretty | 14 +++++++++++++ ...nterval_profiling_normal.thapi_text_pretty | 8 +++++++ ...erval_profiling_event-AA.thapi_text_pretty | 3 +++ ...rval_profiling_event-AAB.thapi_text_pretty | 5 +++++ ...val_profiling_event-AABB.thapi_text_pretty | 6 ++++++ ...rval_profiling_event-ABA.thapi_text_pretty | 5 +++++ ...val_profiling_event-ABAB.thapi_text_pretty | 6 ++++++ ...val_profiling_event-ABBA.thapi_text_pretty | 6 ++++++ ...l_profiling_event-AsAeAs.thapi_text_pretty | 4 ++++ ...profiling_event-AsAeAsAe.thapi_text_pretty | 5 +++++ ...l_profiling_event-AsAsAe.thapi_text_pretty | 4 ++++ ...profiling_event-AsAsAeAe.thapi_text_pretty | 5 +++++ ...rval_profiling_event-BAA.thapi_text_pretty | 5 +++++ ...nterval_profiling_normal.thapi_text_pretty | 2 ++ ...erval_profiling_task-AAB.thapi_text_pretty | 3 +++ ...rval_profiling_task-AABB.thapi_text_pretty | 4 ++++ ...rval_profiling_task-ABBA.thapi_text_pretty | 4 ++++ ...erval_profiling_task-BAA.thapi_text_pretty | 3 +++ .../omp/tests/data_op_emi.thapi_text_pretty | 6 ++++++ .../omp/tests/submit_emi.thapi_text_pretty | 2 ++ backends/omp/tests/target.thapi_text_pretty | 2 ++ .../omp/tests/target_emi.thapi_text_pretty | 2 ++ .../opencl/tests/api_call.thapi_text_pretty | 2 ++ .../tests/command_queue.thapi_text_pretty | 6 ++++++ ...mmand_queue_other_thread.thapi_text_pretty | 6 ++++++ ...enqueue_read_buffer_fast.thapi_text_pretty | 4 ++++ .../interleave_process.thapi_text_pretty | 8 +++++++ .../tests/interleave_thread.thapi_text_pretty | 8 +++++++ .../tests/nd_range_kernel.thapi_text_pretty | 5 +++++ .../tests/results_first.thapi_text_pretty | 4 ++++ .../opencl/tests/with_error.thapi_text_pretty | 4 ++++ ...erval_profiling_API_call.thapi_text_pretty | 2 ++ ...nterval_profiling_ignore.thapi_text_pretty | 4 ++++ ...iling_interleave_process.thapi_text_pretty | 8 +++++++ ...al_profiling_multithread.thapi_text_pretty | 8 +++++++ ...nterval_profiling_normal.thapi_text_pretty | 4 ++++ ...profiling_resubmit_event.thapi_text_pretty | 8 +++++++ ...l_profiling_shared_event.thapi_text_pretty | 18 ++++++++++++++++ ...ng_shared_event_resubmit.thapi_text_pretty | 13 ++++++++++++ ...ling_shared_event_xphase.thapi_text_pretty | 21 +++++++++++++++++++ 45 files changed, 281 insertions(+) create mode 100644 backends/cuda/tests/interval_profiling_cuda_driver_api.thapi_text_pretty create mode 100644 backends/cuda/tests/interval_profiling_cuda_memcpy.thapi_text_pretty create mode 100644 backends/cuda/tests/interval_profiling_cuda_multictx.thapi_text_pretty create mode 100644 backends/cuda/tests/interval_profiling_cuda_multistream.thapi_text_pretty create mode 100644 backends/cuda/tests/interval_profiling_kernel_name.thapi_text_pretty create mode 100644 backends/cuda/tests/interval_profiling_multithread.thapi_text_pretty create mode 100644 backends/cuda/tests/interval_profiling_normal.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-AA.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-AAB.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-AABB.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-ABA.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-ABAB.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-ABBA.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-AsAeAs.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-AsAeAsAe.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-AsAsAe.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-AsAsAeAe.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_event-BAA.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_normal.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_task-AAB.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_task-AABB.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_task-ABBA.thapi_text_pretty create mode 100644 backends/itt/tests/interval_profiling_task-BAA.thapi_text_pretty create mode 100644 backends/omp/tests/data_op_emi.thapi_text_pretty create mode 100644 backends/omp/tests/submit_emi.thapi_text_pretty create mode 100644 backends/omp/tests/target.thapi_text_pretty create mode 100644 backends/omp/tests/target_emi.thapi_text_pretty create mode 100644 backends/opencl/tests/api_call.thapi_text_pretty create mode 100644 backends/opencl/tests/command_queue.thapi_text_pretty create mode 100644 backends/opencl/tests/command_queue_other_thread.thapi_text_pretty create mode 100644 backends/opencl/tests/enqueue_read_buffer_fast.thapi_text_pretty create mode 100644 backends/opencl/tests/interleave_process.thapi_text_pretty create mode 100644 backends/opencl/tests/interleave_thread.thapi_text_pretty create mode 100644 backends/opencl/tests/nd_range_kernel.thapi_text_pretty create mode 100644 backends/opencl/tests/results_first.thapi_text_pretty create mode 100644 backends/opencl/tests/with_error.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_API_call.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_ignore.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_interleave_process.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_multithread.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_normal.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty diff --git a/backends/cuda/tests/interval_profiling_cuda_driver_api.thapi_text_pretty b/backends/cuda/tests/interval_profiling_cuda_driver_api.thapi_text_pretty new file mode 100644 index 000000000..bf8f1e839 --- /dev/null +++ b/backends/cuda/tests/interval_profiling_cuda_driver_api.thapi_text_pretty @@ -0,0 +1,6 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 3 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } +12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } +12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_cuda_memcpy.thapi_text_pretty b/backends/cuda/tests/interval_profiling_cuda_memcpy.thapi_text_pretty new file mode 100644 index 000000000..0cc2268e4 --- /dev/null +++ b/backends/cuda/tests/interval_profiling_cuda_memcpy.thapi_text_pretty @@ -0,0 +1,6 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 3 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuMemcpyHtoD_v2_entry: { dstDevice: 0x00007f0000000000, srcHost: 0x0000500000000000, ByteCount: 400 } +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } +12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuMemcpyHtoD_v2_exit: { cuResult: CUDA_SUCCESS } +12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_cuda_multictx.thapi_text_pretty b/backends/cuda/tests/interval_profiling_cuda_multictx.thapi_text_pretty new file mode 100644 index 000000000..5a059e3ef --- /dev/null +++ b/backends/cuda/tests/interval_profiling_cuda_multictx.thapi_text_pretty @@ -0,0 +1,14 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 123 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1230000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } +12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } +12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } +12:00:00.060000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxDestroy_v2_entry: { ctx: 0x1230000000000000 } +12:00:00.070000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxDestroy_v2_exit: { cuResult: CUDA_SUCCESS } +12:00:00.080000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x1000000000000000, flags: 0, dev: 456 } +12:00:00.090000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x4560000000000000 } +12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } +12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x3000000000000000, hStop: 0x4000000000000000 } +12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } +12:00:00.130000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x3000000000000000, hStop: 0x4000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_cuda_multistream.thapi_text_pretty b/backends/cuda/tests/interval_profiling_cuda_multistream.thapi_text_pretty new file mode 100644 index 000000000..18fce0964 --- /dev/null +++ b/backends/cuda/tests/interval_profiling_cuda_multistream.thapi_text_pretty @@ -0,0 +1,10 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 3 } +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuStreamCreate_entry: { phStream: 0x0000000000000000, Flags: 0 } +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuStreamCreate_exit: { cuResult: CUDA_SUCCESS, phStream_val: 0x1000000000000000 } +12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 100 } +12:00:00.005000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x2000000000000000 } +12:00:00.006000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } +12:00:00.007000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } +12:00:00.008000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } +12:00:00.009000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_kernel_name.thapi_text_pretty b/backends/cuda/tests/interval_profiling_kernel_name.thapi_text_pretty new file mode 100644 index 000000000..218e51a32 --- /dev/null +++ b/backends/cuda/tests/interval_profiling_kernel_name.thapi_text_pretty @@ -0,0 +1,8 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 3 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuModuleGetFunction_entry: { hfunc: 0x0000000000000000, hmod: 0x0000000000000000, name: 0x0000000000000000, name_val: "testkernel" } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuModuleGetFunction_exit: { cuResult: CUDA_SUCCESS, hfunc_val: 0x1000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchKernel_entry: { f: 0x1000000000000000, gridDimX: 1, gridDimY: 1, gridDimZ: 1, blockDimX: 1, blockDimY: 1, blockDimZ: 1, sharedMemBytes: 0, hStream: 0x0000000000000000, kernelParams: 0x0000000000000000, extra: 0x0000000000000000, extra_vals: [ ] } +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } +12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchKernel_exit: { cuResult: CUDA_SUCCESS } +12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_multithread.thapi_text_pretty b/backends/cuda/tests/interval_profiling_multithread.thapi_text_pretty new file mode 100644 index 000000000..42356344f --- /dev/null +++ b/backends/cuda/tests/interval_profiling_multithread.thapi_text_pretty @@ -0,0 +1,14 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuDevicePrimaryCtxRetain_entry: { pctx: 0x0000000000000000, dev: 3 } +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuDevicePrimaryCtxRetain_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxSetCurrent_entry: { ctx: 0x1000000000000000 } +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxSetCurrent_exit: { cuResult: CUDA_SUCCESS } +12:00:00.004000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda:cuCtxCreate_v2_entry: { pctx: 0x0000000000000000, flags: 0, dev: 7 } +12:00:00.005000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda:cuCtxCreate_v2_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x2000000000000000 } +12:00:00.006000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } +12:00:00.007000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } +12:00:00.008000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } +12:00:00.009000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x3000000000000000, hStop: 0x4000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } +12:00:00.011000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } +12:00:00.012000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x3000000000000000, hStop: 0x4000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/cuda/tests/interval_profiling_normal.thapi_text_pretty b/backends/cuda/tests/interval_profiling_normal.thapi_text_pretty new file mode 100644 index 000000000..f5748b895 --- /dev/null +++ b/backends/cuda/tests/interval_profiling_normal.thapi_text_pretty @@ -0,0 +1,8 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuDevicePrimaryCtxRetain_entry: { pctx: 0x0000000000000000, dev: 3 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuDevicePrimaryCtxRetain_exit: { cuResult: CUDA_SUCCESS, pctx_val: 0x1000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxSetCurrent_entry: { ctx: 0x1000000000000000 } +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuCtxSetCurrent_exit: { cuResult: CUDA_SUCCESS } +12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_entry: { hStream: 0x0000000000000000, fn: 0x0000000000000000, userData: 0x0000000000000000 } +12:00:00.050000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling: { hStart: 0x1000000000000000, hStop: 0x2000000000000000 } +12:00:00.070000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda:cuLaunchHostFunc_exit: { cuResult: CUDA_SUCCESS } +12:00:00.070000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_cuda_profiling:event_profiling_results: { hStart: 0x1000000000000000, hStop: 0x2000000000000000, startStatus: CUDA_SUCCESS, stopStatus: CUDA_SUCCESS, status: CUDA_SUCCESS, milliseconds: 1.0 } diff --git a/backends/itt/tests/interval_profiling_event-AA.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AA.thapi_text_pretty new file mode 100644 index 000000000..d0552fb3c --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-AA.thapi_text_pretty @@ -0,0 +1,3 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AAB.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AAB.thapi_text_pretty new file mode 100644 index 000000000..73cae7134 --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-AAB.thapi_text_pretty @@ -0,0 +1,5 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} +12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} diff --git a/backends/itt/tests/interval_profiling_event-AABB.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AABB.thapi_text_pretty new file mode 100644 index 000000000..58cd66cfd --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-AABB.thapi_text_pretty @@ -0,0 +1,6 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} +12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} +12:00:00.043000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 1} diff --git a/backends/itt/tests/interval_profiling_event-ABA.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-ABA.thapi_text_pretty new file mode 100644 index 000000000..f6eedaf20 --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-ABA.thapi_text_pretty @@ -0,0 +1,5 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} +12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-ABAB.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-ABAB.thapi_text_pretty new file mode 100644 index 000000000..37b8b263e --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-ABAB.thapi_text_pretty @@ -0,0 +1,6 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} +12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} +12:00:00.043000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 1} diff --git a/backends/itt/tests/interval_profiling_event-ABBA.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-ABBA.thapi_text_pretty new file mode 100644 index 000000000..33f5f2e38 --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-ABBA.thapi_text_pretty @@ -0,0 +1,6 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000001, namelen: 7, name_val: "Event B"} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} +12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 1} +12:00:00.043000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AsAeAs.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AsAeAs.thapi_text_pretty new file mode 100644 index 000000000..757ebd6bd --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-AsAeAs.thapi_text_pretty @@ -0,0 +1,4 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AsAeAsAe.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AsAeAsAe.thapi_text_pretty new file mode 100644 index 000000000..0f4f4f9f8 --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-AsAeAsAe.thapi_text_pretty @@ -0,0 +1,5 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AsAsAe.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AsAsAe.thapi_text_pretty new file mode 100644 index 000000000..6c582c7a7 --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-AsAsAe.thapi_text_pretty @@ -0,0 +1,4 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-AsAsAeAe.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-AsAsAeAe.thapi_text_pretty new file mode 100644 index 000000000..217f318fe --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-AsAsAeAe.thapi_text_pretty @@ -0,0 +1,5 @@ +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.013000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} +12:00:00.023000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_event-BAA.thapi_text_pretty b/backends/itt/tests/interval_profiling_event-BAA.thapi_text_pretty new file mode 100644 index 000000000..63aaee275 --- /dev/null +++ b/backends/itt/tests/interval_profiling_event-BAA.thapi_text_pretty @@ -0,0 +1,5 @@ +12:00:00.001000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 0, name: 0x000000, namelen: 7, name_val: "Event A"} +12:00:00.002000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_create: {ittResult: 1, name: 0x000010, namelen: 7, name_val: "Event B"} +12:00:00.003000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 1} +12:00:00.013000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_start: {ittResult: 0, event: 0} +12:00:00.023000000 - hostname - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_event_end: {ittResult: 0, event: 0} diff --git a/backends/itt/tests/interval_profiling_normal.thapi_text_pretty b/backends/itt/tests/interval_profiling_normal.thapi_text_pretty new file mode 100644 index 000000000..925596dc8 --- /dev/null +++ b/backends/itt/tests/interval_profiling_normal.thapi_text_pretty @@ -0,0 +1,2 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain", name__strA_val: "test task"} +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} diff --git a/backends/itt/tests/interval_profiling_task-AAB.thapi_text_pretty b/backends/itt/tests/interval_profiling_task-AAB.thapi_text_pretty new file mode 100644 index 000000000..788dcd456 --- /dev/null +++ b/backends/itt/tests/interval_profiling_task-AAB.thapi_text_pretty @@ -0,0 +1,3 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain.AAB", name__strA_val: "test task A"} +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000100, domain__nameA_val: "test.domain.AAB", name__strA_val: "test task B"} diff --git a/backends/itt/tests/interval_profiling_task-AABB.thapi_text_pretty b/backends/itt/tests/interval_profiling_task-AABB.thapi_text_pretty new file mode 100644 index 000000000..bb41cd7f2 --- /dev/null +++ b/backends/itt/tests/interval_profiling_task-AABB.thapi_text_pretty @@ -0,0 +1,4 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain.AABB", name__strA_val: "test task A"} +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000100, domain__nameA_val: "test.domain.AABB", name__strA_val: "test task B"} +12:00:00.040000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} diff --git a/backends/itt/tests/interval_profiling_task-ABBA.thapi_text_pretty b/backends/itt/tests/interval_profiling_task-ABBA.thapi_text_pretty new file mode 100644 index 000000000..209f65e89 --- /dev/null +++ b/backends/itt/tests/interval_profiling_task-ABBA.thapi_text_pretty @@ -0,0 +1,4 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain.ABBA", name__strA_val: "test task A"} +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000100, domain__nameA_val: "test.domain.ABBA", name__strA_val: "test task B"} +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} diff --git a/backends/itt/tests/interval_profiling_task-BAA.thapi_text_pretty b/backends/itt/tests/interval_profiling_task-BAA.thapi_text_pretty new file mode 100644 index 000000000..42535b680 --- /dev/null +++ b/backends/itt/tests/interval_profiling_task-BAA.thapi_text_pretty @@ -0,0 +1,3 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000010, domain__nameA_val: "test.domain.BAA", name__strA_val: "test task B"} +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_begin: {domain: 0x000000, taskid: __itt_null, parentid: __itt_null, name: 0x000100, domain__nameA_val: "test.domain.BAA", name__strA_val: "test task A"} +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_itt:__itt_task_end: {domain: 0x000000} diff --git a/backends/omp/tests/data_op_emi.thapi_text_pretty b/backends/omp/tests/data_op_emi.thapi_text_pretty new file mode 100644 index 000000000..89ddff7aa --- /dev/null +++ b/backends/omp/tests/data_op_emi.thapi_text_pretty @@ -0,0 +1,6 @@ +12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_begin, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_alloc, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x0000000000000000, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } +12:00:00.001000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_end, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_alloc, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x00007ff074080040, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } +12:00:00.002000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_begin, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_delete, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x0000000000000000, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } +12:00:00.003000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_end, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_delete, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x00007ff074080040, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } +12:00:00.004000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_begin, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_associate, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x0000000000000000, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } +12:00:00.005000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_data_op_emi: { endpoint: ompt_scope_end, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, host_op_id: 0x00007ff07bf6f078, optype: ompt_target_data_associate, src_addr: 0x00007ffd717c787c, src_device_num: 1, dest_addr: 0x00007ff074080040, dest_device_num: 0, bytes: 4, codeptr_ra: 0x00007ff07aeeb2e3 } diff --git a/backends/omp/tests/submit_emi.thapi_text_pretty b/backends/omp/tests/submit_emi.thapi_text_pretty new file mode 100644 index 000000000..91440d5f2 --- /dev/null +++ b/backends/omp/tests/submit_emi.thapi_text_pretty @@ -0,0 +1,2 @@ +12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_submit_emi: { endpoint: ompt_scope_begin, target_data: 0x0000000000000008, host_op_id: 0x0000000000000009, requested_num_teams: 3823427656 } +12:00:00.001000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_submit_emi: { endpoint: ompt_scope_end, target_data: 0x0000000000000008, host_op_id: 0x0000000000000009, requested_num_teams: 3823427656 } diff --git a/backends/omp/tests/target.thapi_text_pretty b/backends/omp/tests/target.thapi_text_pretty new file mode 100644 index 000000000..f5f2c48e4 --- /dev/null +++ b/backends/omp/tests/target.thapi_text_pretty @@ -0,0 +1,2 @@ +12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target: { kind: ompt_target, endpoint: ompt_scope_begin, device_num: 0, task_data: 0x00007ff0755cfe80, task_data: 0x00007ff0755eac98, target_id: 1, codeptr_ra: 0x00007ff07af1643a } +12:00:00.001000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target: { kind: ompt_target, endpoint: ompt_scope_end, device_num: 0, task_data: 0x00007ff0755cfe80, task_data: 0x00007ff0755eac98, target_id: 1, codeptr_ra: 0x00007ff07af1643a } diff --git a/backends/omp/tests/target_emi.thapi_text_pretty b/backends/omp/tests/target_emi.thapi_text_pretty new file mode 100644 index 000000000..36b35c18b --- /dev/null +++ b/backends/omp/tests/target_emi.thapi_text_pretty @@ -0,0 +1,2 @@ +12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_emi: { kind: ompt_target, endpoint: ompt_scope_begin, device_num: 0, task_data: 0x00007ff0755cfe80, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, codeptr_ra: 0x00007ff07af1643a } +12:00:00.001000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ompt:ompt_callback_target_emi: { kind: ompt_target, endpoint: ompt_scope_end, device_num: 0, task_data: 0x00007ff0755cfe80, target_task_data: 0x00007ff0755eac98, target_data: 0x00007ff07bf6f060, codeptr_ra: 0x00007ff07af1643a } diff --git a/backends/opencl/tests/api_call.thapi_text_pretty b/backends/opencl/tests/api_call.thapi_text_pretty new file mode 100644 index 000000000..f1af96107 --- /dev/null +++ b/backends/opencl/tests/api_call.thapi_text_pretty @@ -0,0 +1,2 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clLinkProgram_entry: { context: 1, num_devices: 1, device_list: 0, options: 0, num_input_programs: 0, input_programs: 0, pfn_notify: 0, user_data: 0, errcode_ret: 0, _device_list_vals_length: 0, device_list_vals: 0, options_val: "", _input_programs_vals_length: 0, input_programs_vals: 0 } +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clLinkProgram_exit: { program: 1, errcode_ret_val: 0 } diff --git a/backends/opencl/tests/command_queue.thapi_text_pretty b/backends/opencl/tests/command_queue.thapi_text_pretty new file mode 100644 index 000000000..e0947f8ce --- /dev/null +++ b/backends/opencl/tests/command_queue.thapi_text_pretty @@ -0,0 +1,6 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clCreateCommandQueue_entry: { context: 0, device: 10, properties: 0, errcode_ret: 0 } +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clCreateCommandQueue_exit: { command_queue: 23, errcode_ret_val: 0 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 23, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } +12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } +12:00:00.005000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 65 } diff --git a/backends/opencl/tests/command_queue_other_thread.thapi_text_pretty b/backends/opencl/tests/command_queue_other_thread.thapi_text_pretty new file mode 100644 index 000000000..66d09928e --- /dev/null +++ b/backends/opencl/tests/command_queue_other_thread.thapi_text_pretty @@ -0,0 +1,6 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl:clCreateCommandQueue_entry: { context: 0, device: 10, properties: 0, errcode_ret: 0 } +12:00:00.001000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl:clCreateCommandQueue_exit: { command_queue: 23, errcode_ret_val: 0 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 23, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } +12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } +12:00:00.005000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 65 } diff --git a/backends/opencl/tests/enqueue_read_buffer_fast.thapi_text_pretty b/backends/opencl/tests/enqueue_read_buffer_fast.thapi_text_pretty new file mode 100644 index 000000000..af2e61f07 --- /dev/null +++ b/backends/opencl/tests/enqueue_read_buffer_fast.thapi_text_pretty @@ -0,0 +1,4 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 10 } +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } diff --git a/backends/opencl/tests/interleave_process.thapi_text_pretty b/backends/opencl/tests/interleave_process.thapi_text_pretty new file mode 100644 index 000000000..799317e71 --- /dev/null +++ b/backends/opencl/tests/interleave_process.thapi_text_pretty @@ -0,0 +1,8 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 11, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } +12:00:00.001000000 - testhost - vpid: 20, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 12, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 11 } +12:00:00.003000000 - testhost - vpid: 20, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 12 } +12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 11 } +12:00:00.005000000 - testhost - vpid: 20, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 12 } +12:00:00.006000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 11, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 10 } +12:00:00.007000000 - testhost - vpid: 20, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 12, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 20 } diff --git a/backends/opencl/tests/interleave_thread.thapi_text_pretty b/backends/opencl/tests/interleave_thread.thapi_text_pretty new file mode 100644 index 000000000..1cdcefeeb --- /dev/null +++ b/backends/opencl/tests/interleave_thread.thapi_text_pretty @@ -0,0 +1,8 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 11, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } +12:00:00.001000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 12, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 11 } +12:00:00.003000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 12 } +12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 11 } +12:00:00.005000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 12 } +12:00:00.006000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 11, event_command_exec_status: 0, queued_status: 0, queued: 10, submit_status: 0, submit: 0, start_status: 0, start: 15, end_status: 0, end: 25 } +12:00:00.007000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_opencl_profiling:event_profiling_results: { event: 12, event_command_exec_status: 0, queued_status: 0, queued: 100, submit_status: 0, submit: 0, start_status: 0, start: 115, end_status: 0, end: 135 } diff --git a/backends/opencl/tests/nd_range_kernel.thapi_text_pretty b/backends/opencl/tests/nd_range_kernel.thapi_text_pretty new file mode 100644 index 000000000..1820d0550 --- /dev/null +++ b/backends/opencl/tests/nd_range_kernel.thapi_text_pretty @@ -0,0 +1,5 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_arguments:kernel_info: { kernel: 12, function_name: "foo", num_args: 0, context: 1, program: 1, attributes: "" } +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueNDRangeKernel_entry: { command_queue: 10, kernel: 12, work_dim: 1, global_work_offset: 0, global_work_size: 1024, local_work_size: 256, num_events_in_wait_list: 0, event_wait_list: 0, event: 0, _event_wait_list_vals_length: 0, event_wait_list_vals: 0, _global_work_offset_vals_length: 0, global_work_offset_vals: 0, _global_work_size_vals_length: 0, global_work_size_vals: 0, _local_work_size_vals_length: 0, local_work_size_vals: 0 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueNDRangeKernel_exit: { errcode_ret_val: 0, event_val: 1 } +12:00:00.004000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 10 } diff --git a/backends/opencl/tests/results_first.thapi_text_pretty b/backends/opencl/tests/results_first.thapi_text_pretty new file mode 100644 index 000000000..05b255e3e --- /dev/null +++ b/backends/opencl/tests/results_first.thapi_text_pretty @@ -0,0 +1,4 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: 0, queued: 10, submit_status: 0, submit: 0, start_status: 0, start: 20, end_status: 0, end: 30 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } diff --git a/backends/opencl/tests/with_error.thapi_text_pretty b/backends/opencl/tests/with_error.thapi_text_pretty new file mode 100644 index 000000000..cd0557fd3 --- /dev/null +++ b/backends/opencl/tests/with_error.thapi_text_pretty @@ -0,0 +1,4 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_entry: { command_queue: 10, buffer: 0, blocking_read: 0, offset: 0, size: 1024, ptr: 0, num_events_in_wait_list: 0, event_wait_list: 0, event: 17, _event_wait_list_vals_length: 0, event_wait_list_vals: 0 } +12:00:00.001000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling_results: { event: 17, event_command_exec_status: 0, queued_status: -58, queued: 0, submit_status: 0, submit: 0, start_status: 0, start: 0, end_status: 0, end: 10 } +12:00:00.002000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl:clEnqueueReadBuffer_exit: { errcode_ret_val: 0, event_val: 17 } +12:00:00.003000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_opencl_profiling:event_profiling: { status: 0, event: 17 } diff --git a/backends/ze/tests/interval_profiling_API_call.thapi_text_pretty b/backends/ze/tests/interval_profiling_API_call.thapi_text_pretty new file mode 100644 index 000000000..d5dba5b84 --- /dev/null +++ b/backends/ze/tests/interval_profiling_API_call.thapi_text_pretty @@ -0,0 +1,2 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeDriverGet_entry: { pCount: 1, phDrivers: 0, pCount_val: 1 } +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeDriverGet_exit: { zeResult: ZE_RESULT_SUCCESS, pCount_val: 1, _phDrivers_vals_length: 0, phDrivers_vals: 0 } diff --git a/backends/ze/tests/interval_profiling_ignore.thapi_text_pretty b/backends/ze/tests/interval_profiling_ignore.thapi_text_pretty new file mode 100644 index 000000000..816a893b4 --- /dev/null +++ b/backends/ze/tests/interval_profiling_ignore.thapi_text_pretty @@ -0,0 +1,4 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeEventHostSignal_entry: { hEvent: 0x1000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x2000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeEventHostSignal_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x2000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } diff --git a/backends/ze/tests/interval_profiling_interleave_process.thapi_text_pretty b/backends/ze/tests/interval_profiling_interleave_process.thapi_text_pretty new file mode 100644 index 000000000..aaa1cda2c --- /dev/null +++ b/backends/ze/tests/interval_profiling_interleave_process.thapi_text_pretty @@ -0,0 +1,8 @@ +12:00:00.000000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 2, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.020000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x1000000000000000 } +12:00:00.030000000 - testhost - vpid: 2, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x2000000000000000 } +12:00:00.040000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.050000000 - testhost - vpid: 2, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.060000000 - testhost - vpid: 1, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x1000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } +12:00:00.070000000 - testhost - vpid: 2, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x2000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 20, contextStart: 0, contextEnd: 10 } diff --git a/backends/ze/tests/interval_profiling_multithread.thapi_text_pretty b/backends/ze/tests/interval_profiling_multithread.thapi_text_pretty new file mode 100644 index 000000000..aa0ec4468 --- /dev/null +++ b/backends/ze/tests/interval_profiling_multithread.thapi_text_pretty @@ -0,0 +1,8 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.000000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x2000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x1000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x2000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.030000000 - testhost - vpid: 10, vtid: 2 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x2000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 20, contextStart: 0, contextEnd: 20 } +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x1000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } diff --git a/backends/ze/tests/interval_profiling_normal.thapi_text_pretty b/backends/ze/tests/interval_profiling_normal.thapi_text_pretty new file mode 100644 index 000000000..f6db269ab --- /dev/null +++ b/backends/ze/tests/interval_profiling_normal.thapi_text_pretty @@ -0,0 +1,4 @@ +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x1000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x1000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } diff --git a/backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty b/backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty new file mode 100644 index 000000000..b4c3ca9b4 --- /dev/null +++ b/backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty @@ -0,0 +1,8 @@ +# 1 Append, but the underlying cl is Executed twice in a real run, so +# 2 results arrive for the same hEvent. Both are attributed to that one +# Append. +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } +12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } diff --git a/backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty b/backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty new file mode 100644 index 000000000..64199d25b --- /dev/null +++ b/backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty @@ -0,0 +1,18 @@ +# 4 Appends share one hEvent. Each Append's result is attributed back to +# its own Append, in submission order. +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.210000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.220000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.300000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.310000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.320000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.400000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } +12:00:00.410000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } +12:00:00.420000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 200, globalEnd: 240, contextStart: 200, contextEnd: 240 } +12:00:00.430000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 300, globalEnd: 350, contextStart: 300, contextEnd: 350 } diff --git a/backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty b/backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty new file mode 100644 index 000000000..fb64b5d7a --- /dev/null +++ b/backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty @@ -0,0 +1,13 @@ +# 2 Appends share one hEvent, then the underlying cl is Executed twice, +# so 4 results arrive. Each submission's pair of results is attributed to +# the two Appends in submission order. +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } +12:00:00.210000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } +12:00:00.300000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 200, globalEnd: 240, contextStart: 200, contextEnd: 240 } +12:00:00.310000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 300, globalEnd: 350, contextStart: 300, contextEnd: 350 } diff --git a/backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty b/backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty new file mode 100644 index 000000000..e9d336d89 --- /dev/null +++ b/backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty @@ -0,0 +1,21 @@ +# Two build phases on the same cl, both reusing the same hEvent. +# Phase 1: 2 Appends, cl Executed twice -> 4 results. +# Phase 2: cl is Reset, then 1 Append, cl Executed three times -> 3 results. +# Results from a phase are attributed only to that phase's Appends; the +# phase-1 Appends do not bleed into the phase-2 results. +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } +12:00:00.210000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } +12:00:00.300000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 200, globalEnd: 240, contextStart: 200, contextEnd: 240 } +12:00:00.310000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 300, globalEnd: 350, contextStart: 300, contextEnd: 350 } +12:00:00.400000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.410000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.420000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.500000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 400, globalEnd: 410, contextStart: 400, contextEnd: 410 } +12:00:00.510000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 500, globalEnd: 520, contextStart: 500, contextEnd: 520 } +12:00:00.520000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 600, globalEnd: 640, contextStart: 600, contextEnd: 640 } From e20f39e7ee84b812730eb2e30e1adc7fdb83c883 Mon Sep 17 00:00:00 2001 From: rsc18 Date: Fri, 16 Oct 2026 17:10:54 +0000 Subject: [PATCH 6/7] Remove references to unimplemented libfabric tests --- backends/libfabric/Makefile.am | 68 +--------------------------------- 1 file changed, 1 insertion(+), 67 deletions(-) diff --git a/backends/libfabric/Makefile.am b/backends/libfabric/Makefile.am index 01f34e267..657fe9486 100644 --- a/backends/libfabric/Makefile.am +++ b/backends/libfabric/Makefile.am @@ -166,70 +166,4 @@ CLEANFILES += \ babeltrace_libfabric_lib.rb data_DATA = \ - babeltrace_libfabric_lib.rb - -# -# ---------------------------------------------------------------------- -# Tests -# ---------------------------------------------------------------------- -# -# Offline unit tests: a text dump of a raw trace is compiled into a synthetic -# babeltrace source plugin, piped through the real libFIInterval.so, and the -# output diffed against the committed golden. No NIC, no LTTng, no -# application required -- which is the whole point, since none of those exist -# on CI. -# -# The five cases below are not arbitrary; each one encodes a caveat the -# standalone README documents in prose, which is exactly the kind of thing -# that rots without a test. -# - -TRACE_COMMON = \ - tests/send_matched-post_complete.thapi_text_pretty \ - tests/send_null_context-no_join.thapi_text_pretty \ - tests/send_eagain-no_transfer.thapi_text_pretty \ - tests/completion_orphan-no_posting.thapi_text_pretty \ - tests/recv_wildcard-posted_vs_actual.thapi_text_pretty \ - tests/progress_thread-cross_thread_join.thapi_text_pretty \ - tests/ctrl_nested-domain_ep_cq.thapi_text_pretty - -BTX_FI_GENERATED_SOURCE_TEST = \ - btx_source_fi_test/metababel/metababel.h \ - btx_source_fi_test/metababel/btx_component.h \ - btx_source_fi_test/metababel/btx_component.c \ - btx_source_fi_test/metababel/btx_downstream.h \ - btx_source_fi_test/metababel/btx_downstream.c \ - btx_source_fi_test/btx_main.c - -$(abs_builddir)/btx_libfabric_model.yaml: $(srcdir)/btx_libfabric_model.yaml - cp $< $@ - -$(BTX_FI_GENERATED_SOURCE_TEST) &: $(abs_builddir)/btx_libfabric_model.yaml - $(METABABEL) -d $(abs_builddir)/btx_libfabric_model.yaml -t SOURCE -o btx_source_fi_test -p fitest -c test - -noinst_LTLIBRARIES += libtestfisource.la -nodist_libtestfisource_la_SOURCES = $(BTX_FI_GENERATED_SOURCE_TEST) -libtestfisource_la_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I./btx_source_fi_test/ -libtestfisource_la_CFLAGS = -fPIC -Wall -Wextra -Wno-unused-parameter $(WERROR) $(BABELTRACE2_CFLAGS) - -TRACE_OUT = $(TRACE_COMMON:.thapi_text_pretty=.bt_text_pretty) - -TESTS = $(TRACE_COMMON) - -TEST_EXTENSIONS = .thapi_text_pretty -THAPI_TEXT_PRETTY_LOG_COMPILER = $(top_builddir)/utils/test_wrapper_thapi_text_pretty.sh -AM_THAPI_TEXT_PRETTY_LOG_FLAGS = backends/libfabric fi ./btx_source_fi_test/ .libs/libtestfisource.a - -check_DATA = \ - $(TRACE_COMMON) \ - $(TRACE_OUT) - -EXTRA_DIST += \ - $(TRACE_COMMON) \ - $(TRACE_OUT) \ - tests/mock_provider.c \ - tests/testapp.c - -CLEANFILES += \ - $(BTX_FI_GENERATED_SOURCE_TEST) \ - $(abs_builddir)/btx_libfabric_model.yaml + babeltrace_libfabric_lib.rb \ No newline at end of file From 80fb7a43e539388245293297501f71b5ca4d187a Mon Sep 17 00:00:00 2001 From: rsc18 Date: Fri, 16 Oct 2026 18:34:18 +0000 Subject: [PATCH 7/7] fix clang-format, rubocop, yamlfmt --- .../libfabric/btx_fiinterval_callbacks.cpp | 188 +- backends/libfabric/btx_libfabric_model.yaml | 2 +- backends/libfabric/libfabric_events.yaml | 5 - backends/libfabric/src/fi_trace.h | 264 +- backends/libfabric/src/fit_addr.c | 610 ++-- backends/libfabric/src/fit_hook.c | 2529 ++++++++--------- backends/libfabric/src/fit_lttng.c | 306 +- xprof/xprof.rb.in | 8 +- 8 files changed, 1921 insertions(+), 1991 deletions(-) diff --git a/backends/libfabric/btx_fiinterval_callbacks.cpp b/backends/libfabric/btx_fiinterval_callbacks.cpp index 018b759a4..c8b82f583 100644 --- a/backends/libfabric/btx_fiinterval_callbacks.cpp +++ b/backends/libfabric/btx_fiinterval_callbacks.cpp @@ -36,19 +36,19 @@ /* ------------------------------------------------------------------ */ struct posting_s { - int64_t ts; /* when the application posted */ - std::string op; /* "fi_send", "fi_trecv", ... */ - std::string peer; /* as known at posting time; a wildcard - receive will be corrected by the - completion, which knows the real sender */ - uint64_t posted_bytes; /* buffer length, not transferred length */ + int64_t ts; /* when the application posted */ + std::string op; /* "fi_send", "fi_trecv", ... */ + std::string peer; /* as known at posting time; a wildcard + receive will be corrected by the + completion, which knows the real sender */ + uint64_t posted_bytes; /* buffer length, not transferred length */ }; struct ep_info_s { std::string src_addr; std::string domain_name; - bool topo_valid; - uint32_t group, switch_id, port; + bool topo_valid; + uint32_t group, switch_id, port; }; struct data_s { @@ -75,29 +75,24 @@ struct data_s { */ std::unordered_map outstanding; - std::unordered_map endpoints; /* key: host/pid/ep_id */ + std::unordered_map endpoints; /* key: host/pid/ep_id */ }; typedef struct data_s data_t; static void btx_initialize_component(void **usr_data) { *usr_data = new data_t; } -static void btx_finalize_component(void *usr_data) { - delete static_cast(usr_data); -} +static void btx_finalize_component(void *usr_data) { delete static_cast(usr_data); } /* ------------------------------------------------------------------ */ /* helpers */ /* ------------------------------------------------------------------ */ -static std::string ctx_key(const char *hostname, int64_t vpid, - uintptr_t op_context) { - return std::string(hostname) + "/" + std::to_string(vpid) + "/" + - std::to_string(op_context); +static std::string ctx_key(const char *hostname, int64_t vpid, uintptr_t op_context) { + return std::string(hostname) + "/" + std::to_string(vpid) + "/" + std::to_string(op_context); } static std::string ep_key(const char *hostname, int64_t vpid, uint32_t ep_id) { - return std::string(hostname) + "/" + std::to_string(vpid) + "/" + - std::to_string(ep_id); + return std::string(hostname) + "/" + std::to_string(vpid) + "/" + std::to_string(ep_id); } /* @@ -105,19 +100,16 @@ static std::string ep_key(const char *hostname, int64_t vpid, uint32_t ep_id) { * NIC when we know it. This is the string that shows up in iprof's traffic * table, so it wants to be stable and greppable rather than pretty. */ -static std::string traffic_metadata(data_t *s, const char *hostname, - int64_t vpid, uint32_t ep_id, - const std::string &peer) { +static std::string traffic_metadata( + data_t *s, const char *hostname, int64_t vpid, uint32_t ep_id, const std::string &peer) { auto it = s->endpoints.find(ep_key(hostname, vpid, ep_id)); if (it == s->endpoints.end()) return peer; std::string m = it->second.src_addr + "->" + peer; if (it->second.topo_valid) - m += " [" + it->second.domain_name + " g" + - std::to_string(it->second.group) + "s" + - std::to_string(it->second.switch_id) + "p" + - std::to_string(it->second.port) + "]"; + m += " [" + it->second.domain_name + " g" + std::to_string(it->second.group) + "s" + + std::to_string(it->second.switch_id) + "p" + std::to_string(it->second.port) + "]"; return m; } @@ -125,19 +117,34 @@ static std::string traffic_metadata(data_t *s, const char *hostname, /* data path */ /* ------------------------------------------------------------------ */ -static void op_entry_callback(void *btx_handle, void *usr_data, int64_t ts, - const char *hostname, int64_t vpid, uint64_t vtid, - char *op, uint32_t ep_id, uintptr_t op_context) { +static void op_entry_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t ep_id, + uintptr_t op_context) { auto *s = static_cast(usr_data); s->op_stack[{hostname, vpid, vtid}].push_back({ts, op}); } -static void op_exit_callback(void *btx_handle, void *usr_data, int64_t ts, - const char *hostname, int64_t vpid, uint64_t vtid, - char *op, uint32_t ep_id, char *src_addr, - char *peer_addr, char *direction, - uint64_t iov_count, uint64_t size_bytes, - int64_t ret, uintptr_t op_context) { +static void op_exit_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t ep_id, + char *src_addr, + char *peer_addr, + char *direction, + uint64_t iov_count, + uint64_t size_bytes, + int64_t ret, + uintptr_t op_context) { auto *s = static_cast(usr_data); auto &stack = s->op_stack[{hostname, vpid, vtid}]; @@ -149,7 +156,7 @@ static void op_exit_callback(void *btx_handle, void *usr_data, int64_t ts, if (stack.empty()) return; - const int64_t entry_ts = stack.back().first; + const int64_t entry_ts = stack.back().first; const std::string entry_op = stack.back().second; stack.pop_back(); @@ -159,9 +166,8 @@ static void op_exit_callback(void *btx_handle, void *usr_data, int64_t ts, * failure, but it is also not work the application wanted, and flagging it * makes it visible in the tally instead of inflating the success counts. */ - btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, entry_ts, - BACKEND_FI, entry_op.c_str(), (ts - entry_ts), - (ret < 0)); + btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, entry_ts, BACKEND_FI, + entry_op.c_str(), (ts - entry_ts), (ret < 0)); if (ret < 0) return; /* nothing became outstanding; the tracer already nulled context */ @@ -181,15 +187,21 @@ static void op_exit_callback(void *btx_handle, void *usr_data, int64_t ts, * is recycled the instant an operation completes, so at most one live entry * per key is correct; an overwrite means we missed a completion. */ - s->outstanding[ctx_key(hostname, vpid, op_context)] = {entry_ts, entry_op, - peer_addr, size_bytes}; + s->outstanding[ctx_key(hostname, vpid, op_context)] = {entry_ts, entry_op, peer_addr, size_bytes}; } -static void cq_completion_callback(void *btx_handle, void *usr_data, int64_t ts, - const char *hostname, int64_t vpid, - uint64_t vtid, char *op, uint32_t obj_id, - char *src_addr, char *peer_addr, - uint64_t size_bytes, int64_t ret, +static void cq_completion_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t obj_id, + char *src_addr, + char *peer_addr, + uint64_t size_bytes, + int64_t ret, uintptr_t op_context) { auto *s = static_cast(usr_data); @@ -214,9 +226,8 @@ static void cq_completion_callback(void *btx_handle, void *usr_data, int64_t ts, * that. Compare against the CXI counters (iprof --backends fi,cxi) when you * need the hardware view. */ - btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, post.ts, - BACKEND_FI, (post.op + ".transfer").c_str(), - (ts - post.ts), (ret < 0)); + btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, post.ts, BACKEND_FI, + (post.op + ".transfer").c_str(), (ts - post.ts), (ret < 0)); if (ret < 0) return; @@ -228,30 +239,39 @@ static void cq_completion_callback(void *btx_handle, void *usr_data, int64_t ts, * left. Likewise peer_addr: a wildcard receive posts FI_ADDR_UNSPEC and * only fi_cq_readfrom() reveals who actually sent. */ - const std::string peer = - (peer_addr && peer_addr[0] && std::string(peer_addr) != "fi_addr_unspec") - ? peer_addr - : post.peer; - - btx_push_message_lttng_traffic( - btx_handle, hostname, vpid, vtid, post.ts, BACKEND_FI, post.op.c_str(), - size_bytes, traffic_metadata(s, hostname, vpid, obj_id, peer).c_str()); + const std::string peer = (peer_addr && peer_addr[0] && std::string(peer_addr) != "fi_addr_unspec") + ? peer_addr + : post.peer; + + btx_push_message_lttng_traffic(btx_handle, hostname, vpid, vtid, post.ts, BACKEND_FI, + post.op.c_str(), size_bytes, + traffic_metadata(s, hostname, vpid, obj_id, peer).c_str()); } /* ------------------------------------------------------------------ */ /* control path */ /* ------------------------------------------------------------------ */ -static void ctrl_entry_callback(void *btx_handle, void *usr_data, int64_t ts, - const char *hostname, int64_t vpid, - uint64_t vtid, char *op, uint32_t obj_id) { +static void ctrl_entry_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t obj_id) { auto *s = static_cast(usr_data); s->ctrl_stack[{hostname, vpid, vtid}].push_back({ts, op}); } -static void ctrl_exit_callback(void *btx_handle, void *usr_data, int64_t ts, - const char *hostname, int64_t vpid, - uint64_t vtid, char *op, uint32_t obj_id, +static void ctrl_exit_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + char *op, + uint32_t obj_id, int64_t ret) { auto *s = static_cast(usr_data); auto &stack = s->ctrl_stack[{hostname, vpid, vtid}]; @@ -263,44 +283,46 @@ static void ctrl_exit_callback(void *btx_handle, void *usr_data, int64_t ts, * which opens an endpoint, which binds a CQ. A separate stack from the data * path keeps that nesting from interleaving with postings. */ - const int64_t entry_ts = stack.back().first; + const int64_t entry_ts = stack.back().first; const std::string entry_op = stack.back().second; stack.pop_back(); - btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, entry_ts, - BACKEND_FI, entry_op.c_str(), (ts - entry_ts), - (ret < 0)); + btx_push_message_lttng_host(btx_handle, hostname, vpid, vtid, entry_ts, BACKEND_FI, + entry_op.c_str(), (ts - entry_ts), (ret < 0)); } /* ------------------------------------------------------------------ */ /* endpoint description */ /* ------------------------------------------------------------------ */ -static void ep_info_callback(void *btx_handle, void *usr_data, int64_t ts, - const char *hostname, int64_t vpid, uint64_t vtid, - uint32_t ep_id, char *src_addr, char *domain_name, - uint32_t topo_valid, uint32_t group, - uint32_t switch_id, uint32_t port) { +static void ep_info_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + uint32_t ep_id, + char *src_addr, + char *domain_name, + uint32_t topo_valid, + uint32_t group, + uint32_t switch_id, + uint32_t port) { auto *s = static_cast(usr_data); - s->endpoints[ep_key(hostname, vpid, ep_id)] = { - src_addr, domain_name, topo_valid != 0, group, switch_id, port}; + s->endpoints[ep_key(hostname, vpid, ep_id)] = {src_addr, domain_name, topo_valid != 0, + group, switch_id, port}; } /* ------------------------------------------------------------------ */ void btx_register_usr_callbacks(void *btx_handle) { - btx_register_callbacks_initialize_component(btx_handle, - &btx_initialize_component); - btx_register_callbacks_finalize_component(btx_handle, - &btx_finalize_component); + btx_register_callbacks_initialize_component(btx_handle, &btx_initialize_component); + btx_register_callbacks_finalize_component(btx_handle, &btx_finalize_component); btx_register_callbacks_lttng_ust_fi_op_entry(btx_handle, &op_entry_callback); btx_register_callbacks_lttng_ust_fi_op_exit(btx_handle, &op_exit_callback); - btx_register_callbacks_lttng_ust_fi_cq_completion(btx_handle, - &cq_completion_callback); - btx_register_callbacks_lttng_ust_fi_ctrl_entry(btx_handle, - &ctrl_entry_callback); - btx_register_callbacks_lttng_ust_fi_ctrl_exit(btx_handle, - &ctrl_exit_callback); + btx_register_callbacks_lttng_ust_fi_cq_completion(btx_handle, &cq_completion_callback); + btx_register_callbacks_lttng_ust_fi_ctrl_entry(btx_handle, &ctrl_entry_callback); + btx_register_callbacks_lttng_ust_fi_ctrl_exit(btx_handle, &ctrl_exit_callback); btx_register_callbacks_lttng_ust_fi_ep_info(btx_handle, &ep_info_callback); } diff --git a/backends/libfabric/btx_libfabric_model.yaml b/backends/libfabric/btx_libfabric_model.yaml index a8369b694..59c346e1c 100644 --- a/backends/libfabric/btx_libfabric_model.yaml +++ b/backends/libfabric/btx_libfabric_model.yaml @@ -196,4 +196,4 @@ :field_class: :type: integer_unsigned :cast_type: uint32_t - :field_value_range: 32 \ No newline at end of file + :field_value_range: 32 diff --git a/backends/libfabric/libfabric_events.yaml b/backends/libfabric/libfabric_events.yaml index fe9e13a12..0570b9e05 100644 --- a/backends/libfabric/libfabric_events.yaml +++ b/backends/libfabric/libfabric_events.yaml @@ -9,7 +9,6 @@ lttng_ust_fi: - [ctf_string, op, op] - [ctf_integer, uint32_t, ep_id, ep_id] - [ctf_integer_hex, uintptr_t, op_context, "(uintptr_t)op_context"] - - name: op_exit args: - ["const char *", op] @@ -31,7 +30,6 @@ lttng_ust_fi: - [ctf_integer, uint64_t, size_bytes, size_bytes] - [ctf_integer, int64_t, ret, ret] - [ctf_integer_hex, uintptr_t, op_context, "(uintptr_t)op_context"] - - name: cq_completion args: - ["const char *", op] @@ -49,7 +47,6 @@ lttng_ust_fi: - [ctf_integer, uint64_t, size_bytes, size_bytes] - [ctf_integer, int64_t, ret, ret] - [ctf_integer_hex, uintptr_t, op_context, "(uintptr_t)op_context"] - - name: ctrl_entry args: - ["const char *", op] @@ -57,7 +54,6 @@ lttng_ust_fi: fields: - [ctf_string, op, op] - [ctf_integer, uint32_t, obj_id, obj_id] - - name: ctrl_exit args: - ["const char *", op] @@ -67,7 +63,6 @@ lttng_ust_fi: - [ctf_string, op, op] - [ctf_integer, uint32_t, obj_id, obj_id] - [ctf_integer, int64_t, ret, ret] - - name: ep_info args: - [uint32_t, ep_id] diff --git a/backends/libfabric/src/fi_trace.h b/backends/libfabric/src/fi_trace.h index 7976e6518..b00e62d43 100644 --- a/backends/libfabric/src/fi_trace.h +++ b/backends/libfabric/src/fi_trace.h @@ -16,31 +16,31 @@ #include #include +#include #include -#include #include -#include -#include -#include +#include #include +#include +#include -#define FIT_ADDR_MAX 96 /* "fi_addr_cxi://0x04656600" and friends */ +#define FIT_ADDR_MAX 96 /* "fi_addr_cxi://0x04656600" and friends */ #define FIT_RAW_ADDR_MAX 64 -#define FIT_REC_MAX 1024 /* worst-case bytes of one CSV line */ +#define FIT_REC_MAX 1024 /* worst-case bytes of one CSV line */ #ifndef fit_container_of -#define fit_container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define fit_container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif struct fit_ep; struct fit_cq; struct fit_topo; -void fit_record_entry(struct fit_ep *h, const char *op, char dir, - const void *context); -void fit_record_ep_info(unsigned int ep_id, const char *src, - size_t src_len, const struct fit_topo *t); +void fit_record_entry(struct fit_ep *h, const char *op, char dir, const void *context); +void fit_record_ep_info(unsigned int ep_id, + const char *src, + size_t src_len, + const struct fit_topo *t); void fit_record_ctrl_entry(const char *op, unsigned int obj_id); void fit_record_ctrl_exit(const char *op, unsigned int obj_id, int ret); @@ -49,21 +49,21 @@ void fit_record_ctrl_exit(const char *op, unsigned int obj_id, int ret); /* ------------------------------------------------------------------ */ struct fit_config { - int enabled; /* FI_TRACE_ENABLE */ - int resolve; /* FI_TRACE_RESOLVE */ - int trace_msg; /* FI_TRACE_OPS */ - int trace_tagged; - int trace_rma; - int trace_rx; /* FI_TRACE_RX */ - int trace_srx; /* FI_TRACE_SRX */ - int trace_eagain; /* FI_TRACE_EAGAIN */ - int trace_cq; /* FI_TRACE_CQ */ - int verbose; /* FI_TRACE_VERBOSE */ - size_t bufsz; /* FI_TRACE_BUFSZ */ - uint64_t max_records; /* FI_TRACE_MAX_RECORDS */ - char dir[4096]; /* FI_TRACE_DIR */ - char host[256]; - size_t host_len; + int enabled; /* FI_TRACE_ENABLE */ + int resolve; /* FI_TRACE_RESOLVE */ + int trace_msg; /* FI_TRACE_OPS */ + int trace_tagged; + int trace_rma; + int trace_rx; /* FI_TRACE_RX */ + int trace_srx; /* FI_TRACE_SRX */ + int trace_eagain; /* FI_TRACE_EAGAIN */ + int trace_cq; /* FI_TRACE_CQ */ + int verbose; /* FI_TRACE_VERBOSE */ + size_t bufsz; /* FI_TRACE_BUFSZ */ + uint64_t max_records; /* FI_TRACE_MAX_RECORDS */ + char dir[4096]; /* FI_TRACE_DIR */ + char host[256]; + size_t host_len; }; /* CSV fields are sanitised and measured once, where they are produced, @@ -86,15 +86,15 @@ void fit_log(const char *fmt, ...); /* ------------------------------------------------------------------ */ struct fit_peer { - fi_addr_t key; - size_t len; /* strlen(str), precomputed */ - char str[FIT_ADDR_MAX]; + fi_addr_t key; + size_t len; /* strlen(str), precomputed */ + char str[FIT_ADDR_MAX]; }; struct fit_peer_tab { - size_t cap; /* power of two */ - struct fit_peer_tab *older; /* retired tables, freed at ep close */ - _Atomic(struct fit_peer *) slot[]; + size_t cap; /* power of two */ + struct fit_peer_tab *older; /* retired tables, freed at ep close */ + _Atomic(struct fit_peer *) slot[]; }; /* ------------------------------------------------------------------ */ @@ -113,18 +113,17 @@ struct fit_peer_tab { /* ------------------------------------------------------------------ */ struct fit_topo { - int valid; /* 0 if the provider would not say */ - unsigned group; /* dragonfly group */ - unsigned sw; /* switch within the group */ - unsigned port; /* port on that switch */ - char name[64]; /* domain name, e.g. "cxi2" */ - size_t name_len; + int valid; /* 0 if the provider would not say */ + unsigned group; /* dragonfly group */ + unsigned sw; /* switch within the group */ + unsigned port; /* port on that switch */ + char name[64]; /* domain name, e.g. "cxi2" */ + size_t name_len; }; /* Ask the provider where this domain's NIC is. Leaves t->valid 0 on * anything that is not CXI, or a CXI too old to answer. */ -void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, - const struct fi_info *info); +void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, const struct fi_info *info); /* Record one endpoint's location, once, when its address is known. */ @@ -138,37 +137,37 @@ void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, /* ------------------------------------------------------------------ */ struct fit_ep { - /* our vtables -- must stay first-class members for container_of */ - struct fi_ops_msg msg; - struct fi_ops_tagged tagged; - struct fi_ops_rma rma; - struct fi_ops fid_ops; - struct fi_ops_ep ep_ops; - - /* the provider's originals */ - const struct fi_ops_msg *o_msg; - const struct fi_ops_tagged *o_tagged; - const struct fi_ops_rma *o_rma; - const struct fi_ops *o_fid; - const struct fi_ops_ep *o_ep; - - struct fid_ep *ep; - struct fid_av *av; /* bound AV, used to render addresses */ - unsigned int ep_id; - - /* Points into the owning domain's hook state, which outlives every - * endpoint on it. NULL for an endpoint we met without its domain. */ - const struct fit_topo *topo; - int topo_written; - - char src[FIT_ADDR_MAX]; - size_t src_len; - int src_valid; - - pthread_mutex_t lock; /* serialises inserts + src resolution */ - _Atomic(struct fit_peer_tab *) peers; - size_t peer_cnt; /* written under lock */ - unsigned int peer_gen; /* fit_av_gen when the table was built */ + /* our vtables -- must stay first-class members for container_of */ + struct fi_ops_msg msg; + struct fi_ops_tagged tagged; + struct fi_ops_rma rma; + struct fi_ops fid_ops; + struct fi_ops_ep ep_ops; + + /* the provider's originals */ + const struct fi_ops_msg *o_msg; + const struct fi_ops_tagged *o_tagged; + const struct fi_ops_rma *o_rma; + const struct fi_ops *o_fid; + const struct fi_ops_ep *o_ep; + + struct fid_ep *ep; + struct fid_av *av; /* bound AV, used to render addresses */ + unsigned int ep_id; + + /* Points into the owning domain's hook state, which outlives every + * endpoint on it. NULL for an endpoint we met without its domain. */ + const struct fit_topo *topo; + int topo_written; + + char src[FIT_ADDR_MAX]; + size_t src_len; + int src_valid; + + pthread_mutex_t lock; /* serialises inserts + src resolution */ + _Atomic(struct fit_peer_tab *) peers; + size_t peer_cnt; /* written under lock */ + unsigned int peer_gen; /* fit_av_gen when the table was built */ }; /* @@ -184,29 +183,29 @@ extern _Atomic unsigned int fit_av_gen; void fit_ep_peers_free(struct fit_ep *h); struct fit_domain { - struct fi_ops_domain ops; - struct fi_ops fid_ops; - const struct fi_ops_domain *orig; - const struct fi_ops *o_fid; - - /* Resolved once at hook time and handed to every endpoint this - * domain creates. The address of this member is what endpoints - * hold, so the struct must not move. */ - struct fit_topo topo; + struct fi_ops_domain ops; + struct fi_ops fid_ops; + const struct fi_ops_domain *orig; + const struct fi_ops *o_fid; + + /* Resolved once at hook time and handed to every endpoint this + * domain creates. The address of this member is what endpoints + * hold, so the struct must not move. */ + struct fit_topo topo; }; struct fit_fabric { - struct fi_ops_fabric ops; - struct fi_ops fid_ops; - const struct fi_ops_fabric *orig; - const struct fi_ops *o_fid; + struct fi_ops_fabric ops; + struct fi_ops fid_ops; + const struct fi_ops_fabric *orig; + const struct fi_ops *o_fid; }; struct fit_av { - struct fi_ops_av ops; - struct fi_ops fid_ops; - const struct fi_ops_av *orig; - const struct fi_ops *o_fid; + struct fi_ops_av ops; + struct fi_ops fid_ops; + const struct fi_ops_av *orig; + const struct fi_ops *o_fid; }; /* @@ -216,21 +215,21 @@ struct fit_av { * subtracting their timestamps gives the latency the application saw. */ struct fit_cq { - struct fi_ops_cq ops; - struct fi_ops fid_ops; - const struct fi_ops_cq *orig; - const struct fi_ops *o_fid; - - struct fid_cq *cq; - enum fi_cq_format format; - size_t entry_sz; /* 0 if the format is unknown */ - - /* The endpoint this CQ was bound to, so a completion can be - * attributed to a local NIC address. Left NULL if more than one - * endpoint binds the same CQ, because then it would be a guess. */ - struct fit_ep *ep; - int ep_ambiguous; - unsigned int cq_id; + struct fi_ops_cq ops; + struct fi_ops fid_ops; + const struct fi_ops_cq *orig; + const struct fi_ops *o_fid; + + struct fid_cq *cq; + enum fi_cq_format format; + size_t entry_sz; /* 0 if the format is unknown */ + + /* The endpoint this CQ was bound to, so a completion can be + * attributed to a local NIC address. Left NULL if more than one + * endpoint binds the same CQ, because then it would be a guess. */ + struct fit_ep *ep; + int ep_ambiguous; + unsigned int cq_id; }; /* ------------------------------------------------------------------ */ @@ -246,14 +245,23 @@ void fit_writer_flush_all(void); * application's op_context, recorded so the completion can be matched to * this record; it is dropped when the provider refused the operation, * since nothing became outstanding. */ -void fit_record(struct fit_ep *h, const char *op, char dir, fi_addr_t peer, - size_t bytes, size_t iov_count, ssize_t ret, - const void *context); +void fit_record(struct fit_ep *h, + const char *op, + char dir, + fi_addr_t peer, + size_t bytes, + size_t iov_count, + ssize_t ret, + const void *context); /* Emit one completion record. now is the time the batch was reaped. */ -void fit_record_cq(struct fit_cq *c, const char *op, fi_addr_t peer, - size_t bytes, ssize_t ret, const void *context, - uint64_t now); +void fit_record_cq(struct fit_cq *c, + const char *op, + fi_addr_t peer, + size_t bytes, + ssize_t ret, + const void *context, + uint64_t now); uint64_t fit_now_us(void); @@ -261,13 +269,13 @@ uint64_t fit_now_us(void); /* address rendering */ /* ------------------------------------------------------------------ */ -void fit_ep_resolve_src(struct fit_ep *h); +void fit_ep_resolve_src(struct fit_ep *h); /* Returns a sanitised, NUL-terminated peer string and its length. The * result may point into the endpoint's peer cache (stable until the * endpoint closes) or into the caller's scratch buffer. */ -const char *fit_ep_peer_str(struct fit_ep *h, fi_addr_t peer, char *scratch, - size_t scratch_len, size_t *len_out); +const char *fit_ep_peer_str( + struct fit_ep *h, fi_addr_t peer, char *scratch, size_t scratch_len, size_t *len_out); /* ------------------------------------------------------------------ */ /* installers */ @@ -281,26 +289,24 @@ void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo); void fit_hook_av(struct fid_av *av); void fit_hook_cq(struct fid_cq *cq, enum fi_cq_format format); -static inline size_t fit_iov_len(const struct iovec *iov, size_t count) -{ - size_t i, total = 0; +static inline size_t fit_iov_len(const struct iovec *iov, size_t count) { + size_t i, total = 0; - if (!iov) - return 0; - for (i = 0; i < count; i++) - total += iov[i].iov_len; - return total; + if (!iov) + return 0; + for (i = 0; i < count; i++) + total += iov[i].iov_len; + return total; } -static inline size_t fit_rma_iov_len(const struct fi_rma_iov *iov, size_t count) -{ - size_t i, total = 0; +static inline size_t fit_rma_iov_len(const struct fi_rma_iov *iov, size_t count) { + size_t i, total = 0; - if (!iov) - return 0; - for (i = 0; i < count; i++) - total += iov[i].len; - return total; + if (!iov) + return 0; + for (i = 0; i < count; i++) + total += iov[i].len; + return total; } #endif /* FI_TRACE_H */ diff --git a/backends/libfabric/src/fit_addr.c b/backends/libfabric/src/fit_addr.c index a7ba1b259..58ccb250a 100644 --- a/backends/libfabric/src/fit_addr.c +++ b/backends/libfabric/src/fit_addr.c @@ -24,86 +24,83 @@ #define FIT_PEER_CAP_INIT 64 -static void raw_to_hex(const void *addr, size_t len, char *out, size_t out_len) -{ - static const char hex[] = "0123456789abcdef"; - const unsigned char *b = addr; - size_t i, n = 0; - - if (out_len < 4) { - if (out_len) - out[0] = '\0'; - return; - } - out[n++] = '0'; - out[n++] = 'x'; - for (i = 0; i < len && n + 2 < out_len; i++) { - out[n++] = hex[b[i] >> 4]; - out[n++] = hex[b[i] & 0xf]; - } - out[n] = '\0'; +static void raw_to_hex(const void *addr, size_t len, char *out, size_t out_len) { + static const char hex[] = "0123456789abcdef"; + const unsigned char *b = addr; + size_t i, n = 0; + + if (out_len < 4) { + if (out_len) + out[0] = '\0'; + return; + } + out[n++] = '0'; + out[n++] = 'x'; + for (i = 0; i < len && n + 2 < out_len; i++) { + out[n++] = hex[b[i] >> 4]; + out[n++] = hex[b[i] & 0xf]; + } + out[n] = '\0'; } /* Format a raw provider address, preferring the provider's own * stringifier so the output matches fi_info / existing telemetry. */ -static void format_raw(struct fid_av *av, const void *raw, size_t raw_len, - char *out, size_t out_len) -{ - if (av && av->ops && av->ops->straddr) { - size_t len = out_len; - const char *s = fi_av_straddr(av, raw, out, &len); - - if (s) { - out[out_len - 1] = '\0'; - return; - } - } - raw_to_hex(raw, raw_len, out, out_len); +static void +format_raw(struct fid_av *av, const void *raw, size_t raw_len, char *out, size_t out_len) { + if (av && av->ops && av->ops->straddr) { + size_t len = out_len; + const char *s = fi_av_straddr(av, raw, out, &len); + + if (s) { + out[out_len - 1] = '\0'; + return; + } + } + raw_to_hex(raw, raw_len, out, out_len); } -void fit_ep_resolve_src(struct fit_ep *h) -{ - char raw[FIT_RAW_ADDR_MAX]; - size_t raw_len = sizeof(raw); - int ret; - - pthread_mutex_lock(&h->lock); - if (h->src_valid) - goto out; - - /* Mark resolved up front: if the provider cannot answer we do not - * want to retry fi_getname() on every single operation. */ - h->src_valid = 1; - snprintf(h->src, sizeof(h->src), "unknown"); - - if (!fit_cfg.resolve || !h->ep || !h->ep->cm || !h->ep->cm->getname) - goto out; - - ret = fi_getname(&h->ep->fid, raw, &raw_len); - if (ret) { - /* Not enabled yet -- allow a later retry. */ - if (ret == -FI_EOPBADSTATE || ret == -FI_ENOENT) - h->src_valid = 0; - goto out; - } - if (raw_len > sizeof(raw)) - goto out; - - format_raw(h->av, raw, raw_len, h->src, sizeof(h->src)); +void fit_ep_resolve_src(struct fit_ep *h) { + char raw[FIT_RAW_ADDR_MAX]; + size_t raw_len = sizeof(raw); + int ret; + + pthread_mutex_lock(&h->lock); + if (h->src_valid) + goto out; + + /* Mark resolved up front: if the provider cannot answer we do not + * want to retry fi_getname() on every single operation. */ + h->src_valid = 1; + snprintf(h->src, sizeof(h->src), "unknown"); + + if (!fit_cfg.resolve || !h->ep || !h->ep->cm || !h->ep->cm->getname) + goto out; + + ret = fi_getname(&h->ep->fid, raw, &raw_len); + if (ret) { + /* Not enabled yet -- allow a later retry. */ + if (ret == -FI_EOPBADSTATE || ret == -FI_ENOENT) + h->src_valid = 0; + goto out; + } + if (raw_len > sizeof(raw)) + goto out; + + format_raw(h->av, raw, raw_len, h->src, sizeof(h->src)); out: - fit_sanitize(h->src, &h->src_len); - - /* The two halves of "where is this endpoint" become available at - * different moments: the dragonfly coordinates at domain creation, - * the NIC address only after FI_ENABLE. This is the first point - * where both are known, so it is where the pair gets written -- - * once per endpoint, never on the hot path. */ - if (h->src_valid && !h->topo_written) { - h->topo_written = 1; - fit_record_ep_info(h->ep_id, h->src, h->src_len, h->topo); - } - - pthread_mutex_unlock(&h->lock); + fit_sanitize(h->src, &h->src_len); + + /* The two halves of "where is this endpoint" become available at + * different moments: the dragonfly coordinates at domain creation, + * the NIC address only after FI_ENABLE. This is the first point + * where both are known, so it is where the pair gets written -- + * once per endpoint, never on the hot path. */ + if (h->src_valid && !h->topo_written) { + h->topo_written = 1; + fit_record_ep_info(h->ep_id, h->src, h->src_len, h->topo); + } + + pthread_mutex_unlock(&h->lock); } /* ------------------------------------------------------------------ */ @@ -117,72 +114,64 @@ void fit_ep_resolve_src(struct fit_ep *h) /* ------------------------------------------------------------------ */ #ifdef HAVE_FI_CXI_TOPOLOGY -#include #include +#include #endif -void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, - const struct fi_info *info) -{ - memset(t, 0, sizeof(*t)); - - /* The NIC's name ("cxi3") is worth keeping even when the provider - * will not give coordinates: it is how the node's own tooling and - * the counter telemetry refer to the device. */ - if (info && info->domain_attr && info->domain_attr->name) - snprintf(t->name, sizeof(t->name), "%s", - info->domain_attr->name); - else - snprintf(t->name, sizeof(t->name), "unknown"); - fit_sanitize(t->name, &t->name_len); +void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, const struct fi_info *info) { + memset(t, 0, sizeof(*t)); + + /* The NIC's name ("cxi3") is worth keeping even when the provider + * will not give coordinates: it is how the node's own tooling and + * the counter telemetry refer to the device. */ + if (info && info->domain_attr && info->domain_attr->name) + snprintf(t->name, sizeof(t->name), "%s", info->domain_attr->name); + else + snprintf(t->name, sizeof(t->name), "unknown"); + fit_sanitize(t->name, &t->name_len); #ifdef HAVE_FI_CXI_TOPOLOGY - { - static const char *const versions[] = { - "dom_ops_v6", "dom_ops_v5", "dom_ops_v4", NULL - }; - int i; - - if (!domain || !domain->fid.ops || !domain->fid.ops->ops_open) { - fit_log("topology %s: domain has no ops_open", t->name); - return; - } - - /* Keep the table local to this domain; never reuse another NIC's - * extension pointer. Try the same versions as bidir_intra.cpp. */ - for (i = 0; versions[i]; i++) { - struct fi_cxi_dom_ops *ops = NULL; - unsigned grp = 0, sw = 0, port = 0; - int ret = fi_open_ops(&domain->fid, versions[i], 0, - (void **)&ops, NULL); - - if (ret || !ops || !ops->topology) { - fit_log("topology %s: %s unavailable (ret=%d)", - t->name, versions[i], ret); - continue; - } - - ret = ops->topology(&domain->fid, &grp, &sw, &port); - if (ret) { - fit_log("topology %s: %s query failed (ret=%d)", - t->name, versions[i], ret); - continue; - } - - t->group = grp; - t->sw = sw; - t->port = port; - t->valid = 1; - fit_log("topology %s: %s group=%u switch=%u port=%u", - t->name, versions[i], grp, sw, port); - return; - } - fit_log("topology %s: no supported query succeeded", t->name); - } + { + static const char *const versions[] = {"dom_ops_v6", "dom_ops_v5", "dom_ops_v4", NULL}; + int i; + + if (!domain || !domain->fid.ops || !domain->fid.ops->ops_open) { + fit_log("topology %s: domain has no ops_open", t->name); + return; + } + + /* Keep the table local to this domain; never reuse another NIC's + * extension pointer. Try the same versions as bidir_intra.cpp. */ + for (i = 0; versions[i]; i++) { + struct fi_cxi_dom_ops *ops = NULL; + unsigned grp = 0, sw = 0, port = 0; + int ret = fi_open_ops(&domain->fid, versions[i], 0, (void **)&ops, NULL); + + if (ret || !ops || !ops->topology) { + fit_log("topology %s: %s unavailable (ret=%d)", t->name, versions[i], ret); + continue; + } + + ret = ops->topology(&domain->fid, &grp, &sw, &port); + if (ret) { + fit_log("topology %s: %s query failed (ret=%d)", t->name, versions[i], ret); + continue; + } + + t->group = grp; + t->sw = sw; + t->port = port; + t->valid = 1; + fit_log("topology %s: %s group=%u switch=%u port=%u", t->name, versions[i], grp, sw, port); + return; + } + fit_log("topology %s: no supported query succeeded", t->name); + } #else - (void)domain; - fit_log("topology %s: disabled at build time; run make check-topology " - "with the target libfabric headers, then rebuild", t->name); + (void)domain; + fit_log("topology %s: disabled at build time; run make check-topology " + "with the target libfabric headers, then rebuild", + t->name); #endif } @@ -206,139 +195,125 @@ void fit_topo_resolve(struct fit_topo *t, struct fid_domain *domain, /* peer cache */ /* ------------------------------------------------------------------ */ -static inline size_t peer_hash(fi_addr_t key, size_t cap) -{ - uint64_t h = (uint64_t)key; +static inline size_t peer_hash(fi_addr_t key, size_t cap) { + uint64_t h = (uint64_t)key; - h ^= h >> 33; - h *= 0xff51afd7ed558ccdull; - h ^= h >> 33; - return (size_t)(h & (cap - 1)); + h ^= h >> 33; + h *= 0xff51afd7ed558ccdull; + h ^= h >> 33; + return (size_t)(h & (cap - 1)); } /* Lock-free lookup. Entry pointers are stable for the life of the * endpoint, so the caller may keep the returned string. */ -static const char *peer_find(struct fit_ep *h, fi_addr_t key, size_t *len_out) -{ - struct fit_peer_tab *tab; - size_t i, slot, cap; - - tab = atomic_load_explicit(&h->peers, memory_order_acquire); - if (!tab) - return NULL; - - cap = tab->cap; - slot = peer_hash(key, cap); - for (i = 0; i < cap; i++) { - struct fit_peer *e = atomic_load_explicit( - &tab->slot[(slot + i) & (cap - 1)], - memory_order_acquire); - - if (!e) - return NULL; - if (e->key == key) { - *len_out = e->len; - return e->str; - } - } - return NULL; +static const char *peer_find(struct fit_ep *h, fi_addr_t key, size_t *len_out) { + struct fit_peer_tab *tab; + size_t i, slot, cap; + + tab = atomic_load_explicit(&h->peers, memory_order_acquire); + if (!tab) + return NULL; + + cap = tab->cap; + slot = peer_hash(key, cap); + for (i = 0; i < cap; i++) { + struct fit_peer *e = + atomic_load_explicit(&tab->slot[(slot + i) & (cap - 1)], memory_order_acquire); + + if (!e) + return NULL; + if (e->key == key) { + *len_out = e->len; + return e->str; + } + } + return NULL; } /* Caller holds h->lock. */ -static void tab_place(struct fit_peer_tab *tab, struct fit_peer *e) -{ - size_t slot = peer_hash(e->key, tab->cap); - size_t i; - - for (i = 0; i < tab->cap; i++) { - _Atomic(struct fit_peer *) *s = - &tab->slot[(slot + i) & (tab->cap - 1)]; - - if (!atomic_load_explicit(s, memory_order_relaxed)) { - atomic_store_explicit(s, e, memory_order_release); - return; - } - } +static void tab_place(struct fit_peer_tab *tab, struct fit_peer *e) { + size_t slot = peer_hash(e->key, tab->cap); + size_t i; + + for (i = 0; i < tab->cap; i++) { + _Atomic(struct fit_peer *) *s = &tab->slot[(slot + i) & (tab->cap - 1)]; + + if (!atomic_load_explicit(s, memory_order_relaxed)) { + atomic_store_explicit(s, e, memory_order_release); + return; + } + } } /* Caller holds h->lock. The old table is retired rather than freed so * that a concurrent reader still walking it stays safe. */ -static int peer_grow(struct fit_ep *h) -{ - struct fit_peer_tab *old = - atomic_load_explicit(&h->peers, memory_order_relaxed); - size_t new_cap = old ? old->cap * 2 : FIT_PEER_CAP_INIT; - struct fit_peer_tab *fresh; - size_t i; - - fresh = calloc(1, sizeof(*fresh) + new_cap * sizeof(fresh->slot[0])); - if (!fresh) - return -1; - - fresh->cap = new_cap; - fresh->older = old; - - if (old) { - for (i = 0; i < old->cap; i++) { - struct fit_peer *e = atomic_load_explicit( - &old->slot[i], memory_order_relaxed); - - if (e) - tab_place(fresh, e); - } - } - - atomic_store_explicit(&h->peers, fresh, memory_order_release); - return 0; +static int peer_grow(struct fit_ep *h) { + struct fit_peer_tab *old = atomic_load_explicit(&h->peers, memory_order_relaxed); + size_t new_cap = old ? old->cap * 2 : FIT_PEER_CAP_INIT; + struct fit_peer_tab *fresh; + size_t i; + + fresh = calloc(1, sizeof(*fresh) + new_cap * sizeof(fresh->slot[0])); + if (!fresh) + return -1; + + fresh->cap = new_cap; + fresh->older = old; + + if (old) { + for (i = 0; i < old->cap; i++) { + struct fit_peer *e = atomic_load_explicit(&old->slot[i], memory_order_relaxed); + + if (e) + tab_place(fresh, e); + } + } + + atomic_store_explicit(&h->peers, fresh, memory_order_release); + return 0; } /* Caller holds h->lock. Returns the stable string, or NULL on OOM. */ -static const char *peer_insert(struct fit_ep *h, fi_addr_t key, - const char *str, size_t *len_out) -{ - struct fit_peer_tab *tab = - atomic_load_explicit(&h->peers, memory_order_relaxed); - struct fit_peer *e; - - /* Keep the load factor at or below 1/2 so probing stays short. */ - if (!tab || (h->peer_cnt + 1) * 2 > tab->cap) { - if (peer_grow(h)) - return NULL; - tab = atomic_load_explicit(&h->peers, memory_order_relaxed); - } - - e = calloc(1, sizeof(*e)); - if (!e) - return NULL; - - e->key = key; - snprintf(e->str, sizeof(e->str), "%s", str); - fit_sanitize(e->str, &e->len); - - tab_place(tab, e); - h->peer_cnt++; - *len_out = e->len; - return e->str; +static const char *peer_insert(struct fit_ep *h, fi_addr_t key, const char *str, size_t *len_out) { + struct fit_peer_tab *tab = atomic_load_explicit(&h->peers, memory_order_relaxed); + struct fit_peer *e; + + /* Keep the load factor at or below 1/2 so probing stays short. */ + if (!tab || (h->peer_cnt + 1) * 2 > tab->cap) { + if (peer_grow(h)) + return NULL; + tab = atomic_load_explicit(&h->peers, memory_order_relaxed); + } + + e = calloc(1, sizeof(*e)); + if (!e) + return NULL; + + e->key = key; + snprintf(e->str, sizeof(e->str), "%s", str); + fit_sanitize(e->str, &e->len); + + tab_place(tab, e); + h->peer_cnt++; + *len_out = e->len; + return e->str; } -void fit_ep_peers_free(struct fit_ep *h) -{ - struct fit_peer_tab *tab = - atomic_load_explicit(&h->peers, memory_order_relaxed); - size_t i; - - if (tab) { - for (i = 0; i < tab->cap; i++) - free(atomic_load_explicit(&tab->slot[i], - memory_order_relaxed)); - } - while (tab) { - struct fit_peer_tab *older = tab->older; - - free(tab); - tab = older; - } - atomic_store_explicit(&h->peers, NULL, memory_order_relaxed); +void fit_ep_peers_free(struct fit_ep *h) { + struct fit_peer_tab *tab = atomic_load_explicit(&h->peers, memory_order_relaxed); + size_t i; + + if (tab) { + for (i = 0; i < tab->cap; i++) + free(atomic_load_explicit(&tab->slot[i], memory_order_relaxed)); + } + while (tab) { + struct fit_peer_tab *older = tab->older; + + free(tab); + tab = older; + } + atomic_store_explicit(&h->peers, NULL, memory_order_relaxed); } /* @@ -348,78 +323,71 @@ void fit_ep_peers_free(struct fit_ep *h) * retired table stays reachable through ->older so a concurrent reader * walking it is still safe. Caller holds h->lock. */ -static void peer_retire_stale(struct fit_ep *h, unsigned int gen) -{ - struct fit_peer_tab *old, *fresh; - - if (h->peer_gen == gen) - return; - - old = atomic_load_explicit(&h->peers, memory_order_relaxed); - if (old) { - fresh = calloc(1, sizeof(*fresh) + - FIT_PEER_CAP_INIT * sizeof(fresh->slot[0])); - if (!fresh) - return; /* keep the stale table over none */ - fresh->cap = FIT_PEER_CAP_INIT; - fresh->older = old; - atomic_store_explicit(&h->peers, fresh, memory_order_release); - fit_log("address vector changed; dropped %u cached peers", - (unsigned)h->peer_cnt); - h->peer_cnt = 0; - } - h->peer_gen = gen; +static void peer_retire_stale(struct fit_ep *h, unsigned int gen) { + struct fit_peer_tab *old, *fresh; + + if (h->peer_gen == gen) + return; + + old = atomic_load_explicit(&h->peers, memory_order_relaxed); + if (old) { + fresh = calloc(1, sizeof(*fresh) + FIT_PEER_CAP_INIT * sizeof(fresh->slot[0])); + if (!fresh) + return; /* keep the stale table over none */ + fresh->cap = FIT_PEER_CAP_INIT; + fresh->older = old; + atomic_store_explicit(&h->peers, fresh, memory_order_release); + fit_log("address vector changed; dropped %u cached peers", (unsigned)h->peer_cnt); + h->peer_cnt = 0; + } + h->peer_gen = gen; } -const char *fit_ep_peer_str(struct fit_ep *h, fi_addr_t peer, char *scratch, - size_t scratch_len, size_t *len_out) -{ - char raw[FIT_RAW_ADDR_MAX]; - size_t raw_len = sizeof(raw); - const char *cached; - unsigned int gen; - int resolvable; - - if (peer == FI_ADDR_UNSPEC) { - *len_out = sizeof("fi_addr_unspec") - 1; - return "fi_addr_unspec"; - } - if (peer == FI_ADDR_NOTAVAIL) { - *len_out = sizeof("fi_addr_notavail") - 1; - return "fi_addr_notavail"; - } - - /* The common case: already seen this peer, and no fi_av_remove() - * has invalidated the mapping since. Both resolved and unresolved - * renderings are cached, so neither path formats a string more than - * once per peer. */ - gen = atomic_load_explicit(&fit_av_gen, memory_order_acquire); - if (gen == h->peer_gen) { - cached = peer_find(h, peer, len_out); - if (cached) - return cached; - } - - resolvable = fit_cfg.resolve && h->av && h->av->ops && - h->av->ops->lookup; - - if (resolvable && !fi_av_lookup(h->av, peer, raw, &raw_len) && - raw_len <= sizeof(raw)) - format_raw(h->av, raw, raw_len, scratch, scratch_len); - else - snprintf(scratch, scratch_len, "fi_addr://0x%016llx", - (unsigned long long)peer); - - pthread_mutex_lock(&h->lock); - peer_retire_stale(h, gen); - cached = peer_find(h, peer, len_out); /* lost a race? */ - if (!cached) - cached = peer_insert(h, peer, scratch, len_out); - pthread_mutex_unlock(&h->lock); - - if (cached) - return cached; - - fit_sanitize(scratch, len_out); - return scratch; +const char *fit_ep_peer_str( + struct fit_ep *h, fi_addr_t peer, char *scratch, size_t scratch_len, size_t *len_out) { + char raw[FIT_RAW_ADDR_MAX]; + size_t raw_len = sizeof(raw); + const char *cached; + unsigned int gen; + int resolvable; + + if (peer == FI_ADDR_UNSPEC) { + *len_out = sizeof("fi_addr_unspec") - 1; + return "fi_addr_unspec"; + } + if (peer == FI_ADDR_NOTAVAIL) { + *len_out = sizeof("fi_addr_notavail") - 1; + return "fi_addr_notavail"; + } + + /* The common case: already seen this peer, and no fi_av_remove() + * has invalidated the mapping since. Both resolved and unresolved + * renderings are cached, so neither path formats a string more than + * once per peer. */ + gen = atomic_load_explicit(&fit_av_gen, memory_order_acquire); + if (gen == h->peer_gen) { + cached = peer_find(h, peer, len_out); + if (cached) + return cached; + } + + resolvable = fit_cfg.resolve && h->av && h->av->ops && h->av->ops->lookup; + + if (resolvable && !fi_av_lookup(h->av, peer, raw, &raw_len) && raw_len <= sizeof(raw)) + format_raw(h->av, raw, raw_len, scratch, scratch_len); + else + snprintf(scratch, scratch_len, "fi_addr://0x%016llx", (unsigned long long)peer); + + pthread_mutex_lock(&h->lock); + peer_retire_stale(h, gen); + cached = peer_find(h, peer, len_out); /* lost a race? */ + if (!cached) + cached = peer_insert(h, peer, scratch, len_out); + pthread_mutex_unlock(&h->lock); + + if (cached) + return cached; + + fit_sanitize(scratch, len_out); + return scratch; } diff --git a/backends/libfabric/src/fit_hook.c b/backends/libfabric/src/fit_hook.c index 96749adda..84b0b7a2f 100644 --- a/backends/libfabric/src/fit_hook.c +++ b/backends/libfabric/src/fit_hook.c @@ -36,415 +36,416 @@ static void fit_cq_attach_ep(struct fid *bfid, struct fit_ep *h); /* recovering hook state from a vtable pointer */ /* ------------------------------------------------------------------ */ -#define EP_FROM_MSG(ep) fit_container_of((ep)->msg, struct fit_ep, msg) -#define EP_FROM_TAGGED(ep) fit_container_of((ep)->tagged, struct fit_ep, tagged) -#define EP_FROM_RMA(ep) fit_container_of((ep)->rma, struct fit_ep, rma) -#define EP_FROM_FID(fid) fit_container_of((fid)->ops, struct fit_ep, fid_ops) -#define EP_FROM_EPOPS(ep) fit_container_of((ep)->ops, struct fit_ep, ep_ops) +#define EP_FROM_MSG(ep) fit_container_of((ep)->msg, struct fit_ep, msg) +#define EP_FROM_TAGGED(ep) fit_container_of((ep)->tagged, struct fit_ep, tagged) +#define EP_FROM_RMA(ep) fit_container_of((ep)->rma, struct fit_ep, rma) +#define EP_FROM_FID(fid) fit_container_of((fid)->ops, struct fit_ep, fid_ops) +#define EP_FROM_EPOPS(ep) fit_container_of((ep)->ops, struct fit_ep, ep_ops) -#define CALL_OR_ENOSYS(fn, args) ((fn) ? (fn) args : -FI_ENOSYS) +#define CALL_OR_ENOSYS(fn, args) ((fn) ? (fn)args : -FI_ENOSYS) /* ================================================================== */ /* message operations */ /* ================================================================== */ -static ssize_t fit_msg_send(struct fid_ep *ep, const void *buf, size_t len, - void *desc, fi_addr_t dest_addr, void *context) -{ - struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret; - - fit_record_entry(h, "fi_send", 'T', context); - ret = CALL_OR_ENOSYS(h->o_msg->send, - (ep, buf, len, desc, dest_addr, context)); - fit_record(h, "fi_send", 'T', dest_addr, len, 1, ret, context); - return ret; -} - -static ssize_t fit_msg_sendv(struct fid_ep *ep, const struct iovec *iov, - void **desc, size_t count, fi_addr_t dest_addr, - void *context) -{ - struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret; - - fit_record_entry(h, "fi_sendv", 'T', context); - ret = CALL_OR_ENOSYS(h->o_msg->sendv, - (ep, iov, desc, count, dest_addr, context)); - fit_record(h, "fi_sendv", 'T', dest_addr, fit_iov_len(iov, count), - count, ret, context); - return ret; -} - -static ssize_t fit_msg_sendmsg(struct fid_ep *ep, const struct fi_msg *msg, - uint64_t flags) -{ - struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret; - - fit_record_entry(h, "fi_sendmsg", 'T', msg ? msg->context : NULL); - ret = CALL_OR_ENOSYS(h->o_msg->sendmsg, (ep, msg, flags)); - fit_record(h, "fi_sendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, - msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, - msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); - return ret; -} - -static ssize_t fit_msg_inject(struct fid_ep *ep, const void *buf, size_t len, - fi_addr_t dest_addr) -{ - struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret; - - fit_record_entry(h, "fi_inject", 'T', NULL); - ret = CALL_OR_ENOSYS(h->o_msg->inject, (ep, buf, len, dest_addr)); - fit_record(h, "fi_inject", 'T', dest_addr, len, 1, ret, NULL); - return ret; -} - -static ssize_t fit_msg_senddata(struct fid_ep *ep, const void *buf, size_t len, - void *desc, uint64_t data, fi_addr_t dest_addr, - void *context) -{ - struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret; - - fit_record_entry(h, "fi_senddata", 'T', context); - ret = CALL_OR_ENOSYS(h->o_msg->senddata, - (ep, buf, len, desc, data, dest_addr, - context)); - fit_record(h, "fi_senddata", 'T', dest_addr, len, 1, ret, context); - return ret; -} - -static ssize_t fit_msg_injectdata(struct fid_ep *ep, const void *buf, - size_t len, uint64_t data, - fi_addr_t dest_addr) -{ - struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret; - - fit_record_entry(h, "fi_injectdata", 'T', NULL); - ret = CALL_OR_ENOSYS(h->o_msg->injectdata, - (ep, buf, len, data, dest_addr)); - fit_record(h, "fi_injectdata", 'T', dest_addr, len, 1, ret, NULL); - return ret; -} - -static ssize_t fit_msg_recv(struct fid_ep *ep, void *buf, size_t len, - void *desc, fi_addr_t src_addr, void *context) -{ - struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret; - - fit_record_entry(h, "fi_recv", 'R', context); - ret = CALL_OR_ENOSYS(h->o_msg->recv, - (ep, buf, len, desc, src_addr, context)); - fit_record(h, "fi_recv", 'R', src_addr, len, 1, ret, context); - return ret; -} - -static ssize_t fit_msg_recvv(struct fid_ep *ep, const struct iovec *iov, - void **desc, size_t count, fi_addr_t src_addr, - void *context) -{ - struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret; - - fit_record_entry(h, "fi_recvv", 'R', context); - ret = CALL_OR_ENOSYS(h->o_msg->recvv, - (ep, iov, desc, count, src_addr, context)); - fit_record(h, "fi_recvv", 'R', src_addr, fit_iov_len(iov, count), - count, ret, context); - return ret; -} - -static ssize_t fit_msg_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, - uint64_t flags) -{ - struct fit_ep *h = EP_FROM_MSG(ep); - ssize_t ret; - - fit_record_entry(h, "fi_recvmsg", 'R', msg ? msg->context : NULL); - ret = CALL_OR_ENOSYS(h->o_msg->recvmsg, (ep, msg, flags)); - fit_record(h, "fi_recvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, - msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, - msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); - return ret; +static ssize_t fit_msg_send(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + fi_addr_t dest_addr, + void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_send", 'T', context); + ret = CALL_OR_ENOSYS(h->o_msg->send, (ep, buf, len, desc, dest_addr, context)); + fit_record(h, "fi_send", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_msg_sendv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t dest_addr, + void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_sendv", 'T', context); + ret = CALL_OR_ENOSYS(h->o_msg->sendv, (ep, iov, desc, count, dest_addr, context)); + fit_record(h, "fi_sendv", 'T', dest_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_msg_sendmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_sendmsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_msg->sendmsg, (ep, msg, flags)); + fit_record(h, "fi_sendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; +} + +static ssize_t fit_msg_inject(struct fid_ep *ep, const void *buf, size_t len, fi_addr_t dest_addr) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_inject", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_msg->inject, (ep, buf, len, dest_addr)); + fit_record(h, "fi_inject", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_msg_senddata(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + uint64_t data, + fi_addr_t dest_addr, + void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_senddata", 'T', context); + ret = CALL_OR_ENOSYS(h->o_msg->senddata, (ep, buf, len, desc, data, dest_addr, context)); + fit_record(h, "fi_senddata", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_msg_injectdata( + struct fid_ep *ep, const void *buf, size_t len, uint64_t data, fi_addr_t dest_addr) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_injectdata", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_msg->injectdata, (ep, buf, len, data, dest_addr)); + fit_record(h, "fi_injectdata", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_msg_recv( + struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_recv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_msg->recv, (ep, buf, len, desc, src_addr, context)); + fit_record(h, "fi_recv", 'R', src_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_msg_recvv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t src_addr, + void *context) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_recvv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_msg->recvv, (ep, iov, desc, count, src_addr, context)); + fit_record(h, "fi_recvv", 'R', src_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_msg_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_MSG(ep); + ssize_t ret; + + fit_record_entry(h, "fi_recvmsg", 'R', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_msg->recvmsg, (ep, msg, flags)); + fit_record(h, "fi_recvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; } /* ================================================================== */ /* tagged operations */ /* ================================================================== */ -static ssize_t fit_tag_send(struct fid_ep *ep, const void *buf, size_t len, - void *desc, fi_addr_t dest_addr, uint64_t tag, - void *context) -{ - struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret; - - fit_record_entry(h, "fi_tsend", 'T', context); - ret = CALL_OR_ENOSYS(h->o_tagged->send, - (ep, buf, len, desc, dest_addr, tag, - context)); - fit_record(h, "fi_tsend", 'T', dest_addr, len, 1, ret, context); - return ret; -} - -static ssize_t fit_tag_sendv(struct fid_ep *ep, const struct iovec *iov, - void **desc, size_t count, fi_addr_t dest_addr, - uint64_t tag, void *context) -{ - struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret; - - fit_record_entry(h, "fi_tsendv", 'T', context); - ret = CALL_OR_ENOSYS(h->o_tagged->sendv, - (ep, iov, desc, count, dest_addr, tag, - context)); - fit_record(h, "fi_tsendv", 'T', dest_addr, fit_iov_len(iov, count), - count, ret, context); - return ret; -} - -static ssize_t fit_tag_sendmsg(struct fid_ep *ep, - const struct fi_msg_tagged *msg, uint64_t flags) -{ - struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret; - - fit_record_entry(h, "fi_tsendmsg", 'T', msg ? msg->context : NULL); - ret = CALL_OR_ENOSYS(h->o_tagged->sendmsg, (ep, msg, flags)); - fit_record(h, "fi_tsendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, - msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, - msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); - return ret; -} - -static ssize_t fit_tag_inject(struct fid_ep *ep, const void *buf, size_t len, - fi_addr_t dest_addr, uint64_t tag) -{ - struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret; - - fit_record_entry(h, "fi_tinject", 'T', NULL); - ret = CALL_OR_ENOSYS(h->o_tagged->inject, - (ep, buf, len, dest_addr, tag)); - fit_record(h, "fi_tinject", 'T', dest_addr, len, 1, ret, NULL); - return ret; -} - -static ssize_t fit_tag_senddata(struct fid_ep *ep, const void *buf, size_t len, - void *desc, uint64_t data, fi_addr_t dest_addr, - uint64_t tag, void *context) -{ - struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret; - - fit_record_entry(h, "fi_tsenddata", 'T', context); - ret = CALL_OR_ENOSYS(h->o_tagged->senddata, - (ep, buf, len, desc, data, dest_addr, tag, - context)); - fit_record(h, "fi_tsenddata", 'T', dest_addr, len, 1, ret, context); - return ret; -} - -static ssize_t fit_tag_injectdata(struct fid_ep *ep, const void *buf, - size_t len, uint64_t data, - fi_addr_t dest_addr, uint64_t tag) -{ - struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret; - - fit_record_entry(h, "fi_tinjectdata", 'T', NULL); - ret = CALL_OR_ENOSYS(h->o_tagged->injectdata, - (ep, buf, len, data, dest_addr, tag)); - fit_record(h, "fi_tinjectdata", 'T', dest_addr, len, 1, ret, NULL); - return ret; -} - -static ssize_t fit_tag_recv(struct fid_ep *ep, void *buf, size_t len, - void *desc, fi_addr_t src_addr, uint64_t tag, - uint64_t ignore, void *context) -{ - struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret; - - fit_record_entry(h, "fi_trecv", 'R', context); - ret = CALL_OR_ENOSYS(h->o_tagged->recv, - (ep, buf, len, desc, src_addr, tag, ignore, - context)); - fit_record(h, "fi_trecv", 'R', src_addr, len, 1, ret, context); - return ret; -} - -static ssize_t fit_tag_recvv(struct fid_ep *ep, const struct iovec *iov, - void **desc, size_t count, fi_addr_t src_addr, - uint64_t tag, uint64_t ignore, void *context) -{ - struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret; - - fit_record_entry(h, "fi_trecvv", 'R', context); - ret = CALL_OR_ENOSYS(h->o_tagged->recvv, - (ep, iov, desc, count, src_addr, tag, - ignore, context)); - fit_record(h, "fi_trecvv", 'R', src_addr, fit_iov_len(iov, count), - count, ret, context); - return ret; -} - -static ssize_t fit_tag_recvmsg(struct fid_ep *ep, - const struct fi_msg_tagged *msg, uint64_t flags) -{ - struct fit_ep *h = EP_FROM_TAGGED(ep); - ssize_t ret; - - fit_record_entry(h, "fi_trecvmsg", 'R', msg ? msg->context : NULL); - ret = CALL_OR_ENOSYS(h->o_tagged->recvmsg, (ep, msg, flags)); - fit_record(h, "fi_trecvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, - msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, - msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); - return ret; +static ssize_t fit_tag_send(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + fi_addr_t dest_addr, + uint64_t tag, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tsend", 'T', context); + ret = CALL_OR_ENOSYS(h->o_tagged->send, (ep, buf, len, desc, dest_addr, tag, context)); + fit_record(h, "fi_tsend", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_tag_sendv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t dest_addr, + uint64_t tag, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tsendv", 'T', context); + ret = CALL_OR_ENOSYS(h->o_tagged->sendv, (ep, iov, desc, count, dest_addr, tag, context)); + fit_record(h, "fi_tsendv", 'T', dest_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_tag_sendmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tsendmsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->sendmsg, (ep, msg, flags)); + fit_record(h, "fi_tsendmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; +} + +static ssize_t +fit_tag_inject(struct fid_ep *ep, const void *buf, size_t len, fi_addr_t dest_addr, uint64_t tag) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tinject", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->inject, (ep, buf, len, dest_addr, tag)); + fit_record(h, "fi_tinject", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_tag_senddata(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + uint64_t data, + fi_addr_t dest_addr, + uint64_t tag, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tsenddata", 'T', context); + ret = CALL_OR_ENOSYS(h->o_tagged->senddata, (ep, buf, len, desc, data, dest_addr, tag, context)); + fit_record(h, "fi_tsenddata", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_tag_injectdata(struct fid_ep *ep, + const void *buf, + size_t len, + uint64_t data, + fi_addr_t dest_addr, + uint64_t tag) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_tinjectdata", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->injectdata, (ep, buf, len, data, dest_addr, tag)); + fit_record(h, "fi_tinjectdata", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_tag_recv(struct fid_ep *ep, + void *buf, + size_t len, + void *desc, + fi_addr_t src_addr, + uint64_t tag, + uint64_t ignore, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_trecv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_tagged->recv, (ep, buf, len, desc, src_addr, tag, ignore, context)); + fit_record(h, "fi_trecv", 'R', src_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_tag_recvv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t src_addr, + uint64_t tag, + uint64_t ignore, + void *context) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_trecvv", 'R', context); + ret = CALL_OR_ENOSYS(h->o_tagged->recvv, (ep, iov, desc, count, src_addr, tag, ignore, context)); + fit_record(h, "fi_trecvv", 'R', src_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_tag_recvmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_TAGGED(ep); + ssize_t ret; + + fit_record_entry(h, "fi_trecvmsg", 'R', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_tagged->recvmsg, (ep, msg, flags)); + fit_record(h, "fi_trecvmsg", 'R', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; } /* ================================================================== */ /* RMA operations */ /* ================================================================== */ -static ssize_t fit_rma_write(struct fid_ep *ep, const void *buf, size_t len, - void *desc, fi_addr_t dest_addr, uint64_t addr, - uint64_t key, void *context) -{ - struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret; - - fit_record_entry(h, "fi_write", 'T', context); - ret = CALL_OR_ENOSYS(h->o_rma->write, - (ep, buf, len, desc, dest_addr, addr, key, - context)); - fit_record(h, "fi_write", 'T', dest_addr, len, 1, ret, context); - return ret; -} - -static ssize_t fit_rma_writev(struct fid_ep *ep, const struct iovec *iov, - void **desc, size_t count, fi_addr_t dest_addr, - uint64_t addr, uint64_t key, void *context) -{ - struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret; - - fit_record_entry(h, "fi_writev", 'T', context); - ret = CALL_OR_ENOSYS(h->o_rma->writev, - (ep, iov, desc, count, dest_addr, addr, - key, context)); - fit_record(h, "fi_writev", 'T', dest_addr, fit_iov_len(iov, count), - count, ret, context); - return ret; -} - -static ssize_t fit_rma_writemsg(struct fid_ep *ep, - const struct fi_msg_rma *msg, uint64_t flags) -{ - struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret; - - fit_record_entry(h, "fi_writemsg", 'T', msg ? msg->context : NULL); - ret = CALL_OR_ENOSYS(h->o_rma->writemsg, (ep, msg, flags)); - fit_record(h, "fi_writemsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, - msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, - msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); - return ret; -} - -static ssize_t fit_rma_inject(struct fid_ep *ep, const void *buf, size_t len, - fi_addr_t dest_addr, uint64_t addr, uint64_t key) -{ - struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret; - - fit_record_entry(h, "fi_inject_write", 'T', NULL); - ret = CALL_OR_ENOSYS(h->o_rma->inject, - (ep, buf, len, dest_addr, addr, key)); - fit_record(h, "fi_inject_write", 'T', dest_addr, len, 1, ret, NULL); - return ret; -} - -static ssize_t fit_rma_writedata(struct fid_ep *ep, const void *buf, - size_t len, void *desc, uint64_t data, - fi_addr_t dest_addr, uint64_t addr, - uint64_t key, void *context) -{ - struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret; - - fit_record_entry(h, "fi_writedata", 'T', context); - ret = CALL_OR_ENOSYS(h->o_rma->writedata, - (ep, buf, len, desc, data, dest_addr, addr, - key, context)); - fit_record(h, "fi_writedata", 'T', dest_addr, len, 1, ret, context); - return ret; -} - -static ssize_t fit_rma_injectdata(struct fid_ep *ep, const void *buf, - size_t len, uint64_t data, - fi_addr_t dest_addr, uint64_t addr, - uint64_t key) -{ - struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret; - - fit_record_entry(h, "fi_inject_writedata", 'T', NULL); - ret = CALL_OR_ENOSYS(h->o_rma->injectdata, - (ep, buf, len, data, dest_addr, addr, key)); - fit_record(h, "fi_inject_writedata", 'T', dest_addr, len, 1, ret, NULL); - return ret; -} - -static ssize_t fit_rma_read(struct fid_ep *ep, void *buf, size_t len, - void *desc, fi_addr_t src_addr, uint64_t addr, - uint64_t key, void *context) -{ - struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret; - - fit_record_entry(h, "fi_read", 'T', context); - ret = CALL_OR_ENOSYS(h->o_rma->read, - (ep, buf, len, desc, src_addr, addr, key, - context)); - fit_record(h, "fi_read", 'T', src_addr, len, 1, ret, context); - return ret; -} - -static ssize_t fit_rma_readv(struct fid_ep *ep, const struct iovec *iov, - void **desc, size_t count, fi_addr_t src_addr, - uint64_t addr, uint64_t key, void *context) -{ - struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret; - - fit_record_entry(h, "fi_readv", 'T', context); - ret = CALL_OR_ENOSYS(h->o_rma->readv, - (ep, iov, desc, count, src_addr, addr, key, - context)); - fit_record(h, "fi_readv", 'T', src_addr, fit_iov_len(iov, count), - count, ret, context); - return ret; -} - -static ssize_t fit_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg, - uint64_t flags) -{ - struct fit_ep *h = EP_FROM_RMA(ep); - ssize_t ret; - - fit_record_entry(h, "fi_readmsg", 'T', msg ? msg->context : NULL); - ret = CALL_OR_ENOSYS(h->o_rma->readmsg, (ep, msg, flags)); - fit_record(h, "fi_readmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, - msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, - msg ? msg->iov_count : 0, ret, msg ? msg->context : NULL); - return ret; +static ssize_t fit_rma_write(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_write", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->write, (ep, buf, len, desc, dest_addr, addr, key, context)); + fit_record(h, "fi_write", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_rma_writev(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_writev", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->writev, (ep, iov, desc, count, dest_addr, addr, key, context)); + fit_record(h, "fi_writev", 'T', dest_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_rma_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_writemsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_rma->writemsg, (ep, msg, flags)); + fit_record(h, "fi_writemsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; +} + +static ssize_t fit_rma_inject(struct fid_ep *ep, + const void *buf, + size_t len, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_inject_write", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_rma->inject, (ep, buf, len, dest_addr, addr, key)); + fit_record(h, "fi_inject_write", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_rma_writedata(struct fid_ep *ep, + const void *buf, + size_t len, + void *desc, + uint64_t data, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_writedata", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->writedata, + (ep, buf, len, desc, data, dest_addr, addr, key, context)); + fit_record(h, "fi_writedata", 'T', dest_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_rma_injectdata(struct fid_ep *ep, + const void *buf, + size_t len, + uint64_t data, + fi_addr_t dest_addr, + uint64_t addr, + uint64_t key) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_inject_writedata", 'T', NULL); + ret = CALL_OR_ENOSYS(h->o_rma->injectdata, (ep, buf, len, data, dest_addr, addr, key)); + fit_record(h, "fi_inject_writedata", 'T', dest_addr, len, 1, ret, NULL); + return ret; +} + +static ssize_t fit_rma_read(struct fid_ep *ep, + void *buf, + size_t len, + void *desc, + fi_addr_t src_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_read", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->read, (ep, buf, len, desc, src_addr, addr, key, context)); + fit_record(h, "fi_read", 'T', src_addr, len, 1, ret, context); + return ret; +} + +static ssize_t fit_rma_readv(struct fid_ep *ep, + const struct iovec *iov, + void **desc, + size_t count, + fi_addr_t src_addr, + uint64_t addr, + uint64_t key, + void *context) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_readv", 'T', context); + ret = CALL_OR_ENOSYS(h->o_rma->readv, (ep, iov, desc, count, src_addr, addr, key, context)); + fit_record(h, "fi_readv", 'T', src_addr, fit_iov_len(iov, count), count, ret, context); + return ret; +} + +static ssize_t fit_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg, uint64_t flags) { + struct fit_ep *h = EP_FROM_RMA(ep); + ssize_t ret; + + fit_record_entry(h, "fi_readmsg", 'T', msg ? msg->context : NULL); + ret = CALL_OR_ENOSYS(h->o_rma->readmsg, (ep, msg, flags)); + fit_record(h, "fi_readmsg", 'T', msg ? msg->addr : FI_ADDR_UNSPEC, + msg ? fit_iov_len(msg->msg_iov, msg->iov_count) : 0, msg ? msg->iov_count : 0, ret, + msg ? msg->context : NULL); + return ret; } /* ================================================================== */ @@ -467,253 +468,237 @@ static ssize_t fit_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg, * Re-arming is cheap (three pointer compares when nothing changed) and * happens only on the control path, never per operation. */ -static void fit_ep_arm_ops(struct fit_ep *h, const char *when) -{ - struct fid_ep *ep = h->ep; - - if (fit_cfg.trace_msg && ep->msg && ep->msg != &h->msg) { - if (h->o_msg) - fit_log("endpoint %u: provider replaced msg ops at %s;" - " re-arming", h->ep_id, when); - h->o_msg = ep->msg; - h->msg = *ep->msg; - h->msg.send = fit_msg_send; - h->msg.sendv = fit_msg_sendv; - h->msg.sendmsg = fit_msg_sendmsg; - h->msg.inject = fit_msg_inject; - h->msg.senddata = fit_msg_senddata; - h->msg.injectdata = fit_msg_injectdata; - h->msg.recv = fit_msg_recv; - h->msg.recvv = fit_msg_recvv; - h->msg.recvmsg = fit_msg_recvmsg; - ep->msg = &h->msg; - } - - if (fit_cfg.trace_tagged && ep->tagged && ep->tagged != &h->tagged) { - if (h->o_tagged) - fit_log("endpoint %u: provider replaced tagged ops at" - " %s; re-arming", h->ep_id, when); - h->o_tagged = ep->tagged; - h->tagged = *ep->tagged; - h->tagged.send = fit_tag_send; - h->tagged.sendv = fit_tag_sendv; - h->tagged.sendmsg = fit_tag_sendmsg; - h->tagged.inject = fit_tag_inject; - h->tagged.senddata = fit_tag_senddata; - h->tagged.injectdata = fit_tag_injectdata; - h->tagged.recv = fit_tag_recv; - h->tagged.recvv = fit_tag_recvv; - h->tagged.recvmsg = fit_tag_recvmsg; - ep->tagged = &h->tagged; - } - - if (fit_cfg.trace_rma && ep->rma && ep->rma != &h->rma) { - if (h->o_rma) - fit_log("endpoint %u: provider replaced rma ops at %s;" - " re-arming", h->ep_id, when); - h->o_rma = ep->rma; - h->rma = *ep->rma; - h->rma.write = fit_rma_write; - h->rma.writev = fit_rma_writev; - h->rma.writemsg = fit_rma_writemsg; - h->rma.inject = fit_rma_inject; - h->rma.writedata = fit_rma_writedata; - h->rma.injectdata = fit_rma_injectdata; - h->rma.read = fit_rma_read; - h->rma.readv = fit_rma_readv; - h->rma.readmsg = fit_rma_readmsg; - ep->rma = &h->rma; - } -} - -static int fit_ep_bind_inner(struct fid *fid, struct fid *bfid, uint64_t flags) -{ - struct fit_ep *h = EP_FROM_FID(fid); - int ret = h->o_fid->bind ? h->o_fid->bind(fid, bfid, flags) : -FI_ENOSYS; - - /* Binding a CQ or a counter is exactly the sort of event that tells - * a provider which data path this endpoint will use. */ - if (!ret) - fit_ep_arm_ops(h, "bind"); - - if (ret || !bfid) - return ret; - - /* Remember the AV so remote fi_addr_t values can be reverse-mapped. */ - if (bfid->fclass == FI_CLASS_AV) - h->av = (struct fid_av *)bfid; - - /* And note which endpoint owns the CQ, so its completions can be - * attributed to a local NIC address. */ - if (bfid->fclass == FI_CLASS_CQ) - fit_cq_attach_ep(bfid, h); - - return ret; -} - -static int fit_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); - int fit_ret; - - fit_record_ctrl_entry("fi_ep_bind", fit_obj); - fit_ret = fit_ep_bind_inner(fid, bfid, flags); - fit_record_ctrl_exit("fi_ep_bind", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_ep_control_inner(struct fid *fid, int command, void *arg) -{ - struct fit_ep *h = EP_FROM_FID(fid); - int ret = h->o_fid->control ? h->o_fid->control(fid, command, arg) : - -FI_ENOSYS; - - /* The local address only becomes available once enabled -- and so, - * for some providers, does the real data path. */ - if (!ret && command == FI_ENABLE) { - fit_ep_arm_ops(h, "enable"); - fit_ep_resolve_src(h); - } - - return ret; -} - -static int fit_ep_control(struct fid *fid, int command, void *arg) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); - int fit_ret; - - fit_record_ctrl_entry("fi_ep_control", fit_obj); - fit_ret = fit_ep_control_inner(fid, command, arg); - fit_record_ctrl_exit("fi_ep_control", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_ep_close_inner(struct fid *fid) -{ - struct fit_ep *h = EP_FROM_FID(fid); - int ret; - - ret = h->o_fid->close ? h->o_fid->close(fid) : -FI_ENOSYS; - if (ret) - return ret; - - /* The provider object is gone; our side state can go too. */ - fit_ep_peers_free(h); - pthread_mutex_destroy(&h->lock); - free(h); - return 0; -} - -static int fit_ep_close(struct fid *fid) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); - int fit_ret; - - fit_record_ctrl_entry("fi_close_ep", fit_obj); - fit_ret = fit_ep_close_inner(fid); - fit_record_ctrl_exit("fi_close_ep", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_ep_tx_ctx_inner(struct fid_ep *sep, int index, - struct fi_tx_attr *attr, struct fid_ep **tx_ep, - void *context) -{ - struct fit_ep *h = EP_FROM_EPOPS(sep); - int ret = h->o_ep->tx_ctx ? - h->o_ep->tx_ctx(sep, index, attr, tx_ep, context) : -FI_ENOSYS; - - if (!ret && tx_ep && *tx_ep) - fit_hook_ep(*tx_ep, h->topo); - return ret; -} - -static int fit_ep_tx_ctx(struct fid_ep *sep, int index, - struct fi_tx_attr *attr, struct fid_ep **tx_ep, - void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(EP_FROM_EPOPS(sep)->ep_id); - int fit_ret; - - fit_record_ctrl_entry("fi_tx_context", fit_obj); - fit_ret = fit_ep_tx_ctx_inner(sep, index, attr, tx_ep, context); - fit_record_ctrl_exit("fi_tx_context", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_ep_rx_ctx_inner(struct fid_ep *sep, int index, - struct fi_rx_attr *attr, struct fid_ep **rx_ep, - void *context) -{ - struct fit_ep *h = EP_FROM_EPOPS(sep); - int ret = h->o_ep->rx_ctx ? - h->o_ep->rx_ctx(sep, index, attr, rx_ep, context) : -FI_ENOSYS; - - if (!ret && rx_ep && *rx_ep) - fit_hook_ep(*rx_ep, h->topo); - return ret; -} - -static int fit_ep_rx_ctx(struct fid_ep *sep, int index, - struct fi_rx_attr *attr, struct fid_ep **rx_ep, - void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(EP_FROM_EPOPS(sep)->ep_id); - int fit_ret; - - fit_record_ctrl_entry("fi_rx_context", fit_obj); - fit_ret = fit_ep_rx_ctx_inner(sep, index, attr, rx_ep, context); - fit_record_ctrl_exit("fi_rx_context", fit_obj, fit_ret); - return fit_ret; -} - -void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo) -{ - struct fit_ep *h; - - if (!ep) - return; - - h = calloc(1, sizeof(*h)); - if (!h) { - fit_log("out of memory hooking endpoint"); - return; - } - - pthread_mutex_init(&h->lock, NULL); - h->ep = ep; - h->ep_id = atomic_fetch_add(&fit_ep_counter, 1); - /* Borrowed from the domain, which outlives every endpoint on it. */ - h->topo = topo; - snprintf(h->src, sizeof(h->src), "unknown"); - fit_sanitize(h->src, &h->src_len); - - fit_ep_arm_ops(h, "create"); - - if (ep->fid.ops) { - h->o_fid = ep->fid.ops; - h->fid_ops = *ep->fid.ops; - h->fid_ops.bind = fit_ep_bind; - h->fid_ops.control = fit_ep_control; - h->fid_ops.close = fit_ep_close; - ep->fid.ops = &h->fid_ops; - } - - if (ep->ops) { - h->o_ep = ep->ops; - h->ep_ops = *ep->ops; - h->ep_ops.tx_ctx = fit_ep_tx_ctx; - h->ep_ops.rx_ctx = fit_ep_rx_ctx; - ep->ops = &h->ep_ops; - } - - fit_log("hooked endpoint %u (%p)", h->ep_id, (void *)ep); +static void fit_ep_arm_ops(struct fit_ep *h, const char *when) { + struct fid_ep *ep = h->ep; + + if (fit_cfg.trace_msg && ep->msg && ep->msg != &h->msg) { + if (h->o_msg) + fit_log("endpoint %u: provider replaced msg ops at %s;" + " re-arming", + h->ep_id, when); + h->o_msg = ep->msg; + h->msg = *ep->msg; + h->msg.send = fit_msg_send; + h->msg.sendv = fit_msg_sendv; + h->msg.sendmsg = fit_msg_sendmsg; + h->msg.inject = fit_msg_inject; + h->msg.senddata = fit_msg_senddata; + h->msg.injectdata = fit_msg_injectdata; + h->msg.recv = fit_msg_recv; + h->msg.recvv = fit_msg_recvv; + h->msg.recvmsg = fit_msg_recvmsg; + ep->msg = &h->msg; + } + + if (fit_cfg.trace_tagged && ep->tagged && ep->tagged != &h->tagged) { + if (h->o_tagged) + fit_log("endpoint %u: provider replaced tagged ops at" + " %s; re-arming", + h->ep_id, when); + h->o_tagged = ep->tagged; + h->tagged = *ep->tagged; + h->tagged.send = fit_tag_send; + h->tagged.sendv = fit_tag_sendv; + h->tagged.sendmsg = fit_tag_sendmsg; + h->tagged.inject = fit_tag_inject; + h->tagged.senddata = fit_tag_senddata; + h->tagged.injectdata = fit_tag_injectdata; + h->tagged.recv = fit_tag_recv; + h->tagged.recvv = fit_tag_recvv; + h->tagged.recvmsg = fit_tag_recvmsg; + ep->tagged = &h->tagged; + } + + if (fit_cfg.trace_rma && ep->rma && ep->rma != &h->rma) { + if (h->o_rma) + fit_log("endpoint %u: provider replaced rma ops at %s;" + " re-arming", + h->ep_id, when); + h->o_rma = ep->rma; + h->rma = *ep->rma; + h->rma.write = fit_rma_write; + h->rma.writev = fit_rma_writev; + h->rma.writemsg = fit_rma_writemsg; + h->rma.inject = fit_rma_inject; + h->rma.writedata = fit_rma_writedata; + h->rma.injectdata = fit_rma_injectdata; + h->rma.read = fit_rma_read; + h->rma.readv = fit_rma_readv; + h->rma.readmsg = fit_rma_readmsg; + ep->rma = &h->rma; + } +} + +static int fit_ep_bind_inner(struct fid *fid, struct fid *bfid, uint64_t flags) { + struct fit_ep *h = EP_FROM_FID(fid); + int ret = h->o_fid->bind ? h->o_fid->bind(fid, bfid, flags) : -FI_ENOSYS; + + /* Binding a CQ or a counter is exactly the sort of event that tells + * a provider which data path this endpoint will use. */ + if (!ret) + fit_ep_arm_ops(h, "bind"); + + if (ret || !bfid) + return ret; + + /* Remember the AV so remote fi_addr_t values can be reverse-mapped. */ + if (bfid->fclass == FI_CLASS_AV) + h->av = (struct fid_av *)bfid; + + /* And note which endpoint owns the CQ, so its completions can be + * attributed to a local NIC address. */ + if (bfid->fclass == FI_CLASS_CQ) + fit_cq_attach_ep(bfid, h); + + return ret; +} + +static int fit_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_ep_bind", fit_obj); + fit_ret = fit_ep_bind_inner(fid, bfid, flags); + fit_record_ctrl_exit("fi_ep_bind", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_control_inner(struct fid *fid, int command, void *arg) { + struct fit_ep *h = EP_FROM_FID(fid); + int ret = h->o_fid->control ? h->o_fid->control(fid, command, arg) : -FI_ENOSYS; + + /* The local address only becomes available once enabled -- and so, + * for some providers, does the real data path. */ + if (!ret && command == FI_ENABLE) { + fit_ep_arm_ops(h, "enable"); + fit_ep_resolve_src(h); + } + + return ret; +} + +static int fit_ep_control(struct fid *fid, int command, void *arg) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_ep_control", fit_obj); + fit_ret = fit_ep_control_inner(fid, command, arg); + fit_record_ctrl_exit("fi_ep_control", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_close_inner(struct fid *fid) { + struct fit_ep *h = EP_FROM_FID(fid); + int ret; + + ret = h->o_fid->close ? h->o_fid->close(fid) : -FI_ENOSYS; + if (ret) + return ret; + + /* The provider object is gone; our side state can go too. */ + fit_ep_peers_free(h); + pthread_mutex_destroy(&h->lock); + free(h); + return 0; +} + +static int fit_ep_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_FID(fid)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_close_ep", fit_obj); + fit_ret = fit_ep_close_inner(fid); + fit_record_ctrl_exit("fi_close_ep", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_tx_ctx_inner( + struct fid_ep *sep, int index, struct fi_tx_attr *attr, struct fid_ep **tx_ep, void *context) { + struct fit_ep *h = EP_FROM_EPOPS(sep); + int ret = h->o_ep->tx_ctx ? h->o_ep->tx_ctx(sep, index, attr, tx_ep, context) : -FI_ENOSYS; + + if (!ret && tx_ep && *tx_ep) + fit_hook_ep(*tx_ep, h->topo); + return ret; +} + +static int fit_ep_tx_ctx( + struct fid_ep *sep, int index, struct fi_tx_attr *attr, struct fid_ep **tx_ep, void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_EPOPS(sep)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_tx_context", fit_obj); + fit_ret = fit_ep_tx_ctx_inner(sep, index, attr, tx_ep, context); + fit_record_ctrl_exit("fi_tx_context", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_ep_rx_ctx_inner( + struct fid_ep *sep, int index, struct fi_rx_attr *attr, struct fid_ep **rx_ep, void *context) { + struct fit_ep *h = EP_FROM_EPOPS(sep); + int ret = h->o_ep->rx_ctx ? h->o_ep->rx_ctx(sep, index, attr, rx_ep, context) : -FI_ENOSYS; + + if (!ret && rx_ep && *rx_ep) + fit_hook_ep(*rx_ep, h->topo); + return ret; +} + +static int fit_ep_rx_ctx( + struct fid_ep *sep, int index, struct fi_rx_attr *attr, struct fid_ep **rx_ep, void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(EP_FROM_EPOPS(sep)->ep_id); + int fit_ret; + + fit_record_ctrl_entry("fi_rx_context", fit_obj); + fit_ret = fit_ep_rx_ctx_inner(sep, index, attr, rx_ep, context); + fit_record_ctrl_exit("fi_rx_context", fit_obj, fit_ret); + return fit_ret; +} + +void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo) { + struct fit_ep *h; + + if (!ep) + return; + + h = calloc(1, sizeof(*h)); + if (!h) { + fit_log("out of memory hooking endpoint"); + return; + } + + pthread_mutex_init(&h->lock, NULL); + h->ep = ep; + h->ep_id = atomic_fetch_add(&fit_ep_counter, 1); + /* Borrowed from the domain, which outlives every endpoint on it. */ + h->topo = topo; + snprintf(h->src, sizeof(h->src), "unknown"); + fit_sanitize(h->src, &h->src_len); + + fit_ep_arm_ops(h, "create"); + + if (ep->fid.ops) { + h->o_fid = ep->fid.ops; + h->fid_ops = *ep->fid.ops; + h->fid_ops.bind = fit_ep_bind; + h->fid_ops.control = fit_ep_control; + h->fid_ops.close = fit_ep_close; + ep->fid.ops = &h->fid_ops; + } + + if (ep->ops) { + h->o_ep = ep->ops; + h->ep_ops = *ep->ops; + h->ep_ops.tx_ctx = fit_ep_tx_ctx; + h->ep_ops.rx_ctx = fit_ep_rx_ctx; + ep->ops = &h->ep_ops; + } + + fit_log("hooked endpoint %u (%p)", h->ep_id, (void *)ep); } /* ================================================================== */ @@ -726,79 +711,72 @@ void fit_hook_ep(struct fid_ep *ep, const struct fit_topo *topo) #define AV_FROM_OPS(av) fit_container_of((av)->ops, struct fit_av, ops) -static int fit_av_remove_inner(struct fid_av *av, fi_addr_t *fi_addr, size_t count, - uint64_t flags) -{ - struct fit_av *a = AV_FROM_OPS(av); - int ret = a->orig->remove ? - a->orig->remove(av, fi_addr, count, flags) : -FI_ENOSYS; +static int +fit_av_remove_inner(struct fid_av *av, fi_addr_t *fi_addr, size_t count, uint64_t flags) { + struct fit_av *a = AV_FROM_OPS(av); + int ret = a->orig->remove ? a->orig->remove(av, fi_addr, count, flags) : -FI_ENOSYS; - if (!ret) - atomic_fetch_add_explicit(&fit_av_gen, 1, memory_order_release); - return ret; + if (!ret) + atomic_fetch_add_explicit(&fit_av_gen, 1, memory_order_release); + return ret; } -static int fit_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count, - uint64_t flags) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; +static int fit_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count, uint64_t flags) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; - fit_record_ctrl_entry("fi_av_remove", fit_obj); - fit_ret = fit_av_remove_inner(av, fi_addr, count, flags); - fit_record_ctrl_exit("fi_av_remove", fit_obj, fit_ret); - return fit_ret; + fit_record_ctrl_entry("fi_av_remove", fit_obj); + fit_ret = fit_av_remove_inner(av, fi_addr, count, flags); + fit_record_ctrl_exit("fi_av_remove", fit_obj, fit_ret); + return fit_ret; } -static int fit_av_close_inner(struct fid *fid) -{ - struct fit_av *a = fit_container_of(fid->ops, struct fit_av, fid_ops); - int ret = a->o_fid->close ? a->o_fid->close(fid) : -FI_ENOSYS; +static int fit_av_close_inner(struct fid *fid) { + struct fit_av *a = fit_container_of(fid->ops, struct fit_av, fid_ops); + int ret = a->o_fid->close ? a->o_fid->close(fid) : -FI_ENOSYS; - if (!ret) { - atomic_fetch_add_explicit(&fit_av_gen, 1, memory_order_release); - free(a); - } - return ret; + if (!ret) { + atomic_fetch_add_explicit(&fit_av_gen, 1, memory_order_release); + free(a); + } + return ret; } -static int fit_av_close(struct fid *fid) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; +static int fit_av_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; - fit_record_ctrl_entry("fi_close_av", fit_obj); - fit_ret = fit_av_close_inner(fid); - fit_record_ctrl_exit("fi_close_av", fit_obj, fit_ret); - return fit_ret; + fit_record_ctrl_entry("fi_close_av", fit_obj); + fit_ret = fit_av_close_inner(fid); + fit_record_ctrl_exit("fi_close_av", fit_obj, fit_ret); + return fit_ret; } -void fit_hook_av(struct fid_av *av) -{ - struct fit_av *a; +void fit_hook_av(struct fid_av *av) { + struct fit_av *a; - if (!av || !av->ops || av->ops->remove == fit_av_remove) - return; + if (!av || !av->ops || av->ops->remove == fit_av_remove) + return; - a = calloc(1, sizeof(*a)); - if (!a) - return; + a = calloc(1, sizeof(*a)); + if (!a) + return; - if (av->fid.ops) { - a->o_fid = av->fid.ops; - a->fid_ops = *av->fid.ops; - a->fid_ops.close = fit_av_close; - av->fid.ops = &a->fid_ops; - } + if (av->fid.ops) { + a->o_fid = av->fid.ops; + a->fid_ops = *av->fid.ops; + a->fid_ops.close = fit_av_close; + av->fid.ops = &a->fid_ops; + } - a->orig = av->ops; - a->ops = *av->ops; - a->ops.remove = fit_av_remove; - av->ops = &a->ops; + a->orig = av->ops; + a->ops = *av->ops; + a->ops.remove = fit_av_remove; + av->ops = &a->ops; - fit_log("hooked address vector %p", (void *)av); + fit_log("hooked address vector %p", (void *)av); } /* ================================================================== */ @@ -823,20 +801,24 @@ static atomic_uint fit_cq_counter; * so one accessor serves all of them. FI_CQ_FORMAT_CONTEXT stops after * op_context, which is why entry_sz decides how much may be read. */ struct fit_cqe_head { - void *op_context; - uint64_t flags; - size_t len; + void *op_context; + uint64_t flags; + size_t len; }; -static size_t fit_cq_entry_size(enum fi_cq_format format) -{ - switch (format) { - case FI_CQ_FORMAT_CONTEXT: return sizeof(struct fi_cq_entry); - case FI_CQ_FORMAT_MSG: return sizeof(struct fi_cq_msg_entry); - case FI_CQ_FORMAT_DATA: return sizeof(struct fi_cq_data_entry); - case FI_CQ_FORMAT_TAGGED: return sizeof(struct fi_cq_tagged_entry); - default: return 0; - } +static size_t fit_cq_entry_size(enum fi_cq_format format) { + switch (format) { + case FI_CQ_FORMAT_CONTEXT: + return sizeof(struct fi_cq_entry); + case FI_CQ_FORMAT_MSG: + return sizeof(struct fi_cq_msg_entry); + case FI_CQ_FORMAT_DATA: + return sizeof(struct fi_cq_data_entry); + case FI_CQ_FORMAT_TAGGED: + return sizeof(struct fi_cq_tagged_entry); + default: + return 0; + } } /* @@ -845,27 +827,25 @@ static size_t fit_cq_entry_size(enum fi_cq_format format) * tells us nothing but the context, and the posting record it joins to * has to supply the operation type. */ -static const char *fit_cq_opname(uint64_t flags, int have_flags) -{ - if (!have_flags) - return "cq_completion"; - - if (flags & FI_ATOMIC) - return (flags & FI_REMOTE_WRITE) ? "cq_remote_atomic" : - "cq_atomic"; - if (flags & FI_REMOTE_WRITE) - return "cq_remote_write"; - if (flags & FI_REMOTE_READ) - return "cq_remote_read"; - if (flags & FI_WRITE) - return "cq_write"; - if (flags & FI_READ) - return "cq_read"; - if (flags & FI_SEND) - return (flags & FI_TAGGED) ? "cq_tsend" : "cq_send"; - if (flags & FI_RECV) - return (flags & FI_TAGGED) ? "cq_trecv" : "cq_recv"; - return "cq_completion"; +static const char *fit_cq_opname(uint64_t flags, int have_flags) { + if (!have_flags) + return "cq_completion"; + + if (flags & FI_ATOMIC) + return (flags & FI_REMOTE_WRITE) ? "cq_remote_atomic" : "cq_atomic"; + if (flags & FI_REMOTE_WRITE) + return "cq_remote_write"; + if (flags & FI_REMOTE_READ) + return "cq_remote_read"; + if (flags & FI_WRITE) + return "cq_write"; + if (flags & FI_READ) + return "cq_read"; + if (flags & FI_SEND) + return (flags & FI_TAGGED) ? "cq_tsend" : "cq_send"; + if (flags & FI_RECV) + return (flags & FI_TAGGED) ? "cq_trecv" : "cq_recv"; + return "cq_completion"; } /* Walk a batch the provider has just filled in and emit one record per @@ -884,89 +864,83 @@ static const char *fit_cq_opname(uint64_t flags, int have_flags) */ static __thread unsigned fit_cq_depth; -static void fit_cq_report(struct fit_cq *c, const void *buf, ssize_t n, - const fi_addr_t *src) -{ - int have_flags = c->entry_sz >= sizeof(struct fi_cq_msg_entry); - uint64_t now = fit_now_us(); - ssize_t i; +static void fit_cq_report(struct fit_cq *c, const void *buf, ssize_t n, const fi_addr_t *src) { + int have_flags = c->entry_sz >= sizeof(struct fi_cq_msg_entry); + uint64_t now = fit_now_us(); + ssize_t i; - if (!c->entry_sz) - return; + if (!c->entry_sz) + return; - for (i = 0; i < n; i++) { - const struct fit_cqe_head *e = - (const struct fit_cqe_head *)((const char *)buf + - (size_t)i * c->entry_sz); - uint64_t flags = have_flags ? e->flags : 0; - size_t len = have_flags ? e->len : 0; + for (i = 0; i < n; i++) { + const struct fit_cqe_head *e = + (const struct fit_cqe_head *)((const char *)buf + (size_t)i * c->entry_sz); + uint64_t flags = have_flags ? e->flags : 0; + size_t len = have_flags ? e->len : 0; - fit_record_cq(c, fit_cq_opname(flags, have_flags), - src ? src[i] : FI_ADDR_UNSPEC, len, 0, - e->op_context, now); - } + fit_record_cq(c, fit_cq_opname(flags, have_flags), src ? src[i] : FI_ADDR_UNSPEC, len, 0, + e->op_context, now); + } } -static ssize_t fit_cq_read(struct fid_cq *cq, void *buf, size_t count) -{ - struct fit_cq *c = CQ_FROM_OPS(cq); - unsigned outer = fit_cq_depth++; - ssize_t ret; +static ssize_t fit_cq_read(struct fid_cq *cq, void *buf, size_t count) { + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; - ret = CALL_OR_ENOSYS(c->orig->read, (cq, buf, count)); - fit_cq_depth = outer; + ret = CALL_OR_ENOSYS(c->orig->read, (cq, buf, count)); + fit_cq_depth = outer; - /* A polling loop calls this constantly and almost always gets - * -FI_EAGAIN. Nothing happened, so nothing is recorded. */ - if (ret > 0 && !outer) - fit_cq_report(c, buf, ret, NULL); - return ret; + /* A polling loop calls this constantly and almost always gets + * -FI_EAGAIN. Nothing happened, so nothing is recorded. */ + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, NULL); + return ret; } -static ssize_t fit_cq_readfrom(struct fid_cq *cq, void *buf, size_t count, - fi_addr_t *src_addr) -{ - struct fit_cq *c = CQ_FROM_OPS(cq); - unsigned outer = fit_cq_depth++; - ssize_t ret; +static ssize_t fit_cq_readfrom(struct fid_cq *cq, void *buf, size_t count, fi_addr_t *src_addr) { + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; - ret = CALL_OR_ENOSYS(c->orig->readfrom, (cq, buf, count, src_addr)); - fit_cq_depth = outer; + ret = CALL_OR_ENOSYS(c->orig->readfrom, (cq, buf, count, src_addr)); + fit_cq_depth = outer; - if (ret > 0 && !outer) - fit_cq_report(c, buf, ret, src_addr); - return ret; + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, src_addr); + return ret; } -static ssize_t fit_cq_sread(struct fid_cq *cq, void *buf, size_t count, - const void *cond, int timeout) -{ - struct fit_cq *c = CQ_FROM_OPS(cq); - unsigned outer = fit_cq_depth++; - ssize_t ret; +static ssize_t +fit_cq_sread(struct fid_cq *cq, void *buf, size_t count, const void *cond, int timeout) { + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; - ret = CALL_OR_ENOSYS(c->orig->sread, (cq, buf, count, cond, timeout)); - fit_cq_depth = outer; + ret = CALL_OR_ENOSYS(c->orig->sread, (cq, buf, count, cond, timeout)); + fit_cq_depth = outer; - if (ret > 0 && !outer) - fit_cq_report(c, buf, ret, NULL); - return ret; + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, NULL); + return ret; } -static ssize_t fit_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count, - fi_addr_t *src_addr, const void *cond, - int timeout) -{ - struct fit_cq *c = CQ_FROM_OPS(cq); - unsigned outer = fit_cq_depth++; - ssize_t ret; +static ssize_t fit_cq_sreadfrom(struct fid_cq *cq, + void *buf, + size_t count, + fi_addr_t *src_addr, + const void *cond, + int timeout) { + struct fit_cq *c = CQ_FROM_OPS(cq); + unsigned outer = fit_cq_depth++; + ssize_t ret; - ret = CALL_OR_ENOSYS(c->orig->sreadfrom, (cq, buf, count, src_addr, cond, timeout)); - fit_cq_depth = outer; + ret = CALL_OR_ENOSYS(c->orig->sreadfrom, (cq, buf, count, src_addr, cond, timeout)); + fit_cq_depth = outer; - if (ret > 0 && !outer) - fit_cq_report(c, buf, ret, src_addr); - return ret; + if (ret > 0 && !outer) + fit_cq_report(c, buf, ret, src_addr); + return ret; } /* @@ -974,117 +948,110 @@ static ssize_t fit_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count, * without this a transfer that errored would show a posting record and * no completion, indistinguishable from one still in flight. */ -static ssize_t fit_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf, - uint64_t flags) -{ - struct fit_cq *c = CQ_FROM_OPS(cq); - ssize_t ret = CALL_OR_ENOSYS(c->orig->readerr, (cq, buf, flags)); - - if (ret > 0 && buf) - fit_record_cq(c, fit_cq_opname(buf->flags, 1), +static ssize_t fit_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf, uint64_t flags) { + struct fit_cq *c = CQ_FROM_OPS(cq); + ssize_t ret = CALL_OR_ENOSYS(c->orig->readerr, (cq, buf, flags)); + + if (ret > 0 && buf) + fit_record_cq(c, fit_cq_opname(buf->flags, 1), #ifdef HAVE_FI_CQ_ERR_SRC_ADDR - buf->src_addr, + buf->src_addr, #else - /* Older libfabric has no src_addr here. */ - FI_ADDR_UNSPEC, + /* Older libfabric has no src_addr here. */ + FI_ADDR_UNSPEC, #endif - buf->len, -buf->err, buf->op_context, - fit_now_us()); - return ret; -} - -static int fit_cq_close_inner(struct fid *fid) -{ - struct fit_cq *c = fit_container_of(fid->ops, struct fit_cq, fid_ops); - int ret; - - ret = c->o_fid->close ? c->o_fid->close(fid) : -FI_ENOSYS; - if (!ret) - free(c); - return ret; -} - -static int fit_cq_close(struct fid *fid) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(fit_container_of(fid->ops, struct fit_cq, fid_ops)->cq_id); - int fit_ret; - - fit_record_ctrl_entry("fi_close_cq", fit_obj); - fit_ret = fit_cq_close_inner(fid); - fit_record_ctrl_exit("fi_close_cq", fit_obj, fit_ret); - return fit_ret; -} - -void fit_hook_cq(struct fid_cq *cq, enum fi_cq_format format) -{ - struct fit_cq *c; - - if (!cq || !cq->ops || cq->ops->read == fit_cq_read) - return; - - c = calloc(1, sizeof(*c)); - if (!c) - return; - - c->cq = cq; - c->cq_id = atomic_fetch_add(&fit_cq_counter, 1); - c->format = format; - c->entry_sz = fit_cq_entry_size(format); - - /* - * FI_CQ_FORMAT_UNSPEC leaves the choice to the provider and there - * is no way to ask what it picked. Every layout starts with - * op_context, so fall back to that much: completions are still - * timestamped and still join to their posting record, they just - * carry no length of their own. - */ - if (!c->entry_sz) { - c->entry_sz = sizeof(struct fi_cq_entry); - fit_log("cq %u opened with FI_CQ_FORMAT_UNSPEC; " - "completion lengths unavailable", c->cq_id); - } - - if (cq->fid.ops) { - c->o_fid = cq->fid.ops; - c->fid_ops = *cq->fid.ops; - c->fid_ops.close = fit_cq_close; - cq->fid.ops = &c->fid_ops; - } - - c->orig = cq->ops; - c->ops = *cq->ops; - c->ops.read = fit_cq_read; - c->ops.readfrom = fit_cq_readfrom; - c->ops.readerr = fit_cq_readerr; - c->ops.sread = fit_cq_sread; - c->ops.sreadfrom = fit_cq_sreadfrom; - cq->ops = &c->ops; - - fit_log("hooked cq %u (%p) format %d", c->cq_id, (void *)cq, - (int)format); + buf->len, -buf->err, buf->op_context, fit_now_us()); + return ret; +} + +static int fit_cq_close_inner(struct fid *fid) { + struct fit_cq *c = fit_container_of(fid->ops, struct fit_cq, fid_ops); + int ret; + + ret = c->o_fid->close ? c->o_fid->close(fid) : -FI_ENOSYS; + if (!ret) + free(c); + return ret; +} + +static int fit_cq_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(fit_container_of(fid->ops, struct fit_cq, fid_ops)->cq_id); + int fit_ret; + + fit_record_ctrl_entry("fi_close_cq", fit_obj); + fit_ret = fit_cq_close_inner(fid); + fit_record_ctrl_exit("fi_close_cq", fit_obj, fit_ret); + return fit_ret; +} + +void fit_hook_cq(struct fid_cq *cq, enum fi_cq_format format) { + struct fit_cq *c; + + if (!cq || !cq->ops || cq->ops->read == fit_cq_read) + return; + + c = calloc(1, sizeof(*c)); + if (!c) + return; + + c->cq = cq; + c->cq_id = atomic_fetch_add(&fit_cq_counter, 1); + c->format = format; + c->entry_sz = fit_cq_entry_size(format); + + /* + * FI_CQ_FORMAT_UNSPEC leaves the choice to the provider and there + * is no way to ask what it picked. Every layout starts with + * op_context, so fall back to that much: completions are still + * timestamped and still join to their posting record, they just + * carry no length of their own. + */ + if (!c->entry_sz) { + c->entry_sz = sizeof(struct fi_cq_entry); + fit_log("cq %u opened with FI_CQ_FORMAT_UNSPEC; " + "completion lengths unavailable", + c->cq_id); + } + + if (cq->fid.ops) { + c->o_fid = cq->fid.ops; + c->fid_ops = *cq->fid.ops; + c->fid_ops.close = fit_cq_close; + cq->fid.ops = &c->fid_ops; + } + + c->orig = cq->ops; + c->ops = *cq->ops; + c->ops.read = fit_cq_read; + c->ops.readfrom = fit_cq_readfrom; + c->ops.readerr = fit_cq_readerr; + c->ops.sread = fit_cq_sread; + c->ops.sreadfrom = fit_cq_sreadfrom; + cq->ops = &c->ops; + + fit_log("hooked cq %u (%p) format %d", c->cq_id, (void *)cq, (int)format); } /* Called when an endpoint binds a CQ, so a completion can be attributed * to a local NIC address. Only safe if we hooked that CQ ourselves -- * the read slot is the tell. */ -static void fit_cq_attach_ep(struct fid *bfid, struct fit_ep *h) -{ - struct fid_cq *cq = (struct fid_cq *)bfid; - struct fit_cq *c; +static void fit_cq_attach_ep(struct fid *bfid, struct fit_ep *h) { + struct fid_cq *cq = (struct fid_cq *)bfid; + struct fit_cq *c; - if (!cq->ops || cq->ops->read != fit_cq_read) - return; + if (!cq->ops || cq->ops->read != fit_cq_read) + return; - c = CQ_FROM_OPS(cq); - if (c->ep && c->ep != h) { - /* Shared across endpoints; guessing one would be wrong. */ - c->ep_ambiguous = 1; - c->ep = NULL; - return; - } - if (!c->ep_ambiguous) - c->ep = h; + c = CQ_FROM_OPS(cq); + if (c->ep && c->ep != h) { + /* Shared across endpoints; guessing one would be wrong. */ + c->ep_ambiguous = 1; + c->ep = NULL; + return; + } + if (!c->ep_ambiguous) + c->ep = h; } /* ================================================================== */ @@ -1093,225 +1060,227 @@ static void fit_cq_attach_ep(struct fid *bfid, struct fit_ep *h) #define DOM_FROM_OPS(dom) fit_container_of((dom)->ops, struct fit_domain, ops) -static int fit_dom_endpoint_inner(struct fid_domain *domain, struct fi_info *info, - struct fid_ep **ep, void *context) -{ - struct fit_domain *d = DOM_FROM_OPS(domain); - int ret = d->orig->endpoint ? - d->orig->endpoint(domain, info, ep, context) : -FI_ENOSYS; - - if (!ret && ep && *ep) - fit_hook_ep(*ep, &d->topo); - return ret; -} - -static int fit_dom_endpoint(struct fid_domain *domain, struct fi_info *info, - struct fid_ep **ep, void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; - - fit_record_ctrl_entry("fi_endpoint", fit_obj); - fit_ret = fit_dom_endpoint_inner(domain, info, ep, context); - fit_record_ctrl_exit("fi_endpoint", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_dom_endpoint2_inner(struct fid_domain *domain, struct fi_info *info, - struct fid_ep **ep, uint64_t flags, void *context) -{ - struct fit_domain *d = DOM_FROM_OPS(domain); - int ret = d->orig->endpoint2 ? - d->orig->endpoint2(domain, info, ep, flags, context) : - -FI_ENOSYS; - - if (!ret && ep && *ep) - fit_hook_ep(*ep, &d->topo); - return ret; -} - -static int fit_dom_endpoint2(struct fid_domain *domain, struct fi_info *info, - struct fid_ep **ep, uint64_t flags, void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; - - fit_record_ctrl_entry("fi_endpoint2", fit_obj); - fit_ret = fit_dom_endpoint2_inner(domain, info, ep, flags, context); - fit_record_ctrl_exit("fi_endpoint2", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_dom_scalable_ep_inner(struct fid_domain *domain, struct fi_info *info, - struct fid_ep **sep, void *context) -{ - struct fit_domain *d = DOM_FROM_OPS(domain); - int ret = d->orig->scalable_ep ? - d->orig->scalable_ep(domain, info, sep, context) : -FI_ENOSYS; - - if (!ret && sep && *sep) - fit_hook_ep(*sep, &d->topo); - return ret; -} - -static int fit_dom_scalable_ep(struct fid_domain *domain, struct fi_info *info, - struct fid_ep **sep, void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; - - fit_record_ctrl_entry("fi_scalable_ep", fit_obj); - fit_ret = fit_dom_scalable_ep_inner(domain, info, sep, context); - fit_record_ctrl_exit("fi_scalable_ep", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_dom_srx_ctx_inner(struct fid_domain *domain, struct fi_rx_attr *attr, - struct fid_ep **rx_ep, void *context) -{ - struct fit_domain *d = DOM_FROM_OPS(domain); - int ret = d->orig->srx_ctx ? - d->orig->srx_ctx(domain, attr, rx_ep, context) : -FI_ENOSYS; - - /* Off by default. Several providers (tcp among them) route an - * endpoint's receives through an internal shared receive context, so - * hooking both records every fi_recv twice -- once on the endpoint - * the application called, once on the srx underneath it. Set - * FI_TRACE_SRX=1 if the application posts to an srx directly. */ - if (!ret && rx_ep && *rx_ep && fit_cfg.trace_srx) - fit_hook_ep(*rx_ep, &d->topo); - return ret; -} - -static int fit_dom_srx_ctx(struct fid_domain *domain, struct fi_rx_attr *attr, - struct fid_ep **rx_ep, void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; - - fit_record_ctrl_entry("fi_srx_context", fit_obj); - fit_ret = fit_dom_srx_ctx_inner(domain, attr, rx_ep, context); - fit_record_ctrl_exit("fi_srx_context", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_dom_cq_open_inner(struct fid_domain *domain, struct fi_cq_attr *attr, - struct fid_cq **cq, void *context) -{ - struct fit_domain *d = DOM_FROM_OPS(domain); - int ret = d->orig->cq_open ? - d->orig->cq_open(domain, attr, cq, context) : -FI_ENOSYS; - - if (!ret && cq && *cq && fit_cfg.trace_cq) - fit_hook_cq(*cq, attr ? attr->format : FI_CQ_FORMAT_UNSPEC); - return ret; -} - -static int fit_dom_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, - struct fid_cq **cq, void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; - - fit_record_ctrl_entry("fi_cq_open", fit_obj); - fit_ret = fit_dom_cq_open_inner(domain, attr, cq, context); - fit_record_ctrl_exit("fi_cq_open", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_dom_av_open_inner(struct fid_domain *domain, struct fi_av_attr *attr, - struct fid_av **av, void *context) -{ - struct fit_domain *d = DOM_FROM_OPS(domain); - int ret = d->orig->av_open ? - d->orig->av_open(domain, attr, av, context) : -FI_ENOSYS; - - if (!ret && av && *av) - fit_hook_av(*av); - return ret; -} - -static int fit_dom_av_open(struct fid_domain *domain, struct fi_av_attr *attr, - struct fid_av **av, void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; - - fit_record_ctrl_entry("fi_av_open", fit_obj); - fit_ret = fit_dom_av_open_inner(domain, attr, av, context); - fit_record_ctrl_exit("fi_av_open", fit_obj, fit_ret); - return fit_ret; -} - -static int fit_dom_close_inner(struct fid *fid) -{ - struct fit_domain *d = fit_container_of(fid->ops, struct fit_domain, - fid_ops); - int ret = d->o_fid->close ? d->o_fid->close(fid) : -FI_ENOSYS; - - if (!ret) - free(d); - return ret; -} - -static int fit_dom_close(struct fid *fid) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; - - fit_record_ctrl_entry("fi_close_domain", fit_obj); - fit_ret = fit_dom_close_inner(fid); - fit_record_ctrl_exit("fi_close_domain", fit_obj, fit_ret); - return fit_ret; -} - -void fit_hook_domain(struct fid_domain *domain, const struct fi_info *info) -{ - struct fit_domain *d; - - if (!domain || !domain->ops) - return; - - d = calloc(1, sizeof(*d)); - if (!d) - return; - - /* Before patching anything: the extension interface is reached - * through the domain's own ops, and asking while they are still - * entirely the provider's keeps this independent of what we are - * about to change. */ - fit_topo_resolve(&d->topo, domain, info); - - if (domain->fid.ops) { - d->o_fid = domain->fid.ops; - d->fid_ops = *domain->fid.ops; - d->fid_ops.close = fit_dom_close; - domain->fid.ops = &d->fid_ops; - } - - d->orig = domain->ops; - d->ops = *domain->ops; - d->ops.endpoint = fit_dom_endpoint; - d->ops.endpoint2 = fit_dom_endpoint2; - d->ops.scalable_ep = fit_dom_scalable_ep; - d->ops.srx_ctx = fit_dom_srx_ctx; - d->ops.cq_open = fit_dom_cq_open; - d->ops.av_open = fit_dom_av_open; - domain->ops = &d->ops; - - if (d->topo.valid) - fit_log("hooked domain %p (%s: group %u switch %u port %u)", - (void *)domain, d->topo.name, d->topo.group, - d->topo.sw, d->topo.port); - else - fit_log("hooked domain %p", (void *)domain); +static int fit_dom_endpoint_inner(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **ep, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->endpoint ? d->orig->endpoint(domain, info, ep, context) : -FI_ENOSYS; + + if (!ret && ep && *ep) + fit_hook_ep(*ep, &d->topo); + return ret; +} + +static int fit_dom_endpoint(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **ep, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_endpoint", fit_obj); + fit_ret = fit_dom_endpoint_inner(domain, info, ep, context); + fit_record_ctrl_exit("fi_endpoint", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_endpoint2_inner(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **ep, + uint64_t flags, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->endpoint2 ? d->orig->endpoint2(domain, info, ep, flags, context) : -FI_ENOSYS; + + if (!ret && ep && *ep) + fit_hook_ep(*ep, &d->topo); + return ret; +} + +static int fit_dom_endpoint2(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **ep, + uint64_t flags, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_endpoint2", fit_obj); + fit_ret = fit_dom_endpoint2_inner(domain, info, ep, flags, context); + fit_record_ctrl_exit("fi_endpoint2", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_scalable_ep_inner(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **sep, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->scalable_ep ? d->orig->scalable_ep(domain, info, sep, context) : -FI_ENOSYS; + + if (!ret && sep && *sep) + fit_hook_ep(*sep, &d->topo); + return ret; +} + +static int fit_dom_scalable_ep(struct fid_domain *domain, + struct fi_info *info, + struct fid_ep **sep, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_scalable_ep", fit_obj); + fit_ret = fit_dom_scalable_ep_inner(domain, info, sep, context); + fit_record_ctrl_exit("fi_scalable_ep", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_srx_ctx_inner(struct fid_domain *domain, + struct fi_rx_attr *attr, + struct fid_ep **rx_ep, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->srx_ctx ? d->orig->srx_ctx(domain, attr, rx_ep, context) : -FI_ENOSYS; + + /* Off by default. Several providers (tcp among them) route an + * endpoint's receives through an internal shared receive context, so + * hooking both records every fi_recv twice -- once on the endpoint + * the application called, once on the srx underneath it. Set + * FI_TRACE_SRX=1 if the application posts to an srx directly. */ + if (!ret && rx_ep && *rx_ep && fit_cfg.trace_srx) + fit_hook_ep(*rx_ep, &d->topo); + return ret; +} + +static int fit_dom_srx_ctx(struct fid_domain *domain, + struct fi_rx_attr *attr, + struct fid_ep **rx_ep, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_srx_context", fit_obj); + fit_ret = fit_dom_srx_ctx_inner(domain, attr, rx_ep, context); + fit_record_ctrl_exit("fi_srx_context", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_cq_open_inner(struct fid_domain *domain, + struct fi_cq_attr *attr, + struct fid_cq **cq, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->cq_open ? d->orig->cq_open(domain, attr, cq, context) : -FI_ENOSYS; + + if (!ret && cq && *cq && fit_cfg.trace_cq) + fit_hook_cq(*cq, attr ? attr->format : FI_CQ_FORMAT_UNSPEC); + return ret; +} + +static int fit_dom_cq_open(struct fid_domain *domain, + struct fi_cq_attr *attr, + struct fid_cq **cq, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_cq_open", fit_obj); + fit_ret = fit_dom_cq_open_inner(domain, attr, cq, context); + fit_record_ctrl_exit("fi_cq_open", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_av_open_inner(struct fid_domain *domain, + struct fi_av_attr *attr, + struct fid_av **av, + void *context) { + struct fit_domain *d = DOM_FROM_OPS(domain); + int ret = d->orig->av_open ? d->orig->av_open(domain, attr, av, context) : -FI_ENOSYS; + + if (!ret && av && *av) + fit_hook_av(*av); + return ret; +} + +static int fit_dom_av_open(struct fid_domain *domain, + struct fi_av_attr *attr, + struct fid_av **av, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_av_open", fit_obj); + fit_ret = fit_dom_av_open_inner(domain, attr, av, context); + fit_record_ctrl_exit("fi_av_open", fit_obj, fit_ret); + return fit_ret; +} + +static int fit_dom_close_inner(struct fid *fid) { + struct fit_domain *d = fit_container_of(fid->ops, struct fit_domain, fid_ops); + int ret = d->o_fid->close ? d->o_fid->close(fid) : -FI_ENOSYS; + + if (!ret) + free(d); + return ret; +} + +static int fit_dom_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; + + fit_record_ctrl_entry("fi_close_domain", fit_obj); + fit_ret = fit_dom_close_inner(fid); + fit_record_ctrl_exit("fi_close_domain", fit_obj, fit_ret); + return fit_ret; +} + +void fit_hook_domain(struct fid_domain *domain, const struct fi_info *info) { + struct fit_domain *d; + + if (!domain || !domain->ops) + return; + + d = calloc(1, sizeof(*d)); + if (!d) + return; + + /* Before patching anything: the extension interface is reached + * through the domain's own ops, and asking while they are still + * entirely the provider's keeps this independent of what we are + * about to change. */ + fit_topo_resolve(&d->topo, domain, info); + + if (domain->fid.ops) { + d->o_fid = domain->fid.ops; + d->fid_ops = *domain->fid.ops; + d->fid_ops.close = fit_dom_close; + domain->fid.ops = &d->fid_ops; + } + + d->orig = domain->ops; + d->ops = *domain->ops; + d->ops.endpoint = fit_dom_endpoint; + d->ops.endpoint2 = fit_dom_endpoint2; + d->ops.scalable_ep = fit_dom_scalable_ep; + d->ops.srx_ctx = fit_dom_srx_ctx; + d->ops.cq_open = fit_dom_cq_open; + d->ops.av_open = fit_dom_av_open; + domain->ops = &d->ops; + + if (d->topo.valid) + fit_log("hooked domain %p (%s: group %u switch %u port %u)", (void *)domain, d->topo.name, + d->topo.group, d->topo.sw, d->topo.port); + else + fit_log("hooked domain %p", (void *)domain); } /* ================================================================== */ @@ -1320,120 +1289,115 @@ void fit_hook_domain(struct fid_domain *domain, const struct fi_info *info) #define FAB_FROM_OPS(fab) fit_container_of((fab)->ops, struct fit_fabric, ops) -static int fit_fab_domain_inner(struct fid_fabric *fabric, struct fi_info *info, - struct fid_domain **dom, void *context) -{ - struct fit_fabric *f = FAB_FROM_OPS(fabric); - int ret = f->orig->domain ? - f->orig->domain(fabric, info, dom, context) : -FI_ENOSYS; +static int fit_fab_domain_inner(struct fid_fabric *fabric, + struct fi_info *info, + struct fid_domain **dom, + void *context) { + struct fit_fabric *f = FAB_FROM_OPS(fabric); + int ret = f->orig->domain ? f->orig->domain(fabric, info, dom, context) : -FI_ENOSYS; - if (!ret && dom && *dom) - fit_hook_domain(*dom, info); - return ret; + if (!ret && dom && *dom) + fit_hook_domain(*dom, info); + return ret; } -static int fit_fab_domain(struct fid_fabric *fabric, struct fi_info *info, - struct fid_domain **dom, void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; +static int fit_fab_domain(struct fid_fabric *fabric, + struct fi_info *info, + struct fid_domain **dom, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; - fit_record_ctrl_entry("fi_domain", fit_obj); - fit_ret = fit_fab_domain_inner(fabric, info, dom, context); - fit_record_ctrl_exit("fi_domain", fit_obj, fit_ret); - return fit_ret; + fit_record_ctrl_entry("fi_domain", fit_obj); + fit_ret = fit_fab_domain_inner(fabric, info, dom, context); + fit_record_ctrl_exit("fi_domain", fit_obj, fit_ret); + return fit_ret; } -static int fit_fab_domain2_inner(struct fid_fabric *fabric, struct fi_info *info, - struct fid_domain **dom, uint64_t flags, - void *context) -{ - struct fit_fabric *f = FAB_FROM_OPS(fabric); - int ret = f->orig->domain2 ? - f->orig->domain2(fabric, info, dom, flags, context) : - -FI_ENOSYS; +static int fit_fab_domain2_inner(struct fid_fabric *fabric, + struct fi_info *info, + struct fid_domain **dom, + uint64_t flags, + void *context) { + struct fit_fabric *f = FAB_FROM_OPS(fabric); + int ret = f->orig->domain2 ? f->orig->domain2(fabric, info, dom, flags, context) : -FI_ENOSYS; - if (!ret && dom && *dom) - fit_hook_domain(*dom, info); - return ret; + if (!ret && dom && *dom) + fit_hook_domain(*dom, info); + return ret; } -static int fit_fab_domain2(struct fid_fabric *fabric, struct fi_info *info, - struct fid_domain **dom, uint64_t flags, - void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; +static int fit_fab_domain2(struct fid_fabric *fabric, + struct fi_info *info, + struct fid_domain **dom, + uint64_t flags, + void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; - fit_record_ctrl_entry("fi_domain2", fit_obj); - fit_ret = fit_fab_domain2_inner(fabric, info, dom, flags, context); - fit_record_ctrl_exit("fi_domain2", fit_obj, fit_ret); - return fit_ret; + fit_record_ctrl_entry("fi_domain2", fit_obj); + fit_ret = fit_fab_domain2_inner(fabric, info, dom, flags, context); + fit_record_ctrl_exit("fi_domain2", fit_obj, fit_ret); + return fit_ret; } -static int fit_fab_close_inner(struct fid *fid) -{ - struct fit_fabric *f = fit_container_of(fid->ops, struct fit_fabric, - fid_ops); - int ret; +static int fit_fab_close_inner(struct fid *fid) { + struct fit_fabric *f = fit_container_of(fid->ops, struct fit_fabric, fid_ops); + int ret; - ret = f->o_fid->close ? f->o_fid->close(fid) : -FI_ENOSYS; - if (!ret) - free(f); - return ret; + ret = f->o_fid->close ? f->o_fid->close(fid) : -FI_ENOSYS; + if (!ret) + free(f); + return ret; } -static int fit_fab_close(struct fid *fid) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; +static int fit_fab_close(struct fid *fid) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; - fit_record_ctrl_entry("fi_close_fabric", fit_obj); - fit_ret = fit_fab_close_inner(fid); - fit_record_ctrl_exit("fi_close_fabric", fit_obj, fit_ret); - return fit_ret; + fit_record_ctrl_entry("fi_close_fabric", fit_obj); + fit_ret = fit_fab_close_inner(fid); + fit_record_ctrl_exit("fi_close_fabric", fit_obj, fit_ret); + return fit_ret; } -void fit_hook_fabric(struct fid_fabric *fabric) -{ - struct fit_fabric *f; +void fit_hook_fabric(struct fid_fabric *fabric) { + struct fit_fabric *f; - if (!fabric || !fabric->ops) - return; + if (!fabric || !fabric->ops) + return; - f = calloc(1, sizeof(*f)); - if (!f) - return; + f = calloc(1, sizeof(*f)); + if (!f) + return; - if (fabric->fid.ops) { - f->o_fid = fabric->fid.ops; - f->fid_ops = *fabric->fid.ops; - f->fid_ops.close = fit_fab_close; - fabric->fid.ops = &f->fid_ops; - } + if (fabric->fid.ops) { + f->o_fid = fabric->fid.ops; + f->fid_ops = *fabric->fid.ops; + f->fid_ops.close = fit_fab_close; + fabric->fid.ops = &f->fid_ops; + } - f->orig = fabric->ops; - f->ops = *fabric->ops; - f->ops.domain = fit_fab_domain; - f->ops.domain2 = fit_fab_domain2; - fabric->ops = &f->ops; - - fit_log("hooked fabric %p", (void *)fabric); + f->orig = fabric->ops; + f->ops = *fabric->ops; + f->ops.domain = fit_fab_domain; + f->ops.domain2 = fit_fab_domain2; + fabric->ops = &f->ops; + + fit_log("hooked fabric %p", (void *)fabric); } /* ================================================================== */ /* interposed exported symbols */ /* ================================================================== */ -typedef int (*fi_fabric_fn)(struct fi_fabric_attr *, struct fid_fabric **, - void *); -typedef int (*fi_fabric2_fn)(struct fi_info *, struct fid_fabric **, uint64_t, - void *); +typedef int (*fi_fabric_fn)(struct fi_fabric_attr *, struct fid_fabric **, void *); +typedef int (*fi_fabric2_fn)(struct fi_info *, struct fid_fabric **, uint64_t, void *); -static fi_fabric_fn real_fi_fabric; +static fi_fabric_fn real_fi_fabric; static fi_fabric2_fn real_fi_fabric2; /* @@ -1454,119 +1418,104 @@ dlvsym() to pick the right versioned fi_fabric, and gettid(2). \ On macOS or elsewhere, run it in a container: ./build/docker-demo.sh" #endif -static void *fit_dlnext(const char *name) -{ - static const char *const vers[] = { - "FABRIC_1.9", "FABRIC_1.8", "FABRIC_1.7", "FABRIC_1.6", - "FABRIC_1.5", "FABRIC_1.4", "FABRIC_1.3", "FABRIC_1.2", - "FABRIC_1.1", "FABRIC_1.0", - }; - size_t i; - void *sym; - - for (i = 0; i < sizeof(vers) / sizeof(vers[0]); i++) { - sym = dlvsym(RTLD_NEXT, name, vers[i]); - if (sym) - return sym; - } - - (void) dlerror(); - return dlsym(RTLD_NEXT, name); -} - -__attribute__((constructor)) -static void fit_init(void) -{ - fit_config_init(); - - if (!fit_cfg.enabled) { - fit_log("disabled via FI_TRACE_ENABLE"); - return; - } - - real_fi_fabric = (fi_fabric_fn)fit_dlnext("fi_fabric"); - real_fi_fabric2 = (fi_fabric2_fn)fit_dlnext("fi_fabric2"); - - if (!real_fi_fabric && !real_fi_fabric2) { - fit_log("cannot find libfabric fi_fabric(); tracing off"); - return; - } - - // fit_writer_init(); - fit_armed = 1; - fit_log("armed: dir=%s ops=%s%s%s rx=%d resolve=%d", fit_cfg.dir, - fit_cfg.trace_msg ? "msg " : "", - fit_cfg.trace_tagged ? "tagged " : "", - fit_cfg.trace_rma ? "rma" : "", - fit_cfg.trace_rx, fit_cfg.resolve); +static void *fit_dlnext(const char *name) { + static const char *const vers[] = { + "FABRIC_1.9", "FABRIC_1.8", "FABRIC_1.7", "FABRIC_1.6", "FABRIC_1.5", + "FABRIC_1.4", "FABRIC_1.3", "FABRIC_1.2", "FABRIC_1.1", "FABRIC_1.0", + }; + size_t i; + void *sym; + + for (i = 0; i < sizeof(vers) / sizeof(vers[0]); i++) { + sym = dlvsym(RTLD_NEXT, name, vers[i]); + if (sym) + return sym; + } + + (void)dlerror(); + return dlsym(RTLD_NEXT, name); +} + +__attribute__((constructor)) static void fit_init(void) { + fit_config_init(); + + if (!fit_cfg.enabled) { + fit_log("disabled via FI_TRACE_ENABLE"); + return; + } + + real_fi_fabric = (fi_fabric_fn)fit_dlnext("fi_fabric"); + real_fi_fabric2 = (fi_fabric2_fn)fit_dlnext("fi_fabric2"); + + if (!real_fi_fabric && !real_fi_fabric2) { + fit_log("cannot find libfabric fi_fabric(); tracing off"); + return; + } + + // fit_writer_init(); + fit_armed = 1; + fit_log("armed: dir=%s ops=%s%s%s rx=%d resolve=%d", fit_cfg.dir, fit_cfg.trace_msg ? "msg " : "", + fit_cfg.trace_tagged ? "tagged " : "", fit_cfg.trace_rma ? "rma" : "", fit_cfg.trace_rx, + fit_cfg.resolve); } /* These must be exported even though the library is built with * -fvisibility=hidden -- they are the whole point of the preload. */ #define FIT_PUBLIC __attribute__((visibility("default"))) -FIT_PUBLIC int fi_fabric(struct fi_fabric_attr *attr, - struct fid_fabric **fabric, void *context); -FIT_PUBLIC int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, - uint64_t flags, void *context); +FIT_PUBLIC int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, void *context); +FIT_PUBLIC int +fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, void *context); -static int fi_fabric_inner(struct fi_fabric_attr *attr, struct fid_fabric **fabric, - void *context) -{ - int ret; +static int fi_fabric_inner(struct fi_fabric_attr *attr, struct fid_fabric **fabric, void *context) { + int ret; - if (!real_fi_fabric) - real_fi_fabric = (fi_fabric_fn)fit_dlnext("fi_fabric"); - if (!real_fi_fabric) - return -FI_ENOSYS; + if (!real_fi_fabric) + real_fi_fabric = (fi_fabric_fn)fit_dlnext("fi_fabric"); + if (!real_fi_fabric) + return -FI_ENOSYS; - ret = real_fi_fabric(attr, fabric, context); - if (!ret && fit_armed && fabric && *fabric) - fit_hook_fabric(*fabric); - return ret; + ret = real_fi_fabric(attr, fabric, context); + if (!ret && fit_armed && fabric && *fabric) + fit_hook_fabric(*fabric); + return ret; } FIT_PUBLIC -int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, - void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; +int fi_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; - fit_record_ctrl_entry("fi_fabric", fit_obj); - fit_ret = fi_fabric_inner(attr, fabric, context); - fit_record_ctrl_exit("fi_fabric", fit_obj, fit_ret); - return fit_ret; + fit_record_ctrl_entry("fi_fabric", fit_obj); + fit_ret = fi_fabric_inner(attr, fabric, context); + fit_record_ctrl_exit("fi_fabric", fit_obj, fit_ret); + return fit_ret; } -static int fi_fabric2_inner(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, - void *context) -{ - int ret; +static int +fi_fabric2_inner(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, void *context) { + int ret; - if (!real_fi_fabric2) - real_fi_fabric2 = (fi_fabric2_fn)fit_dlnext("fi_fabric2"); - if (!real_fi_fabric2) - return -FI_ENOSYS; + if (!real_fi_fabric2) + real_fi_fabric2 = (fi_fabric2_fn)fit_dlnext("fi_fabric2"); + if (!real_fi_fabric2) + return -FI_ENOSYS; - ret = real_fi_fabric2(info, fabric, flags, context); - if (!ret && fit_armed && fabric && *fabric) - fit_hook_fabric(*fabric); - return ret; + ret = real_fi_fabric2(info, fabric, flags, context); + if (!ret && fit_armed && fabric && *fabric) + fit_hook_fabric(*fabric); + return ret; } FIT_PUBLIC -int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, - void *context) -{ - /* Captured before the call: the close wrappers free this object. */ - uint32_t fit_obj = (uint32_t)(0); - int fit_ret; +int fi_fabric2(struct fi_info *info, struct fid_fabric **fabric, uint64_t flags, void *context) { + /* Captured before the call: the close wrappers free this object. */ + uint32_t fit_obj = (uint32_t)(0); + int fit_ret; - fit_record_ctrl_entry("fi_fabric2", fit_obj); - fit_ret = fi_fabric2_inner(info, fabric, flags, context); - fit_record_ctrl_exit("fi_fabric2", fit_obj, fit_ret); - return fit_ret; + fit_record_ctrl_entry("fi_fabric2", fit_obj); + fit_ret = fi_fabric2_inner(info, fabric, flags, context); + fit_record_ctrl_exit("fi_fabric2", fit_obj, fit_ret); + return fit_ret; } - diff --git a/backends/libfabric/src/fit_lttng.c b/backends/libfabric/src/fit_lttng.c index 358d58d42..5adf5fbe5 100644 --- a/backends/libfabric/src/fit_lttng.c +++ b/backends/libfabric/src/fit_lttng.c @@ -50,29 +50,27 @@ _Atomic unsigned int fit_av_gen = 0; /* logging (stderr, not the trace -- this is for debugging the tracer) */ /* ------------------------------------------------------------------ */ -void fit_log(const char *fmt, ...) -{ - va_list ap; - - if (!fit_cfg.verbose) - return; - - fprintf(stderr, "[fi_trace] "); - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - fputc('\n', stderr); +void fit_log(const char *fmt, ...) { + va_list ap; + + if (!fit_cfg.verbose) + return; + + fprintf(stderr, "[fi_trace] "); + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fputc('\n', stderr); } /* ------------------------------------------------------------------ */ /* configuration */ /* ------------------------------------------------------------------ */ -static int env_int(const char *name, int dflt) -{ - const char *v = getenv(name); +static int env_int(const char *name, int dflt) { + const char *v = getenv(name); - return v ? atoi(v) : dflt; + return v ? atoi(v) : dflt; } /* @@ -83,40 +81,38 @@ static int env_int(const char *name, int dflt) * messaging, "LTTNG_UST_FI_OPS=tagged" is measurably cheaper than filtering on * the LTTng side. */ -static void parse_ops(const char *v) -{ - if (!v || !*v) { - fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 1; - return; - } - - fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 0; - - if (strstr(v, "msg")) - fit_cfg.trace_msg = 1; - if (strstr(v, "tagged")) - fit_cfg.trace_tagged = 1; - if (strstr(v, "rma")) - fit_cfg.trace_rma = 1; +static void parse_ops(const char *v) { + if (!v || !*v) { + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 1; + return; + } + + fit_cfg.trace_msg = fit_cfg.trace_tagged = fit_cfg.trace_rma = 0; + + if (strstr(v, "msg")) + fit_cfg.trace_msg = 1; + if (strstr(v, "tagged")) + fit_cfg.trace_tagged = 1; + if (strstr(v, "rma")) + fit_cfg.trace_rma = 1; } -void fit_config_init(void) -{ - fit_cfg.enabled = env_int("LTTNG_UST_FI_ENABLE", 1); - fit_cfg.resolve = env_int("LTTNG_UST_FI_RESOLVE", 1); - fit_cfg.trace_rx = env_int("LTTNG_UST_FI_RX", 1); - fit_cfg.trace_srx = env_int("LTTNG_UST_FI_SRX", 0); - fit_cfg.trace_eagain = env_int("LTTNG_UST_FI_EAGAIN", 1); - fit_cfg.trace_cq = env_int("LTTNG_UST_FI_CQ", 1); - fit_cfg.verbose = env_int("LTTNG_UST_FI_VERBOSE", 0); - parse_ops(getenv("LTTNG_UST_FI_OPS")); - - /* - * fit_cfg.host / host_len / dir / bufsz / max_records are no longer - * read. They stay in struct fit_config only so that fi_trace.h does - * not have to change; drop them once you are sure nothing else refers - * to them. - */ +void fit_config_init(void) { + fit_cfg.enabled = env_int("LTTNG_UST_FI_ENABLE", 1); + fit_cfg.resolve = env_int("LTTNG_UST_FI_RESOLVE", 1); + fit_cfg.trace_rx = env_int("LTTNG_UST_FI_RX", 1); + fit_cfg.trace_srx = env_int("LTTNG_UST_FI_SRX", 0); + fit_cfg.trace_eagain = env_int("LTTNG_UST_FI_EAGAIN", 1); + fit_cfg.trace_cq = env_int("LTTNG_UST_FI_CQ", 1); + fit_cfg.verbose = env_int("LTTNG_UST_FI_VERBOSE", 0); + parse_ops(getenv("LTTNG_UST_FI_OPS")); + + /* + * fit_cfg.host / host_len / dir / bufsz / max_records are no longer + * read. They stay in struct fit_config only so that fi_trace.h does + * not have to change; drop them once you are sure nothing else refers + * to them. + */ } /* ------------------------------------------------------------------ */ @@ -130,23 +126,22 @@ void fit_config_init(void) * field. Measured once where the string is produced, as before, so the hot * path only memcpys. */ -void fit_sanitize(char *s, size_t *len_out) -{ - size_t n = 0; - - if (!s) { - if (len_out) - *len_out = 0; - return; - } - - for (; s[n]; n++) { - if ((unsigned char)s[n] < 0x20 || (unsigned char)s[n] == 0x7f) - s[n] = '_'; - } - - if (len_out) - *len_out = n; +void fit_sanitize(char *s, size_t *len_out) { + size_t n = 0; + + if (!s) { + if (len_out) + *len_out = 0; + return; + } + + for (; s[n]; n++) { + if ((unsigned char)s[n] < 0x20 || (unsigned char)s[n] == 0x7f) + s[n] = '_'; + } + + if (len_out) + *len_out = n; } /* @@ -162,10 +157,7 @@ void fit_sanitize(char *s, size_t *len_out) * spacing between completions from one reap as an artifact of the loop, not a * real arrival pattern. */ -uint64_t fit_now_us(void) -{ - return 0; -} +uint64_t fit_now_us(void) { return 0; } /* ------------------------------------------------------------------ */ /* data-path records */ @@ -178,94 +170,94 @@ uint64_t fit_now_us(void) * payload rides on the exit event, where it was already being computed before * this port. */ -void fit_record_entry(struct fit_ep *h, const char *op, char dir, - const void *context) -{ - if (dir == 'R' && !fit_cfg.trace_rx) - return; +void fit_record_entry(struct fit_ep *h, const char *op, char dir, const void *context) { + if (dir == 'R' && !fit_cfg.trace_rx) + return; - tracepoint(lttng_ust_fi, op_entry, op, h->ep_id, context); + tracepoint(lttng_ust_fi, op_entry, op, h->ep_id, context); } -void fit_record(struct fit_ep *h, const char *op, char dir, fi_addr_t peer, - size_t bytes, size_t iov_count, ssize_t ret, - const void *context) -{ - char scratch[FIT_ADDR_MAX]; - const char *peer_str; - size_t peer_len = 0; - - if (dir == 'R' && !fit_cfg.trace_rx) - return; - - /* - * A -FI_EAGAIN return means the provider refused the operation and the - * application will retry; no bytes moved. Applications that spin on a - * busy endpoint can generate thousands of these, which swamps the real - * transfers, so they can be dropped. - * - * Dropping the exit here leaves the matching op_entry unpaired, and the - * analysis side discards an entry that never closes. That is the - * intended behaviour, but it does mean LTTNG_UST_FI_EAGAIN=0 makes the - * entry/exit counts in a raw trace dump disagree -- do not go looking - * for a bug there. - */ - if (ret == -FI_EAGAIN && !fit_cfg.trace_eagain) - return; - - if (!h->src_valid) - fit_ep_resolve_src(h); - - peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), &peer_len); - - tracepoint(lttng_ust_fi, op_exit, op, h->ep_id, h->src, peer_str, - (dir == 'R') ? "rx" : "tx", (uint64_t)iov_count, - (uint64_t)bytes, (int64_t)ret, - /* An operation the provider refused never became - * outstanding, so it will produce no completion and must - * not be joined to one. */ - ret ? NULL : context); +void fit_record(struct fit_ep *h, + const char *op, + char dir, + fi_addr_t peer, + size_t bytes, + size_t iov_count, + ssize_t ret, + const void *context) { + char scratch[FIT_ADDR_MAX]; + const char *peer_str; + size_t peer_len = 0; + + if (dir == 'R' && !fit_cfg.trace_rx) + return; + + /* + * A -FI_EAGAIN return means the provider refused the operation and the + * application will retry; no bytes moved. Applications that spin on a + * busy endpoint can generate thousands of these, which swamps the real + * transfers, so they can be dropped. + * + * Dropping the exit here leaves the matching op_entry unpaired, and the + * analysis side discards an entry that never closes. That is the + * intended behaviour, but it does mean LTTNG_UST_FI_EAGAIN=0 makes the + * entry/exit counts in a raw trace dump disagree -- do not go looking + * for a bug there. + */ + if (ret == -FI_EAGAIN && !fit_cfg.trace_eagain) + return; + + if (!h->src_valid) + fit_ep_resolve_src(h); + + peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), &peer_len); + + tracepoint(lttng_ust_fi, op_exit, op, h->ep_id, h->src, peer_str, (dir == 'R') ? "rx" : "tx", + (uint64_t)iov_count, (uint64_t)bytes, (int64_t)ret, + /* An operation the provider refused never became + * outstanding, so it will produce no completion and must + * not be joined to one. */ + ret ? NULL : context); } -void fit_record_cq(struct fit_cq *c, const char *op, fi_addr_t peer, - size_t bytes, ssize_t ret, const void *context, - uint64_t now) -{ - char scratch[FIT_ADDR_MAX]; - const char *peer_str = "fi_addr_unspec"; - size_t peer_len = sizeof("fi_addr_unspec") - 1; - const char *src = "unknown"; - struct fit_ep *h = c->ep; - - (void)now; /* LTTng stamps the event; see fit_now_us() */ - - /* The CQ usually serves exactly one endpoint, in which case its local - * address and id describe this record too. */ - if (h) { - if (!h->src_valid) - fit_ep_resolve_src(h); - src = h->src; - peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), - &peer_len); - } - - tracepoint(lttng_ust_fi, cq_completion, op, - h ? h->ep_id : c->cq_id, src, peer_str, - (uint64_t)bytes, (int64_t)ret, context); +void fit_record_cq(struct fit_cq *c, + const char *op, + fi_addr_t peer, + size_t bytes, + ssize_t ret, + const void *context, + uint64_t now) { + char scratch[FIT_ADDR_MAX]; + const char *peer_str = "fi_addr_unspec"; + size_t peer_len = sizeof("fi_addr_unspec") - 1; + const char *src = "unknown"; + struct fit_ep *h = c->ep; + + (void)now; /* LTTng stamps the event; see fit_now_us() */ + + /* The CQ usually serves exactly one endpoint, in which case its local + * address and id describe this record too. */ + if (h) { + if (!h->src_valid) + fit_ep_resolve_src(h); + src = h->src; + peer_str = fit_ep_peer_str(h, peer, scratch, sizeof(scratch), &peer_len); + } + + tracepoint(lttng_ust_fi, cq_completion, op, h ? h->ep_id : c->cq_id, src, peer_str, + (uint64_t)bytes, (int64_t)ret, context); } /* ------------------------------------------------------------------ */ /* control-path records */ /* ------------------------------------------------------------------ */ -void fit_record_ctrl_entry(const char *op, unsigned int obj_id) -{ - tracepoint(lttng_ust_fi, ctrl_entry, op, obj_id); +void fit_record_ctrl_entry(const char *op, unsigned int obj_id) { + tracepoint(lttng_ust_fi, ctrl_entry, op, obj_id); } -void fit_record_ctrl_exit(const char *op, unsigned int obj_id, int ret) -{ - tracepoint(lttng_ust_fi, ctrl_exit, op, obj_id, (int64_t)ret); +void fit_record_ctrl_exit(const char *op, unsigned int obj_id, int ret) { + tracepoint(lttng_ust_fi, ctrl_exit, op, obj_id, (int64_t)ret); } /* ------------------------------------------------------------------ */ @@ -282,20 +274,18 @@ void fit_record_ctrl_exit(const char *op, unsigned int obj_id, int ret) * by NIC address while the interval stream is keyed by ep_id, and the mapping * between them has to come from somewhere. */ -void fit_record_ep_info(unsigned int ep_id, const char *src, size_t src_len, - const struct fit_topo *t) -{ - (void)src_len; - - if (!fit_armed || !src) - return; - - tracepoint(lttng_ust_fi, ep_info, ep_id, src, - (t && t->valid) ? t->name : "", - (t && t->valid) ? 1u : 0u, - (t && t->valid) ? t->group : 0u, - (t && t->valid) ? t->sw : 0u, - (t && t->valid) ? t->port : 0u); +void fit_record_ep_info(unsigned int ep_id, + const char *src, + size_t src_len, + const struct fit_topo *t) { + (void)src_len; + + if (!fit_armed || !src) + return; + + tracepoint(lttng_ust_fi, ep_info, ep_id, src, (t && t->valid) ? t->name : "", + (t && t->valid) ? 1u : 0u, (t && t->valid) ? t->group : 0u, + (t && t->valid) ? t->sw : 0u, (t && t->valid) ? t->port : 0u); } /* diff --git a/xprof/xprof.rb.in b/xprof/xprof.rb.in index b8c1ccd94..3c8909384 100755 --- a/xprof/xprof.rb.in +++ b/xprof/xprof.rb.in @@ -805,7 +805,7 @@ module LocalMaster in: File::NULL, out: log_path, err: [:child, :out]) @sessiond_waiter = Process.detach(pid) deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 20 - + ready = false begin loop do if @sessiond_waiter.join(0) @@ -823,9 +823,8 @@ module LocalMaster end sleep 0.1 end - rescue Exception - lttng_stop_foreground - raise + ensure + lttng_stop_foreground unless ready end end @@ -843,6 +842,7 @@ module LocalMaster begin Process.kill('KILL', waiter.pid) rescue Errno::ESRCH + # The daemon exited between the check and the signal. end waiter.join end