diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc index 27cd87e8..94b232a2 100644 --- a/src/fe/startup/config_mgr.cc +++ b/src/fe/startup/config_mgr.cc @@ -1060,6 +1060,14 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const if (args.opts & OPT_DEBUG) { args.opts |= OPT_REMAPEXEC; } + if (getenv("SPINDLE_NO_CRASH_DEDUP")) { + debug_printf2("Disabling OPT_CRASH due to environment variable\n"); + setopt(args.opts, OPT_CRASH_LOG, false); + setopt(args.opts, OPT_CRASH_HANDLER, false); + setopt(args.opts, OPT_CRASH_ALTSTACK, false); + } + + return true; } diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c index d12967d5..69899248 100644 --- a/src/flux/flux-spindle.c +++ b/src/flux/flux-spindle.c @@ -568,6 +568,7 @@ static int sp_init (flux_plugin_t *p, const char *tmpdir; const char *test; const char *spindle_enabled; + const char *crash_log; if (!(shell = flux_plugin_get_shell (p)) || !(h = flux_shell_get_flux (shell))) @@ -587,6 +588,10 @@ static int sp_init (flux_plugin_t *p, if ((test = flux_shell_getenv (shell, "SPINDLE_TEST"))) setenv ("SPINDLE_TEST", test, 1); + /* Pass through SPINDLE_NO_CRASH_DEDUP for crash handling disabling */ + if ((crash_log = flux_shell_getenv(shell, "SPINDLE_NO_CRASH_DEDUP"))) + setenv("SPINDLE_NO_CRASH_DEDUP", crash_log, 1); + debug_printf(1, "initializing spindle flux plugin\n"); /* Spindle requires that TMPDIR is set. Propagate TMPDIR from job diff --git a/src/server/Makefile.in b/src/server/Makefile.in index 4c6bf3b3..1c288837 100644 --- a/src/server/Makefile.in +++ b/src/server/Makefile.in @@ -298,6 +298,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ diff --git a/src/server/auditserver/Makefile.am b/src/server/auditserver/Makefile.am index 66b1a331..3fc31f10 100644 --- a/src/server/auditserver/Makefile.am +++ b/src/server/auditserver/Makefile.am @@ -1,8 +1,11 @@ #noinst_LTLIBRARIES = libaudit_server_msocket.la libaudit_server_cobo.la libserverbase.la noinst_LTLIBRARIES = libaudit_server_cobo.la libserverbase.la -AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../cobo -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/../utils -DLIBEXECDIR=\"$(pkglibexecdir)\" -DPROGLIBDIR=\"$(pkglibdir)\" -LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt +confdir = $(PKGSYSCONF_DIR) +conf_DATA = spindle_crash_log_header.txt + +AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../cobo -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/../utils -DLIBEXECDIR=\"$(pkglibexecdir)\" -DPROGLIBDIR=\"$(pkglibdir)\" -DPKGSYSCONFDIR=\"$(PKGSYSCONF_DIR)\" +LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt #AM_LDFLAGS = -all-static libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_audit_server_crash_log.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c filemngt_calc_static_tls.c $(top_srcdir)/../utils/fileutil.c $(top_srcdir)/../utils/static_tls.c @@ -15,3 +18,6 @@ LCD = $(top_builddir)/comlib COD = $(top_builddir)/cobo/ #libaudit_server_msocket_la_LIBADD = $(LDADD) libserverbase.la libaudit_server_cobo_la_LIBADD = $(LDADD) libserverbase.la $(COD)/libldcs_cobo.la + +spindle_crash_log_header.txt: $(top_srcdir)/auditserver/spindle_crash_log_header_template.txt $(top_builddir)/Makefile + $(AM_V_GEN)cp $(top_srcdir)/auditserver/spindle_crash_log_header_template.txt $(top_builddir)/auditserver/spindle_crash_log_header.txt diff --git a/src/server/auditserver/Makefile.in b/src/server/auditserver/Makefile.in index d2e4a314..f03b7eb2 100644 --- a/src/server/auditserver/Makefile.in +++ b/src/server/auditserver/Makefile.in @@ -14,6 +14,7 @@ @SET_MAKE@ + VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ @@ -209,6 +210,35 @@ am__can_run_installinfo = \ n|no|NO) false;; \ *) (install-info --version) >/dev/null 2>&1;; \ esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(confdir)" +DATA = $(conf_DATA) am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) # Read a list of newline-separated strings from the standard input, # and print each of them once, without duplicates. Input order is @@ -308,6 +338,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ @@ -385,8 +416,10 @@ top_srcdir = @top_srcdir@ #noinst_LTLIBRARIES = libaudit_server_msocket.la libaudit_server_cobo.la libserverbase.la noinst_LTLIBRARIES = libaudit_server_cobo.la libserverbase.la -AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../cobo -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/../utils -DLIBEXECDIR=\"$(pkglibexecdir)\" -DPROGLIBDIR=\"$(pkglibdir)\" -LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt +confdir = $(PKGSYSCONF_DIR) +conf_DATA = spindle_crash_log_header.txt +AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../cobo -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/../utils -DLIBEXECDIR=\"$(pkglibexecdir)\" -DPROGLIBDIR=\"$(pkglibdir)\" -DPKGSYSCONFDIR=\"$(PKGSYSCONF_DIR)\" +LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt #AM_LDFLAGS = -all-static libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_audit_server_crash_log.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c filemngt_calc_static_tls.c $(top_srcdir)/../utils/fileutil.c $(top_srcdir)/../utils/static_tls.c libserverbase_la_LIBADD = -lpthread @@ -548,6 +581,27 @@ mostlyclean-libtool: clean-libtool: -rm -rf $(top_builddir)/../utils/.libs $(top_builddir)/../utils/_libs -rm -rf .libs _libs +install-confDATA: $(conf_DATA) + @$(NORMAL_INSTALL) + @list='$(conf_DATA)'; test -n "$(confdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(confdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(confdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(confdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(confdir)" || exit $$?; \ + done + +uninstall-confDATA: + @$(NORMAL_UNINSTALL) + @list='$(conf_DATA)'; test -n "$(confdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(confdir)'; $(am__uninstall_files_from_dir) ID: $(am__tagged_files) $(am__define_uniq_tagged_files); mkid -fID $$unique @@ -636,8 +690,11 @@ distdir-am: $(DISTFILES) done check-am: all-am check: check-am -all-am: Makefile $(LTLIBRARIES) +all-am: Makefile $(LTLIBRARIES) $(DATA) installdirs: + for dir in "$(DESTDIR)$(confdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done install: install-am install-exec: install-exec-am install-data: install-data-am @@ -711,7 +768,7 @@ info: info-am info-am: -install-data-am: +install-data-am: install-confDATA install-dvi: install-dvi-am @@ -775,7 +832,7 @@ ps: ps-am ps-am: -uninstall-am: +uninstall-am: uninstall-confDATA .MAKE: install-am install-strip @@ -784,18 +841,22 @@ uninstall-am: cscopelist-am ctags ctags-am distclean distclean-compile \ distclean-generic distclean-libtool distclean-tags distdir dvi \ dvi-am html html-am info info-am install install-am \ - install-data install-data-am install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-info install-info-am install-man install-pdf \ - install-pdf-am install-ps install-ps-am install-strip \ - installcheck installcheck-am installdirs maintainer-clean \ - maintainer-clean-generic mostlyclean mostlyclean-compile \ - mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ - tags tags-am uninstall uninstall-am + install-confDATA install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-confDATA .PRECIOUS: Makefile +spindle_crash_log_header.txt: $(top_srcdir)/auditserver/spindle_crash_log_header_template.txt $(top_builddir)/Makefile + $(AM_V_GEN)cp $(top_srcdir)/auditserver/spindle_crash_log_header_template.txt $(top_builddir)/auditserver/spindle_crash_log_header.txt + # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. .NOEXPORT: diff --git a/src/server/auditserver/ldcs_audit_server_crash_log.c b/src/server/auditserver/ldcs_audit_server_crash_log.c index a519cd3f..9539756f 100644 --- a/src/server/auditserver/ldcs_audit_server_crash_log.c +++ b/src/server/auditserver/ldcs_audit_server_crash_log.c @@ -239,9 +239,9 @@ static void crash_log_merge_entry(ldcs_process_data_t *procdata, e->exemplar_rank = (int) ent->exemplar; size_t pos = 0; for (j = 0; j < (int) ent->nranks; ++j) { - int32_t r, pid, host_len; - int64_t timestamp; - const char *hostname; + int32_t r = 0, pid = 0, host_len = 0; + int64_t timestamp = 0; + const char *hostname = NULL; crash_log_parse_row(ent->ranks, ent->ranks_len, &pos, &r, &pid, ×tamp, &hostname, &host_len); crash_log_append_rank(e, r, pid, timestamp, hostname, (size_t) host_len); @@ -301,7 +301,29 @@ static int rank_cmp(const void *a, const void *b) return 0; } -#define CRASH_LOG_HEADER "rank,hostname,pid,timestamp,exe,site,exemplar,corepath" +#define DEFAULT_CRASH_LOG_HEADER "rank,hostname,pid,timestamp,exe,site,exemplar,corepath" +static void write_crash_log_header(FILE *f) +{ + FILE *header; + char buffer[4096]; + size_t bytes_read; + + header = fopen(PKGSYSCONFDIR "/spindle_crash_log_header.txt", "r"); + + if (header) { + for (;;) { + bytes_read = fread(buffer, 1, sizeof(buffer), header); + if (bytes_read <= 0) + break; + fwrite(buffer, 1, bytes_read, f); + } + fclose(header); + } + else { + fputs(DEFAULT_CRASH_LOG_HEADER "\n", f); + } + +} static void format_timestamp(int64_t timestamp, char *buf, size_t buflen) { @@ -387,8 +409,9 @@ void crash_log_root_write(ldcs_process_data_t *procdata) /* If the crash log is empty (that is, we're the first writer), * write the CSV header. */ - if (sb.st_size == 0) - fputs(CRASH_LOG_HEADER "\n", f); + if (sb.st_size == 0) { + write_crash_log_header(f); + } for (i = 0; i < procdata->crash_sites_count; ++i) { crash_site_entry_t *e = &procdata->crash_sites[i]; diff --git a/src/server/auditserver/spindle_crash_log_header_template.txt b/src/server/auditserver/spindle_crash_log_header_template.txt new file mode 100644 index 00000000..7fcc67e7 --- /dev/null +++ b/src/server/auditserver/spindle_crash_log_header_template.txt @@ -0,0 +1,13 @@ +# This file was created by Spindle as part of core-file management. Each time +# an application under Spindle would fault and dump core, Spindle will identify +# the unique fault locations across a parallel job and allow only one core file +# from each unique fault location. This prevents core file writing from +# overwhelming the file system. This file is a legend of every fault seen, +# and which core files represent which faults. +# +# Disable this by setting the environment variable SPINDLE_NO_CRASH_DEDUP to any +# value, or by adding the line 'crash-dedup=false' to ~/.spindle/spindle.conf. +# +# The key for the below comma-seperated-values is: +# rank,hostname,pid,timestamp,exe,site,exemplar,corepath + diff --git a/src/server/biter/Makefile.in b/src/server/biter/Makefile.in index d57b6e3f..22715c49 100644 --- a/src/server/biter/Makefile.in +++ b/src/server/biter/Makefile.in @@ -277,6 +277,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ diff --git a/src/server/cache/Makefile.in b/src/server/cache/Makefile.in index 3cb81709..0c5df129 100644 --- a/src/server/cache/Makefile.in +++ b/src/server/cache/Makefile.in @@ -279,6 +279,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ diff --git a/src/server/cobo/Makefile.in b/src/server/cobo/Makefile.in index 599b3f5d..fb25289e 100644 --- a/src/server/cobo/Makefile.in +++ b/src/server/cobo/Makefile.in @@ -263,6 +263,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ diff --git a/src/server/comlib/Makefile.in b/src/server/comlib/Makefile.in index 7a2217a6..dd0f5402 100644 --- a/src/server/comlib/Makefile.in +++ b/src/server/comlib/Makefile.in @@ -307,6 +307,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ diff --git a/src/server/configure b/src/server/configure index 713b1158..335362c8 100755 --- a/src/server/configure +++ b/src/server/configure @@ -635,6 +635,7 @@ ac_subst_vars='am__EXEEXT_FALSE am__EXEEXT_TRUE LTLIBOBJS LIBOBJS +PKGSYSCONF_DIR USE_NUMA_FALSE USE_NUMA_TRUE SPINDLEBE_LIB_VERSION @@ -18261,6 +18262,7 @@ else fi + cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure # tests run on this system so they can be shared between configure @@ -20977,4 +20979,3 @@ if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} fi - diff --git a/src/server/configure.ac b/src/server/configure.ac index 4f0de390..e467b7c6 100644 --- a/src/server/configure.ac +++ b/src/server/configure.ac @@ -93,5 +93,5 @@ AC_SUBST([SPINDLEBE_LIB_VERSION],[`$srcdir/../../LIB_VERSION spindlebe`]) AM_CONDITIONAL([USE_NUMA], [test "x$HAVE_NUMA" = "xtrue"]) +AC_SUBST(PKGSYSCONF_DIR) AC_OUTPUT - diff --git a/src/server/launchmon/Makefile.in b/src/server/launchmon/Makefile.in index fcca4c64..d1809d48 100644 --- a/src/server/launchmon/Makefile.in +++ b/src/server/launchmon/Makefile.in @@ -256,6 +256,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ diff --git a/src/server/logging/Makefile.in b/src/server/logging/Makefile.in index 6e5eb206..d2ba0620 100644 --- a/src/server/logging/Makefile.in +++ b/src/server/logging/Makefile.in @@ -288,6 +288,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ diff --git a/src/server/startup/Makefile.in b/src/server/startup/Makefile.in index 32d2421a..dd55f1a4 100644 --- a/src/server/startup/Makefile.in +++ b/src/server/startup/Makefile.in @@ -389,6 +389,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ PKG_CONFIG = @PKG_CONFIG@ PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c index 1d924d64..7f82cce4 100644 --- a/src/slurm_plugin/slurm_plugin.c +++ b/src/slurm_plugin/slurm_plugin.c @@ -333,6 +333,11 @@ static int forward_environment_to_job_control(spank_t spank) err = spank_job_control_setenv(spank, "TMPDIR", "/tmp", 1); if (err != ESPANK_SUCCESS) return -1; } + envVal = getenv("SPINDLE_NO_CRASH_DEDUP"); + if (envVal) { + err = spank_job_control_setenv(spank, "SPINDLE_NO_CRASH_DEDUP", envVal, 1); + if (err != ESPANK_SUCCESS) return -1; + } /* In the job control context, the SLURM_JOB_NODELIST incorrectly * contains the nodes of the STEP rather than the job, so we save diff --git a/testsuite/run_crash_tests_template.sh b/testsuite/run_crash_tests_template.sh index ceebacc3..1e8e2bf9 100755 --- a/testsuite/run_crash_tests_template.sh +++ b/testsuite/run_crash_tests_template.sh @@ -202,7 +202,7 @@ parse_args() { case "$a" in --launcher=*) LAUNCHER="${a#*=}" ;; --nodes=*) NODES="${a#*=}" ;; - --scratch=*) CRASH_TEST_SCRATCH="${a#*=}" ;; + --scratch=*) CRASH_TEST_SCRATCH=`realpath ${a#*=}` ;; --modes=*) MODES="${a#*=}" ;; --session|--cross-exe|--orig-path) [ -z "$SESSION_TEST" ] || die "--session, --cross-exe and --orig-path are mutually exclusive" @@ -388,14 +388,15 @@ log_check_header() { return 1 fi IFS= read -r header <"$log" - if [ "$header" != "rank,hostname,pid,timestamp,exe,site,exemplar,corepath" ]; then + + if [[ "$header" != "# "* ]]; then echo " incorrect crash log header '$header'" >&2 return 1 fi return 0 } -log_rows() { tail -n +2 "$1"; } +log_rows() { grep -v '^#' $1 | grep -v '^$' | tail -n +1 ; } # Split a log entry into ROW_RANK, ROW_HOST, ROW_PID, ROW_TS, ROW_EXE, # ROW_SITE, ROW_EXEMPLAR, ROW_COREPATH. The site and corepath fields may @@ -480,6 +481,12 @@ verify_crash_log() { local rc=0 total=0 line key local -A seen=() site_exemplar=() site_corepath=() site_exemplar_pid=() while IFS= read -r line; do + if [[ "$line" == "#"* ]]; then + continue + fi + if [[ "$line" == "" ]]; then + continue + fi parse_log_row "$line" key="$ROW_EXE|$ROW_SITE" total=$((total + 1))