Protect Spindle's FDs from close_range, closefrom

src/client/client/intercept.c

@@ -30,6 +30,8 @@ struct spindle_binding_t spindle_bindings[] = {
    { "fopen", (void **) &orig_fopen, "rtcache_fopen", (void *) rtcache_fopen },
    { "fopen64", (void **) &orig_fopen64, "rtcache_fopen64", (void *) rtcache_fopen64 },
    { "close", (void **) &orig_close, "rtcache_close", (void *) rtcache_close },
+   { "close_range", (void **) &orig_close_range, "rtcache_close_range", (void *) rtcache_close_range },
+   { "closefrom", (void **) &orig_closefrom, "rtcache_closefrom", (void *) rtcache_closefrom },
    { "stat", (void **) &orig_stat, "rtcache_stat", (void *) rtcache_stat },
    { "lstat", (void **) &orig_lstat, "rtcache_lstat", (void *) rtcache_lstat },
    { "__xstat", (void **) &orig_xstat, "rtcache_xstat", (void *) rtcache_xstat },

src/client/client/intercept.h

@@ -48,6 +48,8 @@ extern int (*orig_open64)(const char *pathname, int flags, ...);
 extern FILE* (*orig_fopen)(const char *pathname, const char *mode);
 extern FILE* (*orig_fopen64)(const char *pathname, const char *mode);
 extern int (*orig_close)(int fd);
+extern int (*orig_close_range)(unsigned int first, unsigned int last, int flags);
+extern void (*orig_closefrom)(int lowfd);
 extern char* (*orig_getenv)(const char *name);
 extern int (*orig_setenv)(const char *name, const char *value, int overwrite);
 extern int (*orig_unsetenv)(const char *name);
@@ -83,6 +85,8 @@ int rtcache_open64(const char *path, int oflag, ...);
 FILE *rtcache_fopen(const char *path, const char *mode);
 FILE *rtcache_fopen64(const char *path, const char *mode);
 int rtcache_close(int fd);
+int rtcache_close_range(unsigned int first, unsigned int last, int flags);
+void rtcache_closefrom(int lowfd);
 
 int execl_wrapper(const char *path, const char *arg0, ...);
 int execv_wrapper(const char *path, char *const argv[]);

src/client/client/intercept_open.c

@@ -43,6 +43,8 @@ int (*orig_open64)(const char *pathname, int flags, ...);
 FILE* (*orig_fopen)(const char *pathname, const char *mode);
 FILE* (*orig_fopen64)(const char *pathname, const char *mode);
 int (*orig_close)(int fd);
+int (*orig_close_range)(unsigned int first, unsigned int last, int flags);
+void (*orig_closefrom)(int lowfd);
 char* (*orig_dlerror)();
 static int handle_proc_pid_maps_open(const char *path, char **newpath);
 
@@ -294,6 +296,42 @@ int rtcache_close(int fd)
    return orig_close(fd);
 }
 
+int rtcache_close_range(unsigned int first, unsigned int last, int flags)
+{
+   int fds[4];
+   int n, i;
+   unsigned int lowfd = first;
+
+   check_for_fork();
+
+   /* Make sure we call the original if we have
+    * invalid arguments so we get EINVAL. */
+   if (first > last)
+      return orig_close_range(first, last, flags);
+
+   /* Convert the close_range into a sequence of close_range calls, 
+    * skipping over any of Spindle's own fds. */
+   n = get_hidden_fds(fds);
+   for (i = 0; i < n; i++) {
+      unsigned int fd = (unsigned int) fds[i];
+      if (fd < lowfd || fd > last)
+         continue;
+      if (fd > lowfd && orig_close_range(lowfd, fd - 1, flags) == -1)
+         return -1;
+      lowfd = fd + 1;
+   }
+   if (lowfd > last)
+      return 0;
+   return orig_close_range(lowfd, last, flags);
+}
+
+void rtcache_closefrom(int lowfd)
+{
+   /* glibc uses MAX(0, lowfd); mimic that behavior */
+   if (rtcache_close_range(lowfd < 0 ? 0 : lowfd, ~0U, 0) == -1)
+      orig_closefrom(lowfd);
+}
+
 char *dlerror_wrapper()
 {
    char *message = NULL;

src/client/client/should_intercept.c

@@ -287,3 +287,30 @@ int fd_filter(int fd)
 
    return ORIG_CALL;
 }
+
+/* Places Spindle's fds into fds. 
+ * fds must be of size at least 4;
+ * returns the number of fds added to fds */
+int get_hidden_fds(int *fds)
+{
+   int candidates[4];
+   int n = 0;
+   int i, j;
+
+   if (opts & OPT_NOHIDE)
+      return 0;
+
+   client_get_raw_fds(ldcsid, &candidates[0], &candidates[1]);
+   get_debug_fds(&candidates[2], &candidates[3]);
+
+   /* Put sorted candidates into fds, dropping -1 values */
+   for (i = 0; i < 4; i++) {
+      if (candidates[i] < 0)
+         continue;
+      for (j = n; j > 0 && fds[j-1] > candidates[i]; j--)
+         fds[j] = fds[j-1];
+      fds[j] = candidates[i];
+      n++;
+   }
+   return n;
+}

src/client/client/should_intercept.h

@@ -32,5 +32,6 @@ int fopen_filter(const char *fname, const char *flags);
 int exec_filter(const char *fname);
 int stat_filter(const char *fname);
 int fd_filter(int fd);
+int get_hidden_fds(int *fds);
 
 #endif

src/logging/spindle_logc.c

@@ -368,6 +368,12 @@ int is_debug_fd(int fd)
    return (fd == debug_fd || fd == test_fd);
 }
 
+void get_debug_fds(int *log_fd, int *testlog_fd)
+{
+   *log_fd = debug_fd;
+   *testlog_fd = test_fd;
+}
+
 int spindle_debug_printf_impl(int priority, const char *file, unsigned int line, const char *func, const char *format, ...)
 {
    va_list ap;

src/logging/spindle_logc.h

@@ -109,5 +109,6 @@ void init_spindle_debugging(char *name, int survive_exec);
 void fini_spindle_debugging();
 void reset_spindle_debugging();
 int is_debug_fd(int fd);
+void get_debug_fds(int *log_fd, int *testlog_fd);
 
 #endif

testsuite/Makefile.am

@@ -1,7 +1,7 @@
 noinst_PROGRAMS = libgenerator
 
 ABS_TEST_DIR = $(abspath $(top_builddir)/testsuite)
-BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh libbigtls1024.so libbigtls204800.so libbigtls1048576.so
+BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh libbigtls1024.so libbigtls204800.so libbigtls1048576.so close_range_test
 
 BUILT_SOURCES += crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh
 
@@ -479,5 +479,8 @@ exec_shell_expected_output: $(srcdir)/exec_shell_expected_output_template
 exec_shell: $(srcdir)/exec_shell.c
 	$(AM_V_CCLD)$(CC) $(CFLAGS) -o $@ $<
 
-CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh libbigtls204800.so libbigtls1048576.so
+close_range_test: $(srcdir)/close_range_test.c
+	$(AM_V_CCLD)$(CC) $(CFLAGS) -o $@ $<
+
+CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh libbigtls204800.so libbigtls1048576.so close_range_test
 

testsuite/Makefile.in

@@ -360,9 +360,9 @@ BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so \
 	alias/aliastest.py exec_shell.sh exec_shell.tcsh \
 	exec_shell_expected_output exec_shell exec_shell_env.sh \
 	libbigtls1024.so libbigtls204800.so libbigtls1048576.so \
-	crash_test crash_test_fixedaddr crash_test_pie \
-	libcrashfuncs.so libcrashctor.so libcrashfixed.so \
-	run_crash_tests.sh
+	close_range_test crash_test crash_test_fixedaddr \
+	crash_test_pie libcrashfuncs.so libcrashctor.so \
+	libcrashfixed.so run_crash_tests.sh
 @BGQ_BLD_FALSE@DYNAMIC_FLAG = 
 @BGQ_BLD_TRUE@DYNAMIC_FLAG = -dynamic
 @BGQ_BLD_FALSE@IS_BLUEGENE = false
@@ -407,7 +407,8 @@ CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c \
 	exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output \
 	exec_shell crash_test crash_test_fixedaddr crash_test_pie \
 	libcrashfuncs.so libcrashctor.so libcrashfixed.so \
-	run_crash_tests.sh libbigtls204800.so libbigtls1048576.so
+	run_crash_tests.sh libbigtls204800.so libbigtls1048576.so \
+	close_range_test
 all: $(BUILT_SOURCES)
 	$(MAKE) $(AM_MAKEFLAGS) all-am
 
@@ -1162,6 +1163,9 @@ exec_shell_expected_output: $(srcdir)/exec_shell_expected_output_template
 exec_shell: $(srcdir)/exec_shell.c
 	$(AM_V_CCLD)$(CC) $(CFLAGS) -o $@ $<
 
+close_range_test: $(srcdir)/close_range_test.c
+	$(AM_V_CCLD)$(CC) $(CFLAGS) -o $@ $<
+
 # Tell versions [3.59,3.63) of GNU make to not export all variables.
 # Otherwise a system limit (for SysV at least) may be exceeded.
 .NOEXPORT:

testsuite/close_range_test.c

@@ -0,0 +1,91 @@
+#define _GNU_SOURCE
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+/* closefrom and close_range were added in glibc 2.34 */
+#if defined(__GLIBC__) && __GLIBC_PREREQ(2, 34)
+
+static int run_child(int use_closefrom)
+{
+   int status;
+   pid_t pid = fork();
+   if (pid == -1) {
+      perror("fork");
+      return -1;
+   }
+   if (pid == 0) {
+      struct stat st;
+      int fd;
+      /* Intercepted stat to allow fork child to reconnect */
+      stat("/", &st);
+      fd = open("/dev/null", O_RDONLY);
+      if (use_closefrom) {
+         /* closefrom(int lowfd) closes all file descriptors >= lowfd. */
+         closefrom(3);
+      }
+      /* close_range(unsigned int first, unsigned int last, int flags)
+       * closes all file descriptors >= first and <= last. */
+      else if (close_range(3, UINT_MAX, 0) == -1) {
+         perror("close_range");
+         _exit(3);
+      }
+      /* Verify that the file we opened ourselves was closed */
+      if (fcntl(fd, F_GETFD) != -1 || errno != EBADF) {
+         fprintf(stderr, "fd %d was not closed\n", fd);
+         _exit(2);
+      }
+      /* Verify that we can still exec after closing fds;
+       * if closefrom/close_range closed Spindle's own fds,
+       * this will fail. */
+      execl("/bin/true", "true", (char *) NULL);
+      perror("execl");
+      _exit(1);
+   }
+   if (waitpid(pid, &status, 0) == -1) {
+      perror("waitpid");
+      return -1;
+   }
+   return status;
+}
+
+int main(void)
+{
+   int close_range_status, closefrom_status;
+
+   /* close_range gives ENOSYS if the kernel version doesn't support
+    * the corresponding syscall.*/
+   if (close_range(INT_MAX, INT_MAX, 0) == -1 && errno == ENOSYS) {
+      printf("Skipping: ./close_range_test (kernel lacks close_range)\n");
+      return 0;
+   }
+
+   close_range_status = run_child(0);
+   closefrom_status = run_child(1);
+   if (close_range_status == 0 && closefrom_status == 0) {
+      printf("PASSED.\n");
+      return 0;
+   }
+
+   printf("FAILED.\n");
+   if (close_range_status != 0)
+      fprintf(stderr, " close_range: child failed, status=%d\n", close_range_status);
+   if (closefrom_status != 0)
+      fprintf(stderr, " closefrom: child failed, status=%d\n", closefrom_status);
+   return 1;
+}
+
+#else
+
+int main(void)
+{
+   printf("Skipping: ./close_range_test (glibc < 2.34 lacks closefrom/close_range)\n");
+   return 0;
+}
+
+#endif

testsuite/runTests_template

@@ -193,6 +193,9 @@ CHECK_RETCODE
 ./run_driver --serial ./exec_shell
 CHECK_RETCODE
 
+./run_driver --serial ./close_range_test
+CHECK_RETCODE
+
 if [[ $GLOBAL_RESULT != 0 ]]; then
     echo SOME TESTS FAILED
     exit -1