sandbox: set number of allowed fd's

The number of file descriptors enforced by setrlimit() can now be set at
compile time using a flag. The flag defaults to 0 on Linux and -1
everywhere else:

    XMPPIPE_SANDBOX=XMPPIPE_SANDBOX_RLIMIT \
    XMPPIPE_SANDBOX_RLIMIT_NOFILE=-1 \
    make

The meaning of the XMPPIPE_SANDBOX_RLIMIT_NOFILE is:

* -1 : set rlim_cur/rlim_max to the lowest allocated file desciptor

* >=0: set rlim_cur/rlim_max to this number

On some platforms, setting rlim_cur below the value of the highest
allocated fd may interfere with polling. See commit a34d5766c5 for
details.

Makefile

@@ -4,6 +4,7 @@ UNAME_SYS := $(shell uname -s)
 ifeq ($(UNAME_SYS), Linux)
 	LDFLAGS += -luuid -lresolv -Wl,-Bsymbolic-functions -Wl,-z,relro
 	CFLAGS ?= -D_FORTIFY_SOURCE=2 -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -fno-strict-aliasing
+	XMPPIPE_SANDBOX_RLIMIT_NOFILE ?= 0
 else ifeq ($(UNAME_SYS), OpenBSD)
 	XMPPIPE_SANDBOX ?= XMPPIPE_SANDBOX_PLEDGE
 else ifeq ($(UNAME_SYS), SunOS)
@@ -13,7 +14,8 @@ else ifeq ($(UNAME_SYS), Darwin)
 endif
 
 XMPPIPE_SANDBOX ?= XMPPIPE_SANDBOX_NULL
-CFLAGS += -D$(XMPPIPE_SANDBOX)
+XMPPIPE_SANDBOX_RLIMIT_NOFILE ?= -1
+CFLAGS += -D$(XMPPIPE_SANDBOX) -DXMPPIPE_SANDBOX_RLIMIT_NOFILE=$(XMPPIPE_SANDBOX_RLIMIT_NOFILE)
 
 all:
 	$(CC) -g -Wall $(CFLAGS) -o xmppipe src/*.c $(LDFLAGS) -lstrophe

src/xmppipe.h

@@ -118,6 +118,7 @@ void xmppipe_stanza_set_type(xmpp_stanza_t * const, const char * const);
 void xmppipe_stanza_add_child(xmpp_stanza_t *, xmpp_stanza_t *);
 
 int xmppipe_sandbox_init(xmppipe_state_t *state);
+int xmppipe_conn_fd(xmppipe_state_t *state);
 
 int b64_ntop(u_char const *src, size_t srclength, char *target,
         size_t targsize);

src/xmppipe_conn.c

@@ -0,0 +1,48 @@
+/* Copyright (c) 2017, Michael Santos <michael.santos@gmail.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#include "xmppipe.h"
+
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <fcntl.h>
+
+/* Retrieve the XMPP socket opened by libstrophe.
+ *
+ * Ideally getting the XMPP socket would be as simple as:
+ *
+ * state->conn->sock
+ *
+ * But xmpp_conn_t is defined as an opaque type.
+ *
+ * The alternative is hardcoding the offsets based on the libstrophe version.
+ */
+    int
+xmppipe_conn_fd(xmppipe_state_t *state)
+{
+    int fd = 0;
+    struct rlimit rl = {0};
+
+    if (getrlimit(RLIMIT_NOFILE, &rl) < 0)
+        return -1;
+
+    for (fd = STDERR_FILENO+1; fd < rl.rlim_cur; fd++) {
+        if (fcntl(fd, F_GETFD, 0) < 0)
+            continue;
+
+        return fd;
+    }
+
+    return -1;
+}

src/xmppipe_sandbox_rlimit.c

@@ -21,15 +21,27 @@
     int
 xmppipe_sandbox_init(xmppipe_state_t *state)
 {
-    struct rlimit rl = {0};
+    struct rlimit rl_zero = {0};
+    struct rlimit rl_nofile = {0};
+
+    rl_zero.rlim_cur = 0;
+    rl_zero.rlim_max = 0;
+
+    rl_nofile.rlim_cur = XMPPIPE_SANDBOX_RLIMIT_NOFILE;
+    rl_nofile.rlim_max = XMPPIPE_SANDBOX_RLIMIT_NOFILE;
 
 #ifdef RLIMIT_NPROC
-    if (setrlimit(RLIMIT_NPROC, &rl) < 0)
+    if (setrlimit(RLIMIT_NPROC, &rl_zero) < 0)
         return -1;
 #endif
 
 #ifdef RLIMIT_NOFILE
-    if (setrlimit(RLIMIT_NOFILE, &rl) < 0)
+    if (rl_nofile.rlim_cur == (rlim_t)-1) {
+        int fd = xmppipe_conn_fd(state);
+        if (fd < 0) return -1;
+        rl_nofile.rlim_cur = rl_nofile.rlim_max = fd + 1;
+    }
+    if (setrlimit(RLIMIT_NOFILE, &rl_nofile) < 0)
         return -1;
 #endif