[PATCH] cursor: use memfd_create or shm_open for anonymous in-memory files

[Simon Ser]

On Linux, try using memfd_create and file sealing. Fallback to
shm_open on old kernels.

On FreeBSD, use shm_open with SHM_ANON.

Otherwise, use shm_open with a random name, making sure the name
isn't already taken.

Signed-off-by: Simon Ser <contact at emersion.fr>
---
 Makefile.am               |   2 +-
 configure.ac              |   4 +-
 cursor/os-compatibility.c | 172 ++++++++++++++++++--------------------
 3 files changed, 86 insertions(+), 92 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 697c517..c612672 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -136,7 +136,7 @@ libwayland_cursor_la_SOURCES =			\
 	cursor/cursor-data.h			\
 	cursor/xcursor.c			\
 	cursor/xcursor.h
-libwayland_cursor_la_LIBADD = libwayland-client.la
+libwayland_cursor_la_LIBADD = libwayland-client.la -lrt
 
 pkgconfig_DATA += cursor/wayland-cursor.pc
 
diff --git a/configure.ac b/configure.ac
index 9419ae3..d59c61d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -62,8 +62,8 @@ if test "x$GCC" = "xyes"; then
 fi
 AC_SUBST(GCC_CFLAGS)
 
-AC_CHECK_HEADERS([sys/prctl.h])
-AC_CHECK_FUNCS([accept4 mkostemp posix_fallocate prctl])
+AC_CHECK_HEADERS([sys/prctl.h linux/memfd.h])
+AC_CHECK_FUNCS([accept4 prctl])
 
 AC_ARG_ENABLE([libraries],
 	      [AC_HELP_STRING([--disable-libraries],
diff --git a/cursor/os-compatibility.c b/cursor/os-compatibility.c
index e972d21..96649e3 100644
--- a/cursor/os-compatibility.c
+++ b/cursor/os-compatibility.c
@@ -25,124 +25,118 @@
 
 #define _GNU_SOURCE
 
-#include <sys/types.h>
-#include <unistd.h>
-#include <fcntl.h>
 #include <errno.h>
-#include <string.h>
+#include <fcntl.h>
 #include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+#ifdef HAVE_LINUX_MEMFD_H
+#include <linux/memfd.h>
+#endif
 
 #include "config.h"
 #include "os-compatibility.h"
 
-#ifndef HAVE_MKOSTEMP
+static void
+randname(char *buf) {
+	struct timespec ts;
+	long r;
+	int i;
+
+	clock_gettime(CLOCK_REALTIME, &ts);
+	r = ts.tv_nsec;
+	for (i = 0; i < 6; ++i) {
+		buf[i] = 'A'+(r&15)+(r&16)*2;
+		r >>= 5;
+	}
+}
+
 static int
-set_cloexec_or_close(int fd)
+anonymous_shm_open(off_t size)
 {
-	long flags;
-
-	if (fd == -1)
-		return -1;
+	char name[] = "/wayland-cursor-XXXXXX";
+	int fd, retries = 100;
 
-	flags = fcntl(fd, F_GETFD);
-	if (flags == -1)
-		goto err;
+	do {
+		randname(name + strlen(name) - 6);
 
-	if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) == -1)
-		goto err;
+		--retries;
+		fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL | O_CLOEXEC, 0600);
+		if (fd >= 0) {
+			shm_unlink(name);
+			return fd;
+		}
+	} while (retries > 0 && errno == EEXIST);
 
-	return fd;
-
-err:
-	close(fd);
 	return -1;
 }
+
+static int
+seal_or_close(int fd) {
+#if defined(F_ADD_SEALS) && defined(F_SEAL_SHRINK)
+	if (fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK) == -1) {
+		close(fd);
+		return -1;
+	}
 #endif
 
+	return 0;
+}
+
 static int
-create_tmpfile_cloexec(char *tmpname)
+create_anonymous_file(off_t size)
 {
-	int fd;
+	int fd, flags;
 
-#ifdef HAVE_MKOSTEMP
-	fd = mkostemp(tmpname, O_CLOEXEC);
-	if (fd >= 0)
-		unlink(tmpname);
-#else
-	fd = mkstemp(tmpname);
+#if defined(__NR_memfd_create) && defined(MFD_CLOEXEC)
+	flags = MFD_CLOEXEC;
+#if defined(MFD_ALLOW_SEALING)
+	flags |= MFD_ALLOW_SEALING;
+#endif
+	fd = syscall(__NR_memfd_create, "wayland-cursor", flags);
 	if (fd >= 0) {
-		fd = set_cloexec_or_close(fd);
-		unlink(tmpname);
-	}
+		if (ftruncate(fd, size) < 0) {
+			close(fd);
+			return -1;
+		}
+
+#if defined(MFD_ALLOW_SEALING)
+		if (seal_or_close(fd) != 0)
+			return -1;
 #endif
 
+		return fd;
+	} else if (errno != ENOSYS)
+		return fd;
+#endif
+
+#if defined(__FreeBSD__)
+	fd = shm_open(SHM_ANON, O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0600);
+#else
+	fd = anonymous_shm_open(size);
+#endif
+
+	if (ftruncate(fd, size) < 0) {
+		close(fd);
+		return -1;
+	}
+
 	return fd;
 }
 
-/*
- * Create a new, unique, anonymous file of the given size, and
- * return the file descriptor for it. The file descriptor is set
- * CLOEXEC. The file is immediately suitable for mmap()'ing
- * the given size at offset zero.
- *
- * The file should not have a permanent backing store like a disk,
- * but may have if XDG_RUNTIME_DIR is not properly implemented in OS.
- *
- * The file name is deleted from the file system.
- *
- * The file is suitable for buffer sharing between processes by
- * transmitting the file descriptor over Unix sockets using the
- * SCM_RIGHTS methods.
- *
- * If the C library implements posix_fallocate(), it is used to
- * guarantee that disk space is available for the file at the
- * given size. If disk space is insufficent, errno is set to ENOSPC.
- * If posix_fallocate() is not supported, program may receive
- * SIGBUS on accessing mmap()'ed file contents instead.
- */
 int
 os_create_anonymous_file(off_t size)
 {
-	static const char template[] = "/wayland-cursor-shared-XXXXXX";
-	const char *path;
-	char *name;
 	int fd;
-	int ret;
-
-	path = getenv("XDG_RUNTIME_DIR");
-	if (!path) {
-		errno = ENOENT;
-		return -1;
-	}
 
-	name = malloc(strlen(path) + sizeof(template));
-	if (!name)
-		return -1;
-
-	strcpy(name, path);
-	strcat(name, template);
-
-	fd = create_tmpfile_cloexec(name);
-
-	free(name);
-
-	if (fd < 0)
-		return -1;
-
-#ifdef HAVE_POSIX_FALLOCATE
-	ret = posix_fallocate(fd, 0, size);
-	if (ret != 0) {
-		close(fd);
-		errno = ret;
-		return -1;
-	}
-#else
-	ret = ftruncate(fd, size);
-	if (ret < 0) {
-		close(fd);
-		return -1;
-	}
-#endif
+	do {
+		fd = create_anonymous_file(size);
+	} while (fd < 0 && errno == EINTR);
 
 	return fd;
 }
-- 
2.18.0