[Xcb] [PATCH] Introduce xcb_aux_alloc_shm() to create anonymous files in RAM

[Alexander Volkov]

... which then can be mapped with mmap().
It is intended to be used in conjunction with xcb_shm_attach_fd()
to access the created shared memory from a client and the X server.

The implementation is based on the code of xshmfence_alloc_shm()
from libxshmfence.

Signed-off-by: Alexander Volkov <a.volkov at rusbitech.ru>
---
 configure.ac  | 43 ++++++++++++++++++++++++++++
 src/xcb_aux.c | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++
 src/xcb_aux.h |  3 ++
 3 files changed, 124 insertions(+)

diff --git a/configure.ac b/configure.ac
index 1fe1561..276f1ee 100644
--- a/configure.ac
+++ b/configure.ac
@@ -7,10 +7,53 @@ AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
 AM_INIT_AUTOMAKE([foreign dist-bzip2])
 
+AC_USE_SYSTEM_EXTENSIONS
+
 XCB_UTIL_COMMON([1.4], [1.6])
 
 AC_CHECK_FUNCS_ONCE(vasprintf)
 
+AC_CHECK_FUNCS(memfd_create mkostemp)
+
+AC_CHECK_DECLS([__NR_memfd_create], [], [], [[#include <asm/unistd.h>]])
+
+AC_CHECK_HEADERS([linux/memfd.h])
+
+AC_ARG_WITH(shared-memory-dir, AS_HELP_STRING([--with-shared-memory-dir=PATH], [Path to directory in a world-writable temporary directory for anonymous shared memory (default: yes)]),
+[],
+[with_shared_memory_dir=yes])
+
+shmdirs="/run/shm /dev/shm /var/tmp /tmp"
+
+case x"$with_shared_memory_dir" in
+xyes)
+	for dir in $shmdirs; do
+		echo Checking temp dir "$dir"
+		if test -d "$dir"; then
+			with_shared_memory_dir="$dir"
+		fi
+	done
+	;;
+x/*)
+	;;
+xno)
+	;;
+*)
+	AC_MSG_ERROR([Invalid directory specified for --with-shared-memory-dir: $with_shared_memory_dir])
+	;;
+esac
+
+case x"$with_shared_memory_dir" in
+xyes)
+	AC_MSG_ERROR([No directory found for shared memory temp files.])
+	;;
+xno)
+	;;
+*)
+	AC_DEFINE_UNQUOTED(SHMDIR, ["$with_shared_memory_dir"], [Directory for shared memory temp files])
+	;;
+esac
+
 AC_CONFIG_FILES([Makefile
 		src/Makefile
 		xcb-atom.pc
diff --git a/src/xcb_aux.c b/src/xcb_aux.c
index b6f64f8..6f06333 100644
--- a/src/xcb_aux.c
+++ b/src/xcb_aux.c
@@ -3,6 +3,8 @@
  * Copyright © 2008 Ian Osgood <iano at quirkster.com>
  * Copyright © 2008 Jamey Sharp <jamey at minilop.net>
  * Copyright © 2008 Josh Triplett <josh at freedesktop.org>
+ * Copyright © 2013 Keith Packard
+ * Copyright © 2018 Alexander Volkov <a.volkov at rusbitech.ru>
  *
  * Permission is hereby granted, free of charge, to any person
  * obtaining a copy of this software and associated documentation
@@ -33,9 +35,24 @@
 #include "config.h"
 #endif
 
+#include <unistd.h>
+#if HAVE_MEMFD_CREATE
+#include <sys/mman.h>
+#elif HAVE_DECL___NR_MEMFD_CREATE && HAVE_LINUX_MEMFD_H
+#include <sys/syscall.h>
+static int memfd_create(const char *name,
+			    unsigned int flags)
+{
+	return syscall(__NR_memfd_create, name, flags);
+}
+#define HAVE_MEMFD_CREATE	1
+#include <linux/memfd.h>
+#endif
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <fcntl.h>
 
 #include <xcb/xcb.h>
 #include "xcb_aux.h"
@@ -376,3 +393,64 @@ xcb_aux_clear_window(xcb_connection_t *  dpy,
 {
     return xcb_clear_area(dpy, 0, w, 0, 0, 0, 0);
 }
+
+/* SHM related functions */
+
+/**
+ * xcb_aux_alloc_shm:
+ *
+ * Creates an anonymous file of the required size in RAM.
+ * It enables the close-on-exec flag for the file
+ * descriptor. The F_SEAL_SHRINK seal is set if it's
+ * supported, thus the file cannot be reduced in size.
+ * This function is intended for use in conjunction with
+ * xcb_shm_attach_fd().
+ *
+ * Return value: the file descriptor, or -1 on failure
+ * (in which case, errno will be set as appropriate).
+ **/
+int
+xcb_aux_alloc_shm(size_t size)
+{
+	char	template[] = SHMDIR "/xcb_aux-XXXXXX";
+	int	fd;
+
+#if HAVE_MEMFD_CREATE
+	fd = memfd_create("xcb_aux", MFD_CLOEXEC|MFD_ALLOW_SEALING);
+	if (fd < 0)
+#endif
+	{
+#ifdef O_TMPFILE
+		fd = open(SHMDIR, O_TMPFILE|O_RDWR|O_CLOEXEC|O_EXCL, 0666);
+		if (fd < 0)
+#endif
+		{
+#ifndef HAVE_MKOSTEMP
+			int flags;
+			fd = mkstemp(template);
+#else
+			fd = mkostemp(template, O_CLOEXEC);
+#endif
+			if (fd < 0)
+				return fd;
+			unlink(template);
+#ifndef HAVE_MKOSTEMP
+			flags = fcntl(fd, F_GETFD);
+			if (flags != -1) {
+				flags |= FD_CLOEXEC;
+				(void) fcntl(fd, F_SETFD, &flags);
+			}
+#endif
+		}
+	}
+	if (ftruncate(fd, size) < 0) {
+		close(fd);
+		return -1;
+	}
+
+#if defined(F_ADD_SEALS) && defined(F_SEAL_SHRINK)
+	fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK);
+#endif
+
+	return fd;
+}
diff --git a/src/xcb_aux.h b/src/xcb_aux.h
index da4fb85..75b6e66 100644
--- a/src/xcb_aux.h
+++ b/src/xcb_aux.h
@@ -206,6 +206,9 @@ xcb_void_cookie_t
 xcb_aux_clear_window(xcb_connection_t *  dpy,
 		     xcb_window_t        w);
 
+int
+xcb_aux_alloc_shm(size_t size);
+
 #ifdef __cplusplus
 }
 #endif
-- 
2.17.0