[systemd-devel] [PATCH 1/2] core: support Distribute=n to distribute to n SO_REUSEPORT workers
Shawn Landden
shawn at churchofgit.com
Thu Nov 14 14:08:13 PST 2013
v3 make each worker its own service
---
TODO | 3 +-
man/systemd.socket.xml | 9 +++
src/core/dbus-socket.c | 2 +
src/core/load-fragment-gperf.gperf.m4 | 1 +
src/core/service.c | 7 +-
src/core/service.h | 1 -
src/core/socket.c | 124 ++++++++++++++++++++--------------
src/core/socket.h | 4 ++
8 files changed, 96 insertions(+), 55 deletions(-)
diff --git a/TODO b/TODO
index 57e1122..733e528 100644
--- a/TODO
+++ b/TODO
@@ -82,7 +82,7 @@ Features:
* rfkill,backlight: we probably should run the load tools inside of the udev rules so that the state is properly initialized by the time other software sees it
-* Add a new Distribute=$NUMBER key to socket units that makes use of SO_REUSEPORT to distribute network traffic on $NUMBER instances
+* respawn Distribute= worker threads when they die unexpectedly
* tmpfiles: when applying ownership to /run/log/journal, also do this for the journal fails contained in it
@@ -261,7 +261,6 @@ Features:
* teach ConditionKernelCommandLine= globs or regexes (in order to match foobar={no,0,off})
* Support SO_REUSEPORT with socket activation:
- - Let systemd maintain a pool of servers.
- Use for seamless upgrades, by running the new server before stopping the
old.
diff --git a/man/systemd.socket.xml b/man/systemd.socket.xml
index 7c10c58..92a9275 100644
--- a/man/systemd.socket.xml
+++ b/man/systemd.socket.xml
@@ -519,6 +519,15 @@
</varlistentry>
<varlistentry>
+ <term><varname>Distribute=</varname></term>
+ <listitem><para>Takes an integer
+ value. If greater than one, systemd will spawn
+ given number of instances of service each
+ listening to the same socket. This option implies
+ <varname>Reuseport=</varname> above.</para></listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><varname>SmackLabel=</varname></term>
<term><varname>SmackLabelIPIn=</varname></term>
<term><varname>SmackLabelIPOut=</varname></term>
diff --git a/src/core/dbus-socket.c b/src/core/dbus-socket.c
index 60a8d05..4644007 100644
--- a/src/core/dbus-socket.c
+++ b/src/core/dbus-socket.c
@@ -68,6 +68,7 @@
" <property name=\"Listen\" type=\"a(ss)\" access=\"read\"/>\n" \
" <property name=\"Result\" type=\"s\" access=\"read\"/>\n" \
" <property name=\"ReusePort\" type=\"b\" access=\"read\"/>\n" \
+ " <property name=\"Distribute\" type=\"u\" access=\"read\"/>\n" \
" <property name=\"SmackLabel\" type=\"s\" access=\"read\"/>\n" \
" <property name=\"SmackLabelIPIn\" type=\"s\" access=\"read\"/>\n" \
" <property name=\"SmackLabelIPOut\" type=\"s\" access=\"read\"/>\n" \
@@ -196,6 +197,7 @@ static const BusProperty bus_socket_properties[] = {
{ "MessageQueueMessageSize", bus_property_append_long, "x", offsetof(Socket, mq_msgsize) },
{ "Result", bus_socket_append_socket_result, "s", offsetof(Socket, result) },
{ "ReusePort", bus_property_append_bool, "b", offsetof(Socket, reuseport) },
+ { "Distribute", bus_property_append_unsigned, "u", offsetof(Socket, distribute) },
{ "SmackLabel", bus_property_append_string, "s", offsetof(Socket, smack), true },
{ "SmackLabelIPIn", bus_property_append_string, "s", offsetof(Socket, smack_ip_in), true },
{ "SmackLabelIPOut",bus_property_append_string, "s", offsetof(Socket, smack_ip_out), true },
diff --git a/src/core/load-fragment-gperf.gperf.m4 b/src/core/load-fragment-gperf.gperf.m4
index b64fdc9..4058a1f 100644
--- a/src/core/load-fragment-gperf.gperf.m4
+++ b/src/core/load-fragment-gperf.gperf.m4
@@ -211,6 +211,7 @@ Socket.PassCredentials, config_parse_bool, 0,
Socket.PassSecurity, config_parse_bool, 0, offsetof(Socket, pass_sec)
Socket.TCPCongestion, config_parse_string, 0, offsetof(Socket, tcp_congestion)
Socket.ReusePort, config_parse_bool, 0, offsetof(Socket, reuseport)
+Socket.Distribute, config_parse_unsigned, 0, offsetof(Socket, distribute)
Socket.MessageQueueMaxMessages, config_parse_long, 0, offsetof(Socket, mq_maxmsg)
Socket.MessageQueueMessageSize, config_parse_long, 0, offsetof(Socket, mq_msgsize)
Socket.Service, config_parse_socket_service, 0, 0
diff --git a/src/core/service.c b/src/core/service.c
index 3da32a1..8fc55a0 100644
--- a/src/core/service.c
+++ b/src/core/service.c
@@ -3663,7 +3663,6 @@ static void service_bus_query_pid_done(
int service_set_socket_fd(Service *s, int fd, Socket *sock) {
assert(s);
- assert(fd >= 0);
/* This is called by the socket code when instantiating a new
* service for a stream socket and the socket needs to be
@@ -3678,8 +3677,10 @@ int service_set_socket_fd(Service *s, int fd, Socket *sock) {
if (s->state != SERVICE_DEAD)
return -EAGAIN;
- s->socket_fd = fd;
- s->got_socket_fd = true;
+ if (fd >= 0) {
+ s->socket_fd = fd;
+ s->got_socket_fd = true;
+ }
unit_ref_set(&s->accept_socket, UNIT(sock));
diff --git a/src/core/service.h b/src/core/service.h
index 37fa6ff..2ffe7d1 100644
--- a/src/core/service.h
+++ b/src/core/service.h
@@ -26,7 +26,6 @@ typedef struct Service Service;
#include "unit.h"
#include "path.h"
#include "ratelimit.h"
-#include "service.h"
#include "kill.h"
#include "exit-status.h"
diff --git a/src/core/socket.c b/src/core/socket.c
index 751f20b..10a0d95 100644
--- a/src/core/socket.c
+++ b/src/core/socket.c
@@ -153,34 +153,30 @@ static void socket_done(Unit *u) {
}
static int socket_instantiate_service(Socket *s) {
- char *prefix, *name;
+ _cleanup_free_ char *prefix = NULL, *name = NULL;
int r;
Unit *u;
assert(s);
/* This fills in s->service if it isn't filled in yet. For
- * Accept=yes sockets we create the next connection service
- * here. For Accept=no this is mostly a NOP since the service
+ * Accept=yes and Distribute=n sockets we create the next connection
+ * service here. For Accept=no this is mostly a NOP since the service
* is figured out at load time anyway. */
- if (UNIT_DEREF(s->service))
+ if (UNIT_DEREF(s->service) && !s->distribute)
return 0;
- assert(s->accept);
+ assert(s->accept || s->distribute);
if (!(prefix = unit_name_to_prefix(UNIT(s)->id)))
return -ENOMEM;
r = asprintf(&name, "%s@%u.service", prefix, s->n_accepted);
- free(prefix);
-
if (r < 0)
return -ENOMEM;
r = manager_load_unit(UNIT(s)->manager, name, NULL, NULL, &u);
- free(name);
-
if (r < 0)
return r;
@@ -513,6 +509,11 @@ static void socket_dump(Unit *u, FILE *f, const char *prefix) {
"%sReusePort: %s\n",
prefix, yes_no(s->reuseport));
+ if (s->distribute)
+ fprintf(f,
+ "%sDistribute: %d\n",
+ prefix, s->distribute);
+
if (s->smack)
fprintf(f,
"%sSmackLabel: %s\n",
@@ -577,9 +578,14 @@ static int instance_from_socket(int fd, unsigned nr, char **instance) {
struct sockaddr_storage storage;
} local, remote;
- assert(fd >= 0);
assert(instance);
+ if (fd < 0) {
+ if(asprintf(&r, "%u", nr) < 0)
+ return -ENOMEM;
+ goto shortcut;
+ }
+
l = sizeof(local);
if (getsockname(fd, &local.sa, &l) < 0)
return -errno;
@@ -663,6 +669,7 @@ static int instance_from_socket(int fd, unsigned nr, char **instance) {
assert_not_reached("Unhandled socket type.");
}
+shortcut:
*instance = r;
return 0;
}
@@ -1454,7 +1461,7 @@ static void socket_enter_running(Socket *s, int cfd) {
return;
}
- if (cfd < 0) {
+ if (cfd < 0 && !(s->distribute)) {
Iterator i;
Unit *u;
bool pending = false;
@@ -1486,56 +1493,68 @@ static void socket_enter_running(Socket *s, int cfd) {
return;
}
- r = socket_instantiate_service(s);
- if (r < 0)
- goto fail;
-
- r = instance_from_socket(cfd, s->n_accepted, &instance);
- if (r < 0) {
- if (r != -ENOTCONN)
- goto fail;
-
- /* ENOTCONN is legitimate if TCP RST was received.
- * This connection is over, but the socket unit lives on. */
- close_nointr_nofail(cfd);
- return;
- }
-
prefix = unit_name_to_prefix(UNIT(s)->id);
if (!prefix) {
r = -ENOMEM;
goto fail;
}
- name = unit_name_build(prefix, instance, ".service");
+ /* loop if Distribute= set */
+ do {
+ r = socket_instantiate_service(s);
+ if (r < 0)
+ goto fail;
- if (!name) {
- r = -ENOMEM;
- goto fail;
- }
+ r = instance_from_socket(cfd, s->n_accepted, &instance);
+ if (r < 0) {
+ if (r != -ENOTCONN)
+ goto fail;
- r = unit_add_name(UNIT_DEREF(s->service), name);
- if (r < 0)
- goto fail;
+ /* ENOTCONN is legitimate if TCP RST was received.
+ * This connection is over, but the socket unit lives on. */
+ close_nointr_nofail(cfd);
+ return;
+ }
- service = SERVICE(UNIT_DEREF(s->service));
- unit_ref_unset(&s->service);
- s->n_accepted ++;
+ name = unit_name_build(prefix, instance, ".service");
+ if (!name) {
+ r = -ENOMEM;
+ goto fail;
+ }
- UNIT(service)->no_gc = false;
+ r = unit_add_name(UNIT_DEREF(s->service), name);
+ if (r < 0)
+ goto fail;
- unit_choose_id(UNIT(service), name);
+ service = SERVICE(UNIT_DEREF(s->service));
+ unit_ref_unset(&s->service);
+ s->n_accepted ++;
- r = service_set_socket_fd(service, cfd, s);
- if (r < 0)
- goto fail;
+ UNIT(service)->no_gc = false;
- cfd = -1;
- s->n_connections ++;
+ unit_choose_id(UNIT(service), name);
- r = manager_add_job(UNIT(s)->manager, JOB_START, UNIT(service), JOB_REPLACE, true, &error, NULL);
- if (r < 0)
- goto fail;
+ r = service_set_socket_fd(service, cfd, s);
+ if (r < 0)
+ goto fail;
+
+ cfd = -1;
+ s->n_connections ++;
+
+ r = manager_add_job(UNIT(s)->manager, JOB_START, UNIT(service), JOB_REPLACE, true, &error, NULL);
+ if (r < 0)
+ goto fail;
+
+ if(s->distribute > s->n_connections) {
+ free(instance); free(name);
+ instance = NULL; name = NULL;
+
+ /* distribute implies reuseport */
+ s->reuseport = true;
+
+ socket_enter_listening(s);
+ }
+ } while(s->distribute > s->n_connections);
/* Notify clients about changed counters */
unit_add_to_dbus_queue(UNIT(s));
@@ -2263,14 +2282,21 @@ void socket_connection_unref(Socket *s) {
/* The service is dead. Yay!
*
- * This is strictly for one-instance-per-connection
- * services. */
+ * This is for one-instance-per-connection
+ * and Distribute= services */
assert(s->n_connections > 0);
s->n_connections--;
log_debug_unit(UNIT(s)->id,
"%s: One connection closed, %u left.", UNIT(s)->id, s->n_connections);
+
+ if(s->distribute > s->n_connections && s->state == SOCKET_RUNNING){
+ s->reuseport = true;
+ /* (re)enter systemd into SO_REUSEPORT pool, when it gets a
+ * connection it will reestablish distribute target */
+ socket_enter_listening(s);
+ }
}
static void socket_reset_failed(Unit *u) {
diff --git a/src/core/socket.h b/src/core/socket.h
index 3d7eadc..5928356 100644
--- a/src/core/socket.h
+++ b/src/core/socket.h
@@ -93,6 +93,8 @@ struct Socket {
LIST_HEAD(SocketPort, ports);
unsigned n_accepted;
+ /* when Accept=true this is the number of active connectoins
+ * when Distribute=n this is the number of active workers */
unsigned n_connections;
unsigned max_connections;
@@ -145,6 +147,8 @@ struct Socket {
char *bind_to_device;
char *tcp_congestion;
bool reuseport;
+ /* implies reuseport */
+ unsigned distribute;
long mq_maxmsg;
long mq_msgsize;
--
1.8.4.3
More information about the systemd-devel
mailing list