[systemd-devel] [PATCH 1/3] journald: Add deferred log processing logic
Karol Lewandowski
k.lewandowsk at samsung.com
Thu Dec 12 05:12:54 PST 2013
This commits extends journald with ability to pickup log snippets
from /dev/shm at configured time intervals (PickupIntervalSec).
First pickup is delayed by PickupStartupDelaySec.
This allows logging to be completely async, reducing number of
wakeups drastically under heavy load.
---
src/journal/journald-gperf.gperf | 2 +
src/journal/journald-server.c | 107 +++++++++++++++++++++++++++++++++++++++
src/journal/journald-server.h | 5 ++
src/journal/journald.conf | 2 +
4 files changed, 116 insertions(+)
diff --git a/src/journal/journald-gperf.gperf b/src/journal/journald-gperf.gperf
index 2ecba3b..4cb4682 100644
--- a/src/journal/journald-gperf.gperf
+++ b/src/journal/journald-gperf.gperf
@@ -19,6 +19,8 @@ Journal.Storage, config_parse_storage, 0, offsetof(Server, storage)
Journal.Compress, config_parse_bool, 0, offsetof(Server, compress)
Journal.Seal, config_parse_bool, 0, offsetof(Server, seal)
Journal.SyncIntervalSec, config_parse_sec, 0, offsetof(Server, sync_interval_usec)
+Journal.PickupStartupDelaySec, config_parse_sec, 0, offsetof(Server, pickup_startup_delay_sec)
+Journal.PickupIntervalSec, config_parse_sec, 0, offsetof(Server, pickup_interval_sec)
Journal.RateLimitInterval, config_parse_sec, 0, offsetof(Server, rate_limit_interval)
Journal.RateLimitBurst, config_parse_unsigned, 0, offsetof(Server, rate_limit_burst)
Journal.SystemMaxUse, config_parse_bytes_off, 0, offsetof(Server, system_metrics.max_use)
diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c
index 01e75b6..22fb733 100644
--- a/src/journal/journald-server.c
+++ b/src/journal/journald-server.c
@@ -64,6 +64,9 @@
#define USER_JOURNALS_MAX 1024
+#define PICKUP_STARTUP_DELAY_SEC (5*USEC_PER_SEC)
+#define DEFAULT_PICKUP_INTERVAL_SEC (5*USEC_PER_SEC)
+
#define DEFAULT_SYNC_INTERVAL_USEC (5*USEC_PER_MINUTE)
#define DEFAULT_RATE_LIMIT_INTERVAL (30*USEC_PER_SEC)
#define DEFAULT_RATE_LIMIT_BURST 1000
@@ -332,6 +335,48 @@ void server_rotate(Server *s) {
}
}
+static void pickup_files(Server *s) {
+ static const char dir[] = "/dev/shm";
+ DIR *d;
+ int dfd;
+ struct dirent dentry;
+ struct dirent *de;
+ int fd;
+ int processed = 0;
+
+ d = opendir(dir);
+ if (!d) {
+ log_notice("Unable to open %s", dir);
+ return;
+ }
+
+ dfd = dirfd(d);
+
+ while (readdir_r(d, &dentry, &de) == 0 && de) {
+
+ if (!startswith(dentry.d_name, "journal-snippet."))
+ continue;
+
+ fd = openat(dfd, dentry.d_name, O_RDONLY);
+ if (fd < 0) {
+ log_error("Unable to pickup journald snippet file %s/%s: %m",
+ dir, dentry.d_name);
+ continue;
+ }
+
+ server_process_native_file(s, fd, NULL, NULL, NULL, 0);
+ unlinkat(dfd, dentry.d_name, 0);
+ close(fd);
+
+ ++ processed;
+ }
+
+ closedir(d);
+
+ if (processed)
+ log_debug("Pickup: processed %d files", processed);
+}
+
void server_sync(Server *s) {
static const struct itimerspec sync_timer_disable = {};
JournalFile *f;
@@ -1124,6 +1169,17 @@ int process_event(Server *s, struct epoll_event *ev) {
server_sync(s);
return 1;
+ } else if (ev->data.fd == s->pickup_timer_fd) {
+ int r;
+ uint64_t t;
+
+ r = read(ev->data.fd, (void *)&t, sizeof(t));
+ if (r < 0)
+ return 0;
+
+ pickup_files(s);
+ return 1;
+
} else if (ev->data.fd == s->dev_kmsg_fd) {
int r;
@@ -1409,6 +1465,47 @@ static int server_open_sync_timer(Server *s) {
return 0;
}
+static int server_open_pickup_timer(Server *s) {
+ int r;
+ struct epoll_event ev;
+ struct itimerspec ts = {
+ { s->pickup_interval_sec / USEC_PER_SEC, 0},
+ { s->pickup_startup_delay_sec / USEC_PER_SEC, 0},
+ };
+
+ assert(s);
+
+ log_debug("PickupIntervalSec = %llu, PickupStartupDelaySec = %llu",
+ s->pickup_interval_sec / USEC_PER_SEC,
+ s->pickup_startup_delay_sec / USEC_PER_SEC);
+
+ if (s->pickup_interval_sec == 0)
+ return 1;
+
+ s->pickup_timer_fd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC);
+ if (s->pickup_timer_fd < 0)
+ return -errno;
+
+ zero(ev);
+ ev.events = EPOLLIN;
+ ev.data.fd = s->pickup_timer_fd;
+
+ r = epoll_ctl(s->epoll_fd, EPOLL_CTL_ADD, s->pickup_timer_fd, &ev);
+ if (r < 0) {
+ log_error("Failed to add pickup timer fd to epoll object: %m");
+ return -errno;
+ }
+
+ r = timerfd_settime(s->pickup_timer_fd, 0, &ts, NULL);
+ if (r < 0) {
+ log_error("Failed to setup pickup timer: %m");
+ return -1;
+ }
+
+ return 0;
+}
+
+
int server_schedule_sync(Server *s, int priority) {
int r;
@@ -1452,6 +1549,9 @@ int server_init(Server *s) {
s->sync_interval_usec = DEFAULT_SYNC_INTERVAL_USEC;
s->sync_scheduled = false;
+ s->pickup_interval_sec = DEFAULT_PICKUP_INTERVAL_SEC;
+ s->pickup_startup_delay_sec = PICKUP_STARTUP_DELAY_SEC;
+
s->rate_limit_interval = DEFAULT_RATE_LIMIT_INTERVAL;
s->rate_limit_burst = DEFAULT_RATE_LIMIT_BURST;
@@ -1555,6 +1655,10 @@ int server_init(Server *s) {
if (r < 0)
return r;
+ r = server_open_pickup_timer(s);
+ if (r < 0)
+ return r;
+
r = open_signalfd(s);
if (r < 0)
return r;
@@ -1630,6 +1734,9 @@ void server_done(Server *s) {
if (s->sync_timer_fd >= 0)
close_nointr_nofail(s->sync_timer_fd);
+ if (s->pickup_timer_fd >= 0)
+ close_nointr_nofail(s->pickup_timer_fd);
+
if (s->rate_limit)
journal_rate_limit_free(s->rate_limit);
diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h
index db577b8..5ae4376 100644
--- a/src/journal/journald-server.h
+++ b/src/journal/journald-server.h
@@ -75,6 +75,9 @@ typedef struct Server {
usec_t rate_limit_interval;
unsigned rate_limit_burst;
+ usec_t pickup_interval_sec;
+ usec_t pickup_startup_delay_sec;
+
JournalMetrics runtime_metrics;
JournalMetrics system_metrics;
@@ -118,6 +121,8 @@ typedef struct Server {
struct udev *udev;
+ int pickup_timer_fd;
+
int sync_timer_fd;
bool sync_scheduled;
} Server;
diff --git a/src/journal/journald.conf b/src/journal/journald.conf
index 54f6833..99e7c07 100644
--- a/src/journal/journald.conf
+++ b/src/journal/journald.conf
@@ -13,6 +13,8 @@
#Seal=yes
#SplitMode=login
#SyncIntervalSec=5m
+#PickupIntervalSec=5
+#PickupStartupDelaySec=5
#RateLimitInterval=30s
#RateLimitBurst=1000
#SystemMaxUse=
--
1.8.4.rc3
More information about the systemd-devel
mailing list