[systemd-commits] 2 commits - src/journal

[Michal Schmidt]

 src/journal/sd-journal.c |   47 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

New commits:
commit f8b5a3b75fb55f0acb85c21424b3893c822742e9
Author: Michal Schmidt <mschmidt at redhat.com>
Date:   Tue Dec 9 20:27:26 2014 +0100

    journal: optimize iteration: skip files that cannot improve current candidate entry
    
    Suppose that while iterating we have already looked into a journal file
    and got a candidate for the next entry. And we are considering to look
    into another journal file because it may contain an entry that is nearer
    to the current location than the candidate.
    
    We should skip the whole journal file if we can tell by looking at its
    header that none of its entries can precede the candidate.
    
    Before:
    $ time ./journalctl --since=2014-06-01 --until=2014-07-01 > /dev/null
    
    real    0m20.518s
    user    0m19.989s
    sys     0m0.328s
    
    After:
    $ time ./journalctl --since=2014-06-01 --until=2014-07-01 > /dev/null
    
    real    0m9.445s
    user    0m9.228s
    sys     0m0.213s

diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index b937daf..5e2da99 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -517,6 +517,27 @@ static bool whole_file_precedes_location(JournalFile *f, Location *l, direction_
         return false;
 }
 
+static bool file_may_have_preceding_entry(JournalFile *f, JournalFile *of, uint64_t op, direction_t direction) {
+        Object *o;
+        int r;
+
+        assert(f);
+        assert(of);
+
+        r = journal_file_move_to_object(of, OBJECT_ENTRY, op, &o);
+        if (r < 0)
+                return true;
+
+        if (sd_id128_equal(f->header->seqnum_id, of->header->seqnum_id))
+                return direction == DIRECTION_DOWN ?
+                        le64toh(o->entry.seqnum) >= le64toh(f->header->head_entry_seqnum) :
+                        le64toh(o->entry.seqnum) <= le64toh(f->header->tail_entry_seqnum);
+
+        return direction == DIRECTION_DOWN ?
+                le64toh(o->entry.realtime) >= le64toh(f->header->head_entry_realtime) :
+                le64toh(o->entry.realtime) <= le64toh(f->header->tail_entry_realtime);
+}
+
 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
         uint64_t a;
 
@@ -905,6 +926,9 @@ static int real_journal_next(sd_journal *j, direction_t direction) {
                 if (whole_file_precedes_location(f, &j->current_location, direction))
                         continue;
 
+                if (new_file && !file_may_have_preceding_entry(f, new_file, new_offset, direction))
+                        continue;
+
                 r = next_beyond_location(j, f, direction, &o, &p);
                 if (r < 0) {
                         log_debug_errno(r, "Can't iterate through %s, ignoring: %m", f->path);

commit b7c88ab8cc7d55a43450bf3dea750f95f2e910d6
Author: Michal Schmidt <mschmidt at redhat.com>
Date:   Tue Dec 9 20:27:19 2014 +0100

    journal: optimize iteration: skip whole files behind current location
    
    Interleaving of entries from many journal files is expensive. But there
    is room for optimization.
    
    We can skip looking into journal files whose entries all lie before the
    current iterating location. We can tell if that's the case from looking
    at the journal file header. This saves a huge amount of work if one has
    many of mostly not interleaved journal files.
    
    On my workstation with 90 journal files in /var/log/journal/ID/
    totalling 3.4 GB I get these results:
    
    Before:
    $ time ./journalctl --since=2014-06-01 --until=2014-07-01 > /dev/null
    
    real    5m54.258s
    user    2m4.263s
    sys     3m48.965s
    
    After:
    $ time ./journalctl --since=2014-06-01 --until=2014-07-01 > /dev/null
    
    real    0m20.518s
    user    0m19.989s
    sys     0m0.328s
    
    The high "sys" time in the original was caused by putting more stress on
    the mmap-cache than it could handle. With the patch the working set
    now consists of fewer mmap windows and mmap-cache is not thrashing.

diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index bb410ed..b937daf 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -497,6 +497,26 @@ static int compare_entry_order(JournalFile *af, Object *_ao,
         return 0;
 }
 
+static bool whole_file_precedes_location(JournalFile *f, Location *l, direction_t direction) {
+        assert(f);
+        assert(l);
+
+        if (l->type != LOCATION_DISCRETE && l->type != LOCATION_SEEK)
+                return false;
+
+        if (l->seqnum_set && sd_id128_equal(l->seqnum_id, f->header->seqnum_id))
+                return direction == DIRECTION_DOWN ?
+                        l->seqnum > le64toh(f->header->tail_entry_seqnum) :
+                        l->seqnum < le64toh(f->header->head_entry_seqnum);
+
+        if (l->realtime_set)
+                return direction == DIRECTION_DOWN ?
+                        l->realtime > le64toh(f->header->tail_entry_realtime) :
+                        l->realtime < le64toh(f->header->head_entry_realtime);
+
+        return false;
+}
+
 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
         uint64_t a;
 
@@ -882,6 +902,9 @@ static int real_journal_next(sd_journal *j, direction_t direction) {
         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
                 bool found;
 
+                if (whole_file_precedes_location(f, &j->current_location, direction))
+                        continue;
+
                 r = next_beyond_location(j, f, direction, &o, &p);
                 if (r < 0) {
                         log_debug_errno(r, "Can't iterate through %s, ignoring: %m", f->path);