[Libreoffice-commits] online.git: 2 commits - common/Authorization.cpp common/Message.hpp common/MessageQueue.cpp common/Protocol.cpp common/Protocol.hpp common/Util.hpp kit/ChildSession.cpp kit/ForKit.cpp kit/Kit.cpp test/helpers.hpp test/TileCacheTests.cpp test/UnitAdmin.cpp test/UnitBadDocLoad.cpp test/UnitClose.cpp test/UnitCursor.cpp test/UnitInsertDelete.cpp test/UnitLoad.cpp test/UnitPasswordProtected.cpp test/UnitRenderingOptions.cpp test/UnitSession.cpp test/UnitTyping.cpp test/UnitWOPIWatermark.cpp test/WhiteBoxTests.cpp tools/Connect.cpp tools/KitClient.cpp tools/WebSocketDump.cpp wsd/Admin.cpp wsd/Auth.cpp wsd/ClientSession.cpp wsd/DocumentBroker.cpp wsd/FileServer.cpp wsd/LOOLWSD.cpp wsd/Storage.cpp wsd/TileCache.cpp wsd/TileDesc.hpp wsd/TraceFile.hpp

Ashod Nakashian (via logerrit) logerrit at kemper.freedesktop.org
Tue Jun 2 17:05:39 UTC 2020


 common/Authorization.cpp       |    2 
 common/Message.hpp             |    6 +-
 common/MessageQueue.cpp        |   12 ++---
 common/Protocol.cpp            |    6 +-
 common/Protocol.hpp            |   96 +++++++----------------------------------
 common/Util.hpp                |   76 ++++++++++++++++++++++++++++++++
 kit/ChildSession.cpp           |   11 ++--
 kit/ForKit.cpp                 |    4 -
 kit/Kit.cpp                    |   10 ++--
 test/TileCacheTests.cpp        |    8 +--
 test/UnitAdmin.cpp             |   16 +++---
 test/UnitBadDocLoad.cpp        |    2 
 test/UnitClose.cpp             |    2 
 test/UnitCursor.cpp            |    2 
 test/UnitInsertDelete.cpp      |    6 +-
 test/UnitLoad.cpp              |    2 
 test/UnitPasswordProtected.cpp |    4 -
 test/UnitRenderingOptions.cpp  |    2 
 test/UnitSession.cpp           |    2 
 test/UnitTyping.cpp            |    2 
 test/UnitWOPIWatermark.cpp     |    2 
 test/WhiteBoxTests.cpp         |   30 ++++++------
 test/helpers.hpp               |    4 -
 tools/Connect.cpp              |    2 
 tools/KitClient.cpp            |    2 
 tools/WebSocketDump.cpp        |    2 
 wsd/Admin.cpp                  |    8 +--
 wsd/Auth.cpp                   |    2 
 wsd/ClientSession.cpp          |    8 +--
 wsd/DocumentBroker.cpp         |    4 -
 wsd/FileServer.cpp             |    2 
 wsd/LOOLWSD.cpp                |    4 -
 wsd/Storage.cpp                |    9 ++-
 wsd/TileCache.cpp              |    2 
 wsd/TileDesc.hpp               |   16 +++---
 wsd/TraceFile.hpp              |    2 
 36 files changed, 196 insertions(+), 174 deletions(-)

New commits:
commit d2d0492245f85ea8e19cdab7b3608f6b4c31212b
Author:     Ashod Nakashian <ashod.nakashian at collabora.co.uk>
AuthorDate: Thu May 21 10:22:49 2020 -0400
Commit:     Michael Meeks <michael.meeks at collabora.com>
CommitDate: Tue Jun 2 18:03:36 2020 +0100

    wsd: move LOOLProtocol::tokenize to Util::tokenize
    
    The tokenizer(s) are more generic than the protocol
    logic, and are used from contexts that don't involve
    the protocol as such.
    
    Change-Id: Ie8c256bf11a91e466bff794021f41603c9596a7f

diff --git a/common/Authorization.cpp b/common/Authorization.cpp
index cb605fd41..abaddae26 100644
--- a/common/Authorization.cpp
+++ b/common/Authorization.cpp
@@ -53,7 +53,7 @@ void Authorization::authorizeRequest(Poco::Net::HTTPRequest& request) const
             //   Authorization: Basic ....
             //   X-Something-Custom: Huh
             // Regular expression evaluates and finds "\n\r" and tokenizes accordingly
-            StringVector tokens(LOOLProtocol::tokenize(_data, "\n\r"));
+            StringVector tokens(Util::tokenize(_data, "\n\r"));
             for (auto it = tokens.begin(); it != tokens.end(); ++it)
             {
                 std::string token = tokens.getParam(*it);
diff --git a/common/Message.hpp b/common/Message.hpp
index 8fc894b96..320760c5b 100644
--- a/common/Message.hpp
+++ b/common/Message.hpp
@@ -31,7 +31,7 @@ public:
             const enum Dir dir) :
         _forwardToken(getForwardToken(message.data(), message.size())),
         _data(skipWhitespace(message.data() + _forwardToken.size()), message.data() + message.size()),
-        _tokens(LOOLProtocol::tokenize(_data.data(), _data.size())),
+        _tokens(Util::tokenize(_data.data(), _data.size())),
         _id(makeId(dir)),
         _firstLine(LOOLProtocol::getFirstLine(_data.data(), _data.size())),
         _abbr(_id + ' ' + LOOLProtocol::getAbbreviatedMessage(_data.data(), _data.size())),
@@ -48,7 +48,7 @@ public:
             const size_t reserve) :
         _forwardToken(getForwardToken(message.data(), message.size())),
         _data(std::max(reserve, message.size())),
-        _tokens(LOOLProtocol::tokenize(message.data() + _forwardToken.size(), message.size() - _forwardToken.size())),
+        _tokens(Util::tokenize(message.data() + _forwardToken.size(), message.size() - _forwardToken.size())),
         _id(makeId(dir)),
         _firstLine(LOOLProtocol::getFirstLine(message)),
         _abbr(_id + ' ' + LOOLProtocol::getAbbreviatedMessage(message)),
@@ -67,7 +67,7 @@ public:
             const enum Dir dir) :
         _forwardToken(getForwardToken(p, len)),
         _data(skipWhitespace(p + _forwardToken.size()), p + len),
-        _tokens(LOOLProtocol::tokenize(_data.data(), _data.size())),
+        _tokens(Util::tokenize(_data.data(), _data.size())),
         _id(makeId(dir)),
         _firstLine(LOOLProtocol::getFirstLine(_data.data(), _data.size())),
         _abbr(_id + ' ' + LOOLProtocol::getAbbreviatedMessage(_data.data(), _data.size())),
diff --git a/common/MessageQueue.cpp b/common/MessageQueue.cpp
index b171ce553..a57525984 100644
--- a/common/MessageQueue.cpp
+++ b/common/MessageQueue.cpp
@@ -30,7 +30,7 @@ void TileQueue::put_impl(const Payload& value)
     {
         LOG_TRC("Processing [" << LOOLProtocol::getAbbreviatedMessage(msg) << "]. Before canceltiles have " << getQueue().size() << " in queue.");
         const std::string seqs = msg.substr(12);
-        StringVector tokens(LOOLProtocol::tokenize(seqs, ','));
+        StringVector tokens(Util::tokenize(seqs, ','));
         getQueue().erase(std::remove_if(getQueue().begin(), getQueue().end(),
                 [&tokens](const Payload& v)
                 {
@@ -186,7 +186,7 @@ std::string TileQueue::removeCallbackDuplicate(const std::string& callbackMsg)
 {
     assert(LOOLProtocol::matchPrefix("callback", callbackMsg, /*ignoreWhitespace*/ true));
 
-    StringVector tokens = LOOLProtocol::tokenize(callbackMsg);
+    StringVector tokens = Util::tokenize(callbackMsg);
 
     if (tokens.size() < 3)
         return std::string();
@@ -211,7 +211,7 @@ std::string TileQueue::removeCallbackDuplicate(const std::string& callbackMsg)
         {
             auto& it = getQueue()[i];
 
-            StringVector queuedTokens = LOOLProtocol::tokenize(it.data(), it.size());
+            StringVector queuedTokens = Util::tokenize(it.data(), it.size());
             if (queuedTokens.size() < 3)
             {
                 ++i;
@@ -321,7 +321,7 @@ std::string TileQueue::removeCallbackDuplicate(const std::string& callbackMsg)
         {
             auto& it = getQueue()[i];
 
-            StringVector queuedTokens = LOOLProtocol::tokenize(it.data(), it.size());
+            StringVector queuedTokens = Util::tokenize(it.data(), it.size());
             if (queuedTokens.size() < 4)
                 continue;
 
@@ -361,13 +361,13 @@ std::string TileQueue::removeCallbackDuplicate(const std::string& callbackMsg)
 
         for (size_t i = 0; i < getQueue().size(); ++i)
         {
-            auto& it = getQueue()[i];
+            const auto& it = getQueue()[i];
 
             // skip non-callbacks quickly
             if (!LOOLProtocol::matchPrefix("callback", it))
                 continue;
 
-            StringVector queuedTokens = LOOLProtocol::tokenize(it.data(), it.size());
+            StringVector queuedTokens = Util::tokenize(it.data(), it.size());
             if (queuedTokens.size() < 3)
                 continue;
 
diff --git a/common/Protocol.cpp b/common/Protocol.cpp
index 7e5208774..771b87b49 100644
--- a/common/Protocol.cpp
+++ b/common/Protocol.cpp
@@ -27,7 +27,7 @@ namespace LOOLProtocol
         int minor = -1;
         std::string patch;
 
-        StringVector firstTokens(tokenize(version, '.'));
+        StringVector firstTokens(Util::tokenize(version, '.'));
         if (firstTokens.size() > 0)
         {
             major = std::stoi(firstTokens[0]);
@@ -35,7 +35,7 @@ namespace LOOLProtocol
             StringVector secondTokens;
             if (firstTokens.size() > 1)
             {
-                secondTokens = tokenize(firstTokens[1], '-');
+                secondTokens = Util::tokenize(firstTokens[1], '-');
             }
             if (secondTokens.size() > 0)
             {
@@ -215,7 +215,7 @@ namespace LOOLProtocol
 
     bool getTokenKeywordFromMessage(const std::string& message, const std::string& name, const std::map<std::string, int>& map, int& value)
     {
-        return getTokenKeyword(tokenize(message), name, map, value);
+        return getTokenKeyword(Util::tokenize(message), name, map, value);
     }
 };
 
diff --git a/common/Protocol.hpp b/common/Protocol.hpp
index f0ad8039f..5eb90941b 100644
--- a/common/Protocol.hpp
+++ b/common/Protocol.hpp
@@ -111,83 +111,6 @@ namespace LOOLProtocol
     bool getTokenStringFromMessage(const std::string& message, const std::string& name, std::string& value);
     bool getTokenKeywordFromMessage(const std::string& message, const std::string& name, const std::map<std::string, int>& map, int& value);
 
-    /// Tokenize space-delimited values until we hit new-line or the end.
-    template <typename T>
-    inline void tokenize(const char* data, const std::size_t size, const T& delimiter,
-                         std::vector<StringToken>& tokens)
-    {
-        if (size == 0 || data == nullptr || *data == '\0')
-            return;
-
-        tokens.reserve(16);
-
-        const char* start = data;
-        const char* end = data;
-        for (std::size_t i = 0; i < size && data[i] != '\n'; ++i, ++end)
-        {
-            if (data[i] == delimiter)
-            {
-                if (start != end && *start != delimiter)
-                    tokens.emplace_back(start - data, end - start);
-
-                start = end;
-            }
-            else if (*start == delimiter)
-                ++start;
-        }
-
-        if (start != end && *start != delimiter && *start != '\n')
-            tokens.emplace_back(start - data, end - start);
-    }
-
-    /// Tokenize single-char delimited values until we hit new-line or the end.
-    inline StringVector tokenize(const char* data, const std::size_t size,
-                                 const char delimiter = ' ')
-    {
-        if (size == 0 || data == nullptr || *data == '\0')
-            return StringVector();
-
-        std::vector<StringToken> tokens;
-        tokenize(data, size, delimiter, tokens);
-        return StringVector(std::string(data, size), tokens);
-    }
-
-    /// Tokenize single-char delimited values until we hit new-line or the end.
-    inline StringVector tokenize(const std::string& s, const char delimiter = ' ')
-    {
-        if (s.empty())
-            return StringVector();
-
-        std::vector<StringToken> tokens;
-        tokenize(s.data(), s.size(), delimiter, tokens);
-        return StringVector(s, tokens);
-    }
-
-    inline
-    StringVector tokenize(const std::string& s, const char* delimiter)
-    {
-        if (s.empty())
-            return StringVector();
-
-        std::size_t start = 0;
-        std::size_t end = s.find(delimiter, start);
-
-        std::vector<StringToken> tokens;
-        tokens.reserve(16);
-
-        tokens.emplace_back(start, end - start);
-        start = end + std::strlen(delimiter);
-
-        while(end != std::string::npos)
-        {
-            end = s.find(delimiter, start);
-            tokens.emplace_back(start, end - start);
-            start = end + std::strlen(delimiter);
-        }
-
-        return StringVector(s, tokens);
-    }
-
     inline
     std::vector<int> tokenizeInts(const char* data, const size_t size, const char delimiter = ',')
     {
@@ -224,7 +147,7 @@ namespace LOOLProtocol
 
     inline bool getTokenIntegerFromMessage(const std::string& message, const std::string& name, int& value)
     {
-        return getTokenInteger(tokenize(message), name, value);
+        return getTokenInteger(Util::tokenize(message), name, value);
     }
 
     /// Returns the first token of a message.
diff --git a/common/Util.hpp b/common/Util.hpp
index 46a88fec0..b4c5c1234 100644
--- a/common/Util.hpp
+++ b/common/Util.hpp
@@ -382,6 +382,82 @@ namespace Util
         return false;
     }
 
+    /// Tokenize space-delimited values until we hit new-line or the end.
+    template <typename T>
+    inline void tokenize(const char* data, const std::size_t size, const T& delimiter,
+                         std::vector<StringToken>& tokens)
+    {
+        if (size == 0 || data == nullptr || *data == '\0')
+            return;
+
+        tokens.reserve(16);
+
+        const char* start = data;
+        const char* end = data;
+        for (std::size_t i = 0; i < size && data[i] != '\n'; ++i, ++end)
+        {
+            if (data[i] == delimiter)
+            {
+                if (start != end && *start != delimiter)
+                    tokens.emplace_back(start - data, end - start);
+
+                start = end;
+            }
+            else if (*start == delimiter)
+                ++start;
+        }
+
+        if (start != end && *start != delimiter && *start != '\n')
+            tokens.emplace_back(start - data, end - start);
+    }
+
+    /// Tokenize single-char delimited values until we hit new-line or the end.
+    inline StringVector tokenize(const char* data, const std::size_t size,
+                                 const char delimiter = ' ')
+    {
+        if (size == 0 || data == nullptr || *data == '\0')
+            return StringVector();
+
+        std::vector<StringToken> tokens;
+        tokenize(data, size, delimiter, tokens);
+        return StringVector(std::string(data, size), std::move(tokens));
+    }
+
+    /// Tokenize single-char delimited values until we hit new-line or the end.
+    inline StringVector tokenize(const std::string& s, const char delimiter = ' ')
+    {
+        if (s.empty())
+            return StringVector();
+
+        std::vector<StringToken> tokens;
+        tokenize(s.data(), s.size(), delimiter, tokens);
+        return StringVector(s, std::move(tokens));
+    }
+
+    inline StringVector tokenize(const std::string& s, const char* delimiter)
+    {
+        if (s.empty())
+            return StringVector();
+
+        std::size_t start = 0;
+        std::size_t end = s.find(delimiter, start);
+
+        std::vector<StringToken> tokens;
+        tokens.reserve(16);
+
+        tokens.emplace_back(start, end - start);
+        start = end + std::strlen(delimiter);
+
+        while (end != std::string::npos)
+        {
+            end = s.find(delimiter, start);
+            tokens.emplace_back(start, end - start);
+            start = end + std::strlen(delimiter);
+        }
+
+        return StringVector(s, std::move(tokens));
+    }
+
 #ifdef IOS
 
     inline void *memrchr(const void *s, int c, size_t n)
diff --git a/kit/ChildSession.cpp b/kit/ChildSession.cpp
index 74e267cde..ac1d1ec4f 100644
--- a/kit/ChildSession.cpp
+++ b/kit/ChildSession.cpp
@@ -114,7 +114,7 @@ bool ChildSession::_handleInput(const char *buffer, int length)
 {
     LOG_TRC(getName() << ": handling [" << getAbbreviatedMessage(buffer, length) << "].");
     const std::string firstLine = getFirstLine(buffer, length);
-    const StringVector tokens = LOOLProtocol::tokenize(firstLine.data(), firstLine.size());
+    const StringVector tokens = Util::tokenize(firstLine.data(), firstLine.size());
 
     if (LOOLProtocol::tokenIndicatesUserInteraction(tokens[0]))
     {
@@ -1572,7 +1572,7 @@ bool ChildSession::renderWindow(const char* /*buffer*/, int /*length*/, const St
         && paintRectangle != "undefined")
     {
         const StringVector rectParts
-            = LOOLProtocol::tokenize(paintRectangle.c_str(), paintRectangle.length(), ',');
+            = Util::tokenize(paintRectangle.c_str(), paintRectangle.length(), ',');
         if (rectParts.size() == 4)
         {
             startX = std::atoi(rectParts[0].c_str());
@@ -2366,7 +2366,7 @@ void ChildSession::loKitCallback(const int type, const std::string& payload)
     {
     case LOK_CALLBACK_INVALIDATE_TILES:
         {
-            StringVector tokens(LOOLProtocol::tokenize(payload, ','));
+            StringVector tokens(Util::tokenize(payload, ','));
             if (tokens.size() == 5)
             {
                 int part, x, y, width, height;
diff --git a/kit/ForKit.cpp b/kit/ForKit.cpp
index c5bd274dc..3a0a0811d 100644
--- a/kit/ForKit.cpp
+++ b/kit/ForKit.cpp
@@ -90,7 +90,7 @@ protected:
         if (UnitKit::get().filterKitMessage(this, message))
             return;
 #endif
-        StringVector tokens = LOOLProtocol::tokenize(message);
+        StringVector tokens = Util::tokenize(message);
         Log::StreamLogger logger = Log::debug();
         if (logger.enabled())
         {
@@ -500,7 +500,7 @@ int main(int argc, char** argv)
         {
             eq = std::strchr(cmd, '=');
             const std::string rlimits = std::string(eq+1);
-            StringVector tokens = LOOLProtocol::tokenize(rlimits, ';');
+            StringVector tokens = Util::tokenize(rlimits, ';');
             for (const auto& cmdLimit : tokens)
             {
                 const std::pair<std::string, std::string> pair = Util::split(tokens.getParam(cmdLimit), ':');
diff --git a/kit/Kit.cpp b/kit/Kit.cpp
index 367ca60c0..9fbc658b4 100644
--- a/kit/Kit.cpp
+++ b/kit/Kit.cpp
@@ -1307,7 +1307,7 @@ public:
 
         if (type == LOK_CALLBACK_CELL_CURSOR)
         {
-            StringVector tokens(LOOLProtocol::tokenize(payload, ','));
+            StringVector tokens(Util::tokenize(payload, ','));
             // Payload may be 'EMPTY'.
             if (tokens.size() == 4)
             {
@@ -1325,7 +1325,7 @@ public:
             const Poco::Dynamic::Var result = parser.parse(payload);
             const auto& command = result.extract<Poco::JSON::Object::Ptr>();
             std::string rectangle = command->get("rectangle").toString();
-            StringVector tokens(LOOLProtocol::tokenize(rectangle, ','));
+            StringVector tokens(Util::tokenize(rectangle, ','));
             // Payload may be 'EMPTY'.
             if (tokens.size() == 4)
             {
@@ -1346,7 +1346,7 @@ public:
             targetViewId = command->get("viewId").toString();
             std::string part = command->get("part").toString();
             std::string text = command->get("rectangle").toString();
-            StringVector tokens(LOOLProtocol::tokenize(text, ','));
+            StringVector tokens(Util::tokenize(text, ','));
             // Payload may be 'EMPTY'.
             if (tokens.size() == 4)
             {
@@ -1941,7 +1941,7 @@ public:
 
                 LOG_TRC("Kit handling queue message: " << LOOLProtocol::getAbbreviatedMessage(input));
 
-                const StringVector tokens = LOOLProtocol::tokenize(input.data(), input.size());
+                const StringVector tokens = Util::tokenize(input.data(), input.size());
 
                 if (tokens.equals(0, "eof"))
                 {
@@ -2259,7 +2259,7 @@ protected:
         if (UnitKit::get().filterKitMessage(this, message))
             return;
 #endif
-        StringVector tokens = LOOLProtocol::tokenize(message);
+        StringVector tokens = Util::tokenize(message);
         Log::StreamLogger logger = Log::debug();
         if (logger.enabled())
         {
diff --git a/test/TileCacheTests.cpp b/test/TileCacheTests.cpp
index 9df566372..5a14a4c09 100644
--- a/test/TileCacheTests.cpp
+++ b/test/TileCacheTests.cpp
@@ -1090,7 +1090,7 @@ void TileCacheTests::checkTiles(std::shared_ptr<LOOLWebSocket>& socket, const st
         std::istringstream istr(response.substr(8));
         std::getline(istr, line);
 
-        StringVector tokens(LOOLProtocol::tokenize(line, ' '));
+        StringVector tokens(Util::tokenize(line, ' '));
 #if defined CPPUNIT_ASSERT_GREATEREQUAL
         if (docType == "presentation")
             CPPUNIT_ASSERT_GREATEREQUAL(static_cast<size_t>(7), tokens.size()); // We have an extra field.
@@ -1200,7 +1200,7 @@ void TileCacheTests::requestTiles(std::shared_ptr<LOOLWebSocket>& socket,
             sendTextFrame(socket, text, name);
             tile = assertResponseString(socket, "tile:", name);
             // expected tile: part= width= height= tileposx= tileposy= tilewidth= tileheight=
-            StringVector tokens(LOOLProtocol::tokenize(tile, ' '));
+            StringVector tokens(Util::tokenize(tile, ' '));
             LOK_ASSERT_EQUAL(std::string("tile:"), tokens[0]);
             LOK_ASSERT_EQUAL(0, std::stoi(tokens[1].substr(std::string("nviewid=").size())));
             LOK_ASSERT_EQUAL(part, std::stoi(tokens[2].substr(std::string("part=").size())));
@@ -1348,7 +1348,7 @@ void TileCacheTests::testTileProcessed()
             ++arrivedTile;
 
             // Store tileID, so we can send it back
-            StringVector tokens(LOOLProtocol::tokenize(tile, ' '));
+            StringVector tokens(Util::tokenize(tile, ' '));
             std::string tileID = tokens[2].substr(std::string("part=").size()) + ':' +
                                  tokens[5].substr(std::string("tileposx=").size()) + ':' +
                                  tokens[6].substr(std::string("tileposy=").size()) + ':' +
@@ -1396,7 +1396,7 @@ void TileCacheTests::testTileInvalidatedOutside()
 
     // First wsd forwards the invalidation
     std::string sInvalidate = assertResponseString(socket, "invalidatetiles:", testname);
-    StringVector tokens(LOOLProtocol::tokenize(sInvalidate, ' '));
+    StringVector tokens(Util::tokenize(sInvalidate, ' '));
     int y = std::stoi(tokens[3].substr(std::string("y=").size()));
     int height = std::stoi(tokens[5].substr(std::string("height=").size()));
 
diff --git a/test/UnitAdmin.cpp b/test/UnitAdmin.cpp
index 7c1eb0eea..d6111df5b 100644
--- a/test/UnitAdmin.cpp
+++ b/test/UnitAdmin.cpp
@@ -142,7 +142,7 @@ private:
         }
         lock.unlock();
 
-        StringVector tokens(LOOLProtocol::tokenize(_messageReceived, ' '));
+        StringVector tokens(Util::tokenize(_messageReceived, ' '));
         if (tokens.size() != 1 ||
             tokens[0] != "NotAuthenticated")
         {
@@ -173,7 +173,7 @@ private:
         }
         lock.unlock();
 
-        StringVector tokens(LOOLProtocol::tokenize(_messageReceived, ' '));
+        StringVector tokens(Util::tokenize(_messageReceived, ' '));
         if (tokens.size() != 1 ||
             tokens[0] != "InvalidAuthToken")
         {
@@ -225,7 +225,7 @@ private:
         lock.unlock();
 
         {
-            StringVector tokens(LOOLProtocol::tokenize(_messageReceived, ' '));
+            StringVector tokens(Util::tokenize(_messageReceived, ' '));
             if (tokens.size() != 5 ||
                 tokens[0] != "adddoc" ||
                 tokens[2] != documentPath1.substr(documentPath1.find_last_of('/') + 1) )
@@ -253,7 +253,7 @@ private:
         lock.unlock();
 
         {
-            StringVector tokens(LOOLProtocol::tokenize(_messageReceived, ' '));
+            StringVector tokens(Util::tokenize(_messageReceived, ' '));
             if (tokens.size() != 5 ||
                 tokens[0] != "adddoc" ||
                 tokens[2] != documentPath1.substr(documentPath1.find_last_of('/') + 1) )
@@ -288,7 +288,7 @@ private:
         lock.unlock();
 
         {
-            StringVector tokens(LOOLProtocol::tokenize(_messageReceived, ' '));
+            StringVector tokens(Util::tokenize(_messageReceived, ' '));
             if (tokens.size() != 5 ||
                 tokens[0] != "adddoc" ||
                 tokens[2] != documentPath2.substr(documentPath2.find_last_of('/') + 1) )
@@ -321,7 +321,7 @@ private:
         }
         lock.unlock();
 
-        StringVector tokens(LOOLProtocol::tokenize(_messageReceived, ' '));
+        StringVector tokens(Util::tokenize(_messageReceived, ' '));
         if (tokens.size() != 2 ||
             tokens[0] != "active_users_count")
         {
@@ -354,7 +354,7 @@ private:
         }
         lock.unlock();
 
-        StringVector tokens(LOOLProtocol::tokenize(_messageReceived, ' '));
+        StringVector tokens(Util::tokenize(_messageReceived, ' '));
         if (tokens.size() != 2 ||
             tokens[0] != "active_docs_count" ||
             std::stoi(tokens[1]) != _docsCount)
@@ -390,7 +390,7 @@ private:
         }
         lock.unlock();
 
-        StringVector tokens(LOOLProtocol::tokenize(_messageReceived, ' '));
+        StringVector tokens(Util::tokenize(_messageReceived, ' '));
         if (tokens.size() != 3 ||
             tokens[0] != "rmdoc" ||
             stoi(tokens[1]) != _docPid1)
diff --git a/test/UnitBadDocLoad.cpp b/test/UnitBadDocLoad.cpp
index 3f4110dfc..550191f69 100644
--- a/test/UnitBadDocLoad.cpp
+++ b/test/UnitBadDocLoad.cpp
@@ -57,7 +57,7 @@ UnitBase::TestResult UnitBadDocLoad::testBadDocLoadFail()
         helpers::sendTextFrame(socket, "load url=" + documentURL, testname);
 
         const auto response = helpers::getResponseString(socket, "error:", testname);
-        StringVector tokens(LOOLProtocol::tokenize(response, ' '));
+        StringVector tokens(Util::tokenize(response, ' '));
         LOK_ASSERT_EQUAL(static_cast<size_t>(3), tokens.size());
 
         std::string errorCommand;
diff --git a/test/UnitClose.cpp b/test/UnitClose.cpp
index bfacf6b8e..eba8378e6 100644
--- a/test/UnitClose.cpp
+++ b/test/UnitClose.cpp
@@ -194,7 +194,7 @@ UnitBase::TestResult UnitClose::testAlertAllUsers()
         {
             const std::string response
                 = helpers::assertResponseString(socket[i], "error:", testname);
-            StringVector tokens(LOOLProtocol::tokenize(response.substr(6), ' '));
+            StringVector tokens(Util::tokenize(response.substr(6), ' '));
             std::string cmd;
             LOOLProtocol::getTokenString(tokens, "cmd", cmd);
             LOK_ASSERT_EQUAL(std::string("internal"), cmd);
diff --git a/test/UnitCursor.cpp b/test/UnitCursor.cpp
index 36a9512b2..c8f177ad5 100644
--- a/test/UnitCursor.cpp
+++ b/test/UnitCursor.cpp
@@ -34,7 +34,7 @@ void getCursor(const std::string& message, int& cursorX, int& cursorY, int& curs
     LOK_ASSERT_EQUAL(std::string(".uno:CellCursor"), text);
     text = command->get("commandValues").toString();
     LOK_ASSERT(!text.empty());
-    StringVector position(LOOLProtocol::tokenize(text, ','));
+    StringVector position(Util::tokenize(text, ','));
     cursorX = std::stoi(position[0]);
     cursorY = std::stoi(position[1]);
     cursorWidth = std::stoi(position[2]);
diff --git a/test/UnitInsertDelete.cpp b/test/UnitInsertDelete.cpp
index e5b3bc6d8..962d70508 100644
--- a/test/UnitInsertDelete.cpp
+++ b/test/UnitInsertDelete.cpp
@@ -35,7 +35,7 @@ void getPartHashCodes(const std::string& testname, const std::string& response,
     TST_LOG("Reading parts from [" << response << "].");
 
     // Expected format is something like 'type= parts= current= width= height= viewid= [hiddenparts=]'.
-    StringVector tokens(LOOLProtocol::tokenize(line, ' '));
+    StringVector tokens(Util::tokenize(line, ' '));
 #if defined CPPUNIT_ASSERT_GREATEREQUAL
     CPPUNIT_ASSERT_GREATEREQUAL(static_cast<size_t>(7), tokens.size());
 #else
@@ -275,7 +275,7 @@ UnitBase::TestResult UnitInsertDelete::testCursorPosition()
         LOK_ASSERT_MESSAGE("missing property rectangle", command0->has("rectangle"));
 
         StringVector cursorTokens(
-            LOOLProtocol::tokenize(command0->get("rectangle").toString(), ','));
+            Util::tokenize(command0->get("rectangle").toString(), ','));
         LOK_ASSERT_EQUAL(static_cast<size_t>(4), cursorTokens.size());
 
         // Create second view
@@ -291,7 +291,7 @@ UnitBase::TestResult UnitInsertDelete::testCursorPosition()
         LOK_ASSERT_MESSAGE("missing property rectangle", command->has("rectangle"));
 
         StringVector viewTokens(
-            LOOLProtocol::tokenize(command->get("rectangle").toString(), ','));
+            Util::tokenize(command->get("rectangle").toString(), ','));
         LOK_ASSERT_EQUAL(static_cast<size_t>(4), viewTokens.size());
 
         // check both cursor should be equal
diff --git a/test/UnitLoad.cpp b/test/UnitLoad.cpp
index 175697487..9179e66cb 100644
--- a/test/UnitLoad.cpp
+++ b/test/UnitLoad.cpp
@@ -159,7 +159,7 @@ UnitBase::TestResult UnitLoad::testExcelLoad()
         const auto status = helpers::assertResponseString(socket, "status:", testname);
 
         // Expected format is something like 'status: type=text parts=2 current=0 width=12808 height=1142 viewid=0\n...'.
-        StringVector tokens(LOOLProtocol::tokenize(status, ' '));
+        StringVector tokens(Util::tokenize(status, ' '));
         LOK_ASSERT_EQUAL(static_cast<size_t>(7), tokens.size());
     }
     catch (const Poco::Exception& exc)
diff --git a/test/UnitPasswordProtected.cpp b/test/UnitPasswordProtected.cpp
index e3ed75ab9..d3126115b 100644
--- a/test/UnitPasswordProtected.cpp
+++ b/test/UnitPasswordProtected.cpp
@@ -51,7 +51,7 @@ UnitBase::TestResult UnitPasswordProtected::testPasswordProtectedDocumentWithout
         helpers::sendTextFrame(socket, "load url=" + documentURL);
 
         const auto response = helpers::getResponseString(socket, "error:", testname);
-        StringVector tokens(LOOLProtocol::tokenize(response, ' '));
+        StringVector tokens(Util::tokenize(response, ' '));
         LOK_ASSERT_EQUAL(static_cast<size_t>(3), tokens.size());
 
         std::string errorCommand;
@@ -87,7 +87,7 @@ UnitBase::TestResult UnitPasswordProtected::testPasswordProtectedDocumentWithWro
         helpers::sendTextFrame(socket, "load url=" + documentURL + " password=2");
 
         const auto response = helpers::getResponseString(socket, "error:", testname);
-        StringVector tokens(LOOLProtocol::tokenize(response, ' '));
+        StringVector tokens(Util::tokenize(response, ' '));
         LOK_ASSERT_EQUAL(static_cast<size_t>(3), tokens.size());
 
         std::string errorCommand;
diff --git a/test/UnitRenderingOptions.cpp b/test/UnitRenderingOptions.cpp
index b0b839b27..9c0a4ddce 100644
--- a/test/UnitRenderingOptions.cpp
+++ b/test/UnitRenderingOptions.cpp
@@ -51,7 +51,7 @@ void UnitRenderingOptions::invokeTest()
 
         // Expected format is something like 'status: type=text parts=2 current=0 width=12808 height=1142'.
 
-        StringVector tokens(LOOLProtocol::tokenize(status, ' '));
+        StringVector tokens(Util::tokenize(status, ' '));
         LOK_ASSERT_EQUAL(static_cast<size_t>(7), tokens.size());
 
         const std::string token = tokens[5];
diff --git a/test/UnitSession.cpp b/test/UnitSession.cpp
index eabd338da..b81bd7cd3 100644
--- a/test/UnitSession.cpp
+++ b/test/UnitSession.cpp
@@ -184,7 +184,7 @@ UnitBase::TestResult UnitSession::testSlideShow()
         LOK_ASSERT_MESSAGE("did not receive a downloadas: message as expected",
                                !response.empty());
 
-        StringVector tokens(LOOLProtocol::tokenize(response.substr(11), ' '));
+        StringVector tokens(Util::tokenize(response.substr(11), ' '));
         // "downloadas: jail= dir= name=slideshow.svg port= id=slideshow"
         const std::string jail = tokens[0].substr(std::string("jail=").size());
         const std::string dir = tokens[1].substr(std::string("dir=").size());
diff --git a/test/UnitTyping.cpp b/test/UnitTyping.cpp
index ae21c8bcc..1b245ea47 100644
--- a/test/UnitTyping.cpp
+++ b/test/UnitTyping.cpp
@@ -229,7 +229,7 @@ public:
                         {
 // 1544818858022 INCOMING: tile: nviewid=0 part=0 width=256 height=256 tileposx=15360 tileposy=38400 tilewidth=3840 tileheight=3840 oldwid=0 wid=232 ver=913 imgsize=1002
 // Socket.js:123 1544818858027 OUTGOING: tileprocessed tile=0:15360:38400:3840:3840
-                            TileDesc desc = TileDesc::parse(LOOLProtocol::tokenize(tile.data(), tile.size()));
+                            TileDesc desc = TileDesc::parse(Util::tokenize(tile.data(), tile.size()));
                             sendTextFrame(sock, "tileprocessed tile=" + desc.generateID(), testname);
                         }
 
diff --git a/test/UnitWOPIWatermark.cpp b/test/UnitWOPIWatermark.cpp
index 042e22fdb..e5ffad862 100644
--- a/test/UnitWOPIWatermark.cpp
+++ b/test/UnitWOPIWatermark.cpp
@@ -136,7 +136,7 @@ public:
 
                 if(!tile.empty())
                 {
-                    StringVector tokens(LOOLProtocol::tokenize(tile, ' '));
+                    StringVector tokens(Util::tokenize(tile, ' '));
                     std::string nviewid = tokens[1].substr(std::string("nviewid=").size());
                     if (!nviewid.empty() && nviewid != "0")
                     {
diff --git a/test/WhiteBoxTests.cpp b/test/WhiteBoxTests.cpp
index 45b846987..a97451172 100644
--- a/test/WhiteBoxTests.cpp
+++ b/test/WhiteBoxTests.cpp
@@ -90,7 +90,7 @@ void WhiteBoxTests::testLOOLProtocolFunctions()
     LOK_ASSERT_EQUAL(2, mumble);
 
     std::string message("hello x=1 y=2 foo=42 bar=hello-sailor mumble='goodbye' zip zap");
-    StringVector tokens(LOOLProtocol::tokenize(message));
+    StringVector tokens(Util::tokenize(message));
 
     LOK_ASSERT(LOOLProtocol::getTokenInteger(tokens, "foo", foo));
     LOK_ASSERT_EQUAL(42, foo);
@@ -322,51 +322,51 @@ void WhiteBoxTests::testTokenizer()
 {
     StringVector tokens;
 
-    tokens = LOOLProtocol::tokenize("");
+    tokens = Util::tokenize("");
     LOK_ASSERT_EQUAL(static_cast<size_t>(0), tokens.size());
 
-    tokens = LOOLProtocol::tokenize("  ");
+    tokens = Util::tokenize("  ");
     LOK_ASSERT_EQUAL(static_cast<size_t>(0), tokens.size());
 
-    tokens = LOOLProtocol::tokenize("A");
+    tokens = Util::tokenize("A");
     LOK_ASSERT_EQUAL(static_cast<size_t>(1), tokens.size());
     LOK_ASSERT_EQUAL(std::string("A"), tokens[0]);
 
-    tokens = LOOLProtocol::tokenize("  A");
+    tokens = Util::tokenize("  A");
     LOK_ASSERT_EQUAL(static_cast<size_t>(1), tokens.size());
     LOK_ASSERT_EQUAL(std::string("A"), tokens[0]);
 
-    tokens = LOOLProtocol::tokenize("A  ");
+    tokens = Util::tokenize("A  ");
     LOK_ASSERT_EQUAL(static_cast<size_t>(1), tokens.size());
     LOK_ASSERT_EQUAL(std::string("A"), tokens[0]);
 
-    tokens = LOOLProtocol::tokenize(" A ");
+    tokens = Util::tokenize(" A ");
     LOK_ASSERT_EQUAL(static_cast<size_t>(1), tokens.size());
     LOK_ASSERT_EQUAL(std::string("A"), tokens[0]);
 
-    tokens = LOOLProtocol::tokenize(" A  Z ");
+    tokens = Util::tokenize(" A  Z ");
     LOK_ASSERT_EQUAL(static_cast<size_t>(2), tokens.size());
     LOK_ASSERT_EQUAL(std::string("A"), tokens[0]);
     LOK_ASSERT_EQUAL(std::string("Z"), tokens[1]);
 
-    tokens = LOOLProtocol::tokenize("\n");
+    tokens = Util::tokenize("\n");
     LOK_ASSERT_EQUAL(static_cast<size_t>(0), tokens.size());
 
-    tokens = LOOLProtocol::tokenize(" A  \nZ ");
+    tokens = Util::tokenize(" A  \nZ ");
     LOK_ASSERT_EQUAL(static_cast<size_t>(1), tokens.size());
     LOK_ASSERT_EQUAL(std::string("A"), tokens[0]);
 
-    tokens = LOOLProtocol::tokenize(" A  Z\n ");
+    tokens = Util::tokenize(" A  Z\n ");
     LOK_ASSERT_EQUAL(static_cast<size_t>(2), tokens.size());
     LOK_ASSERT_EQUAL(std::string("A"), tokens[0]);
     LOK_ASSERT_EQUAL(std::string("Z"), tokens[1]);
 
-    tokens = LOOLProtocol::tokenize(" A  Z  \n ");
+    tokens = Util::tokenize(" A  Z  \n ");
     LOK_ASSERT_EQUAL(static_cast<size_t>(2), tokens.size());
     LOK_ASSERT_EQUAL(std::string("A"), tokens[0]);
     LOK_ASSERT_EQUAL(std::string("Z"), tokens[1]);
 
-    tokens = LOOLProtocol::tokenize("tile nviewid=0 part=0 width=256 height=256 tileposx=0 tileposy=0 tilewidth=3840 tileheight=3840 ver=-1");
+    tokens = Util::tokenize("tile nviewid=0 part=0 width=256 height=256 tileposx=0 tileposy=0 tilewidth=3840 tileheight=3840 ver=-1");
     LOK_ASSERT_EQUAL(static_cast<size_t>(10), tokens.size());
     LOK_ASSERT_EQUAL(std::string("tile"), tokens[0]);
     LOK_ASSERT_EQUAL(std::string("nviewid=0"), tokens[1]);
@@ -380,11 +380,11 @@ void WhiteBoxTests::testTokenizer()
     LOK_ASSERT_EQUAL(std::string("ver=-1"), tokens[9]);
 
     // With custom delimiters
-    tokens = LOOLProtocol::tokenize(std::string("ABC:DEF"), ':');
+    tokens = Util::tokenize(std::string("ABC:DEF"), ':');
     LOK_ASSERT_EQUAL(std::string("ABC"), tokens[0]);
     LOK_ASSERT_EQUAL(std::string("DEF"), tokens[1]);
 
-    tokens = LOOLProtocol::tokenize(std::string("ABC,DEF,XYZ"), ',');
+    tokens = Util::tokenize(std::string("ABC,DEF,XYZ"), ',');
     LOK_ASSERT_EQUAL(std::string("ABC"), tokens[0]);
     LOK_ASSERT_EQUAL(std::string("DEF"), tokens[1]);
     LOK_ASSERT_EQUAL(std::string("XYZ"), tokens[2]);
diff --git a/test/helpers.hpp b/test/helpers.hpp
index d7bd71fdb..e246e5a57 100644
--- a/test/helpers.hpp
+++ b/test/helpers.hpp
@@ -555,7 +555,7 @@ inline
 void parseDocSize(const std::string& message, const std::string& type,
                   int& part, int& parts, int& width, int& height, int& viewid)
 {
-    StringVector tokens(LOOLProtocol::tokenize(message, ' '));
+    StringVector tokens(Util::tokenize(message, ' '));
 
     // Expected format is something like 'type= parts= current= width= height='.
     const std::string text = tokens[0].substr(std::string("type=").size());
@@ -584,7 +584,7 @@ std::vector<char> assertTileMessage(LOOLWebSocket& ws, const std::string& testna
     const std::vector<char> response = getTileMessage(ws, testname);
 
     const std::string firstLine = LOOLProtocol::getFirstLine(response);
-    StringVector tileTokens(LOOLProtocol::tokenize(firstLine, ' '));
+    StringVector tileTokens(Util::tokenize(firstLine, ' '));
     LOK_ASSERT_EQUAL(std::string("tile:"), tileTokens[0]);
     LOK_ASSERT_EQUAL(std::string("part="), tileTokens[1].substr(0, std::string("part=").size()));
     LOK_ASSERT_EQUAL(std::string("width="), tileTokens[2].substr(0, std::string("width=").size()));
diff --git a/tools/Connect.cpp b/tools/Connect.cpp
index 32172da5c..23a8ea0ad 100644
--- a/tools/Connect.cpp
+++ b/tools/Connect.cpp
@@ -88,7 +88,7 @@ public:
                     }
 
                     std::string firstLine = getFirstLine(buffer, n);
-                    StringVector tokens(LOOLProtocol::tokenize(firstLine, ' '));
+                    StringVector tokens(Util::tokenize(firstLine, ' '));
 
                     if (std::getenv("DISPLAY") != nullptr && tokens.equals(0, "tile:"))
                     {
diff --git a/tools/KitClient.cpp b/tools/KitClient.cpp
index b3fa02a62..ab33451d3 100644
--- a/tools/KitClient.cpp
+++ b/tools/KitClient.cpp
@@ -88,7 +88,7 @@ protected:
             std::string line;
             std::getline(std::cin, line);
 
-            StringVector tokens(LOOLProtocol::tokenize(line, ' '));
+            StringVector tokens(Util::tokenize(line, ' '));
 
             if (tokens.size() == 0)
                 continue;
diff --git a/tools/WebSocketDump.cpp b/tools/WebSocketDump.cpp
index 7a592c550..9a62378c5 100644
--- a/tools/WebSocketDump.cpp
+++ b/tools/WebSocketDump.cpp
@@ -141,7 +141,7 @@ private:
             LOG_INF("Incoming websocket request: " << request.getURI());
 
             const std::string& requestURI = request.getURI();
-            StringVector pathTokens(LOOLProtocol::tokenize(requestURI, '/'));
+            StringVector pathTokens(Util::tokenize(requestURI, '/'));
             if (request.find("Upgrade") != request.end() && Poco::icompare(request["Upgrade"], "websocket") == 0)
             {
                 auto dumpHandler = std::make_shared<DumpSocketHandler>(_socket, request);
diff --git a/wsd/Admin.cpp b/wsd/Admin.cpp
index 734672b7c..3f48a0d8a 100644
--- a/wsd/Admin.cpp
+++ b/wsd/Admin.cpp
@@ -51,7 +51,7 @@ void AdminSocketHandler::handleMessage(const std::vector<char> &payload)
 {
     // FIXME: check fin, code etc.
     const std::string firstLine = getFirstLine(payload.data(), payload.size());
-    StringVector tokens(LOOLProtocol::tokenize(firstLine, ' '));
+    StringVector tokens(Util::tokenize(firstLine, ' '));
     LOG_TRC("Recv: " << firstLine << " tokens " << tokens.size());
 
     if (tokens.empty())
@@ -214,7 +214,7 @@ void AdminSocketHandler::handleMessage(const std::vector<char> &payload)
     {
         for (size_t i = 1; i < tokens.size(); i++)
         {
-            StringVector setting(LOOLProtocol::tokenize(tokens[i], '='));
+            StringVector setting(Util::tokenize(tokens[i], '='));
             int settingVal = 0;
             try
             {
@@ -282,7 +282,7 @@ void AdminSocketHandler::handleMessage(const std::vector<char> &payload)
     else if (tokens.equals(0, "update-log-levels") && tokens.size() > 1) {
         for (size_t i = 1; i < tokens.size(); i++)
         {
-            StringVector _channel(LOOLProtocol::tokenize(tokens[i], '='));
+            StringVector _channel(Util::tokenize(tokens[i], '='));
             if (_channel.size() == 2)
             {
                 _admin->setChannelLogLevel((_channel[0] != "?" ? _channel[0]: ""), _channel[1]);
@@ -357,7 +357,7 @@ bool AdminSocketHandler::handleInitialRequest(
     }
 
     const std::string& requestURI = request.getURI();
-    StringVector pathTokens(LOOLProtocol::tokenize(requestURI, '/'));
+    StringVector pathTokens(Util::tokenize(requestURI, '/'));
 
     if (request.find("Upgrade") != request.end() && Poco::icompare(request["Upgrade"], "websocket") == 0)
     {
diff --git a/wsd/Auth.cpp b/wsd/Auth.cpp
index 021257235..03807d81e 100644
--- a/wsd/Auth.cpp
+++ b/wsd/Auth.cpp
@@ -101,7 +101,7 @@ const std::string JWTAuth::getAccessToken()
 
 bool JWTAuth::verify(const std::string& accessToken)
 {
-    StringVector tokens(LOOLProtocol::tokenize(accessToken, '.'));
+    StringVector tokens(Util::tokenize(accessToken, '.'));
 
     try
     {
diff --git a/wsd/ClientSession.cpp b/wsd/ClientSession.cpp
index e3934b1f2..d164a0d45 100644
--- a/wsd/ClientSession.cpp
+++ b/wsd/ClientSession.cpp
@@ -322,7 +322,7 @@ bool ClientSession::_handleInput(const char *buffer, int length)
 {
     LOG_TRC(getName() << ": handling incoming [" << getAbbreviatedMessage(buffer, length) << "].");
     const std::string firstLine = getFirstLine(buffer, length);
-    const StringVector tokens = LOOLProtocol::tokenize(firstLine.data(), firstLine.size());
+    const StringVector tokens = Util::tokenize(firstLine.data(), firstLine.size());
 
     std::shared_ptr<DocumentBroker> docBroker = getDocumentBroker();
     if (!docBroker || docBroker->isMarkedToDestroy())
@@ -938,7 +938,7 @@ bool ClientSession::forwardToChild(const std::string& message,
 bool ClientSession::filterMessage(const std::string& message) const
 {
     bool allowed = true;
-    StringVector tokens(LOOLProtocol::tokenize(message, ' '));
+    StringVector tokens(Util::tokenize(message, ' '));
 
     // Set allowed flag to false depending on if particular WOPI properties are set
     if (tokens.equals(0, "downloadas"))
@@ -1289,7 +1289,7 @@ bool ClientSession::handleKitToClientMessage(const char* buffer, const int lengt
 #endif
     else if (tokens.size() == 2 && tokens.equals(0, "statechanged:"))
     {
-        StringVector stateTokens(LOOLProtocol::tokenize(tokens[1], '='));
+        StringVector stateTokens(Util::tokenize(tokens[1], '='));
         if (stateTokens.size() == 2 && stateTokens.equals(0, ".uno:ModifiedStatus"))
         {
             docBroker->setModified(stateTokens.equals(1, "true"));
@@ -1482,7 +1482,7 @@ bool ClientSession::handleKitToClientMessage(const char* buffer, const int lengt
             const Poco::Dynamic::Var result = parser.parse(stringJSON);
             const auto& object = result.extract<Poco::JSON::Object::Ptr>();
             const std::string rectangle = object->get("rectangle").toString();
-            StringVector rectangleTokens(LOOLProtocol::tokenize(rectangle, ','));
+            StringVector rectangleTokens(Util::tokenize(rectangle, ','));
             int x = 0, y = 0, w = 0, h = 0;
             if (rectangleTokens.size() > 2 &&
                 stringToInteger(rectangleTokens[0], x) &&
diff --git a/wsd/DocumentBroker.cpp b/wsd/DocumentBroker.cpp
index 01bf64955..d13a5d1a3 100644
--- a/wsd/DocumentBroker.cpp
+++ b/wsd/DocumentBroker.cpp
@@ -821,7 +821,7 @@ bool DocumentBroker::load(const std::shared_ptr<ClientSession>& session, const s
                         ++outputs;
                     }
 
-                    StringVector args(LOOLProtocol::tokenize(commandLine, ' '));
+                    StringVector args(Util::tokenize(commandLine, ' '));
                     std::string command(args[0]);
                     args.erase(args.begin()); // strip the command
 
@@ -2062,7 +2062,7 @@ bool DocumentBroker::forwardToChild(const std::string& viewId, const std::string
     {
         assert(!_uriJailed.empty());
 
-        StringVector tokens = LOOLProtocol::tokenize(msg);
+        StringVector tokens = Util::tokenize(msg);
         if (tokens.size() > 1 && tokens.equals(1, "load"))
         {
             // The json options must come last.
diff --git a/wsd/FileServer.cpp b/wsd/FileServer.cpp
index e833a7f82..1c47b0f8d 100644
--- a/wsd/FileServer.cpp
+++ b/wsd/FileServer.cpp
@@ -150,7 +150,7 @@ bool isConfigAuthOk(const std::string& userProvidedUsr, const std::string& userP
 #if HAVE_PKCS5_PBKDF2_HMAC
         // Extract the salt from the config
         std::vector<unsigned char> saltData;
-        StringVector tokens = LOOLProtocol::tokenize(securePass, '.');
+        StringVector tokens = Util::tokenize(securePass, '.');
         if (tokens.size() != 5 ||
             !tokens.equals(0, "pbkdf2") ||
             !tokens.equals(1, "sha512") ||
diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp
index 3c293457d..2b6bb013e 100644
--- a/wsd/LOOLWSD.cpp
+++ b/wsd/LOOLWSD.cpp
@@ -816,7 +816,7 @@ void ForKitProcWSHandler::handleMessage(const std::vector<char> &data)
 {
     LOG_TRC("ForKitProcWSHandler: handling incoming [" << LOOLProtocol::getAbbreviatedMessage(&data[0], data.size()) << "].");
     const std::string firstLine = LOOLProtocol::getFirstLine(&data[0], data.size());
-    const StringVector tokens = LOOLProtocol::tokenize(firstLine.data(), firstLine.size());
+    const StringVector tokens = Util::tokenize(firstLine.data(), firstLine.size());
 
     if (tokens.equals(0, "segfaultcount"))
     {
@@ -2184,7 +2184,7 @@ public:
         if(request.has("X-Forwarded-For"))
         {
             const std::string fowardedData = request.get("X-Forwarded-For");
-            StringVector tokens = LOOLProtocol::tokenize(fowardedData, ',');
+            StringVector tokens = Util::tokenize(fowardedData, ',');
             for (const auto& token : tokens)
             {
                 std::string param = tokens.getParam(token);
diff --git a/wsd/Storage.cpp b/wsd/Storage.cpp
index 00a78ecc5..7e262b1cd 100644
--- a/wsd/Storage.cpp
+++ b/wsd/Storage.cpp
@@ -437,7 +437,7 @@ static void addStorageDebugCookie(Poco::Net::HTTPRequest& request)
     if (std::getenv("LOOL_STORAGE_COOKIE"))
     {
         Poco::Net::NameValueCollection nvcCookies;
-        StringVector cookieTokens = LOOLProtocol::tokenize(std::string(std::getenv("LOOL_STORAGE_COOKIE")), ':');
+        StringVector cookieTokens = Util::tokenize(std::string(std::getenv("LOOL_STORAGE_COOKIE")), ':');
         if (cookieTokens.size() == 2)
         {
             nvcCookies.add(cookieTokens[0], cookieTokens[1]);
@@ -452,13 +452,16 @@ static void addStorageReuseCookie(Poco::Net::HTTPRequest& request, const std::st
 {
     if (!reuseStorageCookies.empty())
     {
+
         Poco::Net::NameValueCollection nvcCookies;
         request.getCookies(nvcCookies); // Preserve existing cookies.
 
-        StringVector cookies = LOOLProtocol::tokenize(reuseStorageCookies, ':');
+        StringVector cookies = Util::tokenize(reuseStorageCookies, ':');
+        LOG_TRC("Parsing reuse cookies to set in Wopi request ["
+                << reuseStorageCookies << "], found " << cookies.size() << " cookies.");
         for (auto cookie : cookies)
         {
-            StringVector cookieTokens = LOOLProtocol::tokenize(cookies.getParam(cookie), '=');
+            StringVector cookieTokens = Util::tokenize(cookies.getParam(cookie), '=');
             if (cookieTokens.size() == 2)
             {
                 nvcCookies.add(cookieTokens[0], cookieTokens[1]);
diff --git a/wsd/TileCache.cpp b/wsd/TileCache.cpp
index aa8ed5fc5..12231a383 100644
--- a/wsd/TileCache.cpp
+++ b/wsd/TileCache.cpp
@@ -321,7 +321,7 @@ void TileCache::invalidateTiles(const std::string& tiles, int normalizedViewId)
 
 std::pair<int, Util::Rectangle> TileCache::parseInvalidateMsg(const std::string& tiles)
 {
-    StringVector tokens = LOOLProtocol::tokenize(tiles);
+    StringVector tokens = Util::tokenize(tiles);
 
     assert(tokens.size() > 0 && tokens.equals(0, "invalidatetiles:"));
 
diff --git a/wsd/TileDesc.hpp b/wsd/TileDesc.hpp
index 60bb67648..dd607f8ff 100644
--- a/wsd/TileDesc.hpp
+++ b/wsd/TileDesc.hpp
@@ -243,7 +243,7 @@ public:
     /// Deserialize a TileDesc from a string format.
     static TileDesc parse(const std::string& message)
     {
-        return parse(LOOLProtocol::tokenize(message.data(), message.size()));
+        return parse(Util::tokenize(message.data(), message.size()));
     }
 
     std::string generateID() const
@@ -299,12 +299,12 @@ private:
             throw BadArgumentException("Invalid tilecombine descriptor.");
         }
 
-        StringVector positionXtokens(LOOLProtocol::tokenize(tilePositionsX, ','));
-        StringVector positionYtokens(LOOLProtocol::tokenize(tilePositionsY, ','));
-        StringVector imgSizeTokens(LOOLProtocol::tokenize(imgSizes, ','));
-        StringVector verTokens(LOOLProtocol::tokenize(vers, ','));
-        StringVector oldWireIdTokens(LOOLProtocol::tokenize(oldWireIds, ','));
-        StringVector wireIdTokens(LOOLProtocol::tokenize(wireIds, ','));
+        StringVector positionXtokens(Util::tokenize(tilePositionsX, ','));
+        StringVector positionYtokens(Util::tokenize(tilePositionsY, ','));
+        StringVector imgSizeTokens(Util::tokenize(imgSizes, ','));
+        StringVector verTokens(Util::tokenize(vers, ','));
+        StringVector oldWireIdTokens(Util::tokenize(oldWireIds, ','));
+        StringVector wireIdTokens(Util::tokenize(wireIds, ','));
 
         const size_t numberOfPositions = positionXtokens.size();
 
@@ -517,7 +517,7 @@ public:
     /// Deserialize a TileDesc from a string format.
     static TileCombined parse(const std::string& message)
     {
-        return parse(LOOLProtocol::tokenize(message.data(), message.size()));
+        return parse(Util::tokenize(message.data(), message.size()));
     }
 
     static TileCombined create(const std::vector<TileDesc>& tiles)
diff --git a/wsd/TraceFile.hpp b/wsd/TraceFile.hpp
index 38e43e839..fee52b516 100644
--- a/wsd/TraceFile.hpp
+++ b/wsd/TraceFile.hpp
@@ -198,7 +198,7 @@ public:
             // Remap the URL to the snapshot.
             if (LOOLProtocol::matchPrefix("load", data))
             {
-                StringVector tokens = LOOLProtocol::tokenize(data);
+                StringVector tokens = Util::tokenize(data);
                 if (tokens.size() >= 2)
                 {
                     std::string url;
commit d89f50960925b303792462898e0bc4d6f52f876b
Author:     Ashod Nakashian <ashod.nakashian at collabora.co.uk>
AuthorDate: Sun May 31 13:44:22 2020 -0400
Commit:     Michael Meeks <michael.meeks at collabora.com>
CommitDate: Tue Jun 2 18:00:20 2020 +0100

    wsd: overload getTokenInteger for string-literals
    
    When parsing, we virtually always know the name
    of the token we are parsing at compile time.
    Taking advantage of that means we also know
    its size at compile time, and can optimize
    std::string allocation, size counting and
    the implementation of getTokenInteger.
    
    Change-Id: I502a643c14cace7dd755df565b3b5c445688faad

diff --git a/common/Protocol.hpp b/common/Protocol.hpp
index 60d1aa1ce..f0ad8039f 100644
--- a/common/Protocol.hpp
+++ b/common/Protocol.hpp
@@ -76,6 +76,23 @@ namespace LOOLProtocol
 
     bool getTokenInteger(const StringVector& tokens, const std::string& name, int& value);
 
+    /// Literal-string token names.
+    template <std::size_t N>
+    inline bool getTokenInteger(const std::string& token, const char (&name)[N], int& value)
+    {
+        // N includes null termination.
+        static_assert(N > 1, "Token name must be at least one character long.");
+        if (token.size() > N && token[N - 1] == '=' && token.compare(0, N - 1, name) == 0)
+        {
+            const char* str = token.data() + N;
+            char* endptr = nullptr;
+            value = std::strtol(str, &endptr, 10);
+            return (endptr > str);
+        }
+
+        return false;
+    }
+
     inline bool getTokenString(const StringVector& tokens,
                                const std::string& name,
                                std::string& value)
diff --git a/kit/ChildSession.cpp b/kit/ChildSession.cpp
index 69ccab599..74e267cde 100644
--- a/kit/ChildSession.cpp
+++ b/kit/ChildSession.cpp
@@ -1349,7 +1349,6 @@ bool ChildSession::mouseEvent(const char* /*buffer*/, int /*length*/,
                               const StringVector& tokens,
                               const LokEventTargetEnum target)
 {
-    int type, x, y, count;
     bool success = true;
 
     // default values for compatibility reasons with older loleaflets
@@ -1371,6 +1370,10 @@ bool ChildSession::mouseEvent(const char* /*buffer*/, int /*length*/,
             minTokens++;
     }
 
+    int type = 0;
+    int x = 0;
+    int y = 0;
+    int count = 0;
     if (tokens.size() < minTokens ||
         !getTokenKeyword(tokens[counter++], "type",
                          {{"buttondown", LOK_MOUSEEVENT_MOUSEBUTTONDOWN},


More information about the Libreoffice-commits mailing list