[Libreoffice-commits] online.git: Branch 'distro/collabora/collabora-online-4-0' - common/Util.hpp net/Socket.cpp net/Socket.hpp net/WebSocketHandler.hpp wsd/LOOLWSD.cpp

Michael Meeks (via logerrit) logerrit at kemper.freedesktop.org
Thu Jun 13 08:15:42 UTC 2019


 common/Util.hpp          |   23 ++++++++
 net/Socket.cpp           |  126 +++++++++++++++++++++++++++++++++++++++++++++--
 net/Socket.hpp           |   22 +++++++-
 net/WebSocketHandler.hpp |    8 +-
 wsd/LOOLWSD.cpp          |   24 +++++---
 5 files changed, 184 insertions(+), 19 deletions(-)

New commits:
commit b726c6c4392df213a4da0a1a8b4ba58654a65e8b
Author:     Michael Meeks <michael.meeks at collabora.com>
AuthorDate: Wed May 22 11:25:28 2019 +0100
Commit:     Andras Timar <andras.timar at collabora.com>
CommitDate: Thu Jun 13 10:15:23 2019 +0200

    Initial chunked transfer encoding.
    
    Important for convert-to on larger documents and/or with newer curls.
    
    Change-Id: Id18be6d22741a3af7cee39a069c509e4f662977b
    Reviewed-on: https://gerrit.libreoffice.org/72748
    Reviewed-by: Andras Timar <andras.timar at collabora.com>
    Tested-by: Andras Timar <andras.timar at collabora.com>

diff --git a/common/Util.hpp b/common/Util.hpp
index 07af2a6a4..0190f953b 100644
--- a/common/Util.hpp
+++ b/common/Util.hpp
@@ -162,6 +162,18 @@ namespace Util
     // Extract all json entries into a map.
     std::map<std::string, std::string> JsonToMap(const std::string& jsonString);
 
+    inline int hexDigitFromChar(char c)
+    {
+        if (c >= '0' && c <= '9')
+            return c - '0';
+        else if (c >= 'a' && c <= 'f')
+            return c - 'a' + 10;
+        else if (c >= 'A' && c <= 'F')
+            return c - 'A' + 10;
+        else
+            return -1;
+    }
+
     /// Dump a lineof data as hex
     inline std::string stringifyHexLine(
                             const std::vector<char> &buffer,
@@ -230,6 +242,17 @@ namespace Util
         os.flush();
     }
 
+    inline std::string dumpHex (const char *legend, const char *prefix,
+                                const std::vector<char>::iterator &startIt,
+                                const std::vector<char>::iterator &endIt,
+                                bool skipDup = true, const unsigned int width = 32)
+    {
+        std::ostringstream oss;
+        std::vector<char> data(startIt, endIt);
+        dumpHex(oss, legend, prefix, data, skipDup, width);
+        return oss.str();
+    }
+
     /// Trim spaces from the left. Just spaces.
     inline std::string& ltrim(std::string& s)
     {
diff --git a/net/Socket.cpp b/net/Socket.cpp
index 2737c6f1c..5b863c8ae 100644
--- a/net/Socket.cpp
+++ b/net/Socket.cpp
@@ -432,14 +432,21 @@ bool ServerSocket::bind(Type type, int port)
 
 #ifndef MOBILEAPP
 
+// For a verbose life, tweak here:
+#if 0
+#  define LOG_CHUNK(X) LOG_TRC(X)
+#else
+#  define LOG_CHUNK(X)
+#endif
+
 bool StreamSocket::parseHeader(const char *clientName,
                                Poco::MemoryInputStream &message,
                                Poco::Net::HTTPRequest &request,
-                               size_t *requestSize)
+                               MessageMap *map)
 {
     LOG_TRC("#" << getFD() << " handling incoming " << _inBuffer.size() << " bytes.");
 
-    assert(!requestSize || *requestSize == 0);
+    assert(!map || (map->_headerSize == 0 && map->_messageSize == 0));
 
     // Find the end of the header, if any.
     static const std::string marker("\r\n\r\n");
@@ -453,8 +460,11 @@ bool StreamSocket::parseHeader(const char *clientName,
 
     // Skip the marker.
     itBody += marker.size();
-    if (requestSize)
-        *requestSize = static_cast<size_t>(itBody - _inBuffer.begin());
+    if (map) // a reasonable guess so far
+    {
+        map->_headerSize = static_cast<size_t>(itBody - _inBuffer.begin());
+        map->_messageSize = map->_headerSize;
+    }
 
     try
     {
@@ -485,6 +495,8 @@ bool StreamSocket::parseHeader(const char *clientName,
             LOG_DBG("Not enough content yet: ContentLength: " << contentLength << ", available: " << available);
             return false;
         }
+        if (map)
+            map->_messageSize += contentLength;
 
         if (request.getExpectContinue() && !_sentHTTPContinue)
         {
@@ -494,6 +506,79 @@ bool StreamSocket::parseHeader(const char *clientName,
                  sizeof("HTTP/1.1 100 Continue\r\n\r\n") - 1);
             _sentHTTPContinue = true;
         }
+
+        if (request.getChunkedTransferEncoding())
+        {
+            // keep the header
+            if (map)
+                map->_spans.push_back(std::pair<size_t, size_t>(0, itBody - _inBuffer.begin()));
+
+            int chunk = 0;
+            while (itBody != _inBuffer.end())
+            {
+                auto chunkStart = itBody;
+
+                // skip whitespace
+                for (; itBody != _inBuffer.end() && isascii(*itBody) && isspace(*itBody); ++itBody)
+                    ; // skip.
+
+                // each chunk is preceeded by its length in hex.
+                size_t chunkLen = 0;
+                for (; itBody != _inBuffer.end(); ++itBody)
+                {
+                    int digit = Util::hexDigitFromChar(*itBody);
+                    if (digit >= 0)
+                        chunkLen = chunkLen * 16 + digit;
+                    else
+                        break;
+                }
+
+                LOG_CHUNK("Chunk of length " << chunkLen);
+
+                for (; itBody != _inBuffer.end() && *itBody != '\n'; ++itBody)
+                    ; // skip to end of line
+
+                if (itBody != _inBuffer.end())
+                    itBody++; /* \n */;
+
+                // skip the chunk.
+                auto chunkOffset = itBody - _inBuffer.begin();
+                auto chunkAvailable = _inBuffer.size() - chunkOffset;
+
+                if (chunkLen == 0) // we're complete.
+                {
+                    map->_messageSize = chunkOffset;
+                    return true;
+                }
+
+                if (chunkLen > chunkAvailable + 2)
+                {
+                    LOG_DBG("Not enough content yet in chunk " << chunk <<
+                            " starting at offset " << (chunkStart - _inBuffer.begin()) <<
+                            " chunk len: " << chunkLen << ", available: " << chunkAvailable);
+                    return false;
+                }
+                itBody += chunkLen;
+
+                map->_spans.push_back(std::pair<size_t,size_t>(chunkOffset, chunkLen));
+
+                if (*itBody != '\r' || *(itBody + 1) != '\n')
+                {
+                    LOG_ERR("Missing \\r\\n at end of chunk " << chunk << " of length " << chunkLen);
+                    LOG_CHUNK("Chunk " << chunk << " is: \n" << Util::dumpHex("", "", chunkStart, itBody + 1, false));
+                    return false; // TODO: throw something sensible in this case
+                }
+                else
+                {
+                    LOG_CHUNK("Chunk " << chunk << " is: \n" << Util::dumpHex("", "", chunkStart, itBody + 1, false));
+                }
+
+                itBody+=2;
+                chunk++;
+            }
+            LOG_TRC("Not enough chunks yet, so far " << chunk << " chunks of total length " << (itBody - _inBuffer.begin()));
+            return false;
+        }
     }
     catch (const Poco::Exception& exc)
     {
@@ -513,6 +598,39 @@ bool StreamSocket::parseHeader(const char *clientName,
     return true;
 }
 
+bool StreamSocket::compactChunks(MessageMap *map)
+{
+    assert (map);
+    if (!map->_spans.size())
+        return false; // single message.
+
+    LOG_CHUNK("Pre-compact " << map->_spans.size() << " chunks: \n" <<
+              Util::dumpHex("", "", _inBuffer.begin(), _inBuffer.end(), false));
+
+    char *first = &_inBuffer[0];
+    char *dest = first;
+    for (auto &span : map->_spans)
+    {
+        std::memmove(dest, &_inBuffer[span.first], span.second);
+        dest += span.second;
+    }
+
+    // Erase the duplicate bits.
+    size_t newEnd = dest - first;
+    size_t gap = map->_messageSize - newEnd;
+    _inBuffer.erase(_inBuffer.begin() + newEnd, _inBuffer.begin() + map->_messageSize);
+
+    LOG_CHUNK("Post-compact with erase of " << newEnd << " to " << map->_messageSize << " giving: \n" <<
+              Util::dumpHex("", "", _inBuffer.begin(), _inBuffer.end(), false));
+
+    // shrink our size to fit
+    map->_messageSize -= gap;
+
+    dumpState(std::cerr);
+
+    return true;
+}
+
 namespace HttpHelper
 {
     void sendUncompressedFileContent(const std::shared_ptr<StreamSocket>& socket,
diff --git a/net/Socket.hpp b/net/Socket.hpp
index de20991e1..49cfeab49 100644
--- a/net/Socket.hpp
+++ b/net/Socket.hpp
@@ -930,9 +930,21 @@ public:
         return socket;
     }
 
+        /// Messages can be in chunks, only parts of message being valid.
+    struct MessageMap {
+        MessageMap() : _headerSize(0), _messageSize(0) {}
+        /// Size of HTTP headers
+        size_t _headerSize;
+        /// Entire size of data associated with this message
+        size_t _messageSize;
+        // offset + lengths to collate into the real stream
+        std::vector<std::pair<size_t, size_t>> _spans;
+    };
+
     /// Remove the first @count bytes from input buffer
-    void eraseFirstInputBytes(size_t count)
+    void eraseFirstInputBytes(const MessageMap &map)
     {
+        size_t count = map._headerSize;
         size_t toErase = std::min(count, _inBuffer.size());
         if (toErase < count)
             LOG_ERR("#" << getFD() << ": attempted to remove: " << count << " which is > size: " << _inBuffer.size() << " clamped to " << toErase);
@@ -940,12 +952,16 @@ public:
             _inBuffer.erase(_inBuffer.begin(), _inBuffer.begin() + count);
     }
 
+    /// Compacts chunk headers away leaving just the data we want
+    /// returns true if we did any re-sizing/movement of _inBuffer.
+    bool compactChunks(MessageMap *map);
+
     /// Detects if we have an HTTP header in the provided message and
     /// populates a request for that.
     bool parseHeader(const char *clientLoggingName,
                      Poco::MemoryInputStream &message,
                      Poco::Net::HTTPRequest &request,
-                     size_t *requestSize = nullptr);
+                     MessageMap *map = nullptr);
 
     /// Get input/output statistics on this stream
     void getIOStats(uint64_t &sent, uint64_t &recv)
@@ -966,6 +982,8 @@ public:
 
 protected:
 
+    std::vector<std::pair<size_t, size_t>> findChunks(Poco::Net::HTTPRequest &request);
+
     /// Called when a polling event is received.
     /// @events is the mask of events that triggered the wake.
     void handlePoll(SocketDisposition &disposition,
diff --git a/net/WebSocketHandler.hpp b/net/WebSocketHandler.hpp
index 472360ab6..f483661be 100644
--- a/net/WebSocketHandler.hpp
+++ b/net/WebSocketHandler.hpp
@@ -683,7 +683,7 @@ protected:
         LOG_TRC("Incoming client websocket upgrade response: " << std::string(&socket->getInBuffer()[0], socket->getInBuffer().size()));
 
         bool bOk = false;
-        size_t responseSize = 0;
+        StreamSocket::MessageMap map;
 
         try
         {
@@ -699,7 +699,7 @@ protected:
                                           marker.begin(), marker.end());
 
                 if (itBody != socket->getInBuffer().end())
-                    responseSize = itBody - socket->getInBuffer().begin() + marker.size();
+                    map._headerSize = itBody - socket->getInBuffer().begin() + marker.size();
             }
 
             if (response.getStatus() == Poco::Net::HTTPResponse::HTTP_SWITCHING_PROTOCOLS &&
@@ -728,7 +728,7 @@ protected:
             LOG_DBG("handleClientUpgrade exception caught.");
         }
 
-        if (!bOk || responseSize == 0)
+        if (!bOk || map._headerSize == 0)
         {
             LOG_ERR("Bad websocker server response.");
 
@@ -737,7 +737,7 @@ protected:
         }
 
         setWebSocket();
-        socket->eraseFirstInputBytes(responseSize);
+        socket->eraseFirstInputBytes(map);
     }
 #endif
 
diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp
index 90659923a..89171b339 100644
--- a/wsd/LOOLWSD.cpp
+++ b/wsd/LOOLWSD.cpp
@@ -1826,9 +1826,7 @@ private:
         try
         {
 #ifndef MOBILEAPP
-            size_t requestSize = 0;
-
-            if (!socket->parseHeader("Prisoner", message, request, &requestSize))
+            if (!socket->parseHeader("Prisoner", message, request))
                 return;
 
             LOG_TRC("Child connection with URI [" << LOOLWSD::anonymizeUrl(request.getURI()) << "].");
@@ -2048,16 +2046,23 @@ private:
             return;
         }
 
-        Poco::MemoryInputStream message(&socket->getInBuffer()[0],
-                                        socket->getInBuffer().size());;
+        Poco::MemoryInputStream startmessage(&socket->getInBuffer()[0],
+                                             socket->getInBuffer().size());;
         Poco::Net::HTTPRequest request;
-        size_t requestSize = 0;
 
-        if (!socket->parseHeader("Client", message, request, &requestSize))
+        StreamSocket::MessageMap map;
+        if (!socket->parseHeader("Client", startmessage, request, &map))
             return;
 
         try
         {
+            // We may need to re-write the chunks moving the inBuffer.
+            socket->compactChunks(&map);
+            Poco::MemoryInputStream message(&socket->getInBuffer()[0],
+                                            socket->getInBuffer().size());
+            // update the read cursor - headers are not altered by chunks.
+            message.seekg(startmessage.tellg(), std::ios::beg);
+
             // Check and remove the ServiceRoot from the request.getURI()
             if (!Util::startsWith(request.getURI(), LOOLWSD::ServiceRoot))
                 throw BadRequestException("The request does not start with prefix: " + LOOLWSD::ServiceRoot);
@@ -2166,7 +2171,7 @@ private:
 
         // if we succeeded - remove the request from our input buffer
         // we expect one request per socket
-        socket->eraseFirstInputBytes(requestSize);
+        socket->eraseFirstInputBytes(map);
 #else
         Poco::Net::HTTPRequest request;
         handleClientWsUpgrade(request, std::string(socket->getInBuffer().data(), socket->getInBuffer().size()), disposition);
@@ -2331,7 +2336,8 @@ private:
         return "application/octet-stream";
     }
 
-    void handlePostRequest(const Poco::Net::HTTPRequest& request, Poco::MemoryInputStream& message,
+    void handlePostRequest(const Poco::Net::HTTPRequest& request,
+                           Poco::MemoryInputStream& message,
                            SocketDisposition &disposition)
     {
         LOG_INF("Post request: [" << LOOLWSD::anonymizeUrl(request.getURI()) << "]");


More information about the Libreoffice-commits mailing list