[Libreoffice-commits] online.git: Branch 'distro/collabora/collabora-online-4-0' - common/Util.hpp net/Socket.cpp net/Socket.hpp net/WebSocketHandler.hpp wsd/LOOLWSD.cpp
Michael Meeks (via logerrit)
logerrit at kemper.freedesktop.org
Thu Jun 13 08:15:42 UTC 2019
common/Util.hpp | 23 ++++++++
net/Socket.cpp | 126 +++++++++++++++++++++++++++++++++++++++++++++--
net/Socket.hpp | 22 +++++++-
net/WebSocketHandler.hpp | 8 +-
wsd/LOOLWSD.cpp | 24 +++++---
5 files changed, 184 insertions(+), 19 deletions(-)
New commits:
commit b726c6c4392df213a4da0a1a8b4ba58654a65e8b
Author: Michael Meeks <michael.meeks at collabora.com>
AuthorDate: Wed May 22 11:25:28 2019 +0100
Commit: Andras Timar <andras.timar at collabora.com>
CommitDate: Thu Jun 13 10:15:23 2019 +0200
Initial chunked transfer encoding.
Important for convert-to on larger documents and/or with newer curls.
Change-Id: Id18be6d22741a3af7cee39a069c509e4f662977b
Reviewed-on: https://gerrit.libreoffice.org/72748
Reviewed-by: Andras Timar <andras.timar at collabora.com>
Tested-by: Andras Timar <andras.timar at collabora.com>
diff --git a/common/Util.hpp b/common/Util.hpp
index 07af2a6a4..0190f953b 100644
--- a/common/Util.hpp
+++ b/common/Util.hpp
@@ -162,6 +162,18 @@ namespace Util
// Extract all json entries into a map.
std::map<std::string, std::string> JsonToMap(const std::string& jsonString);
+ inline int hexDigitFromChar(char c)
+ {
+ if (c >= '0' && c <= '9')
+ return c - '0';
+ else if (c >= 'a' && c <= 'f')
+ return c - 'a' + 10;
+ else if (c >= 'A' && c <= 'F')
+ return c - 'A' + 10;
+ else
+ return -1;
+ }
+
/// Dump a lineof data as hex
inline std::string stringifyHexLine(
const std::vector<char> &buffer,
@@ -230,6 +242,17 @@ namespace Util
os.flush();
}
+ inline std::string dumpHex (const char *legend, const char *prefix,
+ const std::vector<char>::iterator &startIt,
+ const std::vector<char>::iterator &endIt,
+ bool skipDup = true, const unsigned int width = 32)
+ {
+ std::ostringstream oss;
+ std::vector<char> data(startIt, endIt);
+ dumpHex(oss, legend, prefix, data, skipDup, width);
+ return oss.str();
+ }
+
/// Trim spaces from the left. Just spaces.
inline std::string& ltrim(std::string& s)
{
diff --git a/net/Socket.cpp b/net/Socket.cpp
index 2737c6f1c..5b863c8ae 100644
--- a/net/Socket.cpp
+++ b/net/Socket.cpp
@@ -432,14 +432,21 @@ bool ServerSocket::bind(Type type, int port)
#ifndef MOBILEAPP
+// For a verbose life, tweak here:
+#if 0
+# define LOG_CHUNK(X) LOG_TRC(X)
+#else
+# define LOG_CHUNK(X)
+#endif
+
bool StreamSocket::parseHeader(const char *clientName,
Poco::MemoryInputStream &message,
Poco::Net::HTTPRequest &request,
- size_t *requestSize)
+ MessageMap *map)
{
LOG_TRC("#" << getFD() << " handling incoming " << _inBuffer.size() << " bytes.");
- assert(!requestSize || *requestSize == 0);
+ assert(!map || (map->_headerSize == 0 && map->_messageSize == 0));
// Find the end of the header, if any.
static const std::string marker("\r\n\r\n");
@@ -453,8 +460,11 @@ bool StreamSocket::parseHeader(const char *clientName,
// Skip the marker.
itBody += marker.size();
- if (requestSize)
- *requestSize = static_cast<size_t>(itBody - _inBuffer.begin());
+ if (map) // a reasonable guess so far
+ {
+ map->_headerSize = static_cast<size_t>(itBody - _inBuffer.begin());
+ map->_messageSize = map->_headerSize;
+ }
try
{
@@ -485,6 +495,8 @@ bool StreamSocket::parseHeader(const char *clientName,
LOG_DBG("Not enough content yet: ContentLength: " << contentLength << ", available: " << available);
return false;
}
+ if (map)
+ map->_messageSize += contentLength;
if (request.getExpectContinue() && !_sentHTTPContinue)
{
@@ -494,6 +506,79 @@ bool StreamSocket::parseHeader(const char *clientName,
sizeof("HTTP/1.1 100 Continue\r\n\r\n") - 1);
_sentHTTPContinue = true;
}
+
+ if (request.getChunkedTransferEncoding())
+ {
+ // keep the header
+ if (map)
+ map->_spans.push_back(std::pair<size_t, size_t>(0, itBody - _inBuffer.begin()));
+
+ int chunk = 0;
+ while (itBody != _inBuffer.end())
+ {
+ auto chunkStart = itBody;
+
+ // skip whitespace
+ for (; itBody != _inBuffer.end() && isascii(*itBody) && isspace(*itBody); ++itBody)
+ ; // skip.
+
+ // each chunk is preceeded by its length in hex.
+ size_t chunkLen = 0;
+ for (; itBody != _inBuffer.end(); ++itBody)
+ {
+ int digit = Util::hexDigitFromChar(*itBody);
+ if (digit >= 0)
+ chunkLen = chunkLen * 16 + digit;
+ else
+ break;
+ }
+
+ LOG_CHUNK("Chunk of length " << chunkLen);
+
+ for (; itBody != _inBuffer.end() && *itBody != '\n'; ++itBody)
+ ; // skip to end of line
+
+ if (itBody != _inBuffer.end())
+ itBody++; /* \n */;
+
+ // skip the chunk.
+ auto chunkOffset = itBody - _inBuffer.begin();
+ auto chunkAvailable = _inBuffer.size() - chunkOffset;
+
+ if (chunkLen == 0) // we're complete.
+ {
+ map->_messageSize = chunkOffset;
+ return true;
+ }
+
+ if (chunkLen > chunkAvailable + 2)
+ {
+ LOG_DBG("Not enough content yet in chunk " << chunk <<
+ " starting at offset " << (chunkStart - _inBuffer.begin()) <<
+ " chunk len: " << chunkLen << ", available: " << chunkAvailable);
+ return false;
+ }
+ itBody += chunkLen;
+
+ map->_spans.push_back(std::pair<size_t,size_t>(chunkOffset, chunkLen));
+
+ if (*itBody != '\r' || *(itBody + 1) != '\n')
+ {
+ LOG_ERR("Missing \\r\\n at end of chunk " << chunk << " of length " << chunkLen);
+ LOG_CHUNK("Chunk " << chunk << " is: \n" << Util::dumpHex("", "", chunkStart, itBody + 1, false));
+ return false; // TODO: throw something sensible in this case
+ }
+ else
+ {
+ LOG_CHUNK("Chunk " << chunk << " is: \n" << Util::dumpHex("", "", chunkStart, itBody + 1, false));
+ }
+
+ itBody+=2;
+ chunk++;
+ }
+ LOG_TRC("Not enough chunks yet, so far " << chunk << " chunks of total length " << (itBody - _inBuffer.begin()));
+ return false;
+ }
}
catch (const Poco::Exception& exc)
{
@@ -513,6 +598,39 @@ bool StreamSocket::parseHeader(const char *clientName,
return true;
}
+bool StreamSocket::compactChunks(MessageMap *map)
+{
+ assert (map);
+ if (!map->_spans.size())
+ return false; // single message.
+
+ LOG_CHUNK("Pre-compact " << map->_spans.size() << " chunks: \n" <<
+ Util::dumpHex("", "", _inBuffer.begin(), _inBuffer.end(), false));
+
+ char *first = &_inBuffer[0];
+ char *dest = first;
+ for (auto &span : map->_spans)
+ {
+ std::memmove(dest, &_inBuffer[span.first], span.second);
+ dest += span.second;
+ }
+
+ // Erase the duplicate bits.
+ size_t newEnd = dest - first;
+ size_t gap = map->_messageSize - newEnd;
+ _inBuffer.erase(_inBuffer.begin() + newEnd, _inBuffer.begin() + map->_messageSize);
+
+ LOG_CHUNK("Post-compact with erase of " << newEnd << " to " << map->_messageSize << " giving: \n" <<
+ Util::dumpHex("", "", _inBuffer.begin(), _inBuffer.end(), false));
+
+ // shrink our size to fit
+ map->_messageSize -= gap;
+
+ dumpState(std::cerr);
+
+ return true;
+}
+
namespace HttpHelper
{
void sendUncompressedFileContent(const std::shared_ptr<StreamSocket>& socket,
diff --git a/net/Socket.hpp b/net/Socket.hpp
index de20991e1..49cfeab49 100644
--- a/net/Socket.hpp
+++ b/net/Socket.hpp
@@ -930,9 +930,21 @@ public:
return socket;
}
+ /// Messages can be in chunks, only parts of message being valid.
+ struct MessageMap {
+ MessageMap() : _headerSize(0), _messageSize(0) {}
+ /// Size of HTTP headers
+ size_t _headerSize;
+ /// Entire size of data associated with this message
+ size_t _messageSize;
+ // offset + lengths to collate into the real stream
+ std::vector<std::pair<size_t, size_t>> _spans;
+ };
+
/// Remove the first @count bytes from input buffer
- void eraseFirstInputBytes(size_t count)
+ void eraseFirstInputBytes(const MessageMap &map)
{
+ size_t count = map._headerSize;
size_t toErase = std::min(count, _inBuffer.size());
if (toErase < count)
LOG_ERR("#" << getFD() << ": attempted to remove: " << count << " which is > size: " << _inBuffer.size() << " clamped to " << toErase);
@@ -940,12 +952,16 @@ public:
_inBuffer.erase(_inBuffer.begin(), _inBuffer.begin() + count);
}
+ /// Compacts chunk headers away leaving just the data we want
+ /// returns true if we did any re-sizing/movement of _inBuffer.
+ bool compactChunks(MessageMap *map);
+
/// Detects if we have an HTTP header in the provided message and
/// populates a request for that.
bool parseHeader(const char *clientLoggingName,
Poco::MemoryInputStream &message,
Poco::Net::HTTPRequest &request,
- size_t *requestSize = nullptr);
+ MessageMap *map = nullptr);
/// Get input/output statistics on this stream
void getIOStats(uint64_t &sent, uint64_t &recv)
@@ -966,6 +982,8 @@ public:
protected:
+ std::vector<std::pair<size_t, size_t>> findChunks(Poco::Net::HTTPRequest &request);
+
/// Called when a polling event is received.
/// @events is the mask of events that triggered the wake.
void handlePoll(SocketDisposition &disposition,
diff --git a/net/WebSocketHandler.hpp b/net/WebSocketHandler.hpp
index 472360ab6..f483661be 100644
--- a/net/WebSocketHandler.hpp
+++ b/net/WebSocketHandler.hpp
@@ -683,7 +683,7 @@ protected:
LOG_TRC("Incoming client websocket upgrade response: " << std::string(&socket->getInBuffer()[0], socket->getInBuffer().size()));
bool bOk = false;
- size_t responseSize = 0;
+ StreamSocket::MessageMap map;
try
{
@@ -699,7 +699,7 @@ protected:
marker.begin(), marker.end());
if (itBody != socket->getInBuffer().end())
- responseSize = itBody - socket->getInBuffer().begin() + marker.size();
+ map._headerSize = itBody - socket->getInBuffer().begin() + marker.size();
}
if (response.getStatus() == Poco::Net::HTTPResponse::HTTP_SWITCHING_PROTOCOLS &&
@@ -728,7 +728,7 @@ protected:
LOG_DBG("handleClientUpgrade exception caught.");
}
- if (!bOk || responseSize == 0)
+ if (!bOk || map._headerSize == 0)
{
LOG_ERR("Bad websocker server response.");
@@ -737,7 +737,7 @@ protected:
}
setWebSocket();
- socket->eraseFirstInputBytes(responseSize);
+ socket->eraseFirstInputBytes(map);
}
#endif
diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp
index 90659923a..89171b339 100644
--- a/wsd/LOOLWSD.cpp
+++ b/wsd/LOOLWSD.cpp
@@ -1826,9 +1826,7 @@ private:
try
{
#ifndef MOBILEAPP
- size_t requestSize = 0;
-
- if (!socket->parseHeader("Prisoner", message, request, &requestSize))
+ if (!socket->parseHeader("Prisoner", message, request))
return;
LOG_TRC("Child connection with URI [" << LOOLWSD::anonymizeUrl(request.getURI()) << "].");
@@ -2048,16 +2046,23 @@ private:
return;
}
- Poco::MemoryInputStream message(&socket->getInBuffer()[0],
- socket->getInBuffer().size());;
+ Poco::MemoryInputStream startmessage(&socket->getInBuffer()[0],
+ socket->getInBuffer().size());;
Poco::Net::HTTPRequest request;
- size_t requestSize = 0;
- if (!socket->parseHeader("Client", message, request, &requestSize))
+ StreamSocket::MessageMap map;
+ if (!socket->parseHeader("Client", startmessage, request, &map))
return;
try
{
+ // We may need to re-write the chunks moving the inBuffer.
+ socket->compactChunks(&map);
+ Poco::MemoryInputStream message(&socket->getInBuffer()[0],
+ socket->getInBuffer().size());
+ // update the read cursor - headers are not altered by chunks.
+ message.seekg(startmessage.tellg(), std::ios::beg);
+
// Check and remove the ServiceRoot from the request.getURI()
if (!Util::startsWith(request.getURI(), LOOLWSD::ServiceRoot))
throw BadRequestException("The request does not start with prefix: " + LOOLWSD::ServiceRoot);
@@ -2166,7 +2171,7 @@ private:
// if we succeeded - remove the request from our input buffer
// we expect one request per socket
- socket->eraseFirstInputBytes(requestSize);
+ socket->eraseFirstInputBytes(map);
#else
Poco::Net::HTTPRequest request;
handleClientWsUpgrade(request, std::string(socket->getInBuffer().data(), socket->getInBuffer().size()), disposition);
@@ -2331,7 +2336,8 @@ private:
return "application/octet-stream";
}
- void handlePostRequest(const Poco::Net::HTTPRequest& request, Poco::MemoryInputStream& message,
+ void handlePostRequest(const Poco::Net::HTTPRequest& request,
+ Poco::MemoryInputStream& message,
SocketDisposition &disposition)
{
LOG_INF("Post request: [" << LOOLWSD::anonymizeUrl(request.getURI()) << "]");
More information about the Libreoffice-commits
mailing list