[Libreoffice-commits] online.git: common/Util.cpp common/Util.hpp loleaflet/src loolwsd.xml.in wsd/Admin.cpp wsd/Admin.hpp wsd/AdminModel.cpp wsd/AdminModel.hpp wsd/DocumentBroker.cpp wsd/DocumentBroker.hpp wsd/LOOLWSD.cpp wsd/LOOLWSD.hpp
Pranav Kant
pranavk at collabora.co.uk
Fri Jul 7 15:45:20 UTC 2017
common/Util.cpp | 21 +++++++++++++++++++++
common/Util.hpp | 3 +++
loleaflet/src/core/Socket.js | 4 ++--
loolwsd.xml.in | 1 +
wsd/Admin.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++
wsd/Admin.hpp | 5 +++++
wsd/AdminModel.cpp | 19 +++++++++++++++++++
wsd/AdminModel.hpp | 21 +++++++++++++++++++++
wsd/DocumentBroker.cpp | 12 ++++++------
wsd/DocumentBroker.hpp | 4 ++++
wsd/LOOLWSD.cpp | 14 ++++++++++++++
wsd/LOOLWSD.hpp | 4 ++++
12 files changed, 143 insertions(+), 8 deletions(-)
New commits:
commit fde57adbbf9ab2fba80c6b8e0d877c797b55bea5
Author: Pranav Kant <pranavk at collabora.co.uk>
Date: Fri Jul 7 17:12:19 2017 +0530
Introduce hard mode when we are OOM
Start killing documents when memory usage goes above threshold.
Also make it possible to close documents from admin instance.
In DocumentBroker::closeDocument, just set the _stop flag and wake
up the polling thread which will terminate the children, instead of
manually terminating the children.
Change-Id: Ie70e05b3fb6ea816a87b6dcfaed92cdddb94aa90
diff --git a/common/Util.cpp b/common/Util.cpp
index a61fe6d0..a3e5e798 100644
--- a/common/Util.cpp
+++ b/common/Util.cpp
@@ -171,6 +171,27 @@ namespace Util
return nullptr;
}
+ size_t getTotalSystemMemory()
+ {
+ size_t totalMemKb = 0;
+ FILE* file = fopen("/proc/meminfo", "r");
+ if (file != nullptr)
+ {
+ char line[4096] = { 0 };
+ while (fgets(line, sizeof(line), file))
+ {
+ const char* value;
+ if ((value = startsWith(line, "MemTotal:")))
+ {
+ totalMemKb = atoi(value);
+ break;
+ }
+ }
+ }
+
+ return totalMemKb;
+ }
+
std::pair<size_t, size_t> getPssAndDirtyFromSMaps(FILE* file)
{
size_t numPSSKb = 0;
diff --git a/common/Util.hpp b/common/Util.hpp
index 4f9906e9..e97d1808 100644
--- a/common/Util.hpp
+++ b/common/Util.hpp
@@ -95,6 +95,9 @@ namespace Util
#endif
}
+ /// Returns the total physical memory (in kB) available in the system
+ size_t getTotalSystemMemory();
+
/// Returns the process PSS in KB (works only when we have perms for /proc/pid/smaps).
size_t getMemoryUsagePSS(const Poco::Process::PID pid);
diff --git a/loleaflet/src/core/Socket.js b/loleaflet/src/core/Socket.js
index 235a5948..0e23d797 100644
--- a/loleaflet/src/core/Socket.js
+++ b/loleaflet/src/core/Socket.js
@@ -243,7 +243,7 @@ L.Socket = L.Class.extend({
if (textMsg === 'ownertermination') {
msg = _('Session terminated by document owner');
}
- else if (textMsg === 'idle') {
+ else if (textMsg === 'idle' || textMsg === 'oom') {
msg = _('Session was terminated due to idleness - please click to reload');
this._map._documentIdle = true;
}
@@ -334,7 +334,7 @@ L.Socket = L.Class.extend({
});
options.$vex.append(options.$vexContent);
- if (textMsg === 'idle') {
+ if (textMsg === 'idle' || textMsg === 'oom') {
var map = this._map;
options.$vex.bind('click.vex', function(e) {
console.debug('idleness: reactivating');
diff --git a/loolwsd.xml.in b/loolwsd.xml.in
index 8ec29c82..a8b67737 100644
--- a/loolwsd.xml.in
+++ b/loolwsd.xml.in
@@ -11,6 +11,7 @@
<server_name desc="Hostname:port of the server running loolwsd. If empty, it's derived from the request." type="string" default=""></server_name>
<file_server_root_path desc="Path to the directory that should be considered root for the file server. This should be the directory containing loleaflet." type="path" relative="true" default="loleaflet/../"></file_server_root_path>
+ <memproportion desc="The maximum percentage of system memory consumed by all of the LibreOffice Online, after which we start cleaning up idle documents" type="double" default="80.0"></memproportion>
<num_prespawn_children desc="Number of child processes to keep started in advance and waiting for new clients." type="uint" default="1">1</num_prespawn_children>
<per_document desc="Document-specific settings, including LO Core settings.">
<max_concurrency desc="The maximum number of threads to use while processing a document." type="uint" default="4">4</max_concurrency>
diff --git a/wsd/Admin.cpp b/wsd/Admin.cpp
index 04a20ad5..b1de0ba6 100644
--- a/wsd/Admin.cpp
+++ b/wsd/Admin.cpp
@@ -333,6 +333,9 @@ Admin::Admin() :
{
LOG_INF("Admin ctor.");
+ _totalSysMem = Util::getTotalSystemMemory();
+ LOG_TRC("Total system memory : " << _totalSysMem);
+
const auto totalMem = getTotalMemoryUsage();
LOG_TRC("Total memory used: " << totalMem);
_model.addMemStats(totalMem);
@@ -384,6 +387,9 @@ void Admin::pollingThread()
lastMem = now;
memWait += _memStatsTaskIntervalMs;
+
+ // If our total memory consumption is above limit, cleanup
+ triggerMemoryCleanup(totalMem);
}
int netWait = _networkStatsIntervalMs -
@@ -526,6 +532,43 @@ void Admin::notifyForkit()
IoUtil::writeToPipe(_forKitWritePipe, oss.str());
}
+void Admin::triggerMemoryCleanup(size_t totalMem)
+{
+ LOG_TRC("Total memory we are consuming (in kB): " << totalMem);
+ // Trigger mem cleanup when we are consuming too much memory (as configured by sysadmin)
+ const auto memLimit = LOOLWSD::getConfigValue<double>("memproportion", static_cast<double>(80.0));
+ LOG_TRC("Mem proportion for LOOL configured : " << memLimit);
+ float memToFreePercentage = 0;
+ if ( (memToFreePercentage = (totalMem/static_cast<double>(_totalSysMem)) - memLimit/100.) > 0.0 )
+ {
+ int memToFree = memToFreePercentage * totalMem;
+ LOG_TRC("Memory to be freed (in kB) : " << memToFree);
+ // prepare document list sorted by most idle times
+ std::list<DocBasicInfo> docList = _model.getDocumentsSortedByIdle();
+
+ LOG_TRC("Checking saved documents in document list, length: " << docList.size());
+ // Kill the saved documents first
+ std::list<DocBasicInfo>::iterator docIt = docList.begin();
+ while (docIt != docList.end() && memToFree > 0)
+ {
+ LOG_TRC("Document: DocKey[" << docIt->_docKey << "], Idletime[" << docIt->_idleTime << "],"
+ << " Saved: [" << docIt->_saved << "], Mem: [" << docIt->_mem << "].");
+ if (docIt->_saved)
+ {
+ // Kill and remove from list
+ LOG_DBG("OOM: Killing saved document with DocKey " << docIt->_docKey);
+ LOOLWSD::closeDocument(docIt->_docKey, "oom");
+ memToFree -= docIt->_mem;
+ docList.erase(docIt);
+ }
+ else
+ ++docIt;
+ }
+ }
+
+ LOG_TRC("OOM: Memory to free percentage : " << memToFreePercentage);
+}
+
void Admin::dumpState(std::ostream& os)
{
// FIXME: be more helpful ...
diff --git a/wsd/Admin.hpp b/wsd/Admin.hpp
index d5b62cb8..54b9a8e7 100644
--- a/wsd/Admin.hpp
+++ b/wsd/Admin.hpp
@@ -118,6 +118,10 @@ private:
/// Notify Forkit of changed settings.
void notifyForkit();
+ /// Memory consumption has increased, start killing kits etc. till memory consumption gets back
+ /// under @hardModeLimit
+ void triggerMemoryCleanup(size_t hardModeLimit);
+
private:
/// The model is accessed only during startup & in
/// the Admin Poll thread.
@@ -128,6 +132,7 @@ private:
size_t _lastJiffies;
uint64_t _lastSentCount;
uint64_t _lastRecvCount;
+ size_t _totalSysMem;
std::atomic<int> _memStatsTaskIntervalMs;
std::atomic<int> _cpuStatsTaskIntervalMs;
diff --git a/wsd/AdminModel.cpp b/wsd/AdminModel.cpp
index c90dd0eb..8b8f31a7 100644
--- a/wsd/AdminModel.cpp
+++ b/wsd/AdminModel.cpp
@@ -629,6 +629,25 @@ unsigned AdminModel::getTotalActiveViews()
return numTotalViews;
}
+std::list<DocBasicInfo> AdminModel::getDocumentsSortedByIdle() const
+{
+ std::list<DocBasicInfo> docList;
+ for (const auto& it: _documents)
+ {
+ docList.emplace_back(it.second.getDocKey(),
+ it.second.getIdleTime(),
+ !it.second.getModifiedStatus(),
+ it.second.getMemoryDirty());
+ }
+
+ // Sort the list by idle times;
+ docList.sort([](const DocBasicInfo& a, const DocBasicInfo& b) {
+ return a._idleTime > b._idleTime;
+ });
+
+ return docList;
+}
+
std::string AdminModel::getDocuments() const
{
assertCorrectThread();
diff --git a/wsd/AdminModel.hpp b/wsd/AdminModel.hpp
index b217c2be..d56bdb70 100644
--- a/wsd/AdminModel.hpp
+++ b/wsd/AdminModel.hpp
@@ -54,6 +54,22 @@ struct DocProcSettings
size_t LimitFileSizeMb;
};
+/// Containing basic information about document
+struct DocBasicInfo
+{
+ std::string _docKey;
+ std::time_t _idleTime;
+ bool _saved;
+ int _mem;
+
+ DocBasicInfo(const std::string& docKey, std::time_t idleTime, bool saved, int mem)
+ : _docKey(docKey),
+ _idleTime(idleTime),
+ _saved(saved),
+ _mem(mem)
+ { }
+};
+
/// A document in Admin controller.
class Document
{
@@ -73,6 +89,8 @@ public:
{
}
+ const std::string getDocKey() const { return _docKey; }
+
Poco::Process::PID getPid() const { return _pid; }
std::string getFilename() const { return _filename; }
@@ -247,6 +265,9 @@ public:
uint64_t getSentBytesTotal() { return _sentBytesTotal; }
uint64_t getRecvBytesTotal() { return _recvBytesTotal; }
+ /// Document basic info list sorted by most idle time
+ std::list<DocBasicInfo> getDocumentsSortedByIdle() const;
+
private:
std::string getMemStats();
diff --git a/wsd/DocumentBroker.cpp b/wsd/DocumentBroker.cpp
index 9af4ba4a..84fddd68 100644
--- a/wsd/DocumentBroker.cpp
+++ b/wsd/DocumentBroker.cpp
@@ -160,6 +160,7 @@ DocumentBroker::DocumentBroker(const std::string& uri,
_cursorHeight(0),
_poll(new DocumentBrokerPoll("docbroker_" + _docId, *this)),
_stop(false),
+ _closeReason("stopped"),
_tileVersion(0),
_debugRenderedTileCount(0)
{
@@ -229,8 +230,6 @@ void DocumentBroker::pollThread()
static const bool AutoSaveEnabled = !std::getenv("LOOL_NO_AUTOSAVE");
static const size_t IdleDocTimeoutSecs = LOOLWSD::getConfigValue<int>(
"per_document.idle_timeout_secs", 3600);
- std::string closeReason = "stopped";
-
// Used to accumulate B/W deltas.
uint64_t adminSent = 0;
uint64_t adminRecv = 0;
@@ -269,7 +268,7 @@ void DocumentBroker::pollThread()
if (ShutdownRequestFlag)
{
- closeReason = "recycling";
+ _closeReason = "recycling";
_stop = true;
}
else if (AutoSaveEnabled && !_stop &&
@@ -288,7 +287,7 @@ void DocumentBroker::pollThread()
{
LOG_INF("Terminating " << (idle ? "idle" : "dead") <<
" DocumentBroker for docKey [" << getDocKey() << "].");
- closeReason = (idle ? "idle" : "dead");
+ _closeReason = (idle ? "idle" : "dead");
_stop = true;
}
}
@@ -311,7 +310,7 @@ void DocumentBroker::pollThread()
}
// Terminate properly while we can.
- terminateChild(closeReason);
+ terminateChild(_closeReason);
// Stop to mark it done and cleanup.
_poll->stop();
@@ -1462,7 +1461,8 @@ void DocumentBroker::closeDocument(const std::string& reason)
assertCorrectThread();
LOG_DBG("Closing DocumentBroker for docKey [" << _docKey << "] with reason: " << reason);
- terminateChild(reason);
+ _closeReason = reason; // used later in the polling loop
+ stop();
}
void DocumentBroker::broadcastMessage(const std::string& message)
diff --git a/wsd/DocumentBroker.hpp b/wsd/DocumentBroker.hpp
index 98094da2..dd0ace39 100644
--- a/wsd/DocumentBroker.hpp
+++ b/wsd/DocumentBroker.hpp
@@ -342,6 +342,9 @@ public:
/// Sends a message to all sessions
void broadcastMessage(const std::string& message);
+ /// Sets the reason for closing document;
+ void setCloseReason(const std::string& closeReason) { _closeReason = closeReason; }
+
private:
/// Shutdown all client connections with the given reason.
@@ -415,6 +418,7 @@ private:
mutable std::mutex _mutex;
std::unique_ptr<DocumentBrokerPoll> _poll;
std::atomic<bool> _stop;
+ std::string _closeReason;
/// Versioning is used to prevent races between
/// painting and invalidation.
diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp
index 23fbfc81..b21dd3bc 100644
--- a/wsd/LOOLWSD.cpp
+++ b/wsd/LOOLWSD.cpp
@@ -624,6 +624,7 @@ void LOOLWSD::initialize(Application& self)
{ "lo_jail_subpath", "lo" },
{ "server_name", "" },
{ "file_server_root_path", "loleaflet/.." },
+ { "memproportion", "80.0" },
{ "num_prespawn_children", "1" },
{ "per_document.max_concurrency", "4" },
{ "per_document.idle_timeout_secs", "3600" },
@@ -1118,6 +1119,19 @@ void LOOLWSD::doHousekeeping()
PrisonerPoll.wakeup();
}
+void LOOLWSD::closeDocument(const std::string& docKey, const std::string& message)
+{
+ std::unique_lock<std::mutex> docBrokersLock(DocBrokersMutex);
+ auto docBrokerIt = DocBrokers.find(docKey);
+ if (docBrokerIt != DocBrokers.end())
+ {
+ std::shared_ptr<DocumentBroker> docBroker = docBrokerIt->second;
+ docBroker->addCallback([docBroker, message]() {
+ docBroker->closeDocument(message);
+ });
+ }
+}
+
/// Really do the house-keeping
void PrisonerPoll::wakeupHook()
{
diff --git a/wsd/LOOLWSD.hpp b/wsd/LOOLWSD.hpp
index d98ecebe..c26d6858 100644
--- a/wsd/LOOLWSD.hpp
+++ b/wsd/LOOLWSD.hpp
@@ -121,6 +121,9 @@ public:
/// child kit processes and cleans up DocBrokers.
static void doHousekeeping();
+ /// Close document with @docKey and a @message
+ static void closeDocument(const std::string& docKey, const std::string& message);
+
protected:
void initialize(Poco::Util::Application& self) override;
void defineOptions(Poco::Util::OptionSet& options) override;
@@ -155,6 +158,7 @@ private:
void operator()(unsigned int& value) { value = _config.getUInt(_name); }
void operator()(bool& value) { value = _config.getBool(_name); }
void operator()(std::string& value) { value = _config.getString(_name); }
+ void operator()(double& value) { value = _config.getDouble(_name); }
};
template <typename T>
More information about the Libreoffice-commits
mailing list