[Libreoffice-commits] online.git: Branch 'distro/collabora/collabora-online-cd' - common/Util.cpp common/Util.hpp loleaflet/src loolwsd.xml.in wsd/Admin.cpp wsd/Admin.hpp wsd/AdminModel.cpp wsd/AdminModel.hpp wsd/DocumentBroker.cpp wsd/DocumentBroker.hpp wsd/LOOLWSD.cpp wsd/LOOLWSD.hpp
Pranav Kant
pranavk at collabora.co.uk
Fri Jul 7 16:41:07 UTC 2017
common/Util.cpp | 21 ++++++++++++++++++++
common/Util.hpp | 3 ++
loleaflet/src/core/Socket.js | 4 +--
loolwsd.xml.in | 1
wsd/Admin.cpp | 44 +++++++++++++++++++++++++++++++++++++++++++
wsd/Admin.hpp | 8 ++++++-
wsd/AdminModel.cpp | 19 ++++++++++++++++++
wsd/AdminModel.hpp | 21 ++++++++++++++++++++
wsd/DocumentBroker.cpp | 12 +++++------
wsd/DocumentBroker.hpp | 4 +++
wsd/LOOLWSD.cpp | 14 +++++++++++++
wsd/LOOLWSD.hpp | 4 +++
12 files changed, 146 insertions(+), 9 deletions(-)
New commits:
commit 99c65d66434e6c623b0a2d7e5fdc5b46a45ade19
Author: Pranav Kant <pranavk at collabora.co.uk>
Date: Fri Jul 7 17:12:19 2017 +0530
Introduce memproportion of memory to use before shutting down processes.
Start killing documents when memory usage goes above threshold.
Also make it possible to close documents from admin instance.
In DocumentBroker::closeDocument, just set the _stop flag and wake
up the polling thread which will terminate the children, instead of
manually terminating the children.
Change-Id: Ie70e05b3fb6ea816a87b6dcfaed92cdddb94aa90
(cherry picked from commit fde57adbbf9ab2fba80c6b8e0d877c797b55bea5)
diff --git a/common/Util.cpp b/common/Util.cpp
index b3bcb354..bd7209f9 100644
--- a/common/Util.cpp
+++ b/common/Util.cpp
@@ -171,6 +171,27 @@ namespace Util
return nullptr;
}
+ size_t getTotalSystemMemory()
+ {
+ size_t totalMemKb = 0;
+ FILE* file = fopen("/proc/meminfo", "r");
+ if (file != nullptr)
+ {
+ char line[4096] = { 0 };
+ while (fgets(line, sizeof(line), file))
+ {
+ const char* value;
+ if ((value = startsWith(line, "MemTotal:")))
+ {
+ totalMemKb = atoi(value);
+ break;
+ }
+ }
+ }
+
+ return totalMemKb;
+ }
+
std::pair<size_t, size_t> getPssAndDirtyFromSMaps(FILE* file)
{
size_t numPSSKb = 0;
diff --git a/common/Util.hpp b/common/Util.hpp
index 93f25cf0..5480d790 100644
--- a/common/Util.hpp
+++ b/common/Util.hpp
@@ -95,6 +95,9 @@ namespace Util
#endif
}
+ /// Returns the total physical memory (in kB) available in the system
+ size_t getTotalSystemMemory();
+
/// Returns the process PSS in KB (works only when we have perms for /proc/pid/smaps).
size_t getMemoryUsagePSS(const Poco::Process::PID pid);
diff --git a/loleaflet/src/core/Socket.js b/loleaflet/src/core/Socket.js
index e137acca..9843d6e7 100644
--- a/loleaflet/src/core/Socket.js
+++ b/loleaflet/src/core/Socket.js
@@ -231,7 +231,7 @@ L.Socket = L.Class.extend({
if (textMsg === 'ownertermination') {
msg = _('Session terminated by document owner');
}
- else if (textMsg === 'idle') {
+ else if (textMsg === 'idle' || textMsg === 'oom') {
msg = _('Session was terminated due to idleness - please click to reload');
this._map._documentIdle = true;
}
@@ -301,7 +301,7 @@ L.Socket = L.Class.extend({
});
options.$vex.append(options.$vexContent);
- if (textMsg === 'idle') {
+ if (textMsg === 'idle' || textMsg === 'oom') {
var map = this._map;
options.$vex.bind('click.vex', function(e) {
console.debug('idleness: reactivating');
diff --git a/loolwsd.xml.in b/loolwsd.xml.in
index eed51283..6f8d6d35 100644
--- a/loolwsd.xml.in
+++ b/loolwsd.xml.in
@@ -11,6 +11,7 @@
<server_name desc="Hostname:port of the server running loolwsd. If empty, it's derived from the request." type="string" default=""></server_name>
<file_server_root_path desc="Path to the directory that should be considered root for the file server. This should be the directory containing loleaflet." type="path" relative="true" default="loleaflet/../"></file_server_root_path>
+ <memproportion desc="The maximum percentage of system memory consumed by all of the LibreOffice Online, after which we start cleaning up idle documents" type="double" default="80.0"></memproportion>
<num_prespawn_children desc="Number of child processes to keep started in advance and waiting for new clients." type="uint" default="1">1</num_prespawn_children>
<per_document desc="Document-specific settings, including LO Core settings.">
<max_concurrency desc="The maximum number of threads to use while processing a document." type="uint" default="4">4</max_concurrency>
diff --git a/wsd/Admin.cpp b/wsd/Admin.cpp
index 5153586d..3cffc82b 100644
--- a/wsd/Admin.cpp
+++ b/wsd/Admin.cpp
@@ -291,6 +291,9 @@ Admin::Admin() :
{
LOG_INF("Admin ctor.");
+ _totalSysMem = Util::getTotalSystemMemory();
+ LOG_TRC("Total system memory : " << _totalSysMem);
+
const auto totalMem = getTotalMemoryUsage();
LOG_TRC("Total memory used: " << totalMem);
_model.addMemStats(totalMem);
@@ -336,6 +339,9 @@ void Admin::pollingThread()
lastMem = now;
memWait += _memStatsTaskIntervalMs;
+
+ // If our total memory consumption is above limit, cleanup
+ triggerMemoryCleanup(totalMem);
}
// Handle websockets & other work.
@@ -422,6 +428,44 @@ void Admin::updateMemoryDirty(const std::string& docKey, int dirty)
{ _model.updateMemoryDirty(docKey, dirty); });
}
+
+void Admin::triggerMemoryCleanup(size_t totalMem)
+{
+ LOG_TRC("Total memory we are consuming (in kB): " << totalMem);
+ // Trigger mem cleanup when we are consuming too much memory (as configured by sysadmin)
+ const auto memLimit = LOOLWSD::getConfigValue<double>("memproportion", static_cast<double>(80.0));
+ LOG_TRC("Mem proportion for LOOL configured : " << memLimit);
+ float memToFreePercentage = 0;
+ if ( (memToFreePercentage = (totalMem/static_cast<double>(_totalSysMem)) - memLimit/100.) > 0.0 )
+ {
+ int memToFree = memToFreePercentage * totalMem;
+ LOG_TRC("Memory to be freed (in kB) : " << memToFree);
+ // prepare document list sorted by most idle times
+ std::list<DocBasicInfo> docList = _model.getDocumentsSortedByIdle();
+
+ LOG_TRC("Checking saved documents in document list, length: " << docList.size());
+ // Kill the saved documents first
+ std::list<DocBasicInfo>::iterator docIt = docList.begin();
+ while (docIt != docList.end() && memToFree > 0)
+ {
+ LOG_TRC("Document: DocKey[" << docIt->_docKey << "], Idletime[" << docIt->_idleTime << "],"
+ << " Saved: [" << docIt->_saved << "], Mem: [" << docIt->_mem << "].");
+ if (docIt->_saved)
+ {
+ // Kill and remove from list
+ LOG_DBG("OOM: Killing saved document with DocKey " << docIt->_docKey);
+ LOOLWSD::closeDocument(docIt->_docKey, "oom");
+ memToFree -= docIt->_mem;
+ docList.erase(docIt);
+ }
+ else
+ ++docIt;
+ }
+ }
+
+ LOG_TRC("OOM: Memory to free percentage : " << memToFreePercentage);
+}
+
void Admin::dumpState(std::ostream& os)
{
// FIXME: be more helpful ...
diff --git a/wsd/Admin.hpp b/wsd/Admin.hpp
index f9d8bb6a..65ca3243 100644
--- a/wsd/Admin.hpp
+++ b/wsd/Admin.hpp
@@ -103,11 +103,17 @@ public:
void dumpState(std::ostream& os) override;
private:
+ /// Memory consumption has increased, start killing kits etc. till memory consumption gets back
+ /// under @hardModeLimit
+ void triggerMemoryCleanup(size_t hardModeLimit);
+
+private:
/// The model is accessed only during startup & in
/// the Admin Poll thread.
AdminModel _model;
int _forKitPid;
- long _lastTotalMemory;
+ size_t _lastTotalMemory;
+ size_t _totalSysMem;
std::atomic<int> _memStatsTaskIntervalMs;
std::atomic<int> _cpuStatsTaskIntervalMs;
diff --git a/wsd/AdminModel.cpp b/wsd/AdminModel.cpp
index dcb8279f..0afcd342 100644
--- a/wsd/AdminModel.cpp
+++ b/wsd/AdminModel.cpp
@@ -426,6 +426,25 @@ unsigned AdminModel::getTotalActiveViews()
return numTotalViews;
}
+std::list<DocBasicInfo> AdminModel::getDocumentsSortedByIdle() const
+{
+ std::list<DocBasicInfo> docList;
+ for (const auto& it: _documents)
+ {
+ docList.emplace_back(it.second.getDocKey(),
+ it.second.getIdleTime(),
+ !it.second.getModifiedStatus(),
+ it.second.getMemoryDirty());
+ }
+
+ // Sort the list by idle times;
+ docList.sort([](const DocBasicInfo& a, const DocBasicInfo& b) {
+ return a._idleTime > b._idleTime;
+ });
+
+ return docList;
+}
+
std::string AdminModel::getDocuments() const
{
assertCorrectThread();
diff --git a/wsd/AdminModel.hpp b/wsd/AdminModel.hpp
index 5445224b..16a99125 100644
--- a/wsd/AdminModel.hpp
+++ b/wsd/AdminModel.hpp
@@ -43,6 +43,22 @@ private:
std::time_t _end = 0;
};
+/// Containing basic information about document
+struct DocBasicInfo
+{
+ std::string _docKey;
+ std::time_t _idleTime;
+ bool _saved;
+ int _mem;
+
+ DocBasicInfo(const std::string& docKey, std::time_t idleTime, bool saved, int mem)
+ : _docKey(docKey),
+ _idleTime(idleTime),
+ _saved(saved),
+ _mem(mem)
+ { }
+};
+
/// A document in Admin controller.
class Document
{
@@ -59,6 +75,8 @@ public:
{
}
+ const std::string getDocKey() const { return _docKey; }
+
Poco::Process::PID getPid() const { return _pid; }
std::string getFilename() const { return _filename; }
@@ -198,6 +216,9 @@ public:
void updateLastActivityTime(const std::string& docKey);
void updateMemoryDirty(const std::string& docKey, int dirty);
+ /// Document basic info list sorted by most idle time
+ std::list<DocBasicInfo> getDocumentsSortedByIdle() const;
+
private:
std::string getMemStats();
diff --git a/wsd/DocumentBroker.cpp b/wsd/DocumentBroker.cpp
index a993e3d9..818ecfb1 100644
--- a/wsd/DocumentBroker.cpp
+++ b/wsd/DocumentBroker.cpp
@@ -157,6 +157,7 @@ DocumentBroker::DocumentBroker(const std::string& uri,
_cursorHeight(0),
_poll(new DocumentBrokerPoll("docbroker_" + _docId, *this)),
_stop(false),
+ _closeReason("stopped"),
_tileVersion(0),
_debugRenderedTileCount(0)
{
@@ -223,8 +224,6 @@ void DocumentBroker::pollThread()
static const bool AutoSaveEnabled = !std::getenv("LOOL_NO_AUTOSAVE");
static const size_t IdleDocTimeoutSecs = LOOLWSD::getConfigValue<int>(
"per_document.idle_timeout_secs", 3600);
- std::string closeReason = "stopped";
-
// Main polling loop goodness.
while (!_stop && _poll->continuePolling() && !TerminationFlag)
{
@@ -241,7 +240,7 @@ void DocumentBroker::pollThread()
if (ShutdownRequestFlag)
{
- closeReason = "recycling";
+ _closeReason = "recycling";
_stop = true;
}
else if (AutoSaveEnabled && !_stop &&
@@ -260,7 +259,7 @@ void DocumentBroker::pollThread()
{
LOG_INF("Terminating " << (idle ? "idle" : "dead") <<
" DocumentBroker for docKey [" << getDocKey() << "].");
- closeReason = (idle ? "idle" : "dead");
+ _closeReason = (idle ? "idle" : "dead");
_stop = true;
}
}
@@ -283,7 +282,7 @@ void DocumentBroker::pollThread()
}
// Terminate properly while we can.
- terminateChild(closeReason);
+ terminateChild(_closeReason);
// Stop to mark it done and cleanup.
_poll->stop();
@@ -1396,7 +1395,8 @@ void DocumentBroker::closeDocument(const std::string& reason)
assertCorrectThread();
LOG_DBG("Closing DocumentBroker for docKey [" << _docKey << "] with reason: " << reason);
- terminateChild(reason);
+ _closeReason = reason; // used later in the polling loop
+ stop();
}
void DocumentBroker::updateLastActivityTime()
diff --git a/wsd/DocumentBroker.hpp b/wsd/DocumentBroker.hpp
index 11354c0f..d0d0e3a5 100644
--- a/wsd/DocumentBroker.hpp
+++ b/wsd/DocumentBroker.hpp
@@ -337,6 +337,9 @@ public:
/// Sends the .uno:Save command to LoKit.
bool sendUnoSave(const std::string& sessionId, bool dontTerminateEdit = true, bool dontSaveIfUnmodified = true);
+ /// Sets the reason for closing document;
+ void setCloseReason(const std::string& closeReason) { _closeReason = closeReason; }
+
private:
/// Shutdown all client connections with the given reason.
@@ -403,6 +406,7 @@ private:
mutable std::mutex _mutex;
std::unique_ptr<DocumentBrokerPoll> _poll;
std::atomic<bool> _stop;
+ std::string _closeReason;
/// Versioning is used to prevent races between
/// painting and invalidation.
diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp
index 5e3ce0e4..5c8adaf6 100644
--- a/wsd/LOOLWSD.cpp
+++ b/wsd/LOOLWSD.cpp
@@ -617,6 +617,7 @@ void LOOLWSD::initialize(Application& self)
{ "lo_jail_subpath", "lo" },
{ "server_name", "" },
{ "file_server_root_path", "loleaflet/.." },
+ { "memproportion", "80.0" },
{ "num_prespawn_children", "1" },
{ "per_document.max_concurrency", "4" },
{ "per_document.idle_timeout_secs", "3600" },
@@ -1096,6 +1097,19 @@ void LOOLWSD::doHousekeeping()
PrisonerPoll.wakeup();
}
+void LOOLWSD::closeDocument(const std::string& docKey, const std::string& message)
+{
+ std::unique_lock<std::mutex> docBrokersLock(DocBrokersMutex);
+ auto docBrokerIt = DocBrokers.find(docKey);
+ if (docBrokerIt != DocBrokers.end())
+ {
+ std::shared_ptr<DocumentBroker> docBroker = docBrokerIt->second;
+ docBroker->addCallback([docBroker, message]() {
+ docBroker->closeDocument(message);
+ });
+ }
+}
+
/// Really do the house-keeping
void PrisonerPoll::wakeupHook()
{
diff --git a/wsd/LOOLWSD.hpp b/wsd/LOOLWSD.hpp
index a608c6c1..178e80c9 100644
--- a/wsd/LOOLWSD.hpp
+++ b/wsd/LOOLWSD.hpp
@@ -121,6 +121,9 @@ public:
/// child kit processes and cleans up DocBrokers.
static void doHousekeeping();
+ /// Close document with @docKey and a @message
+ static void closeDocument(const std::string& docKey, const std::string& message);
+
protected:
void initialize(Poco::Util::Application& self) override;
void defineOptions(Poco::Util::OptionSet& options) override;
@@ -155,6 +158,7 @@ private:
void operator()(unsigned int& value) { value = _config.getUInt(_name); }
void operator()(bool& value) { value = _config.getBool(_name); }
void operator()(std::string& value) { value = _config.getString(_name); }
+ void operator()(double& value) { value = _config.getDouble(_name); }
};
template <typename T>
More information about the Libreoffice-commits
mailing list