[Libreoffice-commits] core.git: comphelper/Library_comphelper.mk comphelper/source include/comphelper package/source
Dennis Francis (via logerrit)
logerrit at kemper.freedesktop.org
Sat Jan 4 06:58:35 UTC 2020
comphelper/Library_comphelper.mk | 1
comphelper/source/misc/meminfo.cxx | 139 +++++++++++++++++++++++++
include/comphelper/meminfo.hxx | 37 ++++++
package/source/zippackage/ZipPackageStream.cxx | 5
4 files changed, 181 insertions(+), 1 deletion(-)
New commits:
commit 0b8ae8725083eb0526a262d434cc06fb3f3e7336
Author: Dennis Francis <dennis.francis at collabora.com>
AuthorDate: Wed Dec 4 15:29:06 2019 +0530
Commit: Dennis Francis <dennis.francis at collabora.com>
CommitDate: Sat Jan 4 07:58:06 2020 +0100
tdf#125662: disable parallel-zip if the memory...
required to carry it out will cause a breach of addressable memory
limit. This can happen in 32bit OSes, where the physical memory limit is
more than the process addressable memory limit. For example in Win32,
a process can address only upto 2GB.
More specifically, avoid using ZipOutputEntryParallel
which rougly needs twice the memory compared to its counterparts.
Change-Id: I4d1abbf75e928188bcf806fcf1e5872b1e51b502
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/84394
Tested-by: Jenkins
Reviewed-by: Dennis Francis <dennis.francis at collabora.com>
diff --git a/comphelper/Library_comphelper.mk b/comphelper/Library_comphelper.mk
index e0bf1a8574f5..b5de3f58b36a 100644
--- a/comphelper/Library_comphelper.mk
+++ b/comphelper/Library_comphelper.mk
@@ -117,6 +117,7 @@ $(eval $(call gb_Library_add_exception_objects,comphelper,\
comphelper/source/misc/listenernotification \
comphelper/source/misc/logging \
comphelper/source/misc/lok \
+ comphelper/source/misc/meminfo \
comphelper/source/misc/mimeconfighelper \
comphelper/source/misc/namedvaluecollection \
comphelper/source/misc/numberedcollection \
diff --git a/comphelper/source/misc/meminfo.cxx b/comphelper/source/misc/meminfo.cxx
new file mode 100644
index 000000000000..d6508e9de567
--- /dev/null
+++ b/comphelper/source/misc/meminfo.cxx
@@ -0,0 +1,139 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * This file incorporates work covered by the following license notice:
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed
+ * with this work for additional information regarding copyright
+ * ownership. The ASF licenses this file to you under the Apache
+ * License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.apache.org/licenses/LICENSE-2.0 .
+ */
+
+#include <sal/types.h>
+#include <comphelper/meminfo.hxx>
+
+#if defined(_WIN32)
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include "psapi.h"
+
+#elif defined(UNX)
+
+#if defined(MACOSX)
+#include <sys/types.h>
+#include <mach/mach.h>
+#include <mach/host_info.h>
+#include <mach/mach_host.h>
+
+#else // Linux
+#include <fstream>
+#include <unistd.h>
+#endif
+
+#endif
+
+namespace comphelper
+{
+// Returns the virtual memory used currently by the process.
+static sal_Int64 getMemUsedBySelf()
+{
+#if defined(UNX)
+#if defined(MACOSX)
+
+ vm_size_t pageSize;
+ struct task_basic_info tInfo;
+ mach_msg_type_number_t tInfoCount = TASK_BASIC_INFO_COUNT;
+
+ if (host_page_size(mach_host_self(), &pageSize) != KERN_SUCCESS)
+ return -1;
+
+ if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&tInfo, &tInfoCount)
+ != KERN_SUCCESS)
+ return -1;
+
+ return static_cast<sal_Int64>(tInfo.virtual_size) * pageSize;
+
+#elif defined(LINUX)
+
+ sal_Int64 pageSize = sysconf(_SC_PAGESIZE);
+ if (pageSize <= 0)
+ return -1;
+
+ std::ifstream ifs("/proc/self/statm", std::ifstream::in);
+ if (!ifs.is_open())
+ return -1;
+
+ sal_Int64 vmSize;
+ if (ifs >> vmSize)
+ return vmSize * pageSize;
+
+ return -1;
+
+#else
+
+ return -1;
+
+#endif
+
+#elif defined(_WIN32)
+
+ PROCESS_MEMORY_COUNTERS_EX pmc;
+ GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&pmc),
+ sizeof(pmc));
+ return static_cast<sal_Int64>(pmc.PrivateUsage);
+
+#else
+
+ return -1;
+
+#endif
+}
+
+static sal_Int64 getMaxAddressableMemLimit()
+{
+#if defined(_WIN64)
+ return 8796093022208ll; // 8 TB
+#elif defined(_WIN32)
+ return 2147483648ll; // 2 GB
+#elif defined(__x86_64__)
+ // TODO: check for artificial limits imposed by 'ulimit -Sv' too for UNX ?
+ return 140737488355328ll; // 128 TB
+#elif defined(__i386__)
+ return 3221225472ll; // 3 GB
+#else
+ return 2147483648ll; // 2 GB
+#endif
+}
+
+// canAlloc() checks whether allocSize bytes can be allocated without exceeding
+// addressable memory limit. This is useful in case of 32-bit systems where
+// process addressable memory limit is less than physical memory limit.
+bool canAlloc(sal_Int64 allocSize)
+{
+ if (allocSize <= 0)
+ return true;
+
+ sal_Int64 vmSize = getMemUsedBySelf();
+ if (vmSize < 0)
+ return true;
+
+ sal_Int64 maxSize = getMaxAddressableMemLimit();
+
+ if (vmSize + allocSize <= maxSize)
+ return true;
+
+ return false;
+}
+
+} // namespace comphelper
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/include/comphelper/meminfo.hxx b/include/comphelper/meminfo.hxx
new file mode 100644
index 000000000000..795d7a45bba9
--- /dev/null
+++ b/include/comphelper/meminfo.hxx
@@ -0,0 +1,37 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * This file incorporates work covered by the following license notice:
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed
+ * with this work for additional information regarding copyright
+ * ownership. The ASF licenses this file to you under the Apache
+ * License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.apache.org/licenses/LICENSE-2.0 .
+ */
+
+#ifndef INCLUDED_COMPHELPER_MEMINFO_HXX
+#define INCLUDED_COMPHELPER_MEMINFO_HXX
+
+#include <comphelper/comphelperdllapi.h>
+
+namespace comphelper
+{
+/**
+ * canAlloc() checks whether allocSize bytes can be allocated without exceeding
+ * addressable memory limit. This is useful in case of 32-bit systems where
+ * process addressable memory limit is less than physical memory limit.
+ */
+COMPHELPER_DLLPUBLIC bool canAlloc(sal_Int64 allocSize);
+}
+
+#endif // INCLUDED_COMPHELPER_MEMINFO_HXX
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */
diff --git a/package/source/zippackage/ZipPackageStream.cxx b/package/source/zippackage/ZipPackageStream.cxx
index 745157cfd02e..d8f9c594d89f 100644
--- a/package/source/zippackage/ZipPackageStream.cxx
+++ b/package/source/zippackage/ZipPackageStream.cxx
@@ -48,6 +48,7 @@
#include <comphelper/seekableinput.hxx>
#include <comphelper/servicehelper.hxx>
#include <comphelper/storagehelper.hxx>
+#include <comphelper/meminfo.hxx>
#include <cppuhelper/exc_hlp.hxx>
#include <cppuhelper/supportsservice.hxx>
#include <cppuhelper/typeprovider.hxx>
@@ -768,7 +769,9 @@ bool ZipPackageStream::saveChild(
// the full size, but it appears that at this point it usually is.
sal_Int64 estimatedSize = xSeek.is() ? xSeek->getLength() : xStream->available();
- if (estimatedSize > 1000000)
+ // ZipOutputEntryParallel/ThreadedDeflater needs to allocate ~twice the estimatedSize.
+ // Check if we can allocate that much without breaching addressable memory limits.
+ if (estimatedSize > 1000000 && comphelper::canAlloc(2*estimatedSize))
{
// Use ThreadDeflater which will split the stream into blocks and compress
// them in threads, but not in background (i.e. writeStream() will block).
More information about the Libreoffice-commits
mailing list