[Libreoffice-commits] dev-tools.git: 3 commits - perf/compare-logs.py perf/test-startup-time.py
Xisco Fauli (via logerrit)
logerrit at kemper.freedesktop.org
Wed Mar 25 13:26:16 UTC 2020
perf/compare-logs.py | 102 +++++++++++++++++++++++++++
perf/test-startup-time.py | 169 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 271 insertions(+)
New commits:
commit 96b3fbed90f9d48dce4abb4d28d901fce8a07669
Author: Xisco Fauli <xiscofauli at libreoffice.org>
AuthorDate: Tue Mar 24 15:09:19 2020 +0100
Commit: Xisco Fauli <xiscofauli at libreoffice.org>
CommitDate: Wed Mar 25 14:25:42 2020 +0100
check for remaining processes and kill them every now and then
diff --git a/perf/test-startup-time.py b/perf/test-startup-time.py
index 5fbd25b..d5b443c 100755
--- a/perf/test-startup-time.py
+++ b/perf/test-startup-time.py
@@ -29,6 +29,7 @@ import tempfile
import time
import subprocess
import logging
+import signal
from multiprocessing_logging import install_mp_handler
extensions = [ "odt", "doc", "docx", "rtf", "ods", "xls", "xlsx", "odp", "ppt", "pptx" ]
@@ -56,6 +57,15 @@ class DefaultHelpParser(argparse.ArgumentParser):
self.print_help()
sys.exit(2)
+def kill_soffice():
+ p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
+ out, err = p.communicate()
+ for line in out.splitlines():
+ if b'soffice' in line:
+ pid = int(line.split(None, 1)[0])
+ print("Killing process: " + str(pid))
+ os.kill(pid, signal.SIGKILL)
+
def get_file_names(filesPath):
auxNames = []
for fileName in os.listdir(filesPath):
@@ -129,18 +139,26 @@ if __name__ == '__main__':
for line in f:
previousResults.append(line.strip().split(' ')[2])
- install_mp_handler()
- pool = multiprocessing.Pool() # use all CPUs
- manager = multiprocessing.Manager()
+ kill_soffice()
+
totalCount = 0
- for fileName in listFiles:
- if fileName not in previousResults:
- totalCount += 1
- pool.apply_async(launchLibreOffice, args=(fileName, liboPath))
+ cpuCount = multiprocessing.cpu_count() #use all CPUs
+ chunkSplit = cpuCount * 16
+ chunks = [listFiles[x:x+chunkSplit] for x in range(0, len(listFiles), chunkSplit)]
+
+ for chunk in chunks:
+ install_mp_handler()
+ pool = multiprocessing.Pool(cpuCount)
+ for fileName in chunk:
+ if fileName not in previousResults:
+ totalCount += 1
+ pool.apply_async(launchLibreOffice, args=(fileName, liboPath))
+
+ pool.close()
+ pool.join()
- pool.close()
- pool.join()
+ kill_soffice()
print()
if totalCount:
commit db7ca440601e61f13e3a81d7c1a9ff38aabd057a
Author: Xisco Fauli <xiscofauli at libreoffice.org>
AuthorDate: Tue Mar 24 13:47:34 2020 +0100
Commit: Xisco Fauli <xiscofauli at libreoffice.org>
CommitDate: Wed Mar 25 14:25:42 2020 +0100
Add script to compare logs
diff --git a/perf/compare-logs.py b/perf/compare-logs.py
new file mode 100755
index 0000000..82e6d0c
--- /dev/null
+++ b/perf/compare-logs.py
@@ -0,0 +1,102 @@
+#! /usr/bin/python3
+# Version: MPL 1.1 / GPLv3+ / LGPLv3+
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License or as specified alternatively below. You may obtain a copy of
+# the License at http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# Major Contributor(s):
+# Copyright (C) 2012 Red Hat, Inc., Michael Stahl <mstahl at redhat.com>
+# (initial developer)
+#
+# All Rights Reserved.
+#
+# For minor contributions see the git repository.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 3 or later (the "GPLv3+"), or
+# the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"),
+# in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable
+# instead of those above.
+
+import sys, os, getopt
+
+def readFile(url):
+ d = {}
+ with open(url) as f:
+ for line in f:
+ key = line.strip().split(' ')[2]
+ val = line.strip().split('- ')[1]
+
+ d[key] = float(val)
+ return d
+
+def usage():
+ message = """usage: {program} [option]..."
+ -h | --help: print usage information
+ --old=URL path to old file to be compared
+ --new=URL path to new file to be comparted"""
+
+ print(message.format(program = os.path.basename(sys.argv[0])))
+
+
+if __name__ == "__main__":
+
+ try:
+ opts,args = getopt.getopt(sys.argv[1:], "o:n:h",
+ ["old=", "new=", "help"])
+ except getopt.GetoptError:
+ usage()
+ sys.exit(1)
+
+ for opt, arg in opts:
+ if opt in ('-h', '--help'):
+ usage()
+ sys.exit(1)
+ elif opt in ('-o', '--old'):
+ old_url = arg
+ elif opt in ('-n', '--new'):
+ new_url = arg
+ else:
+ usage()
+ sys.exit(1)
+
+ if not os.path.exists(old_url):
+ print("File " + old_url + " doesn't exist!")
+ sys.exit(1)
+ elif not os.path.exists(new_url):
+ print("File " + new_url + " doesn't exist!")
+ sys.exit(1)
+
+ oldValues = readFile(old_url)
+ newValues = readFile(new_url)
+ meanOld = sum(oldValues.values())/len(oldValues)
+ maxValue = max(oldValues.values())
+
+ results = {}
+ print("Mean value: " + str(meanOld))
+
+ for k, v in oldValues.items():
+ if k not in newValues:
+ print("File: " + k + " doesn't exist. Why?")
+ continue
+ diff = newValues[k] / v
+ # check if it's 3 times slower for small values
+ # or 2 times slower for greater values
+ # or timeout is reached
+ if diff >= 3 \
+ or (v > meanOld and diff >= 2 ) \
+ or (v != maxValue and newValues[k] == maxValue):
+ results[k] = [diff, v, newValues[k]]
+
+ sorted_results = sorted(results.items(), key=lambda kv: kv[1], reverse=True)
+ for k, v in sorted_results:
+ print("File " + k + " is " + str('%.3f' % v[0]) + " slower. Before: " + str('%.3f' % v[1]) + ". After: " + str('%.3f' % v[2]))
+
+# vim:set shiftwidth=4 softtabstop=4 expandtab:
commit 0ae2ff8b0004ef5fdeb6c31bc442237902ef7392
Author: Xisco Fauli <xiscofauli at libreoffice.org>
AuthorDate: Fri Mar 20 18:40:28 2020 +0100
Commit: Xisco Fauli <xiscofauli at libreoffice.org>
CommitDate: Wed Mar 25 14:25:42 2020 +0100
Add new script to test startup time using OOO_EXIT_POST_STARTUP
diff --git a/perf/test-startup-time.py b/perf/test-startup-time.py
new file mode 100755
index 0000000..5fbd25b
--- /dev/null
+++ b/perf/test-startup-time.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+# Version: MPL 1.1 / GPLv3+ / LGPLv3+
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License or as specified alternatively below. You may obtain a copy of
+# the License at http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# All Rights Reserved.
+#
+# For minor contributions see the git repository.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 3 or later (the "GPLv3+"), or
+# the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"),
+# in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable
+# instead of those above.
+
+import argparse
+import sys
+import os
+import multiprocessing
+import tempfile
+import time
+import subprocess
+import logging
+from multiprocessing_logging import install_mp_handler
+
+extensions = [ "odt", "doc", "docx", "rtf", "ods", "xls", "xlsx", "odp", "ppt", "pptx" ]
+
+importTimeout = 300
+
+def start_logger(fileName):
+ rootLogger = logging.getLogger()
+ rootLogger.setLevel(os.environ.get("LOGLEVEL", "INFO"))
+
+ logFormatter = logging.Formatter("[%(asctime)s] %(message)s")
+ fileHandler = logging.FileHandler(fileName)
+ fileHandler.setFormatter(logFormatter)
+ rootLogger.addHandler(fileHandler)
+
+ streamHandler = logging.StreamHandler(sys.stdout)
+ streamHandler.setFormatter(logFormatter)
+ rootLogger.addHandler(streamHandler)
+
+ return rootLogger
+
+class DefaultHelpParser(argparse.ArgumentParser):
+ def error(self, message):
+ sys.stderr.write('error: %s\n' % message)
+ self.print_help()
+ sys.exit(2)
+
+def get_file_names(filesPath):
+ auxNames = []
+ for fileName in os.listdir(filesPath):
+ for ext in extensions:
+ if fileName.endswith(ext):
+ auxNames.append("file:///" + filesPath + fileName)
+
+ #Remove previous lock files
+ lockFilePath = filesPath + '.~lock.' + fileName + '#'
+ if os.path.isfile(lockFilePath):
+ os.remove(lockFilePath)
+
+ return auxNames
+
+def launchLibreOffice(fileName, soffice):
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ profilePath = os.path.join(tmpdirname, 'libreoffice/4')
+ userPath = os.path.join(profilePath, 'user')
+ os.makedirs(userPath)
+
+ argv = [ soffice + 'instdir/program/soffice',
+ "-env:UserInstallation=file://" + userPath,
+ "--quickstart=no", "--headless", "--nofirststartwizard",
+ "--norestore", "--nologo"]
+ argv.append(fileName)
+
+ diffTime = None
+ try:
+ start_time = time.time()
+ subprocess.run(argv, stderr=subprocess.DEVNULL, timeout=importTimeout)
+ diffTime = time.time() - start_time
+ except subprocess.TimeoutExpired:
+ diffTime = importTimeout
+
+ logger.info(fileName + ' - ' + str(diffTime))
+
+if __name__ == '__main__':
+ parser = DefaultHelpParser()
+
+ parser.add_argument(
+ '--dir', required=True, help="Path to the files directory")
+ parser.add_argument(
+ '--soffice', required=True, help="Path to the LibreOffice directory")
+ argument = parser.parse_args()
+
+ filesPath = os.path.join(argument.dir, '')
+ if not os.path.exists(filesPath):
+ parser.error(filesPath + " is an invalid directory path")
+
+ liboPath = os.path.join(argument.soffice, '')
+ if not os.path.exists(liboPath) or not os.path.exists(liboPath + "instdir/program/"):
+ parser.error(liboPath + " is an invalid LibreOffice path")
+
+ listFiles = get_file_names(filesPath)
+ listFiles.sort()
+
+ os.environ["OOO_EXIT_POST_STARTUP"] = "1"
+
+ sofficePath = liboPath + "instdir/program/soffice"
+ process = subprocess.Popen([sofficePath, "--version"],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout = process.communicate()[0].decode("utf-8")
+ sourceHash = stdout.split(" ")[2].strip()
+
+ logFile = sourceHash + '.log'
+ logger = start_logger(logFile)
+
+ previousResults = []
+ if os.path.exists(logFile):
+ with open(logFile) as f:
+ for line in f:
+ previousResults.append(line.strip().split(' ')[2])
+
+ install_mp_handler()
+ pool = multiprocessing.Pool() # use all CPUs
+ manager = multiprocessing.Manager()
+ totalCount = 0
+
+ for fileName in listFiles:
+ if fileName not in previousResults:
+ totalCount += 1
+ pool.apply_async(launchLibreOffice, args=(fileName, liboPath))
+
+ pool.close()
+ pool.join()
+
+ print()
+ if totalCount:
+ print(str(totalCount) + " new results added to " + logFile)
+ else:
+ print("No new results added to " + logFile)
+
+# vim:set shiftwidth=4 softtabstop=4 expandtab:
More information about the Libreoffice-commits
mailing list