[Libreoffice-commits] dev-tools.git: 4 commits - uitest/mass-testing
Xisco Fauli (via logerrit)
logerrit at kemper.freedesktop.org
Tue Jun 16 07:32:24 UTC 2020
uitest/mass-testing/run.py | 316 ++++++++++++++++++++----------------------
uitest/mass-testing/writer.py | 48 ------
2 files changed, 153 insertions(+), 211 deletions(-)
New commits:
commit 3a135dbf93e05d29a57aa69c53912b6e27b8113f
Author: Xisco Fauli <xiscofauli at libreoffice.org>
AuthorDate: Fri Jun 12 22:38:13 2020 +0200
Commit: Xisco Fauli <xiscofauli at libreoffice.org>
CommitDate: Tue Jun 16 09:31:43 2020 +0200
mass-uitesting: remove 2 more unneeded tests
Change-Id: I9289311fe02c54a7716939d7b2f172330b16b75b
diff --git a/uitest/mass-testing/writer.py b/uitest/mass-testing/writer.py
index 5603186..104e4c7 100755
--- a/uitest/mass-testing/writer.py
+++ b/uitest/mass-testing/writer.py
@@ -71,30 +71,6 @@ class massTesting(UITestCase):
self.ui_test.close_doc()
- def test_insert_remove_header(self):
- xEdit = self.load_file()
- if xEdit:
- document = self.ui_test.get_component()
-
- #Insert Default header if it doesn't exists
- if not document.StyleFamilies.PageStyles.Standard.HeaderIsOn:
- self.xUITest.executeCommand(
- ".uno:InsertPageHeader?PageStyle:string=Default%20Page%20Style&On:bool=true")
-
- self.assertEqual(document.StyleFamilies.PageStyles.Standard.HeaderIsOn, True)
-
- # Delete the header
- self.ui_test.execute_dialog_through_command(
- ".uno:InsertPageHeader?PageStyle:string=Default%20Page%20Style&On:bool=false")
-
- xDialog = self.xUITest.getTopFocusWindow() #question dialog
- xOption = xDialog.getChild("yes")
- xOption.executeAction("CLICK", tuple())
-
- self.assertEqual(document.StyleFamilies.PageStyles.Standard.HeaderIsOn, False)
-
- self.ui_test.close_doc()
-
def test_copy_all_paste_undo(self):
xEdit = self.load_file()
if xEdit:
@@ -110,30 +86,6 @@ class massTesting(UITestCase):
self.ui_test.close_doc()
- def test_insert_remove_footer(self):
- xEdit = self.load_file()
- if xEdit:
- document = self.ui_test.get_component()
-
- #Insert Default footer if it doesn't exists
- if not document.StyleFamilies.PageStyles.Standard.FooterIsOn:
- self.xUITest.executeCommand(
- ".uno:InsertPageFooter?PageStyle:string=Default%20Page%20Style&On:bool=true")
-
- self.assertEqual(document.StyleFamilies.PageStyles.Standard.FooterIsOn, True)
-
- # Delete the header
- self.ui_test.execute_dialog_through_command(
- ".uno:InsertPageFooter?PageStyle:string=Default%20Page%20Style&On:bool=false")
-
- xDialog = self.xUITest.getTopFocusWindow() #question dialog
- xOption = xDialog.getChild("yes")
- xOption.executeAction("CLICK", tuple())
-
- self.assertEqual(document.StyleFamilies.PageStyles.Standard.FooterIsOn, False)
-
- self.ui_test.close_doc()
-
def test_traverse_all_pages(self):
xEdit = self.load_file()
if xEdit:
commit c5a757a61ab44c766aaacb91bba8cbceef9e5dae
Author: Xisco Fauli <xiscofauli at libreoffice.org>
AuthorDate: Tue May 12 18:14:50 2020 +0200
Commit: Xisco Fauli <xiscofauli at libreoffice.org>
CommitDate: Tue Jun 16 09:31:43 2020 +0200
mass-testing: remove resume argument and print count
Change-Id: I5461d3a778a70b13eaf3c6825e0edcddbb0b325e
diff --git a/uitest/mass-testing/run.py b/uitest/mass-testing/run.py
index 04be40c..a10f82f 100755
--- a/uitest/mass-testing/run.py
+++ b/uitest/mass-testing/run.py
@@ -62,16 +62,20 @@ def start_logger(name):
def get_file_names(filesPath):
auxNames = []
for fileName in os.listdir(filesPath):
- auxNames.append("file:///" + filesPath + fileName)
+ for key, val in extensions.items():
+ extension = os.path.splitext(fileName)[1][1:]
+ if extension in val:
+ auxNames.append("file:///" + filesPath + fileName)
- #Remove previous lock files
- lockFilePath = filesPath + '.~lock.' + fileName + '#'
- if os.path.isfile(lockFilePath):
- os.remove(lockFilePath)
+ #Remove previous lock files
+ lockFilePath = filesPath + '.~lock.' + fileName + '#'
+ if os.path.isfile(lockFilePath):
+ os.remove(lockFilePath)
+ break
return auxNames
-def launchLibreOffice(logger, fileName, sofficePath, component, isDebug):
+def launchLibreOffice(logger, fileName, sofficePath, component, countInfo, isDebug):
#Create temp directory for the user profile
with tempfile.TemporaryDirectory() as tmpdirname:
profilePath = os.path.join(tmpdirname, 'libreoffice/4')
@@ -102,7 +106,7 @@ def launchLibreOffice(logger, fileName, sofficePath, component, isDebug):
# 1. The file can't be loaded in 'fileInterval' seconds
# 2. The test can't be executed in 'testInterval' seconds
fileInterval = 20
- testIternval = 20
+ testInterval = 20
timeout = time.time() + fileInterval
notLoaded = True
while True:
@@ -110,9 +114,9 @@ def launchLibreOffice(logger, fileName, sofficePath, component, isDebug):
if time.time() > timeout:
if notLoaded:
- logger.info("SKIP: " + fileName)
+ logger.info(countInfo + " - SKIP: " + fileName)
else:
- logger.info("TIMEOUT: " + fileName)
+ logger.info(countInfo + " - TIMEOUT: " + fileName)
# kill popen process
os.killpg(process.pid, signal.SIGKILL)
@@ -137,7 +141,7 @@ def launchLibreOffice(logger, fileName, sofficePath, component, isDebug):
if line.startswith("mass-uitesting:"):
message = line.split(":")[1]
if message == 'skipped':
- logger.info("SKIP: " + fileName + " : " + importantInfo)
+ logger.info(countInfo + " - SKIP: " + fileName + " : " + importantInfo)
# kill popen process
os.killpg(process.pid, signal.SIGKILL)
@@ -147,7 +151,7 @@ def launchLibreOffice(logger, fileName, sofficePath, component, isDebug):
notLoaded = False
#Extend timeout
- timeout += testIternval
+ timeout += testInterval
elif 'Execution time' in line:
importantInfo = line.split('for ')[1]
@@ -157,15 +161,15 @@ def launchLibreOffice(logger, fileName, sofficePath, component, isDebug):
if importantInfo:
if isFailure:
- logger.info("FAIL: " + fileName + " : " + importantInfo)
+ logger.info(countInfo + " - FAIL: " + fileName + " : " + importantInfo)
else:
# No error found between the Execution time line and the end of stdout
- logger.info("PASS: " + fileName + " : " + str(importantInfo))
+ logger.info(countInfo + " - PASS: " + fileName + " : " + str(importantInfo))
if process.poll() is not None:
break
-def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
+def run_tests_and_get_results(sofficePath, listFiles, isDebug):
process = Popen([sofficePath, "--version"], stdout=PIPE, stderr=PIPE)
stdout = process.communicate()[0].decode("utf-8")
@@ -177,7 +181,8 @@ def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
logName = './logs/' + sourceHash + ".log"
logger = start_logger(logName)
- if isResume:
+ previousLog = ""
+ if os.path.exists(logName):
with open(logName, 'r') as file:
previousLog = file.read()
@@ -185,28 +190,29 @@ def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
cpuCount = multiprocessing.cpu_count() #use all CPUs
chunkSplit = cpuCount * 16
chunks = [listFiles[x:x+chunkSplit] for x in range(0, len(listFiles), chunkSplit)]
+ totalCount = len(listFiles)
+ count = 0
for chunk in chunks:
install_mp_handler()
pool = multiprocessing.Pool(cpuCount)
for fileName in chunk:
- if isResume:
- if fileName in previousLog:
- print("SKIP: " + fileName)
- continue
+ count += 1
+ countInfo = str(count) + '/' + str(totalCount)
+
+ if fileName in previousLog:
+ print(countInfo + " - SKIP: " + fileName)
+ continue
extension = os.path.splitext(fileName)[1][1:]
- component = ""
for key, val in extensions.items():
if extension in val:
component = key
-
- if not component:
- continue
+ break
pool.apply_async(launchLibreOffice,
- args=(logger, fileName, sofficePath, component, isDebug))
+ args=(logger, fileName, sofficePath, component, countInfo, isDebug))
pool.close()
pool.join()
@@ -230,8 +236,6 @@ if __name__ == '__main__':
'--soffice', required=True, help="Path to the LibreOffice directory")
parser.add_argument(
'--debug', action='store_true', help="Flag to print output")
- parser.add_argument(
- '--resume', action='store_true', help="Flag to resume previous runs")
argument = parser.parse_args()
@@ -250,6 +254,6 @@ if __name__ == '__main__':
listFiles = get_file_names(filesPath)
listFiles.sort()
- run_tests_and_get_results(sofficePath, listFiles, argument.debug, argument.resume)
+ run_tests_and_get_results(sofficePath, listFiles, argument.debug)
# vim: set shiftwidth=4 softtabstop=4 expandtab:
commit ead40d49f1e23c86bc505ec1ec720fa8bbacf126
Author: Xisco Fauli <xiscofauli at libreoffice.org>
AuthorDate: Tue May 12 17:26:50 2020 +0200
Commit: Xisco Fauli <xiscofauli at libreoffice.org>
CommitDate: Tue Jun 16 09:31:43 2020 +0200
mass-testing: support multiprocessing
Change-Id: I749d57abcd946d203e9a12b78633c5448ded680f
diff --git a/uitest/mass-testing/run.py b/uitest/mass-testing/run.py
index d6da66a..04be40c 100755
--- a/uitest/mass-testing/run.py
+++ b/uitest/mass-testing/run.py
@@ -15,6 +15,8 @@ from shutil import copyfile
import time
import fcntl
import tempfile
+import multiprocessing
+from multiprocessing_logging import install_mp_handler
extensions = {
'writer' : [ "odt", "doc", "docx", "rtf" ],
@@ -34,6 +36,15 @@ class DefaultHelpParser(argparse.ArgumentParser):
self.print_help()
sys.exit(2)
+def kill_soffice():
+ p = Popen(['ps', '-A'], stdout=PIPE)
+ out, err = p.communicate()
+ for line in out.splitlines():
+ if b'soffice' in line:
+ pid = int(line.split(None, 1)[0])
+ print("Killing process: " + str(pid))
+ os.kill(pid, signal.SIGKILL)
+
def start_logger(name):
rootLogger = logging.getLogger()
rootLogger.setLevel(os.environ.get("LOGLEVEL", "INFO"))
@@ -60,6 +71,100 @@ def get_file_names(filesPath):
return auxNames
+def launchLibreOffice(logger, fileName, sofficePath, component, isDebug):
+ #Create temp directory for the user profile
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ profilePath = os.path.join(tmpdirname, 'libreoffice/4')
+ userPath = os.path.join(profilePath, 'user')
+ os.makedirs(userPath)
+
+ # Replace the profile file with
+ # 1. DisableMacrosExecution = True
+ # 2. IgnoreProtectedArea = True
+ # 3. AutoPilot = False
+ copyfile(os.getcwd() + '/registrymodifications.xcu', userPath + '/registrymodifications.xcu')
+
+ #TODO: Find a better way to pass fileName parameter
+ os.environ["TESTFILENAME"] = fileName
+
+ process = Popen(["python3",
+ './uitest/test_main.py',
+ "--debug",
+ "--soffice=path:" + sofficePath,
+ "--userdir=file://" + profilePath,
+ "--file=" + component + ".py"], stdin=PIPE, stdout=PIPE, stderr=PIPE,
+ preexec_fn=os.setsid)
+
+ # Do not block on process.stdout
+ fcntl.fcntl(process.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
+
+ # Kill the process if:
+ # 1. The file can't be loaded in 'fileInterval' seconds
+ # 2. The test can't be executed in 'testInterval' seconds
+ fileInterval = 20
+ testIternval = 20
+ timeout = time.time() + fileInterval
+ notLoaded = True
+ while True:
+ time.sleep(1)
+
+ if time.time() > timeout:
+ if notLoaded:
+ logger.info("SKIP: " + fileName)
+ else:
+ logger.info("TIMEOUT: " + fileName)
+
+ # kill popen process
+ os.killpg(process.pid, signal.SIGKILL)
+ break
+
+ try:
+ outputLines = process.stdout.readlines()
+ except IOError:
+ pass
+
+ importantInfo = ''
+ isFailure = False
+ for line in outputLines:
+ line = line.decode("utf-8").strip()
+
+ if not line:
+ continue
+
+ if isDebug:
+ print(line)
+
+ if line.startswith("mass-uitesting:"):
+ message = line.split(":")[1]
+ if message == 'skipped':
+ logger.info("SKIP: " + fileName + " : " + importantInfo)
+
+ # kill popen process
+ os.killpg(process.pid, signal.SIGKILL)
+
+ break
+ elif message == 'loaded':
+ notLoaded = False
+
+ #Extend timeout
+ timeout += testIternval
+
+ elif 'Execution time' in line:
+ importantInfo = line.split('for ')[1]
+
+ elif importantInfo and 'error' == line.lower() or 'fail' == line.lower():
+ isFailure = True
+
+ if importantInfo:
+ if isFailure:
+ logger.info("FAIL: " + fileName + " : " + importantInfo)
+ else:
+ # No error found between the Execution time line and the end of stdout
+ logger.info("PASS: " + fileName + " : " + str(importantInfo))
+
+ if process.poll() is not None:
+ break
+
def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
process = Popen([sofficePath, "--version"], stdout=PIPE, stderr=PIPE)
@@ -76,114 +181,38 @@ def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
with open(logName, 'r') as file:
previousLog = file.read()
- for fileName in listFiles:
- if isResume:
- if fileName in previousLog:
- print("SKIP: " + fileName)
+ kill_soffice()
+ cpuCount = multiprocessing.cpu_count() #use all CPUs
+ chunkSplit = cpuCount * 16
+ chunks = [listFiles[x:x+chunkSplit] for x in range(0, len(listFiles), chunkSplit)]
+
+ for chunk in chunks:
+ install_mp_handler()
+ pool = multiprocessing.Pool(cpuCount)
+ for fileName in chunk:
+ if isResume:
+ if fileName in previousLog:
+ print("SKIP: " + fileName)
+ continue
+
+ extension = os.path.splitext(fileName)[1][1:]
+
+ component = ""
+ for key, val in extensions.items():
+ if extension in val:
+ component = key
+
+ if not component:
continue
- extension = os.path.splitext(fileName)[1][1:]
-
- component = ""
- for key, val in extensions.items():
- if extension in val:
- component = key
-
- if not component:
- continue
-
- #Create temp directory for the user profile
- with tempfile.TemporaryDirectory() as tmpdirname:
- profilePath = os.path.join(tmpdirname, 'libreoffice/4')
- userPath = os.path.join(profilePath, 'user')
- os.makedirs(userPath)
-
- # Replace the profile file with
- # 1. DisableMacrosExecution = True
- # 2. IgnoreProtectedArea = True
- # 3. AutoPilot = False
- copyfile(os.getcwd() + '/registrymodifications.xcu', userPath + '/registrymodifications.xcu')
-
- #TODO: Find a better way to pass fileName parameter
- os.environ["TESTFILENAME"] = fileName
-
- process = Popen(["python3",
- './uitest/test_main.py',
- "--debug",
- "--soffice=path:" + sofficePath,
- "--userdir=file://" + profilePath,
- "--file=" + component + ".py"], stdin=PIPE, stdout=PIPE, stderr=PIPE,
- preexec_fn=os.setsid)
-
- # Do not block on process.stdout
- fcntl.fcntl(process.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
-
- # Kill the process if:
- # 1. The file can't be loaded in 'fileInterval' seconds
- # 2. The test can't be executed in 'testInterval' seconds
- fileInterval = 10
- testIternval = 20
- timeout = time.time() + fileInterval
- notLoaded = True
- while True:
- time.sleep(1)
-
- if time.time() > timeout:
- if notLoaded:
- logger.info("SKIP: " + fileName)
- else:
- logger.info("TIMEOUT: " + fileName)
-
- # kill popen process
- os.killpg(process.pid, signal.SIGKILL)
- break
-
- try:
- outputLines = process.stdout.readlines()
- except IOError:
- pass
-
- importantInfo = ''
- isFailure = False
- for line in outputLines:
- line = line.decode("utf-8").strip()
-
- if not line:
- continue
-
- if isDebug:
- print(line)
-
- if line.startswith("mass-uitesting:"):
- message = line.split(":")[1]
- if message == 'skipped':
- logger.info("SKIP: " + fileName + " : " + importantInfo)
-
- # kill popen process
- os.killpg(process.pid, signal.SIGKILL)
-
- break
- elif message == 'loaded':
- notLoaded = False
-
- #Extend timeout
- timeout += testIternval
-
- elif 'Execution time' in line:
- importantInfo = line.split('for ')[1]
-
- elif importantInfo and 'error' == line.lower() or 'fail' == line.lower():
- isFailure = True
-
- if importantInfo:
- if isFailure:
- logger.info("FAIL: " + fileName + " : " + importantInfo)
- else:
- # No error found between the Execution time line and the end of stdout
- logger.info("PASS: " + fileName + " : " + str(importantInfo))
-
- if process.poll() is not None:
- break
+ pool.apply_async(launchLibreOffice,
+ args=(logger, fileName, sofficePath, component, isDebug))
+
+ pool.close()
+ pool.join()
+
+ kill_soffice()
+
if __name__ == '__main__':
currentPath = os.path.dirname(os.path.realpath(__file__))
commit 07206d1e002a249f7e25e9f723fcb810a8e3ce0c
Author: Xisco Fauli <xiscofauli at libreoffice.org>
AuthorDate: Tue May 12 14:42:03 2020 +0200
Commit: Xisco Fauli <xiscofauli at libreoffice.org>
CommitDate: Tue Jun 16 09:31:43 2020 +0200
mass-uitesting: simplify logs and results
Change-Id: I5074ebff9344d32b9e9a0d5d1619634613023ec6
diff --git a/uitest/mass-testing/run.py b/uitest/mass-testing/run.py
index 92cf32e..d6da66a 100755
--- a/uitest/mass-testing/run.py
+++ b/uitest/mass-testing/run.py
@@ -12,7 +12,6 @@ import sys
import signal
import logging
from shutil import copyfile
-import pickle
import time
import fcntl
import tempfile
@@ -35,12 +34,12 @@ class DefaultHelpParser(argparse.ArgumentParser):
self.print_help()
sys.exit(2)
-def start_logger():
+def start_logger(name):
rootLogger = logging.getLogger()
rootLogger.setLevel(os.environ.get("LOGLEVEL", "INFO"))
logFormatter = logging.Formatter("%(asctime)s %(message)s")
- fileHandler = logging.FileHandler("./log")
+ fileHandler = logging.FileHandler(name)
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
@@ -63,32 +62,26 @@ def get_file_names(filesPath):
def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
- results = {
- 'pass' : 0,
- 'fail' : 0,
- 'timeout' : 0,
- 'skip' : 0}
-
process = Popen([sofficePath, "--version"], stdout=PIPE, stderr=PIPE)
stdout = process.communicate()[0].decode("utf-8")
sourceHash = stdout.split(" ")[2].strip()
- #Keep track of the files run
- filesRun = {}
-
- if isResume:
- pklFile = './resume.pkl'
- if os.path.exists(pklFile):
- with open(pklFile, 'rb') as pickle_in:
- filesRun = pickle.load(pickle_in)
+ if not os.path.exists('./logs'):
+ os.makedirs('./logs')
- if sourceHash not in filesRun:
- filesRun[sourceHash] = {'files': []}
+ logName = './logs/' + sourceHash + ".log"
+ logger = start_logger(logName)
- if 'results' in filesRun[sourceHash]:
- results = filesRun[sourceHash]['results']
+ if isResume:
+ with open(logName, 'r') as file:
+ previousLog = file.read()
for fileName in listFiles:
+ if isResume:
+ if fileName in previousLog:
+ print("SKIP: " + fileName)
+ continue
+
extension = os.path.splitext(fileName)[1][1:]
component = ""
@@ -99,11 +92,6 @@ def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
if not component:
continue
- if isResume:
- if fileName in filesRun[sourceHash]['files']:
- print("SKIP: " + fileName)
- continue
-
#Create temp directory for the user profile
with tempfile.TemporaryDirectory() as tmpdirname:
profilePath = os.path.join(tmpdirname, 'libreoffice/4')
@@ -143,10 +131,8 @@ def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
if time.time() > timeout:
if notLoaded:
logger.info("SKIP: " + fileName)
- results['skip'] += 1
else:
logger.info("TIMEOUT: " + fileName)
- results['timeout'] += 1
# kill popen process
os.killpg(process.pid, signal.SIGKILL)
@@ -172,7 +158,6 @@ def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
message = line.split(":")[1]
if message == 'skipped':
logger.info("SKIP: " + fileName + " : " + importantInfo)
- results['skip'] += 1
# kill popen process
os.killpg(process.pid, signal.SIGKILL)
@@ -193,36 +178,13 @@ def run_tests_and_get_results(sofficePath, listFiles, isDebug, isResume):
if importantInfo:
if isFailure:
logger.info("FAIL: " + fileName + " : " + importantInfo)
- results['fail'] += 1
else:
# No error found between the Execution time line and the end of stdout
logger.info("PASS: " + fileName + " : " + str(importantInfo))
- results['pass'] += 1
if process.poll() is not None:
break
- if isResume:
- filesRun[sourceHash]['files'].append(fileName)
-
- filesRun[sourceHash]['results'] = results
-
- with open(pklFile, 'wb') as pickle_out:
- pickle.dump(filesRun, pickle_out)
-
-
- totalTests = sum(results.values())
- if totalTests > 0:
- logger.info("")
- logger.info("Total Tests: " + str(totalTests))
- logger.info("\tPASS: " + str(results['pass']))
- logger.info("\tSKIP: " + str(results['skip']))
- logger.info("\tTIMEOUT: " + str(results['timeout']))
- logger.info("\tFAIL: " + str(results['fail']))
- logger.info("")
- else:
- print("No test run!")
-
if __name__ == '__main__':
currentPath = os.path.dirname(os.path.realpath(__file__))
uitestPath = os.path.join(currentPath, 'uitest/test_main.py')
@@ -256,11 +218,6 @@ if __name__ == '__main__':
os.environ["URE_BOOTSTRAP"] = "file://" + sofficePath.split('/soffice')[0] + '/fundamentalrc'
os.environ["SAL_USE_VCLPLUGIN"] = "gen"
- if not os.path.exists('./logs'):
- os.makedirs('./logs')
-
- logger = start_logger()
-
listFiles = get_file_names(filesPath)
listFiles.sort()
More information about the Libreoffice-commits
mailing list