[Libreoffice-commits] online.git: 2 commits - cypress_test/Makefile.am cypress_test/run_parallel.sh
Tamás Zolnai (via logerrit)
logerrit at kemper.freedesktop.org
Mon Jul 27 21:19:09 UTC 2020
cypress_test/Makefile.am | 131 ++++++++++++++++++++-----------------------
cypress_test/run_parallel.sh | 95 +++++++++++++++++++++++++++++++
2 files changed, 156 insertions(+), 70 deletions(-)
New commits:
commit 439b4af8ad89ab566c3ac41a955f78223644e633
Author: Tamás Zolnai <tamas.zolnai at collabora.com>
AuthorDate: Thu Jul 23 18:33:09 2020 +0200
Commit: Tamás Zolnai <tamas.zolnai at collabora.com>
CommitDate: Mon Jul 27 23:18:59 2020 +0200
cypress: better way of detecting error during parallel run.
We used to grep for error messages in the output, but
it can change what is written out. Better to catch if
the command fails directly.
Change-Id: I8c3cba212b262227551876bfb23629a243424c39
Reviewed-on: https://gerrit.libreoffice.org/c/online/+/99365
Tested-by: Jenkins
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice at gmail.com>
Reviewed-by: Tamás Zolnai <tamas.zolnai at collabora.com>
diff --git a/cypress_test/run_parallel.sh b/cypress_test/run_parallel.sh
index 38add0215..21f4e6c2a 100755
--- a/cypress_test/run_parallel.sh
+++ b/cypress_test/run_parallel.sh
@@ -43,6 +43,7 @@ while test $# -gt 0; do
esac
shift
done
+TEST_ERROR="${TEST_LOG}.error"
TEST_FILE_PATH=
if [ "${TEST_TYPE}" = "desktop" ]; then
@@ -60,8 +61,6 @@ RUN_COMMAND="${CYPRESS_BINARY} run \
--env ${TEST_ENV}\
--spec=${TEST_FILE_PATH}"
-ERROR_MATCHER="Error:\|Command failed:\|Timed out retrying\|The error was:"
-
print_error() {
echo -e "\n\
CypressError: a test failed, please do one of the following:\n\n\
@@ -71,15 +70,20 @@ print_error() {
\tcd cypress_test && make run-${TEST_TYPE} spec=${TEST_FILE}\n" >> ${ERROR_LOG}
}
+run_command() {
+ rm -rf ${TEST_ERROR}
+ echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND} || touch ${TEST_ERROR}`" > ${TEST_LOG} 2>&1
+}
+
mkdir -p `dirname ${TEST_LOG}`
touch ${TEST_LOG}
-echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND}`" > ${TEST_LOG} 2>&1
-if [ -z `grep -o -m 1 "${ERROR_MATCHER}" ${TEST_LOG}` ];
+run_command
+if [ ! -f ${TEST_ERROR} ];
then cat ${TEST_LOG};
elif [ ${SECOND_CHANCE} = true ];
then echo "Second chance!" > ${TEST_LOG} && \
- echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND}`" >> ${TEST_LOG} 2>&1 && \
- if [ -z `grep -o -m 1 "${ERROR_MATCHER}" ${TEST_LOG}` ];\
+ run_command && \
+ if [ ! -f ${TEST_ERROR} ];\
then cat ${TEST_LOG};\
else cat ${TEST_LOG} >> ${ERROR_LOG} && \
print_error; \
commit 09abe6724f5e13181e7dfeca0ddcb2df131468f4
Author: Tamás Zolnai <tamas.zolnai at collabora.com>
AuthorDate: Thu Jul 23 13:23:14 2020 +0200
Commit: Tamás Zolnai <tamas.zolnai at collabora.com>
CommitDate: Mon Jul 27 23:18:50 2020 +0200
cypress: extract parallel test running code to a shell script file.
Change-Id: Ice856737b5b4bcfc6a202b507a03fdfa16938c25
Reviewed-on: https://gerrit.libreoffice.org/c/online/+/99364
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice at gmail.com>
Tested-by: Jenkins
Reviewed-by: Tamás Zolnai <tamas.zolnai at collabora.com>
diff --git a/cypress_test/Makefile.am b/cypress_test/Makefile.am
index c0dab338b..9f6901bcf 100644
--- a/cypress_test/Makefile.am
+++ b/cypress_test/Makefile.am
@@ -9,6 +9,8 @@ WAIT_ON_BINARY = $(abs_builddir)/node_modules/wait-on/bin/wait-on
GET_PORT_BINARY = $(abs_builddir)/node_modules/get-port-cli/cli.js
NPM_INSTALLED = $(abs_builddir)/workdir/npm_installed
+PARALLEL_SCRIPT = $(abs_srcdir)/run_parallel.sh
+
PID_FILE=$(abs_builddir)/workdir/loolwsd.pid
ERROR_LOG=$(abs_builddir)/workdir/error.log
@@ -202,16 +204,22 @@ define start_Xvfb
endef
DESKTOP_CONFIG = \
- --config integrationFolder=$(DESKTOP_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(DESKTOP_USER_AGENT) \
- --env DATA_FOLDER=$(DESKTOP_DATA_FOLDER),WORKDIR=$(DESKTOP_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)"
+ integrationFolder=$(DESKTOP_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(DESKTOP_USER_AGENT)
+
+DESKTOP_ENV = \
+ DATA_FOLDER=$(DESKTOP_DATA_FOLDER),WORKDIR=$(DESKTOP_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)"
MOBILE_CONFIG = \
- --config integrationFolder=$(MOBILE_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(MOBILE_USER_AGENT) \
- --env DATA_FOLDER=$(MOBILE_DATA_FOLDER),WORKDIR=$(MOBILE_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)"
+ integrationFolder=$(MOBILE_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(MOBILE_USER_AGENT)
+
+MOBILE_ENV = \
+ DATA_FOLDER=$(MOBILE_DATA_FOLDER),WORKDIR=$(MOBILE_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)"
MULTIUSER_CONFIG = \
- --config integrationFolder=$(MULTIUSER_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(DESKTOP_USER_AGENT),defaultCommandTimeout=30000 \
- --env DATA_FOLDER=$(MULTIUSER_DATA_FOLDER),WORKDIR=$(MULTIUSER_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)"
+ integrationFolder=$(MULTIUSER_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(DESKTOP_USER_AGENT),defaultCommandTimeout=30000
+
+MULTIUSER_ENV = \
+ DATA_FOLDER=$(MULTIUSER_DATA_FOLDER),WORKDIR=$(MULTIUSER_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)"
define run_interactive_test
$(if $(2),\
@@ -219,98 +227,81 @@ define run_interactive_test
--browser $(CHROME) \
--headed --no-exit \
$(if $(findstring DESKTOP,$(1)),\
- $(DESKTOP_CONFIG) \
- --spec=$(abs_dir)/integration_tests/desktop/$(2),\
- $(MOBILE_CONFIG) \
+ --config $(DESKTOP_CONFIG) \
+ --env $(DESKTOP_ENV) \
+ --spec=$(abs_dir)/integration_tests/desktop/$(2)\
+ ,\
+ --config $(MOBILE_CONFIG) \
+ --env $(MOBILE_ENV) \
--spec=$(abs_dir)/integration_tests/mobile/$(2)),\
$(CYPRESS_BINARY) open \
- $(if $(findstring DESKTOP,$(1)),$(DESKTOP_CONFIG),$(MOBILE_CONFIG)))
+ $(if $(findstring DESKTOP,$(1)),\
+ --config $(DESKTOP_CONFIG) \
+ --env $(DESKTOP_ENV) \
+ ,\
+ --config $(MOBILE_CONFIG) \
+ --env $(MOBILE_ENV) \))
endef
define run_desktop_tests
@echo $(if $(1),"Running cypress desktop test: $(1)","Running cypress desktop tests...")
@echo
- $(eval RUN_COMMAND = \
+ $(if $(PARALLEL_BUILD),\
+ @$(PARALLEL_SCRIPT) \
+ --browser $(CHROME) \
+ --config $(DESKTOP_CONFIG) \
+ --env $(DESKTOP_ENV)$(if $(3),$(COMMA)$(3)) \
+ --spec $(1) \
+ --type desktop \
+ --log-file $(2) \
+ --second-chance \
+ ,\
$(CYPRESS_BINARY) run \
--browser $(CHROME) \
--headless \
- $(DESKTOP_CONFIG)$(if $(3),$(COMMA)$(3)) \
+ --config $(DESKTOP_CONFIG) \
+ --env $(DESKTOP_ENV)$(if $(3),$(COMMA)$(3)) \
$(if $(1), --spec=$(abs_dir)/integration_tests/desktop/$(1)) \
- $(if $(PARALLEL_BUILD),,|| ($(KILL_COMMAND) && false)))
- $(if $(PARALLEL_BUILD),\
- $(call execute_run_parallel,$(RUN_COMMAND),$(2),desktop,$(1))\
- ,\
- $(RUN_COMMAND)\
+ || ($(KILL_COMMAND) && false) \
)
endef
define run_mobile_tests
@echo $(if $(1),"Running cypress mobile test: $(1)","Running cypress mobile tests...")
@echo
- $(eval RUN_COMMAND = \
+ $(if $(PARALLEL_BUILD),\
+ @$(PARALLEL_SCRIPT) \
+ --browser $(CHROME) \
+ --config $(MOBILE_CONFIG) \
+ --env $(MOBILE_ENV)$(if $(3),$(COMMA)$(3)) \
+ --spec $(1) \
+ --type mobile \
+ --log-file $(2) \
+ --second-chance \
+ ,\
$(CYPRESS_BINARY) run \
--browser $(CHROME) \
--headless \
- $(MOBILE_CONFIG)$(if $(3),$(COMMA)$(3)) \
+ --config $(MOBILE_CONFIG) \
+ --env $(MOBILE_ENV)$(if $(3),$(COMMA)$(3)) \
$(if $(1), --spec=$(abs_dir)/integration_tests/mobile/$(1)) \
- $(if $(PARALLEL_BUILD),,|| ($(KILL_COMMAND) && false)))
- $(if $(PARALLEL_BUILD),\
- $(call execute_run_parallel,$(RUN_COMMAND),$(2),mobile,$(1))\
- ,\
- $(RUN_COMMAND)\
+ || ($(KILL_COMMAND) && false) \
)
endef
define run_multiuser_tests
@echo "Running cypress multi-user test: $(1)"
@echo
- $(eval RUN_COMMAND = \
- $(CYPRESS_BINARY) run \
+ $(if $(PARALLEL_BUILD),\
+ @$(PARALLEL_SCRIPT) \
--browser $(CHROME) \
- --headless \
- $(MULTIUSER_CONFIG)$(if $(3),$(COMMA)$(3)) \
- $(if $(1), --spec=$(abs_dir)/integration_tests/multiuser/$(1)) \
- )
- $(call execute_run_parallel_no_second_chance,$(RUN_COMMAND),$(2),desktop,$(1))
-endef
-
-ERROR_MATCHER = "Error:\|Command failed:\|Timed out retrying\|The error was:"
-
-define execute_run_parallel
- @mkdir -p $(dir $(2)) && touch $(2) && \
- echo "`echo $(1) && $(1)`" > $(2) 2>&1 && \
- if [ -z `grep -o -m 1 $(ERROR_MATCHER) $(2)` ];\
- then cat $(2);\
- elif [ -z $(CYPRESS_NO_SECOND_CHANCE) ]; \
- then echo "Second chance!" > $(2) && \
- echo "`echo $(1) && $(1)`" >> $(2) 2>&1 && \
- if [ -z `grep -o -m 1 $(ERROR_MATCHER) $(2)` ];\
- then cat $(2);\
- else cat $(2) >> $(ERROR_LOG) && \
- echo -e $(call error_output_string,$(3),$(4)) >> $(ERROR_LOG); \
- fi;\
- else cat $(2) >> $(ERROR_LOG) && \
- echo -e $(call error_output_string,$(3),$(4)) >> $(ERROR_LOG); \
- fi;
-endef
-
-define execute_run_parallel_no_second_chance
- @mkdir -p $(dir $(2)) && touch $(2) && \
- echo "`echo $(1) && $(1)`" > $(2) 2>&1 && \
- if [ -z `grep -o -m 1 $(ERROR_MATCHER) $(2)` ];\
- then cat $(2);\
- else cat $(2) >> $(ERROR_LOG) && \
- echo -e $(call error_output_string,$(3),$(4)) >> $(ERROR_LOG); \
- fi;
-endef
-
-define error_output_string
-"\n\
-CypressError: a test failed, please do one of the following:\n\n\
-Run the failing test in headless mode:\n\
-\tcd cypress_test && make check-$(1) spec=$(2)\n\n\
-Open the failing test in the interactive test runner:\n\
-\tcd cypress_test && make run-$(1) spec=$(2)\n"
+ --config $(MULTIUSER_CONFIG) \
+ --env $(MULTIUSER_ENV)$(if $(3),$(COMMA)$(3)) \
+ --spec $(1) \
+ --type multi-user \
+ --log-file $(2) \
+ ,\
+ )
endef
NODE_BINS = \
diff --git a/cypress_test/run_parallel.sh b/cypress_test/run_parallel.sh
new file mode 100755
index 000000000..38add0215
--- /dev/null
+++ b/cypress_test/run_parallel.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CYPRESS_BINARY="${DIR}/node_modules/cypress/bin/cypress"
+DESKTOP_TEST_FOLDER="${DIR}/integration_tests/desktop/"
+MOBILE_TEST_FOLDER="${DIR}/integration_tests/mobile/"
+MULTIUSER_TEST_FOLDER="${DIR}/integration_tests/multiuser/"
+ERROR_LOG="${DIR}/workdir/error.log"
+
+print_help ()
+{
+ echo "Usage: run_parallel.sh --spec <name_spec.js> OPTIONS"
+ echo "Runs a specified cypress test"
+ echo ""
+ echo " --spec <file> The test file we need to run"
+ echo " --log-file <file> Log output to this test"
+ echo " --config <string> Configure options passed to cypress"
+ echo " --env <string> Cypress own environment variables"
+ echo " --type <string> Type of the test (e.g. mobile, desktop)"
+ echo " --browser <file> Path to the browser binary"
+ echo " --second-chance Enable second chance"
+ exit 1
+}
+
+TEST_FILE=
+TEST_LOG=
+TEST_CONFIG=
+TEST_ENV=
+TEST_TYPE=
+BROWSER=
+SECOND_CHANCE=false
+while test $# -gt 0; do
+ case $1 in
+ --spec) TEST_FILE=$2; shift;;
+ --log-file) TEST_LOG=$2; shift;;
+ --config) TEST_CONFIG=$2; shift;;
+ --env) TEST_ENV=$2; shift;;
+ --type) TEST_TYPE=$2; shift;;
+ --browser) BROWSER=$2; shift;;
+ --second-chance) SECOND_CHANCE=true; shift;;
+ --help) print_help ;;
+ -*) ;; # ignore
+ esac
+ shift
+done
+
+TEST_FILE_PATH=
+if [ "${TEST_TYPE}" = "desktop" ]; then
+ TEST_FILE_PATH=${DESKTOP_TEST_FOLDER}${TEST_FILE};
+elif [ "${TEST_TYPE}" = "multi-user" ]; then
+ TEST_FILE_PATH=${MULTIUSER_TEST_FOLDER}${TEST_FILE};
+else
+ TEST_FILE_PATH=${MOBILE_TEST_FOLDER}${TEST_FILE};
+fi
+
+RUN_COMMAND="${CYPRESS_BINARY} run \
+ --browser ${BROWSER} \
+ --headless \
+ --config ${TEST_CONFIG}\
+ --env ${TEST_ENV}\
+ --spec=${TEST_FILE_PATH}"
+
+ERROR_MATCHER="Error:\|Command failed:\|Timed out retrying\|The error was:"
+
+print_error() {
+ echo -e "\n\
+ CypressError: a test failed, please do one of the following:\n\n\
+ Run the failing test in headless mode:\n\
+ \tcd cypress_test && make check-${TEST_TYPE} spec=${TEST_FILE}\n\n\
+ Open the failing test in the interactive test runner:\n\
+ \tcd cypress_test && make run-${TEST_TYPE} spec=${TEST_FILE}\n" >> ${ERROR_LOG}
+}
+
+mkdir -p `dirname ${TEST_LOG}`
+touch ${TEST_LOG}
+echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND}`" > ${TEST_LOG} 2>&1
+if [ -z `grep -o -m 1 "${ERROR_MATCHER}" ${TEST_LOG}` ];
+ then cat ${TEST_LOG};
+ elif [ ${SECOND_CHANCE} = true ];
+ then echo "Second chance!" > ${TEST_LOG} && \
+ echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND}`" >> ${TEST_LOG} 2>&1 && \
+ if [ -z `grep -o -m 1 "${ERROR_MATCHER}" ${TEST_LOG}` ];\
+ then cat ${TEST_LOG};\
+ else cat ${TEST_LOG} >> ${ERROR_LOG} && \
+ print_error; \
+ fi;
+ else cat ${TEST_LOG} >> ${ERROR_LOG} && \
+ print_error;
+fi;
+
+# vim:set shiftwidth=4 expandtab:
More information about the Libreoffice-commits
mailing list