Mesa (master): ci: Consistently use -j4 across x86 build jobs and -j8 on ARM.
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Wed Apr 1 18:52:48 UTC 2020
Module: Mesa
Branch: master
Commit: c1e7e83d52d912b9a1ba5009e8f84d737c1e0d21
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=c1e7e83d52d912b9a1ba5009e8f84d737c1e0d21
Author: Eric Anholt <eric at anholt.net>
Date: Tue Feb 11 15:44:56 2020 -0800
ci: Consistently use -j4 across x86 build jobs and -j8 on ARM.
Our shared runners are set up for concurrent jobs ~= CPUs / 4 (x86) or 8
(ARM). If you use more build processes than that, then jobs may be
fighting each other for shared system resources, possibly to the point of
failure (we've seen one of the runners OOM on some jobs before, though I'm
not sure if this was the cause).
To try to systematically prevent the problem, we make a ninja wrapper in
the containers that passes the -j flags, and set MAKEFLAGS in the
container builds. This doesn't cover make in non-container builds, but I
believe we don't have any of those.
Reviewed-by: Michel Dänzer <mdaenzer at redhat.com>
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3782>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3782>
---
.gitlab-ci.yml | 12 ++++++------
.gitlab-ci/build-apitrace.sh | 2 +-
.gitlab-ci/build-cts-runner.sh | 2 +-
.gitlab-ci/build-deqp-vk.sh | 2 +-
.gitlab-ci/build-fossilize.sh | 2 +-
.gitlab-ci/build-gfxreconstruct.sh | 2 +-
.gitlab-ci/build-piglit.sh | 2 +-
.gitlab-ci/build-renderdoc.sh | 2 +-
.gitlab-ci/build-vulkantools.sh | 2 +-
.gitlab-ci/container/arm_build.sh | 2 +-
.gitlab-ci/container/container_pre_build.sh | 16 ++++++++++++++++
.gitlab-ci/container/lava_arm.sh | 2 +-
.gitlab-ci/container/x86_build.sh | 20 ++++++++++----------
.gitlab-ci/meson-build.sh | 2 +-
14 files changed, 43 insertions(+), 27 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f1697574de2..371c2560b78 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,7 +121,7 @@ x86_build:
- .fdo.container-build at debian
- .container
variables:
- FDO_DISTRIBUTION_TAG: &x86_build "2020-03-13"
+ FDO_DISTRIBUTION_TAG: &x86_build "2020-03-18-jflags"
.use-x86_build:
variables:
@@ -134,19 +134,19 @@ x86_build:
x86_test-gl:
extends: x86_build
variables:
- FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-03-13"
+ FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-03-18-jflags"
# Debian 10 based x86 test image for VK
x86_test-vk:
extends: x86_build
variables:
- FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-03-27"
+ FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-04-01-jflags"
# Debian 9 based x86 build image (old LLVM)
x86_build_old:
extends: x86_build
variables:
- FDO_DISTRIBUTION_TAG: &x86_build_old "2019-09-18"
+ FDO_DISTRIBUTION_TAG: &x86_build_old "2019-03-18-jflags"
FDO_DISTRIBUTION_VERSION: stretch-slim
.use-x86_build_old:
@@ -162,7 +162,7 @@ arm_build:
- .fdo.container-build at debian@arm64v8
- .container
variables:
- FDO_DISTRIBUTION_TAG: &arm_build "2020-03-24"
+ FDO_DISTRIBUTION_TAG: &arm_build "2020-04-01-jflags"
.use-arm_build:
variables:
@@ -175,7 +175,7 @@ arm_build:
arm_test:
extends: arm_build
variables:
- FDO_DISTRIBUTION_TAG: &arm_test "2020-01-30"
+ FDO_DISTRIBUTION_TAG: &arm_test "2020-03-18-jflags"
.use-arm_test:
variables:
diff --git a/.gitlab-ci/build-apitrace.sh b/.gitlab-ci/build-apitrace.sh
index 0361886e7a4..87582f3b9e6 100644
--- a/.gitlab-ci/build-apitrace.sh
+++ b/.gitlab-ci/build-apitrace.sh
@@ -20,7 +20,7 @@ git clone https://github.com/apitrace/apitrace.git --single-branch --no-checkout
pushd /apitrace
git checkout "$APITRACE_VERSION"
cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False -DENABLE_WAFFLE=on -DWaffle_DIR=/usr/local/lib/cmake/Waffle/
-ninja -C _build -j4
+ninja -C _build
mkdir build
cp _build/apitrace build
cp _build/glretrace build
diff --git a/.gitlab-ci/build-cts-runner.sh b/.gitlab-ci/build-cts-runner.sh
index 982e734d73f..82bf91f0c5f 100644
--- a/.gitlab-ci/build-cts-runner.sh
+++ b/.gitlab-ci/build-cts-runner.sh
@@ -5,6 +5,6 @@ set -ex
git clone https://gitlab.freedesktop.org/mesa/parallel-deqp-runner.git --depth 1 -b mesa-ci-2019-12-17 /parallel-deqp-runner
pushd /parallel-deqp-runner
meson build/ $EXTRA_MESON_ARGS
-ninja -C build -j4 install
+ninja -C build install
popd
rm -rf /parallel-deqp-runner
diff --git a/.gitlab-ci/build-deqp-vk.sh b/.gitlab-ci/build-deqp-vk.sh
index f4e2af38796..206ac0109a1 100644
--- a/.gitlab-ci/build-deqp-vk.sh
+++ b/.gitlab-ci/build-deqp-vk.sh
@@ -23,7 +23,7 @@ cmake -G Ninja \
-DDEQP_TARGET=x11_glx \
-DCMAKE_BUILD_TYPE=Release \
/VK-GL-CTS
-ninja -j4
+ninja
# Copy out the mustpass list we want.
mkdir /deqp/mustpass
diff --git a/.gitlab-ci/build-fossilize.sh b/.gitlab-ci/build-fossilize.sh
index 35af32baa5b..21a7838a1b6 100644
--- a/.gitlab-ci/build-fossilize.sh
+++ b/.gitlab-ci/build-fossilize.sh
@@ -9,6 +9,6 @@ git submodule update --init
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release -G Ninja
-ninja -C . -j4 install
+ninja -C . install
cd ../..
rm -rf Fossilize
diff --git a/.gitlab-ci/build-gfxreconstruct.sh b/.gitlab-ci/build-gfxreconstruct.sh
index cb1548ca235..c08959b8a20 100644
--- a/.gitlab-ci/build-gfxreconstruct.sh
+++ b/.gitlab-ci/build-gfxreconstruct.sh
@@ -11,7 +11,7 @@ git checkout "$GFXRECONSTRUCT_VERSION"
git submodule update --init
git submodule update
cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release
-ninja -C _build -j4 gfxrecon-replay
+ninja -C _build gfxrecon-replay
mkdir -p build/bin
install _build/tools/replay/gfxrecon-replay build/bin
strip build/bin/*
diff --git a/.gitlab-ci/build-piglit.sh b/.gitlab-ci/build-piglit.sh
index e76e7ad8c6a..7a0aa608a6d 100644
--- a/.gitlab-ci/build-piglit.sh
+++ b/.gitlab-ci/build-piglit.sh
@@ -7,7 +7,7 @@ pushd /piglit
git checkout 8771c3860505db2bcf4877216221d774bf90af6b
patch -p1 <$OLDPWD/.gitlab-ci/piglit/disable-vs_in.diff
cmake -G Ninja -DCMAKE_BUILD_TYPE=Release
-ninja -j4
+ninja
find -name .git -o -name '*ninja*' -o -iname '*cmake*' -o -name '*.[chao]' | xargs rm -rf
rm -rf target_api
popd
diff --git a/.gitlab-ci/build-renderdoc.sh b/.gitlab-ci/build-renderdoc.sh
index 640386b4422..5d023c73627 100644
--- a/.gitlab-ci/build-renderdoc.sh
+++ b/.gitlab-ci/build-renderdoc.sh
@@ -8,7 +8,7 @@ git clone https://github.com/baldurk/renderdoc.git --single-branch --no-checkout
pushd /renderdoc
git checkout "$RENDERDOC_VERSION"
cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release
-ninja -C _build -j4
+ninja -C _build
mkdir -p build/lib
cp _build/lib/renderdoc.so build/lib
cp _build/lib/librenderdoc.so build/lib
diff --git a/.gitlab-ci/build-vulkantools.sh b/.gitlab-ci/build-vulkantools.sh
index f1fc64aef4c..099d362550f 100644
--- a/.gitlab-ci/build-vulkantools.sh
+++ b/.gitlab-ci/build-vulkantools.sh
@@ -19,7 +19,7 @@ cmake -G Ninja -B_build -H. \
-DBUILD_VIA=OFF \
-DBUILD_VKTRACE_REPLAY=OFF \
-C_build/helper.cmake
-ninja -C _build -j4 VkLayer_screenshot VkLayer_screenshot-staging-json
+ninja -C _build VkLayer_screenshot VkLayer_screenshot-staging-json
mkdir -p build/etc/vulkan/explicit_layer.d
mkdir build/lib
install _build/layersvt/staging-json/VkLayer_screenshot.json build/etc/vulkan/explicit_layer.d
diff --git a/.gitlab-ci/container/arm_build.sh b/.gitlab-ci/container/arm_build.sh
index fc1947fbdbf..74ecd2b1296 100644
--- a/.gitlab-ci/container/arm_build.sh
+++ b/.gitlab-ci/container/arm_build.sh
@@ -59,7 +59,7 @@ export LIBDRM_VERSION=libdrm-2.4.100
wget https://dri.freedesktop.org/libdrm/$LIBDRM_VERSION.tar.bz2
tar -xvf $LIBDRM_VERSION.tar.bz2 && rm $LIBDRM_VERSION.tar.bz2
-cd $LIBDRM_VERSION; meson build -D vc4=true -D freedreno=true -D etnaviv=true; ninja -j4 -C build install; cd ..
+cd $LIBDRM_VERSION; meson build -D vc4=true -D freedreno=true -D etnaviv=true; ninja -C build install; cd ..
rm -rf $LIBDRM_VERSION
############### Generate cross build file for Meson
diff --git a/.gitlab-ci/container/container_pre_build.sh b/.gitlab-ci/container/container_pre_build.sh
index 725a8b5ed4a..97afb0083a4 100755
--- a/.gitlab-ci/container/container_pre_build.sh
+++ b/.gitlab-ci/container/container_pre_build.sh
@@ -1,5 +1,7 @@
#!/bin/sh
+# Common setup among container builds before we get to building code.
+
export CCACHE_COMPILERCHECK=content
export CCACHE_COMPRESS=true
export CCACHE_DIR=/cache/mesa/ccache
@@ -11,3 +13,17 @@ export CC="/usr/lib/ccache/gcc"
export CXX="/usr/lib/ccache/g++"
ccache --show-stats
+
+if uname -m | grep -q arm || uname -m | grep -q aarch64; then
+ export JFLAGS=-j8
+else
+ export JFLAGS=-j4
+fi
+
+# Make a wrapper script for ninja to always include the -j flags
+echo /usr/bin/ninja $JFLAGS '"$@"' > /usr/local/bin/ninja
+chmod +x /usr/local/bin/ninja
+
+# Set MAKEFLAGS so that all make invocations in container builds include the
+# flags (doesn't apply to non-container builds, but we don't run make there)
+export MAKEFLAGS=$JFLAGS
diff --git a/.gitlab-ci/container/lava_arm.sh b/.gitlab-ci/container/lava_arm.sh
index 73a0f6da07d..124ed0792ff 100644
--- a/.gitlab-ci/container/lava_arm.sh
+++ b/.gitlab-ci/container/lava_arm.sh
@@ -45,7 +45,7 @@ mkdir -p kernel
wget -qO- ${KERNEL_URL} | tar -xz --strip-components=1 -C kernel
pushd kernel
./scripts/kconfig/merge_config.sh ${DEFCONFIG} ../.gitlab-ci/${KERNEL_ARCH}.config
-make -j12 ${KERNEL_IMAGE_NAME} dtbs
+make ${KERNEL_IMAGE_NAME} dtbs
cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /lava-files/.
cp ${DEVICE_TREES} /lava-files/.
popd
diff --git a/.gitlab-ci/container/x86_build.sh b/.gitlab-ci/container/x86_build.sh
index 7a14e9315d8..4232e4cc159 100644
--- a/.gitlab-ci/container/x86_build.sh
+++ b/.gitlab-ci/container/x86_build.sh
@@ -170,36 +170,36 @@ export WAYLAND_PROTOCOLS_VERSION=wayland-protocols-1.12
wget $XORG_RELEASES/util/$XORGMACROS_VERSION.tar.bz2
tar -xvf $XORGMACROS_VERSION.tar.bz2 && rm $XORGMACROS_VERSION.tar.bz2
-cd $XORGMACROS_VERSION; ./configure; make -j4 install; cd ..
+cd $XORGMACROS_VERSION; ./configure; make install; cd ..
rm -rf $XORGMACROS_VERSION
wget $XCB_RELEASES/$XCBPROTO_VERSION.tar.bz2
tar -xvf $XCBPROTO_VERSION.tar.bz2 && rm $XCBPROTO_VERSION.tar.bz2
-cd $XCBPROTO_VERSION; ./configure; make -j4 install; cd ..
+cd $XCBPROTO_VERSION; ./configure; make install; cd ..
rm -rf $XCBPROTO_VERSION
wget $XCB_RELEASES/$LIBXCB_VERSION.tar.bz2
tar -xvf $LIBXCB_VERSION.tar.bz2 && rm $LIBXCB_VERSION.tar.bz2
-cd $LIBXCB_VERSION; ./configure; make -j4 install; cd ..
+cd $LIBXCB_VERSION; ./configure; make install; cd ..
rm -rf $LIBXCB_VERSION
wget https://dri.freedesktop.org/libdrm/$LIBDRM_VERSION.tar.bz2
tar -xvf $LIBDRM_VERSION.tar.bz2 && rm $LIBDRM_VERSION.tar.bz2
cd $LIBDRM_VERSION
-meson build -D vc4=true -D freedreno=true -D etnaviv=true -D libdir=lib/x86_64-linux-gnu; ninja -j4 -C build install
-rm -rf build; meson --cross-file=/cross_file-ppc64el.txt build -D libdir=lib/powerpc64le-linux-gnu; ninja -j4 -C build install
-rm -rf build; meson --cross-file=/cross_file-i386.txt build -D libdir=lib/i386-linux-gnu; ninja -j4 -C build install
+meson build -D vc4=true -D freedreno=true -D etnaviv=true -D libdir=lib/x86_64-linux-gnu; ninja -C build install
+rm -rf build; meson --cross-file=/cross_file-ppc64el.txt build -D libdir=lib/powerpc64le-linux-gnu; ninja -C build install
+rm -rf build; meson --cross-file=/cross_file-i386.txt build -D libdir=lib/i386-linux-gnu; ninja -C build install
cd ..
rm -rf $LIBDRM_VERSION
wget $WAYLAND_RELEASES/$LIBWAYLAND_VERSION.tar.xz
tar -xvf $LIBWAYLAND_VERSION.tar.xz && rm $LIBWAYLAND_VERSION.tar.xz
-cd $LIBWAYLAND_VERSION; ./configure --enable-libraries --without-host-scanner --disable-documentation --disable-dtd-validation; make -j4 install; cd ..
+cd $LIBWAYLAND_VERSION; ./configure --enable-libraries --without-host-scanner --disable-documentation --disable-dtd-validation; make install; cd ..
rm -rf $LIBWAYLAND_VERSION
wget $WAYLAND_RELEASES/$WAYLAND_PROTOCOLS_VERSION.tar.xz
tar -xvf $WAYLAND_PROTOCOLS_VERSION.tar.xz && rm $WAYLAND_PROTOCOLS_VERSION.tar.xz
-cd $WAYLAND_PROTOCOLS_VERSION; ./configure; make -j4 install; cd ..
+cd $WAYLAND_PROTOCOLS_VERSION; ./configure; make install; cd ..
rm -rf $WAYLAND_PROTOCOLS_VERSION
@@ -209,7 +209,7 @@ rm -rf $WAYLAND_PROTOCOLS_VERSION
GLVND_VERSION=1.2.0
wget https://gitlab.freedesktop.org/glvnd/libglvnd/-/archive/v$GLVND_VERSION/libglvnd-v$GLVND_VERSION.tar.gz
tar -xvf libglvnd-v$GLVND_VERSION.tar.gz && rm libglvnd-v$GLVND_VERSION.tar.gz
-pushd libglvnd-v$GLVND_VERSION; ./autogen.sh; ./configure; make -j4 install; popd
+pushd libglvnd-v$GLVND_VERSION; ./autogen.sh; ./configure; make install; popd
rm -rf libglvnd-v$GLVND_VERSION
@@ -217,7 +217,7 @@ pushd /usr/local
git clone https://gitlab.freedesktop.org/mesa/shader-db.git --depth 1
rm -rf shader-db/.git
cd shader-db
-make -j4
+make
popd
diff --git a/.gitlab-ci/meson-build.sh b/.gitlab-ci/meson-build.sh
index b16a82472b7..c6affea46da 100755
--- a/.gitlab-ci/meson-build.sh
+++ b/.gitlab-ci/meson-build.sh
@@ -57,7 +57,7 @@ meson _build --native-file=native.file \
${EXTRA_OPTION}
cd _build
meson configure
-ninja -j4
+ninja
LC_ALL=C.UTF-8 ninja test
ninja install
cd ..
More information about the mesa-commit
mailing list