[Mesa-dev] [PATCH] panfrost: ci: Switch from direct Docker use to buildah
Tomeu Vizoso
tomeu.vizoso at collabora.com
Fri Jun 7 11:36:21 UTC 2019
Use the infrastructure in wayland/ci-templates to build the container
images.
This prevents from getting into some situations in which the images
wouldn't be rebuilt, and allows us to share some infrastructure with
other projects in freedesktop.org.
Signed-off-by: Tomeu Vizoso <tomeu.vizoso at collabora.com>
Suggested-by: Michel Dänzer <michel at daenzer.net>
---
src/gallium/drivers/panfrost/ci/Dockerfile | 135 ------------------
.../drivers/panfrost/ci/debian-install.sh | 125 ++++++++++++++++
src/gallium/drivers/panfrost/ci/gitlab-ci.yml | 66 +++++----
3 files changed, 166 insertions(+), 160 deletions(-)
delete mode 100644 src/gallium/drivers/panfrost/ci/Dockerfile
create mode 100644 src/gallium/drivers/panfrost/ci/debian-install.sh
diff --git a/src/gallium/drivers/panfrost/ci/Dockerfile b/src/gallium/drivers/panfrost/ci/Dockerfile
deleted file mode 100644
index fe0e67c27bd1..000000000000
--- a/src/gallium/drivers/panfrost/ci/Dockerfile
+++ /dev/null
@@ -1,135 +0,0 @@
-FROM debian:testing
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN echo 'path-exclude=/usr/share/doc/*' > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
-RUN echo 'path-exclude=/usr/share/man/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
-RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d
-RUN echo 'exit 101' >> /usr/sbin/policy-rc.d
-RUN chmod +x /usr/sbin/policy-rc.d
-
-############### Install packages for building
-
-ARG DEBIAN_ARCH
-RUN dpkg --add-architecture ${DEBIAN_ARCH}
-RUN echo deb-src http://deb.debian.org/debian testing main >> /etc/apt/sources.list
-RUN apt-get update && \
- apt-get -y install ca-certificates && \
- apt-get -y install --no-install-recommends \
- crossbuild-essential-${DEBIAN_ARCH} \
- meson \
- g++ \
- git \
- ccache \
- pkg-config \
- python3-mako \
- python-numpy \
- python-six \
- python-mako \
- python3-pip \
- python3-setuptools \
- python3-six \
- python3-wheel \
- python3-jinja2 \
- bison \
- flex \
- libwayland-dev \
- gettext \
- cmake \
- bc \
- libssl-dev \
- lavacli \
- csvkit \
- curl \
- unzip \
- wget \
- debootstrap \
- procps \
- qemu-user-static \
- cpio \
- \
- libdrm-dev:${DEBIAN_ARCH} \
- libx11-dev:${DEBIAN_ARCH} \
- libxxf86vm-dev:${DEBIAN_ARCH} \
- libexpat1-dev:${DEBIAN_ARCH} \
- libsensors-dev:${DEBIAN_ARCH} \
- libxfixes-dev:${DEBIAN_ARCH} \
- libxdamage-dev:${DEBIAN_ARCH} \
- libxext-dev:${DEBIAN_ARCH} \
- x11proto-dev:${DEBIAN_ARCH} \
- libx11-xcb-dev:${DEBIAN_ARCH} \
- libxcb-dri2-0-dev:${DEBIAN_ARCH} \
- libxcb-glx0-dev:${DEBIAN_ARCH} \
- libxcb-xfixes0-dev:${DEBIAN_ARCH} \
- libxcb-dri3-dev:${DEBIAN_ARCH} \
- libxcb-present-dev:${DEBIAN_ARCH} \
- libxcb-randr0-dev:${DEBIAN_ARCH} \
- libxcb-sync-dev:${DEBIAN_ARCH} \
- libxrandr-dev:${DEBIAN_ARCH} \
- libxshmfence-dev:${DEBIAN_ARCH} \
- libelf-dev:${DEBIAN_ARCH} \
- libwayland-dev:${DEBIAN_ARCH} \
- libwayland-egl-backend-dev:${DEBIAN_ARCH} \
- libclang-7-dev:${DEBIAN_ARCH} \
- zlib1g-dev:${DEBIAN_ARCH} \
- libglvnd-core-dev:${DEBIAN_ARCH} \
- wayland-protocols:${DEBIAN_ARCH} \
- libpng-dev:${DEBIAN_ARCH} && \
- rm -rf /var/lib/apt/lists
-
-############### Cross-build dEQP
-ARG GCC_ARCH
-RUN mkdir -p /artifacts/rootfs/deqp && \
- wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip && \
- unzip -q opengl-es-cts-3.2.5.0.zip -d / && \
- rm opengl-es-cts-3.2.5.0.zip && \
- cd /VK-GL-CTS-opengl-es-cts-3.2.5.0 && \
- python3 external/fetch_sources.py && \
- cd /artifacts/rootfs/deqp && \
- cmake -DDEQP_TARGET=wayland \
- -DCMAKE_BUILD_TYPE=Release \
- -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc \
- -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++ \
- /VK-GL-CTS-opengl-es-cts-3.2.5.0 && \
- make -j$(nproc) && \
- rm -rf /artifacts/rootfs/deqp/external && \
- rm -rf /artifacts/rootfs/deqp/modules/gles3 && \
- rm -rf /artifacts/rootfs/deqp/modules/gles31 && \
- rm -rf /artifacts/rootfs/deqp/modules/internal && \
- rm -rf /artifacts/rootfs/deqp/executor && \
- rm -rf /artifacts/rootfs/deqp/execserver && \
- rm -rf /artifacts/rootfs/deqp/modules/egl && \
- rm -rf /artifacts/rootfs/deqp/framework && \
- find . -name CMakeFiles | xargs rm -rf && \
- find . -name lib\*.a | xargs rm -rf && \
- du -sh * && \
- rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0
-
-
-############### Cross-build kernel
-
-ARG KERNEL_ARCH
-ARG DEFCONFIG
-ARG DEVICE_TREES
-ARG KERNEL_IMAGE_NAME
-ENV KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz"
-
-COPY ${KERNEL_ARCH}.config /panfrost-ci/
-RUN mkdir -p /kernel && \
- wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel && \
- cd /kernel && \
- ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" ./scripts/kconfig/merge_config.sh ${DEFCONFIG} /panfrost-ci/${KERNEL_ARCH}.config && \
- ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" make -j12 ${KERNEL_IMAGE_NAME} dtbs && \
- cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/. && \
- cp ${DEVICE_TREES} /artifacts/. && \
- rm -rf /kernel
-
-
-############### Create rootfs
-
-COPY create-rootfs.sh /artifacts/rootfs/
-RUN debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian && \
- chroot /artifacts/rootfs sh /create-rootfs.sh && \
- rm /artifacts/rootfs/create-rootfs.sh
-
-ENTRYPOINT [""]
diff --git a/src/gallium/drivers/panfrost/ci/debian-install.sh b/src/gallium/drivers/panfrost/ci/debian-install.sh
new file mode 100644
index 000000000000..8a8b4ec9a47c
--- /dev/null
+++ b/src/gallium/drivers/panfrost/ci/debian-install.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+PANFROST_CI_DIR=/tmp/clone/src/gallium/drivers/panfrost/ci
+
+############### Install packages for building
+dpkg --add-architecture ${DEBIAN_ARCH}
+echo 'deb-src https://deb.debian.org/debian testing main' > /etc/apt/sources.list.d/deb-src.list
+apt-get update
+apt-get -y install ca-certificates
+apt-get -y install --no-install-recommends \
+ crossbuild-essential-${DEBIAN_ARCH} \
+ meson \
+ g++ \
+ git \
+ ccache \
+ pkg-config \
+ python3-mako \
+ python-numpy \
+ python-six \
+ python-mako \
+ python3-pip \
+ python3-setuptools \
+ python3-six \
+ python3-wheel \
+ python3-jinja2 \
+ bison \
+ flex \
+ libwayland-dev \
+ gettext \
+ cmake \
+ bc \
+ libssl-dev \
+ lavacli \
+ csvkit \
+ curl \
+ unzip \
+ wget \
+ debootstrap \
+ procps \
+ qemu-user-static \
+ cpio \
+ \
+ libdrm-dev:${DEBIAN_ARCH} \
+ libx11-dev:${DEBIAN_ARCH} \
+ libxxf86vm-dev:${DEBIAN_ARCH} \
+ libexpat1-dev:${DEBIAN_ARCH} \
+ libsensors-dev:${DEBIAN_ARCH} \
+ libxfixes-dev:${DEBIAN_ARCH} \
+ libxdamage-dev:${DEBIAN_ARCH} \
+ libxext-dev:${DEBIAN_ARCH} \
+ x11proto-dev:${DEBIAN_ARCH} \
+ libx11-xcb-dev:${DEBIAN_ARCH} \
+ libxcb-dri2-0-dev:${DEBIAN_ARCH} \
+ libxcb-glx0-dev:${DEBIAN_ARCH} \
+ libxcb-xfixes0-dev:${DEBIAN_ARCH} \
+ libxcb-dri3-dev:${DEBIAN_ARCH} \
+ libxcb-present-dev:${DEBIAN_ARCH} \
+ libxcb-randr0-dev:${DEBIAN_ARCH} \
+ libxcb-sync-dev:${DEBIAN_ARCH} \
+ libxrandr-dev:${DEBIAN_ARCH} \
+ libxshmfence-dev:${DEBIAN_ARCH} \
+ libelf-dev:${DEBIAN_ARCH} \
+ libwayland-dev:${DEBIAN_ARCH} \
+ libwayland-egl-backend-dev:${DEBIAN_ARCH} \
+ libclang-7-dev:${DEBIAN_ARCH} \
+ zlib1g-dev:${DEBIAN_ARCH} \
+ libglvnd-core-dev:${DEBIAN_ARCH} \
+ wayland-protocols:${DEBIAN_ARCH} \
+ libpng-dev:${DEBIAN_ARCH}
+
+############### Cross-build dEQP
+mkdir -p /artifacts/rootfs/deqp
+
+wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip
+unzip -q opengl-es-cts-3.2.5.0.zip -d /
+rm opengl-es-cts-3.2.5.0.zip
+
+cd /VK-GL-CTS-opengl-es-cts-3.2.5.0
+python3 external/fetch_sources.py
+
+cd /artifacts/rootfs/deqp
+cmake -DDEQP_TARGET=wayland \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc \
+ -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++ \
+ /VK-GL-CTS-opengl-es-cts-3.2.5.0
+make -j$(nproc)
+rm -rf /artifacts/rootfs/deqp/external
+rm -rf /artifacts/rootfs/deqp/modules/gles3
+rm -rf /artifacts/rootfs/deqp/modules/gles31
+rm -rf /artifacts/rootfs/deqp/modules/internal
+rm -rf /artifacts/rootfs/deqp/executor
+rm -rf /artifacts/rootfs/deqp/execserver
+rm -rf /artifacts/rootfs/deqp/modules/egl
+rm -rf /artifacts/rootfs/deqp/framework
+find . -name CMakeFiles | xargs rm -rf
+find . -name lib\*.a | xargs rm -rf
+du -sh *
+rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0
+
+
+############### Cross-build kernel
+KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz"
+export ARCH=${KERNEL_ARCH}
+export CROSS_COMPILE="${GCC_ARCH}-"
+
+mkdir -p /kernel
+wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel
+cd /kernel
+./scripts/kconfig/merge_config.sh ${DEFCONFIG} ${PANFROST_CI_DIR}/${KERNEL_ARCH}.config
+make -j12 ${KERNEL_IMAGE_NAME} dtbs
+cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/.
+cp ${DEVICE_TREES} /artifacts/.
+rm -rf /kernel
+
+
+############### Create rootfs
+cp ${PANFROST_CI_DIR}/create-rootfs.sh /artifacts/rootfs/.
+debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian
+chroot /artifacts/rootfs sh /create-rootfs.sh
+rm /artifacts/rootfs/create-rootfs.sh
+
diff --git a/src/gallium/drivers/panfrost/ci/gitlab-ci.yml b/src/gallium/drivers/panfrost/ci/gitlab-ci.yml
index 3491b9f98752..d67c1ff6c946 100644
--- a/src/gallium/drivers/panfrost/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/panfrost/ci/gitlab-ci.yml
@@ -1,39 +1,55 @@
+# DEBIAN_TAG is the tag of the docker image used for the build jobs. If the
+# image doesn't exist yet, the docker-image stage generates it.
+#
+# In order to generate a new image, one should generally change the tag.
+# While removing the image from the registry would also work, that's not
+# recommended except for ephemeral images during development: Replacing an
+# image after a significant amount of time might pull in newer versions of
+# gcc/clang or other packages, which might break the build with older commits
+# using the same tag.
+#
+# After merging a change resulting in generating a new image to the main
+# repository, it's recommended to remove the image from the source repository's
+# container registry, so that the image from the main repository's registry
+# will be used there as well. You can manage your images on your fork of:
+# https://gitlab.freedesktop.org/xorg/xserver/container_registry
variables:
- DOCKER_DRIVER: overlay2
+ UPSTREAM_REPO: mesa/mesa
+ DEBIAN_VERSION: testing-slim
+ DEBIAN_TAG: "2019-06-07-1"
+
+include:
+ - project: 'wayland/ci-templates'
+ ref: c73dae8b84697ef18e2dbbf4fed7386d9652b0cd
+ file: '/templates/debian.yml'
stages:
- containers
- build
- test
+# Retry jobs after runner system failures
+.retry: &retry
+ retry:
+ max: 2
+ when:
+ - runner_system_failure
+
# Build Docker image with deqp, the rootfs and the build deps for Mesa
.container:
+ extends: .debian at container-ifnot-exists
stage: containers
- image: docker:stable
- only:
- changes:
- - src/gallium/drivers/panfrost/ci/gitlab-ci.yml
- - src/gallium/drivers/panfrost/ci/Dockerfile
- - src/gallium/drivers/panfrost/ci/create-rootfs.sh
- - src/gallium/drivers/panfrost/ci/${KERNEL_ARCH}.config
- services:
- - docker:dind
+ <<: *retry
variables:
- DOCKER_HOST: tcp://docker:2375
- DOCKER_DRIVER: overlay2
- script:
- - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - time docker build
- -t $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}
- --build-arg DEBIAN_ARCH=${DEBIAN_ARCH}
- --build-arg GCC_ARCH=${GCC_ARCH}
- --build-arg KERNEL_ARCH=${KERNEL_ARCH}
- --build-arg DEFCONFIG=${DEFCONFIG}
- --build-arg DEVICE_TREES=${DEVICE_TREES}
- --build-arg KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME}
- src/gallium/drivers/panfrost/ci
- - time docker push $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}
- - docker history $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}
+ GIT_STRATEGY: none # no need to pull the whole tree for rebuilding the image
+ IMAGE_LOCAL: "$CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}:latest"
+ DEBIAN_EXEC: 'DEBIAN_ARCH=${DEBIAN_ARCH}
+ GCC_ARCH=${GCC_ARCH}
+ KERNEL_ARCH=${KERNEL_ARCH}
+ DEFCONFIG=${DEFCONFIG}
+ DEVICE_TREES=${DEVICE_TREES}
+ KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME}
+ bash src/gallium/drivers/panfrost/ci/debian-install.sh'
container:armhf:
extends: .container
--
2.20.1
More information about the mesa-dev
mailing list