Mesa (master): panfrost: ci: Switch from direct Docker use to buildah
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Mon Jun 10 06:09:54 UTC 2019
Module: Mesa
Branch: master
Commit: 2fe7f9f2aefc62bcd16e9bcbeaa0c9862d0b006a
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=2fe7f9f2aefc62bcd16e9bcbeaa0c9862d0b006a
Author: Tomeu Vizoso <tomeu.vizoso at collabora.com>
Date: Fri Jun 7 10:20:28 2019 +0200
panfrost: ci: Switch from direct Docker use to buildah
Use the infrastructure in wayland/ci-templates to build the container
images.
This prevents from getting into some situations in which the images
wouldn't be rebuilt, and allows us to share some infrastructure with
other projects in freedesktop.org.
Signed-off-by: Tomeu Vizoso <tomeu.vizoso at collabora.com>
Suggested-by: Michel Dänzer <michel at daenzer.net>
Acked-by: Alyssa Rosenzweig <alyssa.rosenzweig at collabora.com>
---
src/gallium/drivers/panfrost/ci/Dockerfile | 135 ----------------------
src/gallium/drivers/panfrost/ci/debian-install.sh | 125 ++++++++++++++++++++
src/gallium/drivers/panfrost/ci/gitlab-ci.yml | 66 +++++++----
3 files changed, 166 insertions(+), 160 deletions(-)
diff --git a/src/gallium/drivers/panfrost/ci/Dockerfile b/src/gallium/drivers/panfrost/ci/Dockerfile
deleted file mode 100644
index fe0e67c27bd..00000000000
--- a/src/gallium/drivers/panfrost/ci/Dockerfile
+++ /dev/null
@@ -1,135 +0,0 @@
-FROM debian:testing
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN echo 'path-exclude=/usr/share/doc/*' > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
-RUN echo 'path-exclude=/usr/share/man/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
-RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d
-RUN echo 'exit 101' >> /usr/sbin/policy-rc.d
-RUN chmod +x /usr/sbin/policy-rc.d
-
-############### Install packages for building
-
-ARG DEBIAN_ARCH
-RUN dpkg --add-architecture ${DEBIAN_ARCH}
-RUN echo deb-src http://deb.debian.org/debian testing main >> /etc/apt/sources.list
-RUN apt-get update && \
- apt-get -y install ca-certificates && \
- apt-get -y install --no-install-recommends \
- crossbuild-essential-${DEBIAN_ARCH} \
- meson \
- g++ \
- git \
- ccache \
- pkg-config \
- python3-mako \
- python-numpy \
- python-six \
- python-mako \
- python3-pip \
- python3-setuptools \
- python3-six \
- python3-wheel \
- python3-jinja2 \
- bison \
- flex \
- libwayland-dev \
- gettext \
- cmake \
- bc \
- libssl-dev \
- lavacli \
- csvkit \
- curl \
- unzip \
- wget \
- debootstrap \
- procps \
- qemu-user-static \
- cpio \
- \
- libdrm-dev:${DEBIAN_ARCH} \
- libx11-dev:${DEBIAN_ARCH} \
- libxxf86vm-dev:${DEBIAN_ARCH} \
- libexpat1-dev:${DEBIAN_ARCH} \
- libsensors-dev:${DEBIAN_ARCH} \
- libxfixes-dev:${DEBIAN_ARCH} \
- libxdamage-dev:${DEBIAN_ARCH} \
- libxext-dev:${DEBIAN_ARCH} \
- x11proto-dev:${DEBIAN_ARCH} \
- libx11-xcb-dev:${DEBIAN_ARCH} \
- libxcb-dri2-0-dev:${DEBIAN_ARCH} \
- libxcb-glx0-dev:${DEBIAN_ARCH} \
- libxcb-xfixes0-dev:${DEBIAN_ARCH} \
- libxcb-dri3-dev:${DEBIAN_ARCH} \
- libxcb-present-dev:${DEBIAN_ARCH} \
- libxcb-randr0-dev:${DEBIAN_ARCH} \
- libxcb-sync-dev:${DEBIAN_ARCH} \
- libxrandr-dev:${DEBIAN_ARCH} \
- libxshmfence-dev:${DEBIAN_ARCH} \
- libelf-dev:${DEBIAN_ARCH} \
- libwayland-dev:${DEBIAN_ARCH} \
- libwayland-egl-backend-dev:${DEBIAN_ARCH} \
- libclang-7-dev:${DEBIAN_ARCH} \
- zlib1g-dev:${DEBIAN_ARCH} \
- libglvnd-core-dev:${DEBIAN_ARCH} \
- wayland-protocols:${DEBIAN_ARCH} \
- libpng-dev:${DEBIAN_ARCH} && \
- rm -rf /var/lib/apt/lists
-
-############### Cross-build dEQP
-ARG GCC_ARCH
-RUN mkdir -p /artifacts/rootfs/deqp && \
- wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip && \
- unzip -q opengl-es-cts-3.2.5.0.zip -d / && \
- rm opengl-es-cts-3.2.5.0.zip && \
- cd /VK-GL-CTS-opengl-es-cts-3.2.5.0 && \
- python3 external/fetch_sources.py && \
- cd /artifacts/rootfs/deqp && \
- cmake -DDEQP_TARGET=wayland \
- -DCMAKE_BUILD_TYPE=Release \
- -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc \
- -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++ \
- /VK-GL-CTS-opengl-es-cts-3.2.5.0 && \
- make -j$(nproc) && \
- rm -rf /artifacts/rootfs/deqp/external && \
- rm -rf /artifacts/rootfs/deqp/modules/gles3 && \
- rm -rf /artifacts/rootfs/deqp/modules/gles31 && \
- rm -rf /artifacts/rootfs/deqp/modules/internal && \
- rm -rf /artifacts/rootfs/deqp/executor && \
- rm -rf /artifacts/rootfs/deqp/execserver && \
- rm -rf /artifacts/rootfs/deqp/modules/egl && \
- rm -rf /artifacts/rootfs/deqp/framework && \
- find . -name CMakeFiles | xargs rm -rf && \
- find . -name lib\*.a | xargs rm -rf && \
- du -sh * && \
- rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0
-
-
-############### Cross-build kernel
-
-ARG KERNEL_ARCH
-ARG DEFCONFIG
-ARG DEVICE_TREES
-ARG KERNEL_IMAGE_NAME
-ENV KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz"
-
-COPY ${KERNEL_ARCH}.config /panfrost-ci/
-RUN mkdir -p /kernel && \
- wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel && \
- cd /kernel && \
- ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" ./scripts/kconfig/merge_config.sh ${DEFCONFIG} /panfrost-ci/${KERNEL_ARCH}.config && \
- ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" make -j12 ${KERNEL_IMAGE_NAME} dtbs && \
- cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/. && \
- cp ${DEVICE_TREES} /artifacts/. && \
- rm -rf /kernel
-
-
-############### Create rootfs
-
-COPY create-rootfs.sh /artifacts/rootfs/
-RUN debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian && \
- chroot /artifacts/rootfs sh /create-rootfs.sh && \
- rm /artifacts/rootfs/create-rootfs.sh
-
-ENTRYPOINT [""]
diff --git a/src/gallium/drivers/panfrost/ci/debian-install.sh b/src/gallium/drivers/panfrost/ci/debian-install.sh
new file mode 100644
index 00000000000..8a8b4ec9a47
--- /dev/null
+++ b/src/gallium/drivers/panfrost/ci/debian-install.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+PANFROST_CI_DIR=/tmp/clone/src/gallium/drivers/panfrost/ci
+
+############### Install packages for building
+dpkg --add-architecture ${DEBIAN_ARCH}
+echo 'deb-src https://deb.debian.org/debian testing main' > /etc/apt/sources.list.d/deb-src.list
+apt-get update
+apt-get -y install ca-certificates
+apt-get -y install --no-install-recommends \
+ crossbuild-essential-${DEBIAN_ARCH} \
+ meson \
+ g++ \
+ git \
+ ccache \
+ pkg-config \
+ python3-mako \
+ python-numpy \
+ python-six \
+ python-mako \
+ python3-pip \
+ python3-setuptools \
+ python3-six \
+ python3-wheel \
+ python3-jinja2 \
+ bison \
+ flex \
+ libwayland-dev \
+ gettext \
+ cmake \
+ bc \
+ libssl-dev \
+ lavacli \
+ csvkit \
+ curl \
+ unzip \
+ wget \
+ debootstrap \
+ procps \
+ qemu-user-static \
+ cpio \
+ \
+ libdrm-dev:${DEBIAN_ARCH} \
+ libx11-dev:${DEBIAN_ARCH} \
+ libxxf86vm-dev:${DEBIAN_ARCH} \
+ libexpat1-dev:${DEBIAN_ARCH} \
+ libsensors-dev:${DEBIAN_ARCH} \
+ libxfixes-dev:${DEBIAN_ARCH} \
+ libxdamage-dev:${DEBIAN_ARCH} \
+ libxext-dev:${DEBIAN_ARCH} \
+ x11proto-dev:${DEBIAN_ARCH} \
+ libx11-xcb-dev:${DEBIAN_ARCH} \
+ libxcb-dri2-0-dev:${DEBIAN_ARCH} \
+ libxcb-glx0-dev:${DEBIAN_ARCH} \
+ libxcb-xfixes0-dev:${DEBIAN_ARCH} \
+ libxcb-dri3-dev:${DEBIAN_ARCH} \
+ libxcb-present-dev:${DEBIAN_ARCH} \
+ libxcb-randr0-dev:${DEBIAN_ARCH} \
+ libxcb-sync-dev:${DEBIAN_ARCH} \
+ libxrandr-dev:${DEBIAN_ARCH} \
+ libxshmfence-dev:${DEBIAN_ARCH} \
+ libelf-dev:${DEBIAN_ARCH} \
+ libwayland-dev:${DEBIAN_ARCH} \
+ libwayland-egl-backend-dev:${DEBIAN_ARCH} \
+ libclang-7-dev:${DEBIAN_ARCH} \
+ zlib1g-dev:${DEBIAN_ARCH} \
+ libglvnd-core-dev:${DEBIAN_ARCH} \
+ wayland-protocols:${DEBIAN_ARCH} \
+ libpng-dev:${DEBIAN_ARCH}
+
+############### Cross-build dEQP
+mkdir -p /artifacts/rootfs/deqp
+
+wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip
+unzip -q opengl-es-cts-3.2.5.0.zip -d /
+rm opengl-es-cts-3.2.5.0.zip
+
+cd /VK-GL-CTS-opengl-es-cts-3.2.5.0
+python3 external/fetch_sources.py
+
+cd /artifacts/rootfs/deqp
+cmake -DDEQP_TARGET=wayland \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc \
+ -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++ \
+ /VK-GL-CTS-opengl-es-cts-3.2.5.0
+make -j$(nproc)
+rm -rf /artifacts/rootfs/deqp/external
+rm -rf /artifacts/rootfs/deqp/modules/gles3
+rm -rf /artifacts/rootfs/deqp/modules/gles31
+rm -rf /artifacts/rootfs/deqp/modules/internal
+rm -rf /artifacts/rootfs/deqp/executor
+rm -rf /artifacts/rootfs/deqp/execserver
+rm -rf /artifacts/rootfs/deqp/modules/egl
+rm -rf /artifacts/rootfs/deqp/framework
+find . -name CMakeFiles | xargs rm -rf
+find . -name lib\*.a | xargs rm -rf
+du -sh *
+rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0
+
+
+############### Cross-build kernel
+KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz"
+export ARCH=${KERNEL_ARCH}
+export CROSS_COMPILE="${GCC_ARCH}-"
+
+mkdir -p /kernel
+wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel
+cd /kernel
+./scripts/kconfig/merge_config.sh ${DEFCONFIG} ${PANFROST_CI_DIR}/${KERNEL_ARCH}.config
+make -j12 ${KERNEL_IMAGE_NAME} dtbs
+cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/.
+cp ${DEVICE_TREES} /artifacts/.
+rm -rf /kernel
+
+
+############### Create rootfs
+cp ${PANFROST_CI_DIR}/create-rootfs.sh /artifacts/rootfs/.
+debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian
+chroot /artifacts/rootfs sh /create-rootfs.sh
+rm /artifacts/rootfs/create-rootfs.sh
+
diff --git a/src/gallium/drivers/panfrost/ci/gitlab-ci.yml b/src/gallium/drivers/panfrost/ci/gitlab-ci.yml
index 3491b9f9875..d67c1ff6c94 100644
--- a/src/gallium/drivers/panfrost/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/panfrost/ci/gitlab-ci.yml
@@ -1,39 +1,55 @@
+# DEBIAN_TAG is the tag of the docker image used for the build jobs. If the
+# image doesn't exist yet, the docker-image stage generates it.
+#
+# In order to generate a new image, one should generally change the tag.
+# While removing the image from the registry would also work, that's not
+# recommended except for ephemeral images during development: Replacing an
+# image after a significant amount of time might pull in newer versions of
+# gcc/clang or other packages, which might break the build with older commits
+# using the same tag.
+#
+# After merging a change resulting in generating a new image to the main
+# repository, it's recommended to remove the image from the source repository's
+# container registry, so that the image from the main repository's registry
+# will be used there as well. You can manage your images on your fork of:
+# https://gitlab.freedesktop.org/xorg/xserver/container_registry
variables:
- DOCKER_DRIVER: overlay2
+ UPSTREAM_REPO: mesa/mesa
+ DEBIAN_VERSION: testing-slim
+ DEBIAN_TAG: "2019-06-07-1"
+
+include:
+ - project: 'wayland/ci-templates'
+ ref: c73dae8b84697ef18e2dbbf4fed7386d9652b0cd
+ file: '/templates/debian.yml'
stages:
- containers
- build
- test
+# Retry jobs after runner system failures
+.retry: &retry
+ retry:
+ max: 2
+ when:
+ - runner_system_failure
+
# Build Docker image with deqp, the rootfs and the build deps for Mesa
.container:
+ extends: .debian at container-ifnot-exists
stage: containers
- image: docker:stable
- only:
- changes:
- - src/gallium/drivers/panfrost/ci/gitlab-ci.yml
- - src/gallium/drivers/panfrost/ci/Dockerfile
- - src/gallium/drivers/panfrost/ci/create-rootfs.sh
- - src/gallium/drivers/panfrost/ci/${KERNEL_ARCH}.config
- services:
- - docker:dind
+ <<: *retry
variables:
- DOCKER_HOST: tcp://docker:2375
- DOCKER_DRIVER: overlay2
- script:
- - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - time docker build
- -t $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}
- --build-arg DEBIAN_ARCH=${DEBIAN_ARCH}
- --build-arg GCC_ARCH=${GCC_ARCH}
- --build-arg KERNEL_ARCH=${KERNEL_ARCH}
- --build-arg DEFCONFIG=${DEFCONFIG}
- --build-arg DEVICE_TREES=${DEVICE_TREES}
- --build-arg KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME}
- src/gallium/drivers/panfrost/ci
- - time docker push $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}
- - docker history $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}
+ GIT_STRATEGY: none # no need to pull the whole tree for rebuilding the image
+ IMAGE_LOCAL: "$CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}:latest"
+ DEBIAN_EXEC: 'DEBIAN_ARCH=${DEBIAN_ARCH}
+ GCC_ARCH=${GCC_ARCH}
+ KERNEL_ARCH=${KERNEL_ARCH}
+ DEFCONFIG=${DEFCONFIG}
+ DEVICE_TREES=${DEVICE_TREES}
+ KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME}
+ bash src/gallium/drivers/panfrost/ci/debian-install.sh'
container:armhf:
extends: .container
More information about the mesa-commit
mailing list