#!/usr/bin/env python3 # # Re-generate container recipes # # This script uses the "lcitool" available from # # https://gitlab.com/libvirt/libvirt-ci # # Copyright (c) 2020 Red Hat Inc. # # This work is licensed under the terms of the GNU GPL, version 2 # or (at your option) any later version. See the COPYING file in # the top-level directory. import sys import subprocess from pathlib import Path if len(sys.argv) != 1: print("syntax: %s" % sys.argv[0], file=sys.stderr) sys.exit(1) self_dir = Path(__file__).parent src_dir = self_dir.parent.parent dockerfiles_dir = Path(src_dir, "tests", "docker", "dockerfiles") lcitool_path = Path(self_dir, "libvirt-ci", "bin", "lcitool") lcitool_cmd = [lcitool_path, "--data-dir", self_dir] def atomic_write(filename, content): tmp = filename.with_suffix(filename.suffix + ".tmp") try: with tmp.open("w") as fp: print(content, file=fp, end="") tmp.rename(filename) except Exception as ex: tmp.unlink() raise def generate(filename, cmd, trailer): print("Generate %s" % filename) lcitool = subprocess.run(cmd, capture_output=True, encoding='utf8') if lcitool.returncode != 0: raise Exception("Failed to generate %s: %s" % (filename, lcitool.stderr)) content = lcitool.stdout if trailer is not None: content += trailer atomic_write(filename, content) # Optional user setting, this will always be the last thing added # so maximise the number of layers that are cached add_user_mapping = [ "# As a final step configure the user (if env is defined)", "ARG USER", "ARG UID", "RUN if [ \"${USER}\" ]; then \\", " id ${USER} 2>/dev/null || useradd -u ${UID} -U ${USER}; fi\n" ] def generate_dockerfile(host, target, project="qemu", cross=None, trailer=None, enable_rust=True): filename = Path(src_dir, "tests", "docker", "dockerfiles", host + ".docker") cmd = lcitool_cmd + ["dockerfile"] if cross is not None: cmd.extend(["--cross", cross]) cmd.extend([target, project]) if trailer is not None: trailer = trailer.strip() + "\n" trailer += "\n".join(add_user_mapping) else: trailer = "\n".join(add_user_mapping) if enable_rust: trailer += "\nENV ENABLE_RUST=1\n" generate(filename, cmd, trailer) def generate_vars(target, trailer=None): filename = Path(src_dir, ".gitlab-ci.d", target + ".vars") cmd = lcitool_cmd + ["variables", "--format", "shell", target, "qemu"] generate(filename, cmd, trailer) def generate_pkglist(vm, target, project="qemu"): filename = Path(src_dir, "tests", "vm", "generated", vm + ".json") cmd = lcitool_cmd + ["variables", "--format", "json", target, project] generate(filename, cmd, None) def generate_yaml(os, target, arch, trailer=None): filename = Path(src_dir, "scripts", "ci", "setup", os, f"{target}-{arch}.yaml") cmd = lcitool_cmd + ["variables", "--format", "yaml", "-a", arch, target, "qemu"] generate(filename, cmd, trailer) alpine_extras = r""" # https://gitlab.alpinelinux.org/alpine/aports/-/issues/17463 RUN apk add clang19-libclang """ # Netmap still needs to be manually built as it is yet to be packaged # into a distro. We also add cscope and gtags which are used in the CI # test debian13_extras = r""" # netmap/cscope/global RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends \ cscope\ global\ linux-headers-generic RUN git clone https://github.com/luigirizzo/netmap.git /usr/src/netmap RUN cd /usr/src/netmap && git checkout v11.3 RUN cd /usr/src/netmap/LINUX && \ ./configure --no-drivers --no-apps \ --kernel-dir=$(ls -d /usr/src/linux-headers-*-$(dpkg --print-architecture)) \ && make install ENV QEMU_CONFIGURE_OPTS=--enable-netmap """ # Based on the hub.docker.com/library/rust Dockerfiles fedora_rustup_nightly_extras = r""" RUN dnf install -y wget ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo ENV RUSTC=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc ENV RUSTDOC=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustdoc ENV CARGO=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/cargo RUN set -eux && \ rustArch='x86_64-unknown-linux-gnu' && \ rustupSha256='6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d' && \ url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" && \ wget "$url" && \ echo "${rustupSha256} *rustup-init" | sha256sum -c - && \ chmod +x rustup-init && \ ./rustup-init -y --no-modify-path --profile default --default-toolchain nightly --default-host ${rustArch} && \ chmod -R a+w $RUSTUP_HOME $CARGO_HOME && \ /usr/local/cargo/bin/rustup --version && \ /usr/local/cargo/bin/rustup run nightly cargo --version && \ /usr/local/cargo/bin/rustup run nightly rustc --version && \ test "$CARGO" = "$(/usr/local/cargo/bin/rustup +nightly which cargo)" && \ test "$RUSTDOC" = "$(/usr/local/cargo/bin/rustup +nightly which rustdoc)" && \ test "$RUSTC" = "$(/usr/local/cargo/bin/rustup +nightly which rustc)" ENV PATH=$CARGO_HOME/bin:$PATH RUN /usr/local/cargo/bin/rustup run nightly cargo install --locked bindgen-cli RUN $CARGO --list """ ubuntu2404_rust_extras = r""" ENV RUSTC=/usr/bin/rustc-1.83 ENV RUSTDOC=/usr/bin/rustdoc-1.83 ENV CARGO_HOME=/usr/local/cargo ENV PATH=$CARGO_HOME/bin:$PATH RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends cargo RUN cargo install --locked bindgen-cli """ debian_all_test_cross_compilers = r""" # extras for cross and alternate toolchains RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends \ clang\ dpkg-dev\ gdb-multiarch\ libclang-rt-dev ENV AVAILABLE_COMPILERS="\ gcc-aarch64-linux-gnu \ libc6-dev-arm64-cross \ gcc-arm-linux-gnueabihf \ libc6-dev-armhf-cross \ gcc-alpha-linux-gnu \ libc6.1-dev-alpha-cross \ gcc-mips-linux-gnu \ libc6-dev-mips-cross \ gcc-mips64-linux-gnuabi64 \ libc6-dev-mips64-cross \ gcc-mips64el-linux-gnuabi64 \ libc6-dev-mips64el-cross \ gcc-mipsel-linux-gnu \ libc6-dev-mipsel-cross \ gcc-powerpc64le-linux-gnu \ libc6-dev-ppc64el-cross \ gcc-riscv64-linux-gnu \ libc6-dev-riscv64-cross \ gcc-s390x-linux-gnu \ libc6-dev-s390x-cross\ gcc-sh4-linux-gnu \ libc6-dev-sh4-cross \ " RUN if dpkg-architecture -e amd64; then \ export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-hppa-linux-gnu libc6-dev-hppa-cross"; \ export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-m68k-linux-gnu libc6-dev-m68k-cross"; \ export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-powerpc-linux-gnu libc6-dev-powerpc-cross"; \ export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross"; \ export AVAILABLE_COMPILERS="${AVAILABLE_COMPILERS} gcc-sparc64-linux-gnu libc6-dev-sparc64-cross"; \ fi && \ apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends \ ${AVAILABLE_COMPILERS} && \ dpkg-query --showformat '${Package}_${Version}_${Architecture}' --show > /packages.txt ENV QEMU_CONFIGURE_OPTS=--disable-docs ENV DEF_TARGET_LIST=aarch64-linux-user,alpha-linux-user,arm-linux-user,hppa-linux-user,i386-linux-user,m68k-linux-user,mips-linux-user,mips64-linux-user,mips64el-linux-user,mipsel-linux-user,ppc-linux-user,ppc64-linux-user,ppc64le-linux-user,riscv64-linux-user,s390x-linux-user,sh4-linux-user,sparc64-linux-user """ def cross_build(prefix, targets): conf = "ENV QEMU_CONFIGURE_OPTS=--cross-prefix=%s\n" % (prefix) targets = "ENV DEF_TARGET_LIST=%s\n" % (targets) return "".join([conf, targets]) # # Update all the various build configurations. # Please keep each group sorted alphabetically for easy reading. # try: # # Standard native builds # generate_dockerfile("alpine", "alpine-323", trailer=alpine_extras) generate_dockerfile("centos9", "centos-stream-9") generate_dockerfile("debian", "debian-13", trailer=debian13_extras) generate_dockerfile("fedora", "fedora-43") generate_dockerfile("opensuse-leap", "opensuse-leap-16") generate_dockerfile("ubuntu2404", "ubuntu-2404", trailer=ubuntu2404_rust_extras) # # Non-fatal Rust-enabled build # generate_dockerfile("fedora-rust-nightly", "fedora-43", trailer=fedora_rustup_nightly_extras) # # Cross compiling builds # generate_dockerfile("debian-amd64-cross", "debian-13", cross="x86_64", trailer=cross_build("x86_64-linux-gnu-", "x86_64-softmmu," "x86_64-linux-user," "i386-softmmu,i386-linux-user")) generate_dockerfile("debian-arm64-cross", "debian-13", cross="aarch64", trailer=cross_build("aarch64-linux-gnu-", "aarch64-softmmu,aarch64-linux-user")) generate_dockerfile("debian-armhf-cross", "debian-13", cross="armv7l", trailer=cross_build("arm-linux-gnueabihf-", "arm-softmmu,arm-linux-user")) generate_dockerfile("debian-i686-cross", "debian-13", cross="i686", trailer=cross_build("i686-linux-gnu-", "i386-softmmu,i386-linux-user")) generate_dockerfile("debian-ppc64el-cross", "debian-13", cross="ppc64le", trailer=cross_build("powerpc64le-linux-gnu-", "ppc64-softmmu,ppc64-linux-user")) # while not yet a release architecture the packages are still # build while part of testing generate_dockerfile("debian-riscv64-cross", "debian-13", cross="riscv64", trailer=cross_build("riscv64-linux-gnu-", "riscv64-softmmu,riscv64-linux-user")) generate_dockerfile("debian-s390x-cross", "debian-13", cross="s390x", trailer=cross_build("s390x-linux-gnu-", "s390x-softmmu,s390x-linux-user")) generate_dockerfile("fedora-win64-cross", "fedora-43", project='qemu,qemu-win-installer', cross="mingw64", trailer=cross_build("x86_64-w64-mingw32-", "x86_64-softmmu")) # We also generate some docker files with minimal dependencies and # as many cross-compilers as Debian will package for building TCG # tests. generate_dockerfile("debian-all-test-cross", "debian-13", project="qemu-minimal", enable_rust=False, trailer=debian_all_test_cross_compilers) # # GitLab packages lists # generate_vars("macos-14") # # VM packages lists # generate_pkglist("freebsd", "freebsd-14") generate_pkglist("openbsd", "openbsd-78", project="qemu-minimal") # # Ansible package lists # generate_yaml("ubuntu", "ubuntu-2404", "aarch64") generate_yaml("ubuntu", "ubuntu-2404", "s390x") generate_yaml("debian", "debian-13", "ppc64le") sys.exit(0) except Exception as ex: print(str(ex), file=sys.stderr) sys.exit(1)