diff --git a/.ci/Dockerfile b/.ci/Dockerfile deleted file mode 100644 index d33c31e652e..00000000000 --- a/.ci/Dockerfile +++ /dev/null @@ -1,139 +0,0 @@ -# This file is part of darktable. -# copyright (c) 2016-2020 Roman Lebedev. -# -# darktable is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# darktable is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with darktable. If not, see . - -# docker build -t darktable/darktable . - -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -# !!! hub.docker.com will not auto-rebuild the image !!! -# !!! after making changes here, or if you just want to manually refresh !!! -# !!! the image, you need to go to: !!! -# https://hub.docker.com/r/darktable/darktable/~/settings/automated-builds/ !!! -# !!! and press the "Trigger" button. !!! -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -FROM debian:testing -MAINTAINER Roman Lebedev - -# needed at least for python-based jsonschema :( -# see https://github.com/Julian/jsonschema/issues/299 -# and https://github.com/docker-library/python/issues/13 -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 -ENV LC_MESSAGES C.UTF-8 -ENV LANGUAGE C.UTF-8 - -ENV DEBIAN_FRONTEND noninteractive - -# Paper over occasional network flakiness of some mirrors. -RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80retry - -# Do not install recommended packages -RUN echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/80recommends - -# Do not install suggested packages -RUN echo 'APT::Install-Suggests "false";' > /etc/apt/apt.conf.d/80suggests - -# Assume yes -RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/80forceyes - -# Fix broken packages -RUN echo 'APT::Get::Fix-Missing "true";' > /etc/apt/apt.conf.d/80fixmissin - -ENV GCC_VER=9 -ENV LLVM_VER=10 - -# pls keep sorted :) -RUN rm -rf /var/lib/apt/lists/* && apt-get update && \ - apt-get install \ - appstream-util \ - clang-$LLVM_VER \ - cmake \ - desktop-file-utils \ - g++-$GCC_VER \ - gcc-$GCC_VER \ - gettext \ - git \ - intltool \ - libatk1.0-dev \ - libc++-$LLVM_VER-dev \ - libcairo2-dev \ - libcolord-dev \ - libcolord-gtk-dev \ - libcmocka-dev \ - libcups2-dev \ - libcurl4-gnutls-dev \ - libexiv2-dev \ - libgdk-pixbuf2.0-dev \ - libglib2.0-dev \ - libgphoto2-dev \ - libgraphicsmagick1-dev \ - libgtk-3-dev \ - libheif-dev \ - libjpeg-dev \ - libjson-glib-dev \ - liblcms2-dev \ - liblensfun-dev \ - liblua5.2-dev \ - liblua5.3-dev \ - libomp-$LLVM_VER-dev \ - libopencv-calib3d-dev \ - libopencv-core-dev \ - libopencv-features2d-dev \ - libopencv-flann-dev \ - libopencv-imgproc-dev \ - libopenexr-dev \ - libopenjp2-7-dev \ - libosmgpsmap-1.0-dev \ - libpango1.0-dev \ - libpng-dev \ - libpugixml-dev \ - librsvg2-dev \ - libsaxon-java \ - libsecret-1-dev \ - libsqlite3-dev \ - libtiff5-dev \ - libwebp-dev \ - libx11-dev \ - libxml2-dev \ - libxml2-utils \ - make \ - ninja-build \ - perl \ - po4a \ - python3-jsonschema \ - xsltproc \ - zlib1g-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# i'd like to explicitly use ld.gold -# while it may be just immeasurably faster, it is known to cause more issues -# than traditional ld.bfd; plus, at this time, ld.gold seems like the future. -RUN dpkg-divert --add --rename --divert /usr/bin/ld.original /usr/bin/ld && \ - ln -s /usr/bin/ld.gold /usr/bin/ld - -# optional: opencl kernels test-compilation -# pls keep sorted :) -RUN rm -rf /var/lib/apt/lists/* && apt-get update && \ - apt-get install clang-$LLVM_VER libclang-common-$LLVM_VER-dev \ - llvm-$LLVM_VER-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# optional: usermanual deps -# pls keep sorted :) -RUN rm -rf /var/lib/apt/lists/* && apt-get update && \ - apt-get install default-jdk-headless default-jre-headless docbook \ - docbook-xml docbook-xsl docbook-xsl-saxon fop gnome-doc-utils imagemagick \ - libsaxon-java xsltproc && apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 2a916d987b1..89c9e027e74 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -43,24 +43,24 @@ See the [Podman installation guide](https://podman.io/docs/installation). No IDE, no extra tooling — just build and run the container directly. ```bash -# Build the image once (from the repository root) -docker build -t darktable-dev -f .devcontainer/Dockerfile . +# Pull the pre-built CI image +docker pull ghcr.io/darktable-org/darktable-build:latest -# Verify the build compiles (same environment as CI) +# Verify the build compiles cleanly (same environment as CI) docker run --rm --user "$(id -u):$(id -g)" \ -v "$PWD":/workspace -w /workspace \ - darktable-dev \ + ghcr.io/darktable-org/darktable-build:latest \ bash -lc './build.sh --prefix /tmp/dt --build-type Release' # Build an AppImage for GUI testing on the host docker run --rm --user "$(id -u):$(id -g)" \ -v "$PWD":/workspace -w /workspace \ -e APPIMAGE_EXTRACT_AND_RUN=1 \ - darktable-dev \ + ghcr.io/darktable-org/darktable-build:latest \ bash -lc './tools/appimage-build-script.sh' ``` -The AppImage appears in `build/Darktable-*.AppImage` and can be run on the host. +The AppImage appears in `build/Darktable-*.AppImage` and can be run directly on the host. > Replace `docker` with `podman` if you use Podman. @@ -95,10 +95,12 @@ Then: # Start the container devcontainer up --workspace-folder . -# Open a shell +# Open a shell inside it devcontainer exec --workspace-folder . bash ``` +Then build as usual (see [Building](#building)). + > **VS Code and JetBrains bundle their own devcontainer implementation** — you > only need to install the CLI separately when using other editors or working > purely in a terminal. @@ -150,31 +152,32 @@ Using `--configdir` avoids touching your production darktable configuration. cd build && ctest ``` -## CI environment and pre-built images +## CI environment The [Dockerfile](Dockerfile) is the single source of truth for the build -environment. Inspect it for the exact base image, compiler versions, and -package list. +environment. The `.github/workflows/build-docker.yml` workflow implements a +**build → test → push** sequence: it builds a candidate image from the +Dockerfile, runs a smoke-test build of darktable inside it, and only pushes +to GHCR if the build succeeds. Linux CI jobs always pull the last tested +`:latest` image. ### Pre-built images on GHCR -`.github/workflows/build-docker.yml` automatically builds the image and -publishes it to the GitHub Container Registry (GHCR) whenever the `Dockerfile` -changes on the `master` branch. The pre-built image is available at: +The `:latest` tag on `ghcr.io/darktable-org/darktable-build` is updated +whenever `.devcontainer/Dockerfile` changes on `master`, after the candidate +image passes a smoke-test build of darktable. Each release is also tagged +`YYYY-MM-DD-SHORTSHA` for pinned auditing. -``` -ghcr.io/darktable-org/darktable-build:latest -``` +`workflow_dispatch` on `build-docker.yml` lets maintainers trigger a manual +rebuild — useful when the upstream `ubuntu:26.04` base image gains security +patches without any change to the Dockerfile. -Using the pre-built image skips the local build step: +### Customising the build environment -```bash -docker pull ghcr.io/darktable-org/darktable-build:latest -docker run --rm --user "$(id -u):$(id -g)" \ - -v "$PWD":/workspace -w /workspace \ - ghcr.io/darktable-org/darktable-build:latest \ - bash -lc './build.sh --prefix /tmp/dt --build-type Release' -``` +To add or remove packages, edit `.devcontainer/Dockerfile` and submit it as a +normal PR. When the change merges to `master`, `build-docker.yml` runs +automatically, builds and smoke-tests the new image, and pushes it to GHCR +only if the build succeeds. ## Troubleshooting @@ -200,7 +203,7 @@ Always set `APPIMAGE_EXTRACT_AND_RUN=1` — FUSE is not available inside contain git submodule update --init --recursive ``` -### Git says the repository has dubious ownership inside the container +### Git says the mounted repository has dubious ownership inside the container Pass `--user "$(id -u):$(id -g)"` to `docker run` (as shown in the examples above), or mark the path as safe inside the container: @@ -228,5 +231,6 @@ CLI: `devcontainer up --workspace-folder . --remove-existing-container` ├── devcontainer.json # IDE/tooling configuration └── README.md # This file .github/workflows/ -└── build-docker.yml # Publishes the image to GHCR on Dockerfile changes +├── ci.yml # Linux jobs run against the published :latest image +└── build-docker.yml # Build → test → push :latest (on Dockerfile changes or workflow_dispatch) ``` diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e7d3730d53b..a6cc746fd52 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,9 +1,6 @@ { "name": "Darktable Development", - "build": { - "dockerfile": "Dockerfile", - "context": ".." - }, + "image": "ghcr.io/darktable-org/darktable-build:latest", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt=seccomp=unconfined" diff --git a/.github/scripts/test-image.sh b/.github/scripts/test-image.sh new file mode 100755 index 00000000000..a21b431c806 --- /dev/null +++ b/.github/scripts/test-image.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Run all Linux CI matrix configurations against a candidate Docker image. +# Used by build-docker.yml; can also be called locally to validate a new image. +# Usage: test-image.sh +set -euo pipefail + +IMAGE="${1:?Usage: $0 }" +SRC_DIR="${2:?Usage: $0 }" + +run_config() { + local name="$1"; shift + local build_dir="${SRC_DIR}/build/${name}" + local install_dir="${SRC_DIR}/install/${name}" + mkdir -p "${build_dir}" "${install_dir}" + printf '\n=== Testing configuration: %s ===\n\n' "${name}" + docker run --rm \ + --tmpfs /tmp:exec \ + -v "${SRC_DIR}:${SRC_DIR}" \ + -e SRC_DIR="${SRC_DIR}" \ + -e BUILD_DIR="${build_dir}" \ + -e INSTALL_PREFIX="${install_dir}" \ + -e GENERATOR=Ninja \ + "$@" \ + "${IMAGE}" \ + "${SRC_DIR}/.ci/ci-script.sh" + printf '\n=== Configuration %s passed ===\n' "${name}" + printf 'Cleaning up build and install directories for %s\n' "${name}" + docker run --rm \ + -v "${SRC_DIR}:${SRC_DIR}" \ + "${IMAGE}" \ + bash -c 'rm -rf -- "$1" "$2"' _ "${build_dir}" "${install_dir}" +} + +# Mirror all Linux matrix configurations from .github/workflows/ci.yml +run_config GNU16_Release \ + -e CC=gcc-16 -e CXX=g++-16 \ + -e CMAKE_BUILD_TYPE=Release \ + -e TARGET=skiptest \ + -e ECO="-DDONT_USE_INTERNAL_LIBRAW=ON" + +run_config LLVM22_Release \ + -e CC=clang-22 -e CXX=clang++-22 \ + -e CMAKE_BUILD_TYPE=Release \ + -e TARGET=skiptest \ + -e ECO="-DDONT_USE_INTERNAL_LIBRAW=ON" + +run_config GNU16_Debug \ + -e CC=gcc-16 -e CXX=g++-16 \ + -e CMAKE_BUILD_TYPE=Debug \ + -e TARGET=skiptest \ + -e ECO="-DDONT_USE_INTERNAL_LIBRAW=OFF" + +run_config GNU16_Release_tests \ + -e CC=gcc-16 -e CXX=g++-16 \ + -e CMAKE_BUILD_TYPE=Release \ + -e TARGET=build \ + -e ECO="-DDONT_USE_INTERNAL_LIBRAW=ON" + +printf '\n=== All configurations passed ===\n' diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 0855bc2df59..251fced3022 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -6,19 +6,32 @@ on: - master paths: - '.devcontainer/Dockerfile' + pull_request: + branches: + - master + paths: + - '.devcontainer/Dockerfile' workflow_dispatch: -permissions: - contents: read - packages: write - jobs: - build-and-push: + build-test-push: + name: Build, test and push CI Docker image if: github.repository == 'darktable-org/darktable' || github.event_name == 'workflow_dispatch' - name: Build and push darktable-build Docker image runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - uses: actions/checkout@v7 + with: + submodules: false + fetch-depth: 1 + + - name: Get build submodules + run: | + git submodule init + git config submodule.src/tests/integration.update none + git submodule update - name: Log in to GitHub Container Registry uses: docker/login-action@v4 @@ -27,12 +40,29 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push Docker image - uses: docker/build-push-action@v7 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build candidate image + uses: docker/build-push-action@v6 with: context: . file: .devcontainer/Dockerfile - push: true - tags: | - ghcr.io/darktable-org/darktable-build:latest - ghcr.io/darktable-org/darktable-build:${{ github.sha }} + load: true + push: false + tags: darktable-build:candidate + cache-from: type=registry,ref=ghcr.io/darktable-org/darktable-build:latest + + - name: Run CI matrix checks with candidate image + run: .github/scripts/test-image.sh darktable-build:candidate "$GITHUB_WORKSPACE" + + - name: Tag and push tested image to GHCR + if: github.event_name != 'pull_request' + # Retag the already-tested local image — no rebuild, so what CI uses is exactly what was tested. + run: | + SHORT_SHA=$(echo "$GITHUB_SHA" | cut -c1-8) + DATE_TAG="$(date -u +%Y-%m-%d)-${SHORT_SHA}" + docker tag darktable-build:candidate "ghcr.io/darktable-org/darktable-build:latest" + docker tag darktable-build:candidate "ghcr.io/darktable-org/darktable-build:${DATE_TAG}" + docker push "ghcr.io/darktable-org/darktable-build:latest" + docker push "ghcr.io/darktable-org/darktable-build:${DATE_TAG}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d25d03ff2e..0beae94b6bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,26 +44,28 @@ on: permissions: contents: read + packages: read jobs: Linux: if: github.repository == 'darktable-org/darktable' || github.event_name == 'workflow_dispatch' - name: Linux_${{ matrix.distro }}_${{ matrix.compiler.compiler }}_${{ matrix.btype }}${{ matrix.name_suffix }} + name: Linux_${{ matrix.compiler.compiler }}_${{ matrix.btype }}${{ matrix.name_suffix }} runs-on: ubuntu-latest container: - image: ${{ matrix.distro }} + image: ghcr.io/darktable-org/darktable-build:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: --tmpfs /tmp:exec --tmpfs /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}:exec strategy: fail-fast: true matrix: - distro: - - "ubuntu:26.04" compiler: # GCC 16 build is commented out here as we are building the same thing but including unit tests (see the # "include" section below); building a subset of the code using the same compiler serves no purpose. - # - { compiler: GNU16, CC: gcc-16, CXX: g++-16, packages: gcc-16 g++-16 } - - { compiler: LLVM22, CC: clang-22, CXX: clang++-22, packages: clang-22 libomp-22-dev llvm-22-dev libc++-22-dev libc++abi1 lld-22 clang-tools-22 mlir-22-tools libmlir-22-dev} + # - { compiler: GNU16, CC: gcc-16, CXX: g++-16 } + - { compiler: LLVM22, CC: clang-22, CXX: clang++-22 } btype: - Release target: @@ -73,24 +75,21 @@ jobs: eco: [-DDONT_USE_INTERNAL_LIBRAW=ON] include: # We want one run in CI to be Debug to make sure the Debug build isn't broken - - distro: "ubuntu:26.04" - btype: Debug - compiler: { compiler: GNU16, CC: gcc-16, CXX: g++-16, packages: gcc-16 g++-16 } + - btype: Debug + compiler: { compiler: GNU16, CC: gcc-16, CXX: g++-16 } target: skiptest generator: Ninja eco: -DDONT_USE_INTERNAL_LIBRAW=OFF # The other entries use the skiptest target, which never configures # BUILD_TESTING and so never even compiles the unit tests. This one # builds them and runs ctest. - - distro: "ubuntu:26.04" - btype: Release - compiler: { compiler: GNU16, CC: gcc-16, CXX: g++-16, packages: gcc-16 g++-16 } + - btype: Release + compiler: { compiler: GNU16, CC: gcc-16, CXX: g++-16 } target: build generator: Ninja eco: -DDONT_USE_INTERNAL_LIBRAW=ON name_suffix: _tests env: - DISTRO: ${{ matrix.distro }} CC: ${{ matrix.compiler.CC }} CXX: ${{ matrix.compiler.CXX }} SRC_DIR: ${{ github.workspace }}/src @@ -101,91 +100,7 @@ jobs: GENERATOR: ${{ matrix.generator }} TARGET: ${{ matrix.target }} DARKTABLE_CLI: ${{ github.workspace }}/install/bin/darktable-cli - DEBIAN_FRONTEND: noninteractive steps: - - name: Select fallback Ubuntu mirror if requested - if: startsWith(github.ref_name, 'azure-') - # Sometimes the default Ubuntu mirror is unreliable under overload - # and CI fails because of this. We can use a special branch name - # prefix to switch the mirror to the one on Azure. Always using the - # Azure is not a solution as it can also fail from time to time. - run: | - sed -i 's/archive\.ubuntu/azure\.archive\.ubuntu/' /etc/apt/sources.list.d/ubuntu.sources - - name: Update base packages - timeout-minutes: 15 - run: | - set -xe - rm -rf /var/lib/apt/lists/* - apt-get --yes update - apt-get --yes install eatmydata - eatmydata apt-get --yes upgrade - - name: Install compiler ${{ matrix.compiler.compiler }} - run: | - eatmydata apt-get --yes install ${{ matrix.compiler.packages }} - - name: Install Base Dependencies - run: | - eatmydata apt-get --yes install \ - build-essential \ - cmake \ - appstream-util \ - desktop-file-utils \ - gettext \ - git \ - gdb \ - intltool \ - libarchive-dev \ - libatk1.0-dev \ - libavif-dev \ - libcairo2-dev \ - libcmocka-dev \ - libcolord-dev \ - libcolord-gtk-dev \ - libcups2-dev \ - libcurl4-gnutls-dev \ - libexiv2-dev \ - libgdk-pixbuf-2.0-dev \ - libglib2.0-dev \ - libgmic-dev \ - libgphoto2-dev \ - libgraphicsmagick1-dev \ - libgtk-3-dev \ - libheif-dev \ - libjpeg-dev \ - libjson-glib-dev \ - liblcms2-dev \ - liblensfun-dev \ - liblua5.4-dev \ - libonnxruntime-dev \ - libopencv-calib3d-dev \ - libopencv-core-dev \ - libopencv-features2d-dev \ - libopencv-flann-dev \ - libopencv-imgproc-dev \ - libopenexr-dev \ - libopenjp2-7-dev \ - libosmgpsmap-1.0-dev \ - libpango1.0-dev \ - libpng-dev \ - libportmidi-dev \ - libpotrace-dev \ - libpugixml-dev \ - libraw-dev \ - librsvg2-dev \ - libsaxon-java \ - libsdl2-dev \ - libsecret-1-dev \ - libsqlite3-dev \ - libtiff5-dev \ - libwebp-dev \ - libx11-dev \ - libxml2-dev \ - libxml2-utils \ - ninja-build \ - perl \ - po4a \ - python3-jsonschema \ - xsltproc \ - zlib1g-dev; - uses: actions/checkout@v7 with: submodules: false diff --git a/.gitignore b/.gitignore index 4fd5d97fd1a..ec29c2a4bda 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ AppDir/ # personal, per-project agent instructions (see AGENTS.md) AGENTS.local.md +install/