airflow-e2e-tests.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---

name: Airflow E2E Tests

permissions:
  contents: read
on:  # yamllint disable-line rule:truthy
  workflow_dispatch:
    inputs:
      workflow-name:
        description: "Name of the test"
        type: string
        required: true
      runners:
        description: "The array of labels (in json form) determining runners."
        type: string
        default: '["ubuntu-24.04"]'
      platform:
        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
        type: string
        default: 'linux/amd64'
      default-python-version:
        description: "Which version of python should be used by default"
        type: string
        default: '3.11'
      use-uv:
        description: "Whether to use uv to build the image (true/false)"
        type: string
        default: 'true'
      docker-image-tag:
        description: "Tag of the Docker image to test"
        type: string
        required: true
      e2e_test_mode:
        description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, go_sdk, ts_sdk, or yaml_dag"  # yamllint disable-line rule:line-length
        type: string
        default: "basic"
      java-sdk-version:
        description: "JDK version used to build the Java SDK bundles natively (tracks the JAVA_SDK_VERSION breeze constant)"  # yamllint disable-line rule:line-length
        type: string
        default: "17"

  workflow_call:
    inputs:
      workflow-name:
        description: "Name of the test"
        type: string
        required: true
      runners:
        description: "The array of labels (in json form) determining runners."
        required: true
        type: string
      platform:
        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
        required: true
        type: string
      default-python-version:
        description: "Which version of python should be used by default"
        required: true
        type: string
      use-uv:
        description: "Whether to use uv to build the image (true/false)"
        required: true
        type: string
      docker-image-tag:
        description: "Tag of the Docker image to test"
        type: string
        default: ""
      e2e_test_mode:
        description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, go_sdk, ts_sdk, or yaml_dag"  # yamllint disable-line rule:line-length
        type: string
        default: "basic"
      java-sdk-version:
        description: "JDK version used to build the Java SDK bundles natively (tracks the JAVA_SDK_VERSION breeze constant)"  # yamllint disable-line rule:line-length
        type: string
        default: "17"

jobs:
  test-e2e-integration-tests:
    timeout-minutes: 60
    name: ${{ inputs.workflow-name }}
    runs-on: ${{ fromJSON(inputs.runners) }}
    env:
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      VERBOSE: "true"
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: sudo rm -rf ${GITHUB_WORKSPACE}/*
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          fetch-depth: 2
          persist-credentials: false
      - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: ${{ inputs.platform }}
          image-type: "prod"
          python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
          use-uv: ${{ inputs.use-uv }}
          make-mnt-writeable-and-cleanup: true
        id: breeze
      # The java_sdk mode runs the Gradle builds, the go_sdk mode runs the Go
      # build and the ts_sdk mode runs the pnpm build; every other e2e mode
      # skips the toolchain steps below.
      # LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s job uses)
      # makes the e2e conftest build with the host toolchain provisioned here
      # instead of an ephemeral toolchain container, skipping the image pull; local
      # runs keep the containerized builds. The cache keys carry a ``-vN-`` salt
      # (bump to force-invalidate a poisoned cache) and runner.arch keys each cache
      # per architecture, as this job runs on amd64 and arm64. Without its cache
      # each java_sdk run re-downloads the example bundles' full dependency tree
      # (Spark alone is ~1GB) into a fresh ~/.gradle, whose ~/.gradle/jdks holds
      # arch-specific auto-provisioned JDKs (the example pins a Java 11 toolchain).
      - name: "Setup Java for the Java SDK build"
        if: inputs.e2e_test_mode == 'java_sdk'
        uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad  # v6.1.0
        with:
          distribution: 'temurin'
          java-version: ${{ inputs.java-sdk-version }}
      - name: "Restore Java SDK Gradle dependency cache"
        if: inputs.e2e_test_mode == 'java_sdk'
        id: gradle-cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
            ~/.gradle/jdks
            !~/.gradle/caches/*.lock
            !~/.gradle/caches/journal-1
          key: e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }}  # yamllint disable-line rule:line-length
          restore-keys: |
            e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-
      # go-version-file sources the Go toolchain from go-sdk/go.mod, so there is no
      # separate version pin to keep in sync; cache: false because the explicit
      # restore/save pair below persists the module + build caches even on red runs.
      - name: "Setup Go for the Go SDK build"
        if: inputs.e2e_test_mode == 'go_sdk' || inputs.e2e_test_mode == 'yaml_dag'
        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e  # v7.0.0
        with:
          go-version-file: go-sdk/go.mod
          cache: false
      - name: "Restore Go SDK module + build cache"
        if: inputs.e2e_test_mode == 'go_sdk' || inputs.e2e_test_mode == 'yaml_dag'
        id: go-cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go-sdk/go.mod', 'go-sdk/go.sum') }}  # yamllint disable-line rule:line-length
          restore-keys: |
            e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-
      - name: "Setup pnpm for the TypeScript SDK build"
        if: inputs.e2e_test_mode == 'ts_sdk'
        uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413  # v6.1.0
        with:
          package_json_file: ts-sdk/package.json
          run_install: false
      - name: "Setup Node for the TypeScript SDK build"
        if: inputs.e2e_test_mode == 'ts_sdk'
        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1  # v7.1.0
        with:
          node-version: 22
          cache: 'pnpm'
          cache-dependency-path: 'ts-sdk/pnpm-lock.yaml'
      - name: "Test e2e integration tests"
        run: breeze testing airflow-e2e-tests
        env:
          DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
          E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}"
          LANG_SDK_NATIVE_TOOLCHAIN: "${{ (inputs.e2e_test_mode == 'java_sdk' || inputs.e2e_test_mode == 'go_sdk' || inputs.e2e_test_mode == 'ts_sdk' || inputs.e2e_test_mode == 'yaml_dag') && 'true' || 'false' }}"  # yamllint disable-line rule:line-length
      - name: "Save Java SDK Gradle dependency cache"
        # Saved even when the e2e tests fail: the Gradle warm-up is independent of
        # test outcome, and actions/cache's post step would drop it on every red run.
        if: always() && inputs.e2e_test_mode == 'java_sdk' && steps.gradle-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
            ~/.gradle/jdks
            !~/.gradle/caches/*.lock
            !~/.gradle/caches/journal-1
          key: ${{ steps.gradle-cache.outputs.cache-primary-key }}
      - name: "Save Go SDK module + build cache"
        # Saved even when the e2e tests fail, for the same reason as the Gradle cache above.
        if: always() && (inputs.e2e_test_mode == 'go_sdk' || inputs.e2e_test_mode == 'yaml_dag') && steps.go-cache.outputs.cache-hit != 'true'  # yamllint disable-line rule:line-length
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ steps.go-cache.outputs.cache-primary-key }}
      - name: Zip logs
        run: |
          cd ./airflow-e2e-tests && zip -r logs.zip logs
        if: always()
      - name: "Upload logs"
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: "e2e-test-logs-${{ inputs.e2e_test_mode }}"
          path: './airflow-e2e-tests/logs.zip'
          retention-days: '7'
          if-no-files-found: 'error'
        if: always()