diff --git a/.envrc.example b/.envrc.example index 8c58d7f..3fa4412 100644 --- a/.envrc.example +++ b/.envrc.example @@ -9,6 +9,12 @@ # GitLab instance the test runner registers against. export GITLAB_E2E_URL="https://gitlab.com/" +# Optional. Tag this run registers its runner with, and pins its pipeline to via +# the RUNNER_TAG pipeline variable. Leave unset locally: the suite falls back to +# the CI file's default. CI sets a unique value per run so concurrent runs +# cannot pick up each other's jobs. +# export GITLAB_E2E_RUNNER_TAG="e2e-local-$USER" + # Access token needing BOTH `api` and `create_runner` scopes, and Maintainer on # the project: `api` reads the project and triggers pipelines, `create_runner` # mints runners. Use a throwaway/test project, not anything sensitive. diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 081e208..edd7937 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -20,14 +20,10 @@ on: # somewhere else would let a run vouch for a tree it never checked out. workflow_dispatch: -# One global queue, because every run registers runners under the same tag -# against the one shared GitLab project; concurrent runs steal each other's -# build jobs (the same reason the matrix below is max-parallel: 1). Never -# cancel: a superseded run would leave its commit with no e2e evidence, and -# release.yml refuses to publish a tag whose commit has none. -concurrency: - group: e2e-shared-gitlab-project - cancel-in-progress: false +# No concurrency group. Each leg registers its runner under a tag unique to the +# run and the Kubernetes version, and pins its pipeline to that tag, so two runs +# cannot pick up each other's jobs. A shared queue would also let a pending run +# be displaced and cancelled outright, leaving a commit with no e2e evidence. # No workflow-wide token. The e2e job runs checked-out PR code with GitLab # secrets, so write scope is granted only to the authorize job below. @@ -96,7 +92,7 @@ jobs: core.setOutput('ref', pr.data.head.sha); # Docs and workflow-only PRs cannot break the suite, so they skip it rather - # than spending a serialized slot. Non-PR events always run. + # than spending four runners and a few minutes. Non-PR events always run. changes: needs: authorize runs-on: ubuntu-latest @@ -168,11 +164,6 @@ jobs: contents: read strategy: fail-fast: false - # Serialize the matrix: every job registers runners with the same tag - # against the one shared GitLab project, so running them concurrently lets - # a build-job land on a sibling's runner and die (runner_system_failure) - # when that sibling tears down its kind cluster first. - max-parallel: 1 matrix: k8s_version: ${{ fromJson(needs.k8s_versions.outputs.k8s_versions) }} steps: @@ -210,6 +201,10 @@ jobs: GITLAB_E2E_URL: ${{ secrets.GITLAB_E2E_URL }} GITLAB_E2E_TOKEN: ${{ secrets.GITLAB_E2E_TOKEN }} GITLAB_E2E_PROJECT_ID: ${{ secrets.GITLAB_E2E_PROJECT_ID }} + # Unique per run, attempt and matrix leg, so no two legs or runs can + # share a tag. job-index rather than the version string because a + # runner tag cannot contain the dots in e.g. v1.33.7. + GITLAB_E2E_RUNNER_TAG: e2e-${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }} run: make test-e2e # The one context worth marking required. Its name never changes, unlike the diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index cb55bd6..781a187 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -24,13 +24,23 @@ import ( const ( e2eNamespace = "default" - // jobTag must match the tags on build-job in the test project's - // .gitlab-ci.yml so the registered runner picks the job up. - jobTag = "test-gitlab-runner" - timeout = 6 * time.Minute - interval = 5 * time.Second + // defaultJobTag is the RUNNER_TAG default in the test project's + // .gitlab-ci.yml, used when GITLAB_E2E_RUNNER_TAG is unset. + defaultJobTag = "test-gitlab-runner" + timeout = 6 * time.Minute + interval = 5 * time.Second ) +// jobTag is the tag this run registers its runner with and pins its pipeline +// to. CI sets a per-run value so concurrent runs cannot pick up each other's +// jobs; a sibling doing so would kill the job when it tears its cluster down. +var jobTag = func() string { + if t := os.Getenv("GITLAB_E2E_RUNNER_TAG"); t != "" { + return t + } + return defaultJobTag +}() + var ( k8sClient client.Client glab *gitlab.Client diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 7566b91..5661bb1 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -71,7 +71,15 @@ func waitGone(name string) { func triggerPipeline() int64 { GinkgoHelper() - p, _, err := glab.Pipelines.CreatePipeline(projectID, &gitlab.CreatePipelineOptions{Ref: gitlab.Ptr(defaultBranch)}) + // RUNNER_TAG pins build-job to this run's runner. Without it the job takes + // the CI file's default tag, which every concurrent run shares. + p, _, err := glab.Pipelines.CreatePipeline(projectID, &gitlab.CreatePipelineOptions{ + Ref: gitlab.Ptr(defaultBranch), + Variables: &[]*gitlab.PipelineVariableOptions{{ + Key: gitlab.Ptr("RUNNER_TAG"), + Value: gitlab.Ptr(jobTag), + }}, + }) Expect(err).NotTo(HaveOccurred(), "could not trigger a pipeline (token needs api scope + Developer role)") return p.ID } diff --git a/test/e2e/terraform/README.md b/test/e2e/terraform/README.md index 016bff6..d479352 100644 --- a/test/e2e/terraform/README.md +++ b/test/e2e/terraform/README.md @@ -11,7 +11,7 @@ Run everything with `tofu`, not `terraform`. ```text project (private) var.project_name - ├── .gitlab-ci.yml build-job, tagged var.job_tag + ├── .gitlab-ci.yml build-job, tagged $RUNNER_TAG └── project access token scopes: api, create_runner (Maintainer) ``` @@ -25,10 +25,11 @@ simplest and fully-IaC option. - **Project** is a throwaway. Shared runners and Auto DevOps are disabled so the only thing that can run `build-job` is the operator-managed runner the suite - creates. All e2e matrix jobs share this single project, so the CI workflow runs - the k8s matrix with `max-parallel: 1`; parallel jobs would let a build-job land - on a sibling's runner and die when that sibling tears down first. -- **`.gitlab-ci.yml`** defines a single `build-job` tagged with `var.job_tag`. + creates. All e2e matrix jobs share this single project, and they run + concurrently: each pins its pipeline to a tag unique to that run and leg, so a + build-job can only be picked up by the runner its own pipeline named. +- **`.gitlab-ci.yml`** defines a single `build-job` tagged with `$RUNNER_TAG`, + a pipeline variable defaulting to `var.job_tag`. The suite registers its managed runner with `run_untagged = false` and that same tag, then asserts the job ran on *our* runner id. - **Project access token** is what the suite consumes as `GITLAB_E2E_TOKEN`. It @@ -117,12 +118,26 @@ too and use it directly as `GITLAB_E2E_TOKEN`. A personal access token always has a real user owner and will mint the runner. The rest of the fixture (group, project, CI file) is unaffected. -## Keep the CI tag in sync +## How the CI tag is chosen -`var.job_tag` defaults to `test-gitlab-runner`, which must match the `jobTag` -constant in `test/e2e/e2e_suite_test.go`. Change one and you must change the -other, otherwise the managed runner never picks up `build-job` and the suite -times out. +`build-job` is tagged `$RUNNER_TAG`, a pipeline variable whose default comes from +`var.job_tag` (`test-gitlab-runner`). The suite sets that variable when it +triggers a pipeline, using `GITLAB_E2E_RUNNER_TAG` if set and otherwise the +`defaultJobTag` constant in `test/e2e/e2e_suite_test.go`. + +Locally you can ignore all of it: leave `GITLAB_E2E_RUNNER_TAG` unset and the +default applies, in which case `var.job_tag` and `defaultJobTag` must agree or +the managed runner never picks up `build-job` and the suite times out. + +The project sets `ci_pipeline_variables_minimum_override_role = "maintainer"` +because GitLab refuses `RUNNER_TAG` with "Insufficient permissions to set +pipeline variables" otherwise, and new projects default to `no_one_allowed`. +Lowering it further is not needed; raising it breaks the suite. + +CI sets a unique tag per run, attempt and matrix leg. That is what lets the +matrix and separate runs execute concurrently against this one shared project: a +job can only be picked up by the runner its own pipeline named, so a sibling +cannot take it and then kill it by tearing down its cluster. ## Cleanup diff --git a/test/e2e/terraform/main.tf b/test/e2e/terraform/main.tf index 8443141..0e7d7c7 100644 --- a/test/e2e/terraform/main.tf +++ b/test/e2e/terraform/main.tf @@ -21,11 +21,18 @@ locals { stages: - build + variables: + # The e2e suite overrides this per pipeline so a run's job can only be + # picked up by that run's runner. Without it every concurrent run shares + # one tag and a job can land on a sibling's runner, which then tears its + # cluster down mid-job. The default keeps a manual pipeline working. + RUNNER_TAG: ${var.job_tag} + build-job: stage: build image: ${var.ci_job_image} tags: - - ${var.job_tag} + - $RUNNER_TAG script: - echo "e2e build-job on runner $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION)" - echo "commit $CI_COMMIT_SHORT_SHA on ref $CI_COMMIT_REF_NAME" @@ -51,6 +58,13 @@ resource "gitlab_project" "e2e" { initialize_with_readme = true default_branch = "main" + # The suite sets RUNNER_TAG as a pipeline variable so a run's job can only be + # picked up by that run's runner. GitLab refuses that with "Insufficient + # permissions to set pipeline variables" unless this is at or below the + # token's role, and new projects default to no_one_allowed. The e2e token is + # a Maintainer, so maintainer is the least-privilege value that works. + ci_pipeline_variables_minimum_override_role = "maintainer" + # The suite asserts build-job ran on our operator-managed runner. Keep shared # runners out of the picture and silence Auto DevOps pipelines. shared_runners_enabled = false diff --git a/test/e2e/terraform/variables.tf b/test/e2e/terraform/variables.tf index b37d14d..d12c758 100644 --- a/test/e2e/terraform/variables.tf +++ b/test/e2e/terraform/variables.tf @@ -36,9 +36,11 @@ variable "project_name" { variable "job_tag" { description = <<-DESC - Tag applied to build-job in the generated .gitlab-ci.yml. MUST match the - jobTag constant in test/e2e/e2e_suite_test.go (currently "test-gitlab-runner") - or the managed runner never picks the job up and the suite times out. + Default RUNNER_TAG in the generated .gitlab-ci.yml, used when a pipeline + does not set one. MUST match the defaultJobTag constant in + test/e2e/e2e_suite_test.go, or a locally run suite never picks the job up. + CI overrides it per run via GITLAB_E2E_RUNNER_TAG so concurrent runs cannot + steal each other's jobs. DESC type = string default = "test-gitlab-runner"