ci(e2e): give each run a unique runner tag and drop the queue

.envrc.example

@@ -9,6 +9,12 @@
 # GitLab instance the test runner registers against.
 export GITLAB_E2E_URL="https://gitlab.com/"
 
+# Optional. Tag this run registers its runner with, and pins its pipeline to via
+# the RUNNER_TAG pipeline variable. Leave unset locally: the suite falls back to
+# the CI file's default. CI sets a unique value per run so concurrent runs
+# cannot pick up each other's jobs.
+# export GITLAB_E2E_RUNNER_TAG="e2e-local-$USER"
+
 # Access token needing BOTH `api` and `create_runner` scopes, and Maintainer on
 # the project: `api` reads the project and triggers pipelines, `create_runner`
 # mints runners. Use a throwaway/test project, not anything sensitive.

.github/workflows/e2e.yaml

@@ -20,14 +20,10 @@ on:
   # somewhere else would let a run vouch for a tree it never checked out.
   workflow_dispatch:
 
-# One global queue, because every run registers runners under the same tag
-# against the one shared GitLab project; concurrent runs steal each other's
-# build jobs (the same reason the matrix below is max-parallel: 1). Never
-# cancel: a superseded run would leave its commit with no e2e evidence, and
-# release.yml refuses to publish a tag whose commit has none.
-concurrency:
-  group: e2e-shared-gitlab-project
-  cancel-in-progress: false
+# No concurrency group. Each leg registers its runner under a tag unique to the
+# run and the Kubernetes version, and pins its pipeline to that tag, so two runs
+# cannot pick up each other's jobs. A shared queue would also let a pending run
+# be displaced and cancelled outright, leaving a commit with no e2e evidence.
 
 # No workflow-wide token. The e2e job runs checked-out PR code with GitLab
 # secrets, so write scope is granted only to the authorize job below.
@@ -96,7 +92,7 @@ jobs:
             core.setOutput('ref', pr.data.head.sha);
 
   # Docs and workflow-only PRs cannot break the suite, so they skip it rather
-  # than spending a serialized slot. Non-PR events always run.
+  # than spending four runners and a few minutes. Non-PR events always run.
   changes:
     needs: authorize
     runs-on: ubuntu-latest
@@ -168,11 +164,6 @@ jobs:
       contents: read
     strategy:
       fail-fast: false
-      # Serialize the matrix: every job registers runners with the same tag
-      # against the one shared GitLab project, so running them concurrently lets
-      # a build-job land on a sibling's runner and die (runner_system_failure)
-      # when that sibling tears down its kind cluster first.
-      max-parallel: 1
       matrix:
         k8s_version: ${{ fromJson(needs.k8s_versions.outputs.k8s_versions) }}
     steps:
@@ -210,6 +201,10 @@ jobs:
           GITLAB_E2E_URL: ${{ secrets.GITLAB_E2E_URL }}
           GITLAB_E2E_TOKEN: ${{ secrets.GITLAB_E2E_TOKEN }}
           GITLAB_E2E_PROJECT_ID: ${{ secrets.GITLAB_E2E_PROJECT_ID }}
+          # Unique per run, attempt and matrix leg, so no two legs or runs can
+          # share a tag. job-index rather than the version string because a
+          # runner tag cannot contain the dots in e.g. v1.33.7.
+          GITLAB_E2E_RUNNER_TAG: e2e-${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}
         run: make test-e2e
 
   # The one context worth marking required. Its name never changes, unlike the

test/e2e/e2e_suite_test.go

@@ -24,13 +24,23 @@ import (
 
 const (
 	e2eNamespace = "default"
-	// jobTag must match the tags on build-job in the test project's
-	// .gitlab-ci.yml so the registered runner picks the job up.
-	jobTag   = "test-gitlab-runner"
-	timeout  = 6 * time.Minute
-	interval = 5 * time.Second
+	// defaultJobTag is the RUNNER_TAG default in the test project's
+	// .gitlab-ci.yml, used when GITLAB_E2E_RUNNER_TAG is unset.
+	defaultJobTag = "test-gitlab-runner"
+	timeout       = 6 * time.Minute
+	interval      = 5 * time.Second
 )
 
+// jobTag is the tag this run registers its runner with and pins its pipeline
+// to. CI sets a per-run value so concurrent runs cannot pick up each other's
+// jobs; a sibling doing so would kill the job when it tears its cluster down.
+var jobTag = func() string {
+	if t := os.Getenv("GITLAB_E2E_RUNNER_TAG"); t != "" {
+		return t
+	}
+	return defaultJobTag
+}()
+
 var (
 	k8sClient     client.Client
 	glab          *gitlab.Client

test/e2e/e2e_test.go

@@ -71,7 +71,15 @@ func waitGone(name string) {
 
 func triggerPipeline() int64 {
 	GinkgoHelper()
-	p, _, err := glab.Pipelines.CreatePipeline(projectID, &gitlab.CreatePipelineOptions{Ref: gitlab.Ptr(defaultBranch)})
+	// RUNNER_TAG pins build-job to this run's runner. Without it the job takes
+	// the CI file's default tag, which every concurrent run shares.
+	p, _, err := glab.Pipelines.CreatePipeline(projectID, &gitlab.CreatePipelineOptions{
+		Ref: gitlab.Ptr(defaultBranch),
+		Variables: &[]*gitlab.PipelineVariableOptions{{
+			Key:   gitlab.Ptr("RUNNER_TAG"),
+			Value: gitlab.Ptr(jobTag),
+		}},
+	})
 	Expect(err).NotTo(HaveOccurred(), "could not trigger a pipeline (token needs api scope + Developer role)")
 	return p.ID
 }

test/e2e/terraform/README.md

@@ -11,7 +11,7 @@ Run everything with `tofu`, not `terraform`.
 
 ```text
 project (private)                       var.project_name
-  ├── .gitlab-ci.yml                    build-job, tagged var.job_tag
+  ├── .gitlab-ci.yml                    build-job, tagged $RUNNER_TAG
   └── project access token              scopes: api, create_runner (Maintainer)
 ```
 
@@ -25,10 +25,11 @@ simplest and fully-IaC option.
 
 - **Project** is a throwaway. Shared runners and Auto DevOps are disabled so the
   only thing that can run `build-job` is the operator-managed runner the suite
-  creates. All e2e matrix jobs share this single project, so the CI workflow runs
-  the k8s matrix with `max-parallel: 1`; parallel jobs would let a build-job land
-  on a sibling's runner and die when that sibling tears down first.
-- **`.gitlab-ci.yml`** defines a single `build-job` tagged with `var.job_tag`.
+  creates. All e2e matrix jobs share this single project, and they run
+  concurrently: each pins its pipeline to a tag unique to that run and leg, so a
+  build-job can only be picked up by the runner its own pipeline named.
+- **`.gitlab-ci.yml`** defines a single `build-job` tagged with `$RUNNER_TAG`,
+  a pipeline variable defaulting to `var.job_tag`.
   The suite registers its managed runner with `run_untagged = false` and that
   same tag, then asserts the job ran on *our* runner id.
 - **Project access token** is what the suite consumes as `GITLAB_E2E_TOKEN`. It
@@ -117,12 +118,26 @@ too and use it directly as `GITLAB_E2E_TOKEN`. A personal access token always
 has a real user owner and will mint the runner. The rest of the fixture (group,
 project, CI file) is unaffected.
 
-## Keep the CI tag in sync
+## How the CI tag is chosen
 
-`var.job_tag` defaults to `test-gitlab-runner`, which must match the `jobTag`
-constant in `test/e2e/e2e_suite_test.go`. Change one and you must change the
-other, otherwise the managed runner never picks up `build-job` and the suite
-times out.
+`build-job` is tagged `$RUNNER_TAG`, a pipeline variable whose default comes from
+`var.job_tag` (`test-gitlab-runner`). The suite sets that variable when it
+triggers a pipeline, using `GITLAB_E2E_RUNNER_TAG` if set and otherwise the
+`defaultJobTag` constant in `test/e2e/e2e_suite_test.go`.
+
+Locally you can ignore all of it: leave `GITLAB_E2E_RUNNER_TAG` unset and the
+default applies, in which case `var.job_tag` and `defaultJobTag` must agree or
+the managed runner never picks up `build-job` and the suite times out.
+
+The project sets `ci_pipeline_variables_minimum_override_role = "maintainer"`
+because GitLab refuses `RUNNER_TAG` with "Insufficient permissions to set
+pipeline variables" otherwise, and new projects default to `no_one_allowed`.
+Lowering it further is not needed; raising it breaks the suite.
+
+CI sets a unique tag per run, attempt and matrix leg. That is what lets the
+matrix and separate runs execute concurrently against this one shared project: a
+job can only be picked up by the runner its own pipeline named, so a sibling
+cannot take it and then kill it by tearing down its cluster.
 
 ## Cleanup
 

test/e2e/terraform/main.tf

@@ -21,11 +21,18 @@ locals {
     stages:
       - build
 
+    variables:
+      # The e2e suite overrides this per pipeline so a run's job can only be
+      # picked up by that run's runner. Without it every concurrent run shares
+      # one tag and a job can land on a sibling's runner, which then tears its
+      # cluster down mid-job. The default keeps a manual pipeline working.
+      RUNNER_TAG: ${var.job_tag}
+
     build-job:
       stage: build
       image: ${var.ci_job_image}
       tags:
-        - ${var.job_tag}
+        - $RUNNER_TAG
       script:
         - echo "e2e build-job on runner $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION)"
         - echo "commit $CI_COMMIT_SHORT_SHA on ref $CI_COMMIT_REF_NAME"
@@ -51,6 +58,13 @@ resource "gitlab_project" "e2e" {
   initialize_with_readme = true
   default_branch         = "main"
 
+  # The suite sets RUNNER_TAG as a pipeline variable so a run's job can only be
+  # picked up by that run's runner. GitLab refuses that with "Insufficient
+  # permissions to set pipeline variables" unless this is at or below the
+  # token's role, and new projects default to no_one_allowed. The e2e token is
+  # a Maintainer, so maintainer is the least-privilege value that works.
+  ci_pipeline_variables_minimum_override_role = "maintainer"
+
   # The suite asserts build-job ran on our operator-managed runner. Keep shared
   # runners out of the picture and silence Auto DevOps pipelines.
   shared_runners_enabled = false

test/e2e/terraform/variables.tf

@@ -36,9 +36,11 @@ variable "project_name" {
 
 variable "job_tag" {
   description = <<-DESC
-    Tag applied to build-job in the generated .gitlab-ci.yml. MUST match the
-    jobTag constant in test/e2e/e2e_suite_test.go (currently "test-gitlab-runner")
-    or the managed runner never picks the job up and the suite times out.
+    Default RUNNER_TAG in the generated .gitlab-ci.yml, used when a pipeline
+    does not set one. MUST match the defaultJobTag constant in
+    test/e2e/e2e_suite_test.go, or a locally run suite never picks the job up.
+    CI overrides it per run via GITLAB_E2E_RUNNER_TAG so concurrent runs cannot
+    steal each other's jobs.
   DESC
   type        = string
   default     = "test-gitlab-runner"