Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions server/src/__tests__/deployment-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -276,3 +276,50 @@ test('verifier script is old-image-local, read-only, bounded, and uses strict pr
assert.equal(parseSchemaVerifierOutput(output).status, 'unknown')
}
})

test("a proxy probe on a named port is resolved to its number, because the Job has no port list", () => {
// Production's proxy declares `ports: [{containerPort: 9090, name: pg-health}]`
// and probes that name. The Job strips `ports`, so shipping the name verbatim
// leaves kubelet parsing "pg-health" as an integer forever: the sidecar never
// reports `started`, the migrate container never launches, and the Job dies of
// DeadlineExceeded having run nothing.
const deployment = deploymentFixture() as any
const proxy = deployment.spec.template.spec.containers[0]
proxy.ports = [{ containerPort: 9090, name: 'pg-health', protocol: 'TCP' }]
proxy.startupProbe = {
httpGet: { path: '/readiness', port: 'pg-health', scheme: 'HTTP' },
periodSeconds: 2,
failureThreshold: 30,
}
const baseline = extractDeploymentSnapshot(deployment, { capturedAt: '2026-09-10T00:00:00.000Z' })
const job = buildMigrationJob(baseline, {
name: 'cumora-migrate-named-port',
image: `server@sha256:${DIGEST_CANDIDATE}`,
repairMode: 'off',
})

const jobProxy = templateSpec(job).initContainers.find((c: any) => c.name === 'cloud-sql-proxy')
assert.equal(jobProxy.startupProbe.httpGet.port, 9090)
// The rest of the operator's probe is preserved, not replaced by the default.
assert.equal(jobProxy.startupProbe.failureThreshold, 30)
assert.equal(jobProxy.startupProbe.httpGet.path, '/readiness')
// A Job container still carries no port declarations.
assert.equal(jobProxy.ports, undefined)
})

test('an unresolvable named probe port falls back to a probe that can actually pass', () => {
const deployment = deploymentFixture() as any
const proxy = deployment.spec.template.spec.containers[0]
// Name declared nowhere: keeping it would ship a permanently erroring probe.
proxy.startupProbe = { httpGet: { path: '/readiness', port: 'pg-health' }, failureThreshold: 30 }
const baseline = extractDeploymentSnapshot(deployment, { capturedAt: '2026-09-10T00:00:00.000Z' })
const job = buildMigrationJob(baseline, {
name: 'cumora-migrate-unresolvable-port',
image: `server@sha256:${DIGEST_CANDIDATE}`,
repairMode: 'off',
})

const jobProxy = templateSpec(job).initContainers.find((c: any) => c.name === 'cloud-sql-proxy')
assert.equal(jobProxy.startupProbe.httpGet.port, 9090)
assert.equal(jobProxy.startupProbe.failureThreshold, 60)
})
39 changes: 32 additions & 7 deletions server/src/deploy/recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,33 @@ function cleanContainerForJob(container: JsonObject): JsonObject {
return result
}

/**
* A probe copied out of the Deployment may target a *named* port, but the Job's
* containers carry no `ports` list (cleanContainerForJob strips it), so the name
* has nothing to resolve against. kubelet then falls back to parsing the name
* as a number, the probe errors permanently ("strconv.Atoi: parsing ..."), the
* native sidecar never reports `started`, and the migrate container is never
* launched at all — the Job silently burns its whole activeDeadlineSeconds and
* dies as DeadlineExceeded with no logs. Resolve the name to its number here,
* while the source container's port list is still in hand, and return null when
* it cannot be resolved so the caller uses its own numeric probe rather than one
* that can never pass.
*/
function resolveProbePorts(probe: unknown, ports: unknown): JsonValue | null {
if (!isRecord(probe) || !isJsonValue(probe)) return null
const resolved = cloneJson(probe) as JsonObject
for (const key of ['httpGet', 'tcpSocket']) {
const target = resolved[key]
if (!isRecord(target) || typeof target.port !== 'string') continue
const declared = Array.isArray(ports)
? ports.find((entry) => isRecord(entry) && entry.name === target.port && typeof entry.containerPort === 'number')
: undefined
if (!isRecord(declared)) return null
target.port = declared.containerPort
}
return resolved
}

function podSpecForJob(template: JsonObject): { pod: JsonObject; server: JsonObject; proxy: JsonObject; proxyStartupProbe: JsonValue } {
const spec = asJsonObject(getObjectPath(template, ['spec'], 'pod template'), 'pod template spec')
const allInit = Array.isArray(spec.initContainers) ? spec.initContainers : []
Expand All @@ -504,13 +531,11 @@ function podSpecForJob(template: JsonObject): { pod: JsonObject; server: JsonObj
delete pod.containers
delete pod.initContainers
delete pod.ephemeralContainers
const proxyStartupProbe = isJsonValue(proxySource.startupProbe)
? cloneJson(proxySource.startupProbe)
: {
httpGet: { path: '/readiness', port: 9090 },
periodSeconds: 1,
failureThreshold: 60,
}
const proxyStartupProbe = resolveProbePorts(proxySource.startupProbe, proxySource.ports) ?? {
httpGet: { path: '/readiness', port: 9090 },
periodSeconds: 1,
failureThreshold: 60,
}
return { pod, server: cleanContainerForJob(server), proxy: cleanContainerForJob(proxySource), proxyStartupProbe }
}

Expand Down
Loading