diff --git a/deploy/helm/cassandra/helm/scripts/initdb.sh b/deploy/helm/cassandra/helm/scripts/initdb.sh index 800e444fe..02d9b661c 100644 --- a/deploy/helm/cassandra/helm/scripts/initdb.sh +++ b/deploy/helm/cassandra/helm/scripts/initdb.sh @@ -35,6 +35,44 @@ run_cqlsh() { sh -c 'read -r CQLSH_U; read -r CQLSH_P; cqlsh -u "$CQLSH_U" -p "$CQLSH_P" "$@"' -- "$@" } +# A pod can pass its nodetool-based readiness probe before Cassandra starts +# accepting CQL connections. Wait for native transport explicitly so a normal +# startup race is not mistaken for an unknown superuser password. +wait_for_native_transport() { + local pod="$1" + local end="$2" + + until [[ $(kubectl exec "${pod}" -c cassandra -n "${namespace}" -- \ + nodetool statusbinary 2>/dev/null) == "running" ]]; do + if [ $SECONDS -gt "$end" ]; then + echo "Timeout waiting for Cassandra native transport on pod ${pod}" + return 1 + fi + echo "Waiting for Cassandra native transport on pod ${pod}..." + sleep 5 + done +} + +# Native transport can report running before the system_auth roles are ready. +# Accept either the desired credential or Cassandra's bootstrap credential; +# ensure_superuser_password below handles the resulting steady state. +wait_for_superuser_authentication() { + local pod="$1" + local end=$((SECONDS + 120)) + + until run_cqlsh "${pod}" "${CASSANDRA_USER}" "${CASSANDRA_PASSWORD}" \ + localhost -e "SELECT key FROM system.local;" >/dev/null 2>&1 || \ + run_cqlsh "${pod}" "${DEFAULT_CASSANDRA_USER}" "${DEFAULT_CASSANDRA_PASSWORD}" \ + localhost -e "SELECT key FROM system.local;" >/dev/null 2>&1; do + if [ $SECONDS -gt "$end" ]; then + echo "Timeout waiting for Cassandra superuser authentication on pod ${pod}" + return 1 + fi + echo "Waiting for Cassandra superuser authentication on pod ${pod}..." + sleep 5 + done +} + # How many desired replicas # Ensure the "cassandra" superuser has the desired dbUser.password. @@ -112,6 +150,12 @@ initialize_db() { echo "All Cassandra pods are ready" # Always select the 0th pod + if ! wait_for_native_transport "${statefulset}-0" "$end"; then + return 1 + fi + if ! wait_for_superuser_authentication "${statefulset}-0"; then + return 1 + fi if ! ensure_superuser_password "${statefulset}-0"; then return 1 fi diff --git a/deploy/helm/cassandra/helm/values.yaml b/deploy/helm/cassandra/helm/values.yaml index 4993d27ab..caa0d8c1b 100644 --- a/deploy/helm/cassandra/helm/values.yaml +++ b/deploy/helm/cassandra/helm/values.yaml @@ -196,7 +196,7 @@ cassandra: # repository: must be supplied in additional values registry: "" repository: "" - tag: 0.16.0 + tag: 0.17.1 pullPolicy: Always dbUser: diff --git a/deploy/helm/cassandra/tests/test-initdb.sh b/deploy/helm/cassandra/tests/test-initdb.sh new file mode 100755 index 000000000..7c3808215 --- /dev/null +++ b/deploy/helm/cassandra/tests/test-initdb.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +test_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +chart_dir=$(CDPATH='' cd -- "${test_dir}/.." && pwd) +work_dir=$(mktemp -d) +trap 'rm -rf "${work_dir}"' EXIT + +mkdir -p "${work_dir}/bin" + +cat > "${work_dir}/bin/kubectl" <<'EOF' +#!/bin/sh +case "$*" in + *"get statefulset"*) + printf '1' + ;; + *"get pod"*) + printf 'true' + ;; + *"nodetool statusbinary"*) + attempts=0 + if [ -f "${TEST_STATE}" ]; then + attempts=$(cat "${TEST_STATE}") + fi + attempts=$((attempts + 1)) + printf '%s\n' "${attempts}" > "${TEST_STATE}" + if [ "${attempts}" -eq 1 ]; then + printf 'not running\n' + else + printf 'running\n' + fi + ;; + *"cqlsh"*) + cat >/dev/null + attempts=0 + if [ -f "${TEST_AUTH_STATE}" ]; then + attempts=$(cat "${TEST_AUTH_STATE}") + fi + attempts=$((attempts + 1)) + printf '%s\n' "${attempts}" > "${TEST_AUTH_STATE}" + if [ "${attempts}" -le 2 ]; then + exit 1 + fi + ;; + *) + printf 'unexpected kubectl command: %s\n' "$*" >&2 + exit 1 + ;; +esac +EOF + +cat > "${work_dir}/bin/sleep" <<'EOF' +#!/bin/sh +exit 0 +EOF + +chmod +x "${work_dir}/bin/kubectl" "${work_dir}/bin/sleep" + +export TEST_STATE="${work_dir}/native-transport-attempts" +export TEST_AUTH_STATE="${work_dir}/authentication-attempts" +output=$( + PATH="${work_dir}/bin:${PATH}" \ + CASSANDRA_USER=cassandra \ + CASSANDRA_PASSWORD=desired-password \ + bash "${chart_dir}/helm/scripts/initdb.sh" +) + +printf '%s\n' "${output}" | grep -Fq \ + 'Waiting for Cassandra native transport on pod cassandra-0...' +printf '%s\n' "${output}" | grep -Fq \ + 'Waiting for Cassandra superuser authentication on pod cassandra-0...' +printf '%s\n' "${output}" | grep -Fq 'Successfully initialized the db' +[ "$(cat "${TEST_STATE}")" -eq 2 ] +[ "$(cat "${TEST_AUTH_STATE}")" -eq 5 ] diff --git a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl index 2bbfd218b..dfdfaf8cd 100644 --- a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl @@ -156,7 +156,7 @@ releases: {{- end }} - name: cassandra - version: 0.20.1 + version: 0.20.2 condition: cassandra.enabled # From defaults.yaml or env overrides namespace: cassandra-system <<: *dependency # Inherits base values from the dependency template diff --git a/migrations/cassandra/keyspaces/nvct_api/03_init_tables.up.sql b/migrations/cassandra/keyspaces/nvct_api/03_init_tables.up.sql index a547f92f3..6d3729670 100644 --- a/migrations/cassandra/keyspaces/nvct_api/03_init_tables.up.sql +++ b/migrations/cassandra/keyspaces/nvct_api/03_init_tables.up.sql @@ -68,6 +68,7 @@ CREATE TABLE IF NOT EXISTS nvct_api.tasks_v2 ( results_location TEXT, status TEXT, telemetries FROZEN, + health TEXT, health_info FROZEN, percent_complete INT, last_updated_at TIMESTAMP, diff --git a/migrations/cassandra/keyspaces/nvct_api/04_add_task_health.up.sql b/migrations/cassandra/keyspaces/nvct_api/04_add_task_health.up.sql new file mode 100644 index 000000000..648b957d7 --- /dev/null +++ b/migrations/cassandra/keyspaces/nvct_api/04_add_task_health.up.sql @@ -0,0 +1,5 @@ +-- Add the JSON health payload used by current Cloud Tasks releases. +-- Fresh installs receive this column from 03_init_tables.up.sql. This +-- migration upgrades keyspaces created from the earlier canonical schema. + +ALTER TABLE nvct_api.tasks_v2 ADD IF NOT EXISTS health TEXT; diff --git a/migrations/cassandra/tests/test-execute-sqls.sh b/migrations/cassandra/tests/test-execute-sqls.sh index 28601d511..a31595d63 100755 --- a/migrations/cassandra/tests/test-execute-sqls.sh +++ b/migrations/cassandra/tests/test-execute-sqls.sh @@ -90,4 +90,15 @@ if ! grep -q '^until cqlsh ' "${script}"; then fail "execute_sqls.sh must retain the Cassandra authentication readiness check" fi +nvct_schema="${keyspaces}/nvct_api/03_init_tables.up.sql" +nvct_health_migration="${keyspaces}/nvct_api/04_add_task_health.up.sql" +if ! grep -Eq '^[[:space:]]+health[[:space:]]+TEXT,' "${nvct_schema}"; then + fail "the canonical NVCT task schema is missing the health column" +fi +if ! grep -Fq \ + 'ALTER TABLE nvct_api.tasks_v2 ADD IF NOT EXISTS health TEXT;' \ + "${nvct_health_migration}"; then + fail "the NVCT upgrade migration does not add the health column safely" +fi + exit "${status}"