diff --git a/sandbox/sandbox_test.go b/sandbox/sandbox_test.go index a32a2493..c120d7a7 100644 --- a/sandbox/sandbox_test.go +++ b/sandbox/sandbox_test.go @@ -626,7 +626,39 @@ func testCreateReplicationSandbox(t *testing.T) { t.Fatalf(globals.ErrCreatingSandbox, err) } + // #131 fix test: replicas must answer queries immediately (exercises wait in initialize_slaves*) sandboxDir := path.Join(sandboxDef.SandboxDir, defaults.Defaults().MasterSlavePrefix+pathVersion) + for _, n := range []string{"1", "2"} { + use := path.Join(sandboxDir, defaults.Defaults().NodePrefix+n, "use") + if !common.ExecExists(use) { + t.Fatalf("expected replica use script at %s", use) + } + // RunCmdWithArgs (RunCmd takes only the cmd string) + out, err := common.RunCmdWithArgs(use, []string{"-BN", "-e", "SELECT 1;"}); // tests #131 fix: replica must be ready immediately (no sleep) after deploy + if err != nil { + t.Fatalf("replica n%s not ready immediately after deploy replication: %v\noutput: %s", n, err, out) + } + if strings.TrimSpace(out) != "1" { + t.Fatalf("replica n%s: expected '1', got %q", n, out) + } + } + t.Logf("ok - both replicas accepted queries immediately after deploy (no manual sleep)") + + // Directly assert the generated script contains the waits (the actual fix). + initSlavesScript := path.Join(sandboxDir, "initialize_slaves") + initContent, rerr := os.ReadFile(initSlavesScript) + if rerr != nil { + t.Fatalf("could not read initialize_slaves: %v", rerr) + } + initStr := string(initContent) + if !strings.Contains(initStr, `wait_until_replica_ready "$SBDIR/n1/use" 60 1`) { + t.Errorf("initialize_slaves missing wait_until_replica_ready for n1 (the #131 fix)") + } + if !strings.Contains(initStr, `wait_until_replica_ready "$SBDIR/n2/use" 60 1`) { + t.Errorf("initialize_slaves missing wait_until_replica_ready for n2 (the #131 fix)") + } + + sandboxDir = path.Join(sandboxDef.SandboxDir, defaults.Defaults().MasterSlavePrefix+pathVersion) okDirExists(t, sandboxDir) dirs := []string{ defaults.Defaults().MasterName, diff --git a/sandbox/templates/replication/init_slaves.gotxt b/sandbox/templates/replication/init_slaves.gotxt index f1046c6c..7f71f63f 100644 --- a/sandbox/templates/replication/init_slaves.gotxt +++ b/sandbox/templates/replication/init_slaves.gotxt @@ -6,6 +6,26 @@ # This script is called by 'start_all' when needed SBDIR={{.SandboxDir}} cd "$SBDIR" + +# Wait until a replica is ready to accept queries from the sandbox user. +# This is called after START REPLICA (or START SLAVE) during replication +# initialization to prevent the race where the first client command on a +# just-started replica fails with "Access denied" or connection errors. +# Observed especially with MySQL 8.4+ and 9.x. +wait_until_replica_ready() { + local use_cmd=${1:-$SBDIR/use} + local max_attempts=${2:-60} + local sleep_sec=${3:-1} + for i in $(seq 1 $max_attempts); do + if $use_cmd -BN -e "SELECT 1;" >/dev/null 2>&1 ; then + return 0 + fi + sleep $sleep_sec + done + echo "WARNING: replica not ready to accept queries after $((max_attempts * sleep_sec))s (cmd: $use_cmd)" >&2 + return 1 +} + # workaround for Bug#89959 $SBDIR/{{.MasterLabel}}/use -h {{.MasterIp}} -u {{.RplUser}} -p{{.RplPassword}} -e 'set @a=1' if [ ! -f needs_initialization ] @@ -18,6 +38,8 @@ fi echo "initializing {{.SlaveLabel}} {{.Node}}" echo '{{.ChangeMasterTo}} {{.MasterHostParam}}="{{.MasterIp}}", {{.MasterPortParam}}={{.MasterPort}}, {{.MasterUserParam}}="{{.RplUser}}", {{.MasterPasswordParam}}="{{.RplPassword}}" {{.MasterAutoPosition}} {{.ChangeMasterExtra}}' | $SBDIR/{{.NodeLabel}}{{.Node}}/use -u root $SBDIR/{{.NodeLabel}}{{.Node}}/use -u root -e '{{.StartReplica}}' +# Ensure the replica is serving queries before we return from initialization. +wait_until_replica_ready "$SBDIR/{{.NodeLabel}}{{.Node}}/use" 60 1 {{end}} if [ -x ./post_initialization ] then diff --git a/sandbox/templates/replication/init_slaves_84.gotxt b/sandbox/templates/replication/init_slaves_84.gotxt index 0e6ce756..2efd5cbe 100644 --- a/sandbox/templates/replication/init_slaves_84.gotxt +++ b/sandbox/templates/replication/init_slaves_84.gotxt @@ -7,6 +7,26 @@ # Uses MySQL 8.4+ replication syntax (CHANGE REPLICATION SOURCE TO) SBDIR={{.SandboxDir}} cd "$SBDIR" + +# Wait until a replica is ready to accept queries from the sandbox user. +# This is called after START REPLICA (or START SLAVE) during replication +# initialization to prevent the race where the first client command on a +# just-started replica fails with "Access denied" or connection errors. +# Observed especially with MySQL 8.4+ and 9.x. +wait_until_replica_ready() { + local use_cmd=${1:-$SBDIR/use} + local max_attempts=${2:-60} + local sleep_sec=${3:-1} + for i in $(seq 1 $max_attempts); do + if $use_cmd -BN -e "SELECT 1;" >/dev/null 2>&1 ; then + return 0 + fi + sleep $sleep_sec + done + echo "WARNING: replica not ready to accept queries after $((max_attempts * sleep_sec))s (cmd: $use_cmd)" >&2 + return 1 +} + # workaround for Bug#89959 $SBDIR/{{.MasterLabel}}/use -h {{.MasterIp}} -u {{.RplUser}} -p{{.RplPassword}} -e 'set @a=1' if [ ! -f needs_initialization ] @@ -19,6 +39,8 @@ fi echo "initializing {{.SlaveLabel}} {{.Node}}" echo '{{.ChangeMasterTo}} {{.MasterHostParam}}="{{.MasterIp}}", {{.MasterPortParam}}={{.MasterPort}}, {{.MasterUserParam}}="{{.RplUser}}", {{.MasterPasswordParam}}="{{.RplPassword}}" {{.MasterAutoPosition}} {{.ChangeMasterExtra}}' | $SBDIR/{{.NodeLabel}}{{.Node}}/use -u root $SBDIR/{{.NodeLabel}}{{.Node}}/use -u root -e '{{.StartReplica}}' +# Ensure the replica is serving queries before we return from initialization. +wait_until_replica_ready "$SBDIR/{{.NodeLabel}}{{.Node}}/use" 60 1 {{end}} if [ -x ./post_initialization ] then diff --git a/sandbox/templates/single/sb_include.gotxt b/sandbox/templates/single/sb_include.gotxt index eb7d311c..ee8cb2f0 100644 --- a/sandbox/templates/single/sb_include.gotxt +++ b/sandbox/templates/single/sb_include.gotxt @@ -84,3 +84,24 @@ function wait_until_wsrep_ready echo "WARNING: wsrep_ready not ON after $((max_attempts * sleep_sec))s" >&2 return 1 } + +# Wait until a replica is ready to accept queries from the sandbox user. +# This is called after START REPLICA (or START SLAVE) during replication +# initialization to prevent the race where the first client command on a +# just-started replica fails with "Access denied" or connection errors. +# Observed especially with MySQL 8.4+ and 9.x. +# Usage: wait_until_replica_ready "$SBDIR/s1/use" 60 1 +function wait_until_replica_ready +{ + local use_cmd=${1:-$SBDIR/use} + local max_attempts=${2:-60} + local sleep_sec=${3:-1} + for i in $(seq 1 $max_attempts); do + if $use_cmd -BN -e "SELECT 1;" >/dev/null 2>&1 ; then + return 0 + fi + sleep $sleep_sec + done + echo "WARNING: replica not ready to accept queries after $((max_attempts * sleep_sec))s (cmd: $use_cmd)" >&2 + return 1 +} diff --git a/sandbox/templates_flavor_test.go b/sandbox/templates_flavor_test.go index 4e87c357..dd4acd6c 100644 --- a/sandbox/templates_flavor_test.go +++ b/sandbox/templates_flavor_test.go @@ -177,3 +177,48 @@ func TestReplicationStopAndUse_DelegateToSingleTemplates(t *testing.T) { } } } + +// TestInitSlavesTemplates_IncludeReplicaReadyWait ensures that the +// replication initialization templates (used by "deploy replication") +// contain the wait_until_replica_ready logic introduced to fix #131. +// This is a pure template test and does not require any MySQL binaries. +func TestInitSlavesTemplates_IncludeReplicaReadyWait(t *testing.T) { + for name, tmplContent := range map[string]string{ + "init_slaves": initSlavesTemplate, + "init_slaves_84": initSlaves84Template, + } { + data := common.StringMap{ + "ShellPath": "/bin/bash", + "Copyright": "# test", + "AppVersion": "test", + "DateTime": "now", + "TemplateName": name, + "SandboxDir": "/tmp/rsandbox_1234", + "MasterLabel": "master", + "MasterIp": "127.0.0.1", + "RplUser": "rsandbox", + "RplPassword": "rsandbox", + "MasterAutoPosition": "", + "ChangeMasterExtra": "", + "StartReplica": "START REPLICA", + "NodeLabel": "n", + "SlaveLabel": "slave", + "Slaves": []common.StringMap{ + {"NodeLabel": "n", "Node": 1, "SlaveLabel": "slave"}, + {"NodeLabel": "n", "Node": 2, "SlaveLabel": "slave"}, + }, + } + result := renderTemplate(t, tmplContent, data) + + if !strings.Contains(result, "wait_until_replica_ready") { + t.Errorf("%s template must define the wait_until_replica_ready helper (regression for #131)", name) + } + // The call must appear for each slave after the START REPLICA line. + if !strings.Contains(result, `wait_until_replica_ready "$SBDIR/n1/use" 60 1`) { + t.Errorf("%s template must invoke wait for the first replica", name) + } + if !strings.Contains(result, `wait_until_replica_ready "$SBDIR/n2/use" 60 1`) { + t.Errorf("%s template must invoke wait for the second replica", name) + } + } +}