From 64ce332894267f95d0132fb366d28e57b45fb6e2 Mon Sep 17 00:00:00 2001 From: Greg Rychlewski Date: Fri, 19 Dec 2025 15:50:15 -0500 Subject: [PATCH 1/5] use input redirection for mysql structure_load --- Earthfile | 3 +- lib/ecto/adapters/myxql.ex | 90 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/Earthfile b/Earthfile index 136d9a44..a0f97e7f 100644 --- a/Earthfile +++ b/Earthfile @@ -92,7 +92,8 @@ integration-test-mysql: # the default authentication plugin for MySQL 8 is sha 256 but it doesn't come with the docker image. falling back to the 5.7 way --default-authentication-plugin=mysql_native_password; \ # wait for mysql to start - while ! mysqladmin ping --host=127.0.0.1 --port=3306 --protocol=TCP --silent; do \ + while ! mysqladmin ping --host=127.0.0.1 --port=3306 --protocol=TCP; do \ + docker logs mysql; \ test "$(date +%s)" -le "$timeout" || (echo "timed out waiting for mysql"; exit 1); \ echo "waiting for mysql"; \ sleep 1; \ diff --git a/lib/ecto/adapters/myxql.ex b/lib/ecto/adapters/myxql.ex index dc595d06..2fd84d98 100644 --- a/lib/ecto/adapters/myxql.ex +++ b/lib/ecto/adapters/myxql.ex @@ -419,13 +419,15 @@ defmodule Ecto.Adapters.MyXQL do case File.read(path) do {:ok, contents} -> args = [ - "--execute", - "SET FOREIGN_KEY_CHECKS = 0; " <> contents <> "; SET FOREIGN_KEY_CHECKS = 1", + "--silent", + "--batch", + "--unbuffered", + "--init-command=SET FOREIGN_KEY_CHECKS = 0;", "--database", config[:database] ] - case run_with_cmd("mysql", config, args) do + case run_with_port("mysql", config, args, contents) do {_output, 0} -> {:ok, path} {output, _} -> {:error, output} end @@ -497,6 +499,12 @@ defmodule Ecto.Adapters.MyXQL do "please guarantee it is available before running ecto commands" end + {args, cmd_opts} = args_cmd_opts(opts, opt_args, cmd_opts) + + System.cmd(cmd, args, cmd_opts) + end + + defp args_cmd_opts(opts, opt_args, cmd_opts) do env = if password = opts[:password] do [{"MYSQL_PWD", password}] @@ -530,6 +538,80 @@ defmodule Ecto.Adapters.MyXQL do |> Keyword.put_new(:stderr_to_stdout, true) |> Keyword.update(:env, env, &Enum.concat(env, &1)) - System.cmd(cmd, args, cmd_opts) + {args, cmd_opts} + end + + # Ported from Elixir System.cmd implementation with the + # intent of using file redirection for passing dumps + # into the mysql client so that users don't run into + # shell limits when files are too large + defp run_with_port(cmd, opts, opt_args, contents, cmd_opts \\ []) do + abs_cmd = System.find_executable(cmd) + + unless abs_cmd do + raise "could not find executable `#{cmd}` in path, " <> + "please guarantee it is available before running ecto commands" + end + + abs_cmd = String.to_charlist(abs_cmd) + {args, cmd_opts} = args_cmd_opts(opts, opt_args, cmd_opts) + + port_opts = port_opts(cmd_opts, args: args) + port = Port.open({:spawn_executable, abs_cmd}, port_opts) + Port.command(port, contents) + # Use this as a signal to close the port since we cannot + # send an exit command to mysql in batch mode + Port.command(port, ";SELECT '__ECTO_EOF__';\n") + + {initial, fun} = Collectable.into("") + + try do + collect_output(port, initial, fun) + catch + kind, reason -> + fun.(initial, :halt) + :erlang.raise(kind, reason, __STACKTRACE__) + else + {acc, status} -> {fun.(acc, :done), status} + end + end + + defp port_opts([{:stderr_to_stdout, true} | t], acc), + do: port_opts(t, [:stderr_to_stdout | acc]) + + defp port_opts([{:stderr_to_stdout, _} | t], acc), + do: port_opts(t, acc) + + defp port_opts([{:env, enum} | t], acc), + do: port_opts(t, [{:env, validate_env(enum)} | acc]) + + defp port_opts([], acc) do + [:use_stdio, :exit_status, :binary, :hide] ++ acc + end + + defp validate_env(enum) do + Enum.map(enum, fn + {k, nil} -> + {String.to_charlist(k), false} + + {k, v} -> + {String.to_charlist(k), String.to_charlist(v)} + + other -> + raise ArgumentError, "invalid environment key-value #{inspect(other)}" + end) + end + + defp collect_output(port, acc, fun) do + receive do + {^port, {:data, "__ECTO_EOF__" <> _rest}} -> + {acc, 0} + + {^port, {:data, data}} -> + collect_output(port, fun.(acc, {:cont, data}), fun) + + {^port, {:exit_status, status}} -> + {acc, status} + end end end From 1024f58bb2ebed5e44f94a5efd7c04bd58eb4f8c Mon Sep 17 00:00:00 2001 From: Greg Rychlewski Date: Fri, 19 Dec 2025 15:53:18 -0500 Subject: [PATCH 2/5] oops --- Earthfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Earthfile b/Earthfile index a0f97e7f..136d9a44 100644 --- a/Earthfile +++ b/Earthfile @@ -92,8 +92,7 @@ integration-test-mysql: # the default authentication plugin for MySQL 8 is sha 256 but it doesn't come with the docker image. falling back to the 5.7 way --default-authentication-plugin=mysql_native_password; \ # wait for mysql to start - while ! mysqladmin ping --host=127.0.0.1 --port=3306 --protocol=TCP; do \ - docker logs mysql; \ + while ! mysqladmin ping --host=127.0.0.1 --port=3306 --protocol=TCP --silent; do \ test "$(date +%s)" -le "$timeout" || (echo "timed out waiting for mysql"; exit 1); \ echo "waiting for mysql"; \ sleep 1; \ From fe5d756da8cd911fac3071d056fe73039e8ecfb9 Mon Sep 17 00:00:00 2001 From: Greg Rychlewski Date: Sat, 20 Dec 2025 10:01:09 -0500 Subject: [PATCH 3/5] review suggestions --- lib/ecto/adapters/myxql.ex | 106 ++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 61 deletions(-) diff --git a/lib/ecto/adapters/myxql.ex b/lib/ecto/adapters/myxql.ex index 2fd84d98..9e85d71d 100644 --- a/lib/ecto/adapters/myxql.ex +++ b/lib/ecto/adapters/myxql.ex @@ -499,12 +499,46 @@ defmodule Ecto.Adapters.MyXQL do "please guarantee it is available before running ecto commands" end - {args, cmd_opts} = args_cmd_opts(opts, opt_args, cmd_opts) + {args, env} = args_env(opts, opt_args) + + cmd_opts = + cmd_opts + |> Keyword.put_new(:stderr_to_stdout, true) + |> Keyword.update(:env, env, &Enum.concat(env, &1)) System.cmd(cmd, args, cmd_opts) end - defp args_cmd_opts(opts, opt_args, cmd_opts) do + defp run_with_port(cmd, opts, opt_args, contents) do + abs_cmd = System.find_executable(cmd) + + unless abs_cmd do + raise "could not find executable `#{cmd}` in path, " <> + "please guarantee it is available before running ecto commands" + end + + abs_cmd = String.to_charlist(abs_cmd) + {args, env} = args_env(opts, opt_args) + + port_opts = [ + :use_stdio, + :exit_status, + :binary, + :hide, + :stderr_to_stdout, + env: validate_env(env), + args: args + ] + + port = Port.open({:spawn_executable, abs_cmd}, port_opts) + Port.command(port, contents) + # Use this as a signal to close the port since we cannot + # send an exit command to mysql in batch mode + Port.command(port, ";SELECT '__ECTO_EOF__';\n") + collect_output(port, "") + end + + defp args_env(opts, opt_args) do env = if password = opts[:password] do [{"MYSQL_PWD", password}] @@ -533,60 +567,7 @@ defmodule Ecto.Adapters.MyXQL do protocol ] ++ user_args ++ opt_args - cmd_opts = - cmd_opts - |> Keyword.put_new(:stderr_to_stdout, true) - |> Keyword.update(:env, env, &Enum.concat(env, &1)) - - {args, cmd_opts} - end - - # Ported from Elixir System.cmd implementation with the - # intent of using file redirection for passing dumps - # into the mysql client so that users don't run into - # shell limits when files are too large - defp run_with_port(cmd, opts, opt_args, contents, cmd_opts \\ []) do - abs_cmd = System.find_executable(cmd) - - unless abs_cmd do - raise "could not find executable `#{cmd}` in path, " <> - "please guarantee it is available before running ecto commands" - end - - abs_cmd = String.to_charlist(abs_cmd) - {args, cmd_opts} = args_cmd_opts(opts, opt_args, cmd_opts) - - port_opts = port_opts(cmd_opts, args: args) - port = Port.open({:spawn_executable, abs_cmd}, port_opts) - Port.command(port, contents) - # Use this as a signal to close the port since we cannot - # send an exit command to mysql in batch mode - Port.command(port, ";SELECT '__ECTO_EOF__';\n") - - {initial, fun} = Collectable.into("") - - try do - collect_output(port, initial, fun) - catch - kind, reason -> - fun.(initial, :halt) - :erlang.raise(kind, reason, __STACKTRACE__) - else - {acc, status} -> {fun.(acc, :done), status} - end - end - - defp port_opts([{:stderr_to_stdout, true} | t], acc), - do: port_opts(t, [:stderr_to_stdout | acc]) - - defp port_opts([{:stderr_to_stdout, _} | t], acc), - do: port_opts(t, acc) - - defp port_opts([{:env, enum} | t], acc), - do: port_opts(t, [{:env, validate_env(enum)} | acc]) - - defp port_opts([], acc) do - [:use_stdio, :exit_status, :binary, :hide] ++ acc + {args, env} end defp validate_env(enum) do @@ -602,13 +583,16 @@ defmodule Ecto.Adapters.MyXQL do end) end - defp collect_output(port, acc, fun) do + defp collect_output(port, acc) do receive do - {^port, {:data, "__ECTO_EOF__" <> _rest}} -> - {acc, 0} - {^port, {:data, data}} -> - collect_output(port, fun.(acc, {:cont, data}), fun) + acc = acc <> data + + if acc =~ "__ECTO_EOF__" do + {acc, 0} + else + collect_output(port, acc) + end {^port, {:exit_status, status}} -> {acc, status} From ee1ccbcef591e8d5a0433461cfc75a559b1e3b2f Mon Sep 17 00:00:00 2001 From: Greg Rychlewski Date: Sat, 20 Dec 2025 14:13:08 -0500 Subject: [PATCH 4/5] close port --- lib/ecto/adapters/myxql.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ecto/adapters/myxql.ex b/lib/ecto/adapters/myxql.ex index 9e85d71d..5c452873 100644 --- a/lib/ecto/adapters/myxql.ex +++ b/lib/ecto/adapters/myxql.ex @@ -589,11 +589,11 @@ defmodule Ecto.Adapters.MyXQL do acc = acc <> data if acc =~ "__ECTO_EOF__" do - {acc, 0} - else - collect_output(port, acc) + Port.close(port) end + collect_output(port, acc) + {^port, {:exit_status, status}} -> {acc, status} end From 8f24f26fd583d20fbaef3ecdd36fa8134db445cf Mon Sep 17 00:00:00 2001 From: Greg Rychlewski Date: Sat, 20 Dec 2025 14:18:09 -0500 Subject: [PATCH 5/5] close port --- lib/ecto/adapters/myxql.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/ecto/adapters/myxql.ex b/lib/ecto/adapters/myxql.ex index 5c452873..521d97c1 100644 --- a/lib/ecto/adapters/myxql.ex +++ b/lib/ecto/adapters/myxql.ex @@ -590,10 +590,11 @@ defmodule Ecto.Adapters.MyXQL do if acc =~ "__ECTO_EOF__" do Port.close(port) + {acc, 0} + else + collect_output(port, acc) end - collect_output(port, acc) - {^port, {:exit_status, status}} -> {acc, status} end