diff --git a/src/ray/util/BUILD.bazel b/src/ray/util/BUILD.bazel index d143c6210964..21871d3cc52a 100644 --- a/src/ray/util/BUILD.bazel +++ b/src/ray/util/BUILD.bazel @@ -90,6 +90,7 @@ ray_cc_library( ":thread_utils", "@com_github_spdlog//:spdlog", "@com_google_absl//absl/debugging:failure_signal_handler", + "@com_google_absl//absl/log:globals", "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_prod", ], diff --git a/src/ray/util/logging.cc b/src/ray/util/logging.cc index 0b4c4155a5f6..d081b122f4f4 100644 --- a/src/ray/util/logging.cc +++ b/src/ray/util/logging.cc @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,7 @@ #include "absl/debugging/failure_signal_handler.h" #include "absl/debugging/stacktrace.h" #include "absl/debugging/symbolize.h" +#include "absl/log/globals.h" #include "absl/strings/numbers.h" #include "absl/strings/str_format.h" #include "ray/util/thread_utils.h" @@ -345,12 +347,32 @@ void RayLog::InitLogFormat() { return 1; } +namespace { + +// gRPC >= 1.65 logs through absl. When GRPC_VERBOSITY is unset it leaves absl's +// minimum log level alone, and absl defaults to INFO, so gRPC INFO messages (e.g. +// "Got goaway") end up on stderr and are forwarded to the driver. Older gRPC +// defaulted to ERROR; keep that unless the user asks for more with +// GRPC_VERBOSITY. Ray's own logging goes through spdlog and is unaffected. +void InitAbslLogging() { + static std::once_flag once; + std::call_once(once, [] { + const char *grpc_verbosity = std::getenv("GRPC_VERBOSITY"); + if (grpc_verbosity == nullptr || grpc_verbosity[0] == '\0') { + absl::SetMinLogLevel(absl::LogSeverityAtLeast::kError); + } + }); +} + +} // namespace + /*static*/ void RayLog::StartRayLog(const std::string &app_name, RayLogLevel severity_threshold, const std::string &log_filepath, const std::string &err_log_filepath, size_t log_rotation_max_size, size_t log_rotation_file_num) { + InitAbslLogging(); InitSeverityThreshold(severity_threshold); InitLogFormat(); diff --git a/src/ray/util/tests/BUILD.bazel b/src/ray/util/tests/BUILD.bazel index 26e5ac576aff..0ae015a584fe 100644 --- a/src/ray/util/tests/BUILD.bazel +++ b/src/ray/util/tests/BUILD.bazel @@ -117,6 +117,7 @@ ray_cc_test( "//src/ray/util:path_utils", "//src/ray/util:time", "@boost//:asio", + "@com_google_absl//absl/log:globals", "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_main", "@nlohmann_json", diff --git a/src/ray/util/tests/logging_test.cc b/src/ray/util/tests/logging_test.cc index 2ec25ab49717..b31d44921643 100644 --- a/src/ray/util/tests/logging_test.cc +++ b/src/ray/util/tests/logging_test.cc @@ -21,6 +21,7 @@ #include #include +#include "absl/log/globals.h" #include "absl/strings/str_format.h" #include "absl/strings/str_split.h" #include "gtest/gtest.h" @@ -189,6 +190,15 @@ TEST(PrintLogTest, TestRayLogEveryMs) { EXPECT_LT(occurrences, 15); } +// gRPC logs through absl; without GRPC_VERBOSITY its INFO messages would reach +// stderr. StartRayLog keeps absl at ERROR, as older gRPC did. +TEST(PrintLogTest, TestAbslMinLogLevelDefaultsToError) { + ASSERT_EQ(std::getenv("GRPC_VERBOSITY"), nullptr); + RayLog::StartRayLog("/tmp/gcs", RayLogLevel::INFO, /*log_filepath=*/""); + EXPECT_EQ(absl::MinLogLevel(), absl::LogSeverityAtLeast::kError); + RayLog::ShutDownRayLog(); +} + TEST(PrintLogTest, TestTextLogging) { SetEnv("RAY_BACKEND_LOG_JSON", "0"); RayLog::StartRayLog("/tmp/gcs", RayLogLevel::INFO, /*log_filepath=*/"");