Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion conformance/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,18 @@ cc_test(
],
)

# gMock matchers for conformance tests.
cc_library(
name = "global_test_environment",
testonly = 1,
hdrs = ["global_test_environment.h"],
deps = [":test_manager"],
)

# gMock matchers for conformance TestResults.
#
# NOTE: Consumers must link a definition of GetGlobalTestManager(), declared in
# global_test_environment.h. The conformance test environment provides it in
# the real test binary; matchers_test provides an in-memory stand-in.
cc_library(
name = "matchers",
testonly = 1,
Expand All @@ -430,16 +441,23 @@ cc_library(
deps = [
":binary_wireformat",
":conformance_cc_proto",
":global_test_environment",
":test_manager",
":testee",
"//src/google/protobuf",
"//src/google/protobuf:port",
"//src/google/protobuf/util:differencer",
"@abseil-cpp//absl/base:core_headers",
"@abseil-cpp//absl/base:nullability",
"@abseil-cpp//absl/log:absl_check",
"@abseil-cpp//absl/log:absl_log",
"@abseil-cpp//absl/memory",
"@abseil-cpp//absl/status",
"@abseil-cpp//absl/strings",
"@abseil-cpp//absl/strings:str_format",
"@abseil-cpp//absl/strings:string_view",
"@abseil-cpp//absl/synchronization",
"@abseil-cpp//absl/types:optional",
"@googletest//:gtest",
],
)
Expand All @@ -450,14 +468,20 @@ cc_test(
deps = [
":binary_wireformat",
":conformance_cc_proto",
":global_test_environment",
":matchers",
":mock_test_runner",
":test_manager",
":testee",
"//:test_messages_proto2_cc_proto",
"//src/google/protobuf",
"@abseil-cpp//absl/base:log_severity",
"@abseil-cpp//absl/log:absl_check",
"@abseil-cpp//absl/log:scoped_mock_log",
"@abseil-cpp//absl/status:status_matchers",
"@abseil-cpp//absl/strings",
"@abseil-cpp//absl/strings:string_view",
"@abseil-cpp//absl/types:optional",
"@googletest//:gtest",
"@googletest//:gtest_main",
],
Expand Down
36 changes: 36 additions & 0 deletions conformance/global_test_environment.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2026 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd

// Declares the process-wide state that the Yields() matcher depends on (see
// matchers.h). The definition is deliberately not provided here. It is the
// link-time seam between the matchers and the process that hosts them. The
// conformance test binary provides it from its global test environment,
// populated from command-line flags and the failure list on disk. Unit tests
// of the matchers provide a lightweight in-memory stand-in.

#ifndef GOOGLE_PROTOBUF_CONFORMANCE_GLOBAL_TEST_ENVIRONMENT_H__
#define GOOGLE_PROTOBUF_CONFORMANCE_GLOBAL_TEST_ENVIRONMENT_H__

#include "conformance/test_manager.h"

namespace google {
namespace protobuf {
namespace conformance {
namespace internal {

// Returns the process-wide TestManager that records test results against the
// expected-failure list. Yields() reports the outcome of every test here.
// That lets the failure list be validated and regenerated, and the enforcement
// level be applied (see TestManager::set_enforcement_level()).
TestManager& GetGlobalTestManager();

} // namespace internal
} // namespace conformance
} // namespace protobuf
} // namespace google

#endif // GOOGLE_PROTOBUF_CONFORMANCE_GLOBAL_TEST_ENVIRONMENT_H__
152 changes: 152 additions & 0 deletions conformance/matchers.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,22 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "absl/base/nullability.h"
#include "absl/base/thread_annotations.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/strings/ascii.h"
#include "absl/strings/escaping.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/string_view.h"
#include "absl/synchronization/mutex.h"
#include "absl/types/optional.h"
#include "conformance/binary_wireformat.h"
#include "conformance/conformance.pb.h"
#include "conformance/global_test_environment.h"
#include "conformance/test_manager.h"
#include "conformance/testee.h"
#include "google/protobuf/descriptor.h"
#include "google/protobuf/message.h"
Expand All @@ -40,6 +47,8 @@ namespace {

using ::conformance::ConformanceResponse;
using ::conformance::WireFormat;
using ::google::protobuf::conformance::internal::GetGlobalTestManager;
using ::google::protobuf::conformance::internal::TestManager;
using ::google::protobuf::conformance::internal::TestResult;

// Implements EqualsTextProto() and EqualsBinaryProto(): matches a message
Expand Down Expand Up @@ -472,6 +481,144 @@ bool RawPayloadMatcher::MatchPayload(
return false;
}

// The outcome Yields() reached for a result. `matched` says whether the test
// passed once the failure list and the test's priority were taken into
// account. `explanation` is the text to show for it.
struct Verdict {
bool matched = false;
std::string explanation;
};

// Implements Yields(). This is the single matcher that talks to the global
// TestManager; see matchers.h for the policy it applies.
//
// The TestManager must only hear about each test once. It remembers the
// names it was told (TestManager::WasReported()), and a result it already
// knows is not evaluated again. gtest evaluates a failing matcher a second
// time, through the same matcher object, to explain the failure. The matcher
// keeps its last failed verdict for that rerun. Anything else that checks a
// reported result is a test bug and fails.
//
// Matchers are nominally immutable. The cache only affects how a failure
// that has already been decided is explained, never a verdict, and copies of
// a Matcher share one impl, so the matcher behaves the same through every
// copy.
class YieldsMatcherImpl : public testing::MatcherInterface<const TestResult&> {
public:
explicit YieldsMatcherImpl(testing::Matcher<const TestResult&> inner)
: inner_(std::move(inner)) {}

bool MatchAndExplain(const TestResult& result,
testing::MatchResultListener* listener) const override {
{
absl::MutexLock lock(&mutex_);
if (last_failure_.has_value() && last_failure_->result == &result &&
last_failure_->name == result.name()) {
*listener << last_failure_->explanation;
return false;
}
}
if (GetGlobalTestManager().WasReported(result.name())) {
*listener << "TestResult for " << result.name()
<< " was already checked; each result may be checked exactly "
"once";
return false;
}
Verdict verdict = Evaluate(result);
*listener << verdict.explanation;
absl::MutexLock lock(&mutex_);
if (verdict.matched) {
last_failure_.reset();
} else {
last_failure_ = LastFailure{&result, std::string(result.name()),
std::move(verdict.explanation)};
}
return verdict.matched;
}

void DescribeTo(std::ostream* os) const override {
*os << "yields a result that ";
inner_.DescribeTo(os);
*os << " (or is an expected failure)";
}

void DescribeNegationTo(std::ostream* os) const override {
*os << "doesn't yield a result that ";
inner_.DescribeTo(os);
*os << ", nor an expected failure";
}

private:
// Reports `result` to the global TestManager and returns the gtest verdict.
Verdict Evaluate(const TestResult& result) const {
TestManager& manager = GetGlobalTestManager();
const std::string name(result.name());
const ConformanceResponse& response = result.response();

// A skip is decided here; the inner matcher never sees it. The manager
// marks a listed entry as seen and returns an error for it, so this is the
// only place that reports a listed test the testee skipped (stricter than
// legacy, see matchers.h).
if (response.result_case() == ConformanceResponse::kSkipped) {
absl::Status status = manager.ReportSkip(name, response.skipped());
ABSL_LOG(INFO) << "Skipping test " << name << ": " << response.skipped();
if (!status.ok()) {
return {/*matched=*/false,
/*explanation=*/std::string(status.message())};
}
return {/*matched=*/true,
/*explanation=*/absl::StrCat("which was skipped by the testee: ",
response.skipped())};
}

testing::StringMatchResultListener inner_listener;
bool matched = inner_.MatchAndExplain(result, &inner_listener);
std::string message = inner_listener.str();

if (matched) {
absl::Status status = manager.ReportSuccess(name);
if (!status.ok()) {
return {/*matched=*/false,
/*explanation=*/std::string(status.message())};
}
return {/*matched=*/true, /*explanation=*/std::move(message)};
}

if (message.empty()) {
message = absl::StrCat(
"which doesn't match (",
testing::DescribeMatcher<const TestResult&>(inner_), ")");
}

// The manager decides whether the failure is expected (listed), tolerated
// (above the enforcement level) or unexpected, and logs the first two.
absl::Status status =
manager.ReportFailure(name, result.priority(), message);
if (status.ok()) {
return {/*matched=*/true,
/*explanation=*/absl::StrCat(
"which failed, but the failure is expected or tolerated: ",
message)};
}
return {/*matched=*/false,
/*explanation=*/absl::StrCat(message, "\n", status.message())};
}

// The failure this matcher explained last, identified by the result's
// address and name, so that gtest's rerun gets the same explanation. A
// destroyed result's address may be reused, but not its name: Testee::Run()
// rejects duplicate test names.
struct LastFailure {
const TestResult* result;
std::string name;
std::string explanation;
};

testing::Matcher<const TestResult&> inner_;
mutable absl::Mutex mutex_;
mutable absl::optional<LastFailure> last_failure_ ABSL_GUARDED_BY(mutex_);
};

} // namespace

namespace internal {
Expand All @@ -498,6 +645,11 @@ testing::Matcher<const TestResult&> MakeWhenParsedMatcher(
return WhenParsedMatcher(std::move(m), type_override);
}

testing::Matcher<const TestResult&> MakeYieldsMatcher(
testing::Matcher<const TestResult&> inner) {
return testing::MakeMatcher(new YieldsMatcherImpl(std::move(inner)));
}

} // namespace internal

testing::Matcher<const internal::TestResult&> RawPayload(Wire bytes) {
Expand Down
Loading
Loading