From 0ae6223ee58448391600fe5be41d2adfcbe36968 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Fri, 10 Apr 2026 12:41:49 -0400 Subject: [PATCH 1/4] [WIP] Run the trace suite through canonicalisation Signed-off-by: Juan Cruz Viotti --- test/evaluator/CMakeLists.txt | 1 + test/evaluator/tracesuite.cc | 66 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/test/evaluator/CMakeLists.txt b/test/evaluator/CMakeLists.txt index 80a7011d1..d1fd3951a 100644 --- a/test/evaluator/CMakeLists.txt +++ b/test/evaluator/CMakeLists.txt @@ -56,3 +56,4 @@ target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE source target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE sourcemeta::core::jsonschema) target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE sourcemeta::blaze::compiler) target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE sourcemeta::blaze::evaluator) +target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE sourcemeta::blaze::alterschema) diff --git a/test/evaluator/tracesuite.cc b/test/evaluator/tracesuite.cc index 425919519..8fb0ac4b8 100644 --- a/test/evaluator/tracesuite.cc +++ b/test/evaluator/tracesuite.cc @@ -1,5 +1,6 @@ #include +#include #include #include @@ -127,6 +128,54 @@ class TraceTest : public testing::Test { const char *mode_key; }; +class CanonicalizeTraceTest : public testing::Test { +public: + explicit CanonicalizeTraceTest(sourcemeta::core::JSON test_data, + const sourcemeta::blaze::Mode test_mode) + : data{std::move(test_data)}, mode{test_mode} {} + + auto TestBody() -> void override { + auto schema{this->data.at("schema")}; + const auto &instance{this->data.at("instance")}; + const bool expected_valid{this->data.at("valid").to_boolean()}; + + sourcemeta::core::SchemaTransformer bundle; + sourcemeta::blaze::add(bundle, + sourcemeta::blaze::AlterSchemaMode::Canonicalizer); + const auto canonicalize_result{bundle.apply( + schema, sourcemeta::core::schema_walker, + sourcemeta::core::schema_resolver, + [](const auto &pointer, const auto &name, const auto &message, + const auto &, const auto &) { + // TODO: Temporary for debugging purposes. Remove once everything + // passes + std::fprintf(stderr, " canonicalize: %s: %s: %s\n", + sourcemeta::core::to_string(pointer).c_str(), + std::string{name}.c_str(), std::string{message}.c_str()); + })}; + EXPECT_TRUE(canonicalize_result.first); + + const auto compiled_schema{sourcemeta::blaze::compile( + schema, sourcemeta::core::schema_walker, + sourcemeta::core::schema_resolver, + sourcemeta::blaze::default_schema_compiler, this->mode)}; + __ASSERT_TEMPLATE_JSON_SERIALISATION(compiled_schema); + + sourcemeta::blaze::Evaluator evaluator; + const auto result{evaluator.validate(compiled_schema, instance)}; + + if (expected_valid) { + EXPECT_TRUE(result); + } else { + EXPECT_FALSE(result); + } + } + +private: + const sourcemeta::core::JSON data; + const sourcemeta::blaze::Mode mode; +}; + static auto register_tests(const std::filesystem::path &path, const std::string &suite_name) -> void { std::fprintf(stderr, "-- Parsing: %s\n", path.string().c_str()); @@ -157,6 +206,23 @@ static auto register_tests(const std::filesystem::path &path, return new TraceTest(test_case, mode, mode_key); }); } + + for (const auto &[mode_suffix, mode] : + {std::pair{ + "_canonicalize_fast", sourcemeta::blaze::Mode::FastValidation}, + std::pair{ + "_canonicalize_exhaustive", + sourcemeta::blaze::Mode::Exhaustive}}) { + + const auto title{description + mode_suffix}; + + const auto canonicalize_suite_name{suite_name + "_canonicalize"}; + testing::RegisterTest(canonicalize_suite_name.c_str(), title.c_str(), + nullptr, nullptr, __FILE__, __LINE__, + [=]() -> CanonicalizeTraceTest * { + return new CanonicalizeTraceTest(test_case, mode); + }); + } } } From f5f04bdea192db15fa875e28f0d5f055246611b7 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Fri, 10 Apr 2026 13:20:15 -0400 Subject: [PATCH 2/4] Fix names Signed-off-by: Juan Cruz Viotti --- test/evaluator/tracesuite.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/evaluator/tracesuite.cc b/test/evaluator/tracesuite.cc index 8fb0ac4b8..8f942b308 100644 --- a/test/evaluator/tracesuite.cc +++ b/test/evaluator/tracesuite.cc @@ -209,10 +209,9 @@ static auto register_tests(const std::filesystem::path &path, for (const auto &[mode_suffix, mode] : {std::pair{ - "_canonicalize_fast", sourcemeta::blaze::Mode::FastValidation}, + "_fast", sourcemeta::blaze::Mode::FastValidation}, std::pair{ - "_canonicalize_exhaustive", - sourcemeta::blaze::Mode::Exhaustive}}) { + "_exhaustive", sourcemeta::blaze::Mode::Exhaustive}}) { const auto title{description + mode_suffix}; From 4c5901365eed3146e1f355aab99f118643e31175 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Fri, 10 Apr 2026 15:02:04 -0400 Subject: [PATCH 3/4] FIX Signed-off-by: Juan Cruz Viotti --- test/evaluator/tracesuite.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/evaluator/tracesuite.cc b/test/evaluator/tracesuite.cc index 8f942b308..ebcf19b58 100644 --- a/test/evaluator/tracesuite.cc +++ b/test/evaluator/tracesuite.cc @@ -139,7 +139,7 @@ class CanonicalizeTraceTest : public testing::Test { const auto &instance{this->data.at("instance")}; const bool expected_valid{this->data.at("valid").to_boolean()}; - sourcemeta::core::SchemaTransformer bundle; + sourcemeta::blaze::SchemaTransformer bundle; sourcemeta::blaze::add(bundle, sourcemeta::blaze::AlterSchemaMode::Canonicalizer); const auto canonicalize_result{bundle.apply( From 7f47d6f2d3ee7c7b0381b5756e09377f19ce9144 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Fri, 10 Apr 2026 16:12:59 -0400 Subject: [PATCH 4/4] Simpler Signed-off-by: Juan Cruz Viotti --- test/evaluator/CMakeLists.txt | 12 ++- test/evaluator/tracesuite.cc | 65 -------------- test/evaluator/tracesuite_canonical.cc | 118 +++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 66 deletions(-) create mode 100644 test/evaluator/tracesuite_canonical.cc diff --git a/test/evaluator/CMakeLists.txt b/test/evaluator/CMakeLists.txt index d1fd3951a..660700ddd 100644 --- a/test/evaluator/CMakeLists.txt +++ b/test/evaluator/CMakeLists.txt @@ -56,4 +56,14 @@ target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE source target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE sourcemeta::core::jsonschema) target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE sourcemeta::blaze::compiler) target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE sourcemeta::blaze::evaluator) -target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_unit PRIVATE sourcemeta::blaze::alterschema) + +# Evaluator trace suite with canonicalization (JSON-driven) +sourcemeta_googletest(NAMESPACE sourcemeta PROJECT blaze NAME evaluator_trace_suite_canonical + FOLDER "Blaze/Evaluator" SOURCES tracesuite_canonical.cc) +target_compile_definitions(sourcemeta_blaze_evaluator_trace_suite_canonical_unit + PRIVATE TRACE_SUITE_CANONICAL_PATH="${PROJECT_SOURCE_DIR}/test/evaluator") +target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_canonical_unit PRIVATE sourcemeta::core::json) +target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_canonical_unit PRIVATE sourcemeta::core::jsonschema) +target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_canonical_unit PRIVATE sourcemeta::blaze::compiler) +target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_canonical_unit PRIVATE sourcemeta::blaze::evaluator) +target_link_libraries(sourcemeta_blaze_evaluator_trace_suite_canonical_unit PRIVATE sourcemeta::blaze::alterschema) diff --git a/test/evaluator/tracesuite.cc b/test/evaluator/tracesuite.cc index ebcf19b58..425919519 100644 --- a/test/evaluator/tracesuite.cc +++ b/test/evaluator/tracesuite.cc @@ -1,6 +1,5 @@ #include -#include #include #include @@ -128,54 +127,6 @@ class TraceTest : public testing::Test { const char *mode_key; }; -class CanonicalizeTraceTest : public testing::Test { -public: - explicit CanonicalizeTraceTest(sourcemeta::core::JSON test_data, - const sourcemeta::blaze::Mode test_mode) - : data{std::move(test_data)}, mode{test_mode} {} - - auto TestBody() -> void override { - auto schema{this->data.at("schema")}; - const auto &instance{this->data.at("instance")}; - const bool expected_valid{this->data.at("valid").to_boolean()}; - - sourcemeta::blaze::SchemaTransformer bundle; - sourcemeta::blaze::add(bundle, - sourcemeta::blaze::AlterSchemaMode::Canonicalizer); - const auto canonicalize_result{bundle.apply( - schema, sourcemeta::core::schema_walker, - sourcemeta::core::schema_resolver, - [](const auto &pointer, const auto &name, const auto &message, - const auto &, const auto &) { - // TODO: Temporary for debugging purposes. Remove once everything - // passes - std::fprintf(stderr, " canonicalize: %s: %s: %s\n", - sourcemeta::core::to_string(pointer).c_str(), - std::string{name}.c_str(), std::string{message}.c_str()); - })}; - EXPECT_TRUE(canonicalize_result.first); - - const auto compiled_schema{sourcemeta::blaze::compile( - schema, sourcemeta::core::schema_walker, - sourcemeta::core::schema_resolver, - sourcemeta::blaze::default_schema_compiler, this->mode)}; - __ASSERT_TEMPLATE_JSON_SERIALISATION(compiled_schema); - - sourcemeta::blaze::Evaluator evaluator; - const auto result{evaluator.validate(compiled_schema, instance)}; - - if (expected_valid) { - EXPECT_TRUE(result); - } else { - EXPECT_FALSE(result); - } - } - -private: - const sourcemeta::core::JSON data; - const sourcemeta::blaze::Mode mode; -}; - static auto register_tests(const std::filesystem::path &path, const std::string &suite_name) -> void { std::fprintf(stderr, "-- Parsing: %s\n", path.string().c_str()); @@ -206,22 +157,6 @@ static auto register_tests(const std::filesystem::path &path, return new TraceTest(test_case, mode, mode_key); }); } - - for (const auto &[mode_suffix, mode] : - {std::pair{ - "_fast", sourcemeta::blaze::Mode::FastValidation}, - std::pair{ - "_exhaustive", sourcemeta::blaze::Mode::Exhaustive}}) { - - const auto title{description + mode_suffix}; - - const auto canonicalize_suite_name{suite_name + "_canonicalize"}; - testing::RegisterTest(canonicalize_suite_name.c_str(), title.c_str(), - nullptr, nullptr, __FILE__, __LINE__, - [=]() -> CanonicalizeTraceTest * { - return new CanonicalizeTraceTest(test_case, mode); - }); - } } } diff --git a/test/evaluator/tracesuite_canonical.cc b/test/evaluator/tracesuite_canonical.cc new file mode 100644 index 000000000..42527e659 --- /dev/null +++ b/test/evaluator/tracesuite_canonical.cc @@ -0,0 +1,118 @@ +#include + +#include +#include +#include + +#include +#include + +#include "evaluator_utils.h" + +#include // assert +#include // std::fprintf +#include // std::filesystem::path +#include // std::string +#include // std::move + +class CanonicalizeTest : public testing::Test { +public: + explicit CanonicalizeTest(sourcemeta::core::JSON test_data, + const sourcemeta::blaze::Mode test_mode) + : data{std::move(test_data)}, mode{test_mode} {} + + auto TestBody() -> void override { + auto schema{this->data.at("schema")}; + const auto &instance{this->data.at("instance")}; + const bool expected_valid{this->data.at("valid").to_boolean()}; + + sourcemeta::blaze::SchemaTransformer bundle; + sourcemeta::blaze::add(bundle, + sourcemeta::blaze::AlterSchemaMode::Canonicalizer); + const auto canonicalize_result{ + bundle.apply(schema, sourcemeta::core::schema_walker, + sourcemeta::core::schema_resolver, + [](const auto &, const auto &, const auto &, const auto &, + const auto &) {})}; + EXPECT_TRUE(canonicalize_result.first); + + const auto compiled_schema{sourcemeta::blaze::compile( + schema, sourcemeta::core::schema_walker, + sourcemeta::core::schema_resolver, + sourcemeta::blaze::default_schema_compiler, this->mode)}; + __ASSERT_TEMPLATE_JSON_SERIALISATION(compiled_schema); + + sourcemeta::blaze::Evaluator evaluator; + const auto result{evaluator.validate(compiled_schema, instance)}; + + if (expected_valid) { + EXPECT_TRUE(result); + } else { + EXPECT_FALSE(result); + } + } + +private: + const sourcemeta::core::JSON data; + const sourcemeta::blaze::Mode mode; +}; + +static auto register_tests(const std::filesystem::path &path, + const std::string &suite_name) -> void { + std::fprintf(stderr, "-- Parsing: %s\n", path.string().c_str()); + auto suite{sourcemeta::core::read_json(path)}; + assert(suite.is_array()); + + for (const auto &test_case : suite.as_array()) { + assert(test_case.is_object()); + assert(test_case.defines("description")); + const auto &description{test_case.at("description").to_string()}; + + for (const auto &[mode_suffix, mode] : + {std::pair{ + "_fast", sourcemeta::blaze::Mode::FastValidation}, + std::pair{ + "_exhaustive", sourcemeta::blaze::Mode::Exhaustive}}) { + + const auto title{description + mode_suffix}; + + testing::RegisterTest(suite_name.c_str(), title.c_str(), nullptr, nullptr, + __FILE__, __LINE__, [=]() -> CanonicalizeTest * { + return new CanonicalizeTest(test_case, mode); + }); + } + } +} + +auto main(int argc, char **argv) -> int { + testing::InitGoogleTest(&argc, argv); + + try { + register_tests(std::filesystem::path{TRACE_SUITE_CANONICAL_PATH} / + "evaluator_openapi_3_1.json", + "Canonicalize_OpenAPI_3_1"); + register_tests(std::filesystem::path{TRACE_SUITE_CANONICAL_PATH} / + "evaluator_openapi_3_2.json", + "Canonicalize_OpenAPI_3_2"); + register_tests(std::filesystem::path{TRACE_SUITE_CANONICAL_PATH} / + "evaluator_draft7.json", + "Canonicalize_draft7"); + register_tests(std::filesystem::path{TRACE_SUITE_CANONICAL_PATH} / + "evaluator_2019_09.json", + "Canonicalize_2019_09"); + register_tests(std::filesystem::path{TRACE_SUITE_CANONICAL_PATH} / + "evaluator_draft6.json", + "Canonicalize_draft6"); + register_tests(std::filesystem::path{TRACE_SUITE_CANONICAL_PATH} / + "evaluator_2020_12.json", + "Canonicalize_2020_12"); + register_tests(std::filesystem::path{TRACE_SUITE_CANONICAL_PATH} / + "evaluator_draft4.json", + "Canonicalize_draft4"); + } catch (const std::exception &error) { + std::fprintf(stderr, "Error: %s\n", error.what()); + return EXIT_FAILURE; + } + + return RUN_ALL_TESTS(); +}