| // Protocol Buffers - Google's data interchange format |
| // Copyright 2008 Google Inc. All rights reserved. |
| // |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file or at |
| // https://developers.google.com/open-source/licenses/bsd |
| |
| #include "conformance_test.h" |
| |
| #include <stdarg.h> |
| |
| #include <algorithm> |
| #include <cstddef> |
| #include <cstdint> |
| #include <cstdio> |
| #include <fstream> |
| #include <memory> |
| #include <string> |
| #include <utility> |
| |
| #include "google/protobuf/util/field_comparator.h" |
| #include "google/protobuf/util/message_differencer.h" |
| #include "absl/container/btree_map.h" |
| #include "absl/container/flat_hash_set.h" |
| #include "absl/log/absl_check.h" |
| #include "absl/log/absl_log.h" |
| #include "absl/status/status.h" |
| #include "absl/strings/str_cat.h" |
| #include "absl/strings/str_format.h" |
| #include "absl/strings/string_view.h" |
| #include "conformance/conformance.pb.h" |
| #include "failure_list_trie_node.h" |
| #include "google/protobuf/descriptor_legacy.h" |
| #include "google/protobuf/endian.h" |
| #include "google/protobuf/message.h" |
| #include "google/protobuf/text_format.h" |
| |
| using conformance::ConformanceRequest; |
| using conformance::ConformanceResponse; |
| using conformance::TestStatus; |
| using conformance::WireFormat; |
| using google::protobuf::util::DefaultFieldComparator; |
| using google::protobuf::util::MessageDifferencer; |
| using std::string; |
| |
| namespace { |
| |
| static void ReplaceAll(std::string& input, std::string replace_word, |
| std::string replace_by) { |
| size_t pos = input.find(replace_word); |
| while (pos != std::string::npos) { |
| input.replace(pos, replace_word.length(), replace_by); |
| pos = input.find(replace_word, pos + replace_by.length()); |
| } |
| } |
| |
| static std::string ToOctString(const std::string& binary_string) { |
| std::string oct_string; |
| for (size_t i = 0; i < binary_string.size(); i++) { |
| uint8_t c = binary_string.at(i); |
| uint8_t high = c / 64; |
| uint8_t mid = (c % 64) / 8; |
| uint8_t low = c % 8; |
| oct_string.push_back('\\'); |
| oct_string.push_back('0' + high); |
| oct_string.push_back('0' + mid); |
| oct_string.push_back('0' + low); |
| } |
| return oct_string; |
| } |
| |
| // Returns full filename path of written .txt file if successful |
| static std::string ProduceOctalSerialized(const std::string& request, |
| uint32_t len) { |
| char* len_split_bytes = static_cast<char*>(static_cast<void*>(&len)); |
| |
| std::string out; |
| |
| std::string hex_repr; |
| for (int i = 0; i < 4; i++) { |
| auto conversion = (unsigned int)static_cast<uint8_t>(len_split_bytes[i]); |
| std::string hex = absl::StrFormat("\\x%x", conversion); |
| absl::StrAppend(&hex_repr, hex); |
| } |
| |
| absl::StrAppend(&out, hex_repr); |
| |
| absl::StrAppend(&out, ToOctString(request)); |
| |
| return out; |
| } |
| |
| static std::string WriteToFile(const std::string& octal_serialized, |
| const std::string& output_dir, |
| const std::string& test_name) { |
| std::string test_name_txt = test_name; |
| ReplaceAll(test_name_txt, ".", "_"); |
| absl::StrAppend(&test_name_txt, ".txt"); |
| std::string full_filename; |
| if (!output_dir.empty()) { |
| full_filename = output_dir; |
| if (*output_dir.rbegin() != '/') { |
| full_filename.push_back('/'); |
| } |
| absl::StrAppend(&full_filename, test_name_txt); |
| } |
| std::ofstream os{std::string(full_filename)}; |
| if (os) { |
| os << octal_serialized; |
| return full_filename; |
| } else { |
| ABSL_LOG(INFO) << "Failed to open file for debugging: " << full_filename |
| << "\n"; |
| return ""; |
| } |
| } |
| |
| // Removes all newlines. |
| static void Normalize(std::string& input) { |
| input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); |
| } |
| |
| // Sets up a failure message properly for our failure lists. |
| static TestStatus FormatFailureMessage(const TestStatus& input) { |
| // Make copy just this once, as we need to modify it for our failure lists. |
| std::string formatted_failure_message = input.failure_message(); |
| // Remove newlines |
| Normalize(formatted_failure_message); |
| // Truncate failure message if needed |
| if (formatted_failure_message.length() > 128) { |
| formatted_failure_message = formatted_failure_message.substr(0, 128); |
| } |
| TestStatus properly_formatted; |
| properly_formatted.set_name(input.name()); |
| properly_formatted.set_failure_message(formatted_failure_message); |
| return properly_formatted; |
| } |
| |
| bool CheckSetEmpty(const absl::btree_map<std::string, TestStatus>& set_to_check, |
| absl::string_view write_to_file, absl::string_view msg, |
| absl::string_view output_dir, std::string* output) { |
| if (set_to_check.empty()) return true; |
| |
| absl::StrAppendFormat(output, "\n"); |
| absl::StrAppendFormat(output, "%s\n\n", msg); |
| for (const auto& pair : set_to_check) { |
| absl::StrAppendFormat(output, " %s # %s\n", pair.first, |
| pair.second.failure_message()); |
| } |
| absl::StrAppendFormat(output, "\n"); |
| |
| if (!write_to_file.empty()) { |
| std::string full_filename; |
| absl::string_view filename = write_to_file; |
| if (!output_dir.empty()) { |
| full_filename = std::string(output_dir); |
| absl::StrAppend(&full_filename, write_to_file); |
| filename = full_filename; |
| } |
| std::ofstream os{std::string(filename)}; |
| if (os) { |
| for (const auto& pair : set_to_check) { |
| // Additions will not have a 'matched_name' while removals will. |
| string potential_add_or_removal = pair.second.matched_name().empty() |
| ? pair.first |
| : pair.second.matched_name(); |
| os << potential_add_or_removal << " # " << pair.second.failure_message() |
| << "\n"; |
| } |
| } else { |
| absl::StrAppendFormat(output, |
| "Failed to open file: %s\n", |
| filename); |
| } |
| } |
| |
| return false; |
| } |
| |
| } // namespace |
| |
| namespace google { |
| namespace protobuf { |
| |
| constexpr int kMaximumWildcardExpansions = 5; |
| |
| ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting( |
| ConformanceLevel level, conformance::WireFormat input_format, |
| conformance::WireFormat output_format, |
| conformance::TestCategory test_category, const Message& prototype_message, |
| const std::string& test_name, const std::string& input) |
| : level_(level), |
| input_format_(input_format), |
| output_format_(output_format), |
| prototype_message_(prototype_message), |
| prototype_message_for_compare_(prototype_message.New()), |
| test_name_(test_name) { |
| switch (input_format) { |
| case conformance::PROTOBUF: { |
| request_.set_protobuf_payload(input); |
| break; |
| } |
| |
| case conformance::JSON: { |
| request_.set_json_payload(input); |
| break; |
| } |
| |
| case conformance::JSPB: { |
| request_.set_jspb_payload(input); |
| break; |
| } |
| |
| case conformance::TEXT_FORMAT: { |
| request_.set_text_payload(input); |
| break; |
| } |
| |
| default: |
| ABSL_LOG(FATAL) << "Unspecified input format"; |
| } |
| |
| request_.set_test_category(test_category); |
| |
| request_.set_message_type(prototype_message.GetDescriptor()->full_name()); |
| request_.set_requested_output_format(output_format); |
| } |
| |
| std::unique_ptr<Message> |
| ConformanceTestSuite::ConformanceRequestSetting::NewTestMessage() const { |
| return std::unique_ptr<Message>(prototype_message_for_compare_->New()); |
| } |
| |
| std::string |
| ConformanceTestSuite::ConformanceRequestSetting::GetSyntaxIdentifier() const { |
| switch (FileDescriptorLegacy(prototype_message_.GetDescriptor()->file()) |
| .edition()) { |
| case Edition::EDITION_PROTO3: |
| return "Proto3"; |
| case Edition::EDITION_PROTO2: |
| return "Proto2"; |
| default: { |
| std::string id = "Editions"; |
| if (prototype_message_.GetDescriptor()->name() == "TestAllTypesProto2") { |
| absl::StrAppend(&id, "_Proto2"); |
| } else if (prototype_message_.GetDescriptor()->name() == |
| "TestAllTypesProto3") { |
| absl::StrAppend(&id, "_Proto3"); |
| } |
| return id; |
| } |
| } |
| } |
| |
| string ConformanceTestSuite::ConformanceRequestSetting::GetTestName() const { |
| return absl::StrCat(ConformanceLevelToString(level_), ".", |
| GetSyntaxIdentifier(), ".", |
| InputFormatString(input_format_), ".", test_name_, ".", |
| OutputFormatString(output_format_)); |
| } |
| |
| std::string |
| ConformanceTestSuite::ConformanceRequestSetting::ConformanceLevelToString( |
| ConformanceLevel level) const { |
| switch (level) { |
| case REQUIRED: |
| return "Required"; |
| case RECOMMENDED: |
| return "Recommended"; |
| } |
| ABSL_LOG(FATAL) << "Unknown value: " << level; |
| return ""; |
| } |
| |
| std::string ConformanceTestSuite::ConformanceRequestSetting::InputFormatString( |
| conformance::WireFormat format) const { |
| switch (format) { |
| case conformance::PROTOBUF: |
| return "ProtobufInput"; |
| case conformance::JSON: |
| return "JsonInput"; |
| case conformance::TEXT_FORMAT: |
| return "TextFormatInput"; |
| default: |
| ABSL_LOG(FATAL) << "Unspecified output format"; |
| } |
| return ""; |
| } |
| |
| std::string ConformanceTestSuite::ConformanceRequestSetting::OutputFormatString( |
| conformance::WireFormat format) const { |
| switch (format) { |
| case conformance::PROTOBUF: |
| return "ProtobufOutput"; |
| case conformance::JSON: |
| return "JsonOutput"; |
| case conformance::TEXT_FORMAT: |
| return "TextFormatOutput"; |
| default: |
| ABSL_LOG(FATAL) << "Unspecified output format"; |
| } |
| return ""; |
| } |
| |
| void ConformanceTestSuite::TruncateDebugPayload(string* payload) { |
| if (payload != nullptr && payload->size() > 200) { |
| payload->resize(200); |
| payload->append("...(truncated)"); |
| } |
| } |
| |
| ConformanceRequest ConformanceTestSuite::TruncateRequest( |
| const ConformanceRequest& request) { |
| ConformanceRequest debug_request(request); |
| switch (debug_request.payload_case()) { |
| case ConformanceRequest::kProtobufPayload: |
| TruncateDebugPayload(debug_request.mutable_protobuf_payload()); |
| break; |
| case ConformanceRequest::kJsonPayload: |
| TruncateDebugPayload(debug_request.mutable_json_payload()); |
| break; |
| case ConformanceRequest::kTextPayload: |
| TruncateDebugPayload(debug_request.mutable_text_payload()); |
| break; |
| case ConformanceRequest::kJspbPayload: |
| TruncateDebugPayload(debug_request.mutable_jspb_payload()); |
| break; |
| default: |
| // Do nothing. |
| break; |
| } |
| return debug_request; |
| } |
| |
| ConformanceResponse ConformanceTestSuite::TruncateResponse( |
| const ConformanceResponse& response) { |
| ConformanceResponse debug_response(response); |
| switch (debug_response.result_case()) { |
| case ConformanceResponse::kProtobufPayload: |
| TruncateDebugPayload(debug_response.mutable_protobuf_payload()); |
| break; |
| case ConformanceResponse::kJsonPayload: |
| TruncateDebugPayload(debug_response.mutable_json_payload()); |
| break; |
| case ConformanceResponse::kTextPayload: |
| TruncateDebugPayload(debug_response.mutable_text_payload()); |
| break; |
| case ConformanceResponse::kJspbPayload: |
| TruncateDebugPayload(debug_response.mutable_jspb_payload()); |
| break; |
| default: |
| // Do nothing. |
| break; |
| } |
| return debug_response; |
| } |
| |
| void ConformanceTestSuite::ReportSuccess(const TestStatus& test) { |
| if (expected_to_fail_.contains(test.name())) { |
| absl::StrAppendFormat(&output_, |
| "ERROR: test %s (matched to %s) is in the failure " |
| "list, but test succeeded. " |
| "Remove its match from the failure list.\n", |
| test.name(), |
| expected_to_fail_[test.name()].matched_name()); |
| unexpected_succeeding_tests_[test.name()] = expected_to_fail_[test.name()]; |
| } |
| expected_to_fail_.erase(test.name()); |
| successes_++; |
| } |
| |
| void ConformanceTestSuite::ReportFailure(TestStatus& test, |
| ConformanceLevel level, |
| const ConformanceRequest& request, |
| const ConformanceResponse& response) { |
| if (expected_to_fail_.contains(test.name())) { |
| // Make copy just this once, as we need to modify them for comparison. |
| // Failure message from the failure list. |
| string expected_failure_message = |
| expected_to_fail_[test.name()].failure_message(); |
| // Actual failure message from the test run. |
| std::string actual_failure_message = test.failure_message(); |
| |
| Normalize(actual_failure_message); |
| if (actual_failure_message.rfind(expected_failure_message, 0) == 0) { |
| // Our failure messages match. |
| expected_failures_++; |
| } else { |
| // We want to add the test to the failure list with its correct failure |
| // message. |
| unexpected_failure_messages_[test.name()] = FormatFailureMessage(test); |
| // We want to remove the test from the failure list. That means passing |
| // to it the same failure message that was in the list. |
| TestStatus incorrect_failure_message; |
| incorrect_failure_message.set_name(test.name()); |
| incorrect_failure_message.set_failure_message(expected_failure_message); |
| incorrect_failure_message.set_matched_name( |
| expected_to_fail_[test.name()].matched_name()); |
| |
| expected_failure_messages_[test.name()] = incorrect_failure_message; |
| } |
| expected_to_fail_.erase(test.name()); |
| if (!verbose_) return; |
| } else if (level == RECOMMENDED && !enforce_recommended_) { |
| absl::StrAppendFormat(&output_, "WARNING, test=%s: ", test.name()); |
| } else { |
| absl::StrAppendFormat(&output_, "ERROR, test=%s: ", test.name()); |
| |
| unexpected_failing_tests_[test.name()] = FormatFailureMessage(test); |
| } |
| |
| absl::StrAppendFormat(&output_, "%s, request=%s, response=%s\n", |
| test.failure_message(), |
| TruncateRequest(request).ShortDebugString(), |
| TruncateResponse(response).ShortDebugString()); |
| } |
| |
| void ConformanceTestSuite::ReportSkip(const TestStatus& test, |
| const ConformanceRequest& request, |
| const ConformanceResponse& response) { |
| if (verbose_) { |
| absl::StrAppendFormat( |
| &output_, "SKIPPED, test=%s request=%s, response=%s\n", test.name(), |
| request.ShortDebugString(), response.ShortDebugString()); |
| } |
| skipped_[test.name()] = test; |
| } |
| |
| void ConformanceTestSuite::RunValidInputTest( |
| const ConformanceRequestSetting& setting, |
| const std::string& equivalent_text_format) { |
| std::unique_ptr<Message> reference_message(setting.NewTestMessage()); |
| ABSL_CHECK(TextFormat::ParseFromString(equivalent_text_format, |
| reference_message.get())) |
| << "Failed to parse data for test case: " << setting.GetTestName() |
| << ", data: " << equivalent_text_format; |
| const std::string equivalent_wire_format = |
| reference_message->SerializeAsString(); |
| RunValidBinaryInputTest(setting, equivalent_wire_format); |
| } |
| |
| void ConformanceTestSuite::RunValidBinaryInputTest( |
| const ConformanceRequestSetting& setting, |
| const std::string& equivalent_wire_format, bool require_same_wire_format) { |
| const ConformanceRequest& request = setting.GetRequest(); |
| ConformanceResponse response; |
| if (!RunTest(setting.GetTestName(), request, &response)) { |
| return; |
| } |
| |
| VerifyResponse(setting, equivalent_wire_format, response, true, |
| require_same_wire_format); |
| } |
| |
| void ConformanceTestSuite::VerifyResponse( |
| const ConformanceRequestSetting& setting, |
| const std::string& equivalent_wire_format, |
| const ConformanceResponse& response, bool need_report_success, |
| bool require_same_wire_format) { |
| std::unique_ptr<Message> test_message(setting.NewTestMessage()); |
| const ConformanceRequest& request = setting.GetRequest(); |
| const std::string& test_name = setting.GetTestName(); |
| ConformanceLevel level = setting.GetLevel(); |
| std::unique_ptr<Message> reference_message = setting.NewTestMessage(); |
| |
| ABSL_CHECK(reference_message->ParseFromString(equivalent_wire_format)) |
| << "Failed to parse wire data for test case: " << test_name; |
| |
| TestStatus test; |
| test.set_name(test_name); |
| |
| switch (response.result_case()) { |
| case ConformanceResponse::RESULT_NOT_SET: |
| test.set_failure_message( |
| "Response didn't have any field in the Response."); |
| ReportFailure(test, level, request, response); |
| return; |
| |
| case ConformanceResponse::kParseError: |
| case ConformanceResponse::kTimeoutError: |
| case ConformanceResponse::kRuntimeError: |
| case ConformanceResponse::kSerializeError: |
| test.set_failure_message("Failed to parse input or produce output."); |
| ReportFailure(test, level, request, response); |
| return; |
| |
| case ConformanceResponse::kSkipped: |
| ReportSkip(test, request, response); |
| return; |
| |
| default: |
| if (!ParseResponse(response, setting, test_message.get())) return; |
| } |
| |
| MessageDifferencer differencer; |
| DefaultFieldComparator field_comparator; |
| field_comparator.set_treat_nan_as_equal(true); |
| differencer.set_field_comparator(&field_comparator); |
| std::string differences; |
| differencer.ReportDifferencesToString(&differences); |
| |
| bool check = false; |
| |
| if (require_same_wire_format) { |
| ABSL_DCHECK_EQ(response.result_case(), |
| ConformanceResponse::kProtobufPayload); |
| const std::string& protobuf_payload = response.protobuf_payload(); |
| check = equivalent_wire_format == protobuf_payload; |
| differences = absl::StrCat("Expect: ", ToOctString(equivalent_wire_format), |
| ", but got: ", ToOctString(protobuf_payload)); |
| } else { |
| check = differencer.Compare(*reference_message, *test_message); |
| } |
| if (check) { |
| if (need_report_success) { |
| ReportSuccess(test); |
| } |
| } else { |
| test.set_failure_message(absl::StrCat( |
| "Output was not equivalent to reference message: ", differences)); |
| ReportFailure(test, level, request, response); |
| } |
| } |
| |
| bool ConformanceTestSuite::RunTest(const std::string& test_name, |
| const ConformanceRequest& request, |
| ConformanceResponse* response) { |
| if (test_names_ran_.insert(test_name).second == false) { |
| ABSL_LOG(FATAL) << "Duplicated test name: " << test_name; |
| } |
| |
| // In essence, find what wildcarded test names expand to or direct matches |
| // (without wildcards). |
| if (auto result = failure_list_root_.WalkDownMatch(test_name); |
| result.has_value()) { |
| string matched_equivalent = result.value(); |
| unmatched_.erase(matched_equivalent); |
| TestStatus expansion; |
| expansion.set_name(test_name); |
| expansion.set_matched_name(matched_equivalent); |
| expansion.set_failure_message(saved_failure_messages_[matched_equivalent]); |
| expected_to_fail_[test_name] = expansion; |
| |
| if (number_of_matches_.contains(matched_equivalent)) { |
| if (number_of_matches_[matched_equivalent] > kMaximumWildcardExpansions && |
| !exceeded_max_matches_.contains(matched_equivalent)) { |
| exceeded_max_matches_[matched_equivalent] = expansion; |
| } |
| number_of_matches_[matched_equivalent]++; |
| } else { |
| number_of_matches_[matched_equivalent] = 1; |
| } |
| } |
| |
| std::string serialized_request; |
| std::string serialized_response; |
| request.SerializeToString(&serialized_request); |
| |
| uint32_t len = internal::little_endian::FromHost( |
| static_cast<uint32_t>(serialized_request.size())); |
| |
| if (isolated_) { |
| if (names_to_test_.erase(test_name) == |
| 0) { // Tests were asked to be run in isolated mode, but this test was |
| // not asked to be run. |
| expected_to_fail_.erase(test_name); |
| return false; |
| } |
| if (debug_) { |
| std::string octal = ProduceOctalSerialized(serialized_request, len); |
| std::string full_filename = WriteToFile(octal, output_dir_, test_name); |
| if (!full_filename.empty()) { |
| absl::StrAppendFormat( |
| &output_, "Produced octal serialized request file for test %s\n", |
| test_name); |
| absl::StrAppendFormat( |
| &output_, |
| " To pipe the " |
| "serialized request directly to " |
| "the " |
| "testee run from the root of your workspace:\n printf $(" |
| "<\"%s\") | %s\n\n", |
| full_filename, testee_); |
| absl::StrAppendFormat( |
| &output_, |
| " To inspect the wire format of the serialized request with " |
| "protoscope run " |
| "(Disclaimer: This may not work properly on non-Linux " |
| "platforms):\n " |
| " " |
| "contents=$(<\"%s\"); sub=$(cut -d \\\\ -f 6- <<< " |
| "$contents) ; printf \"\\\\${sub}\" | protoscope \n\n\n", |
| full_filename); |
| } |
| } |
| } |
| |
| response->set_protobuf_payload(serialized_request); |
| |
| runner_->RunTest(test_name, len, serialized_request, &serialized_response); |
| |
| if (!response->ParseFromString(serialized_response)) { |
| response->Clear(); |
| response->set_runtime_error("response proto could not be parsed."); |
| } |
| |
| if (verbose_) { |
| absl::StrAppendFormat( |
| &output_, "conformance test: name=%s, request=%s, response=%s\n", |
| test_name, TruncateRequest(request).ShortDebugString(), |
| TruncateResponse(*response).ShortDebugString()); |
| } |
| return true; |
| } |
| |
| std::string ConformanceTestSuite::WireFormatToString(WireFormat wire_format) { |
| switch (wire_format) { |
| case conformance::PROTOBUF: |
| return "PROTOBUF"; |
| case conformance::JSON: |
| return "JSON"; |
| case conformance::JSPB: |
| return "JSPB"; |
| case conformance::TEXT_FORMAT: |
| return "TEXT_FORMAT"; |
| case conformance::UNSPECIFIED: |
| return "UNSPECIFIED"; |
| default: |
| ABSL_LOG(FATAL) << "unknown wire type: " << wire_format; |
| } |
| return ""; |
| } |
| |
| bool ConformanceTestSuite::AddExpectedFailedTest( |
| const TestStatus& expected_failure) { |
| absl::Status attempt = failure_list_root_.Insert(expected_failure.name()); |
| if (!attempt.ok()) { |
| absl::StrAppend(&output_, attempt.message(), "\n\n"); |
| return false; |
| } |
| unmatched_[expected_failure.name()] = expected_failure; |
| saved_failure_messages_[expected_failure.name()] = |
| expected_failure.failure_message(); |
| return true; |
| } |
| |
| bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, |
| std::string* output, |
| const std::string& filename, |
| conformance::FailureSet* failure_list) { |
| runner_ = runner; |
| failure_list_root_ = FailureListTrieNode("root"); |
| successes_ = 0; |
| expected_failures_ = 0; |
| skipped_.clear(); |
| test_names_ran_.clear(); |
| unexpected_failing_tests_.clear(); |
| unexpected_succeeding_tests_.clear(); |
| |
| std::string mode = debug_ ? "DEBUG" : "TEST"; |
| absl::StrAppendFormat( |
| &output_, "CONFORMANCE %s BEGIN ====================================\n\n", |
| mode); |
| |
| failure_list_filename_ = filename; |
| expected_to_fail_.clear(); |
| for (const TestStatus& expected_failure : failure_list->test()) { |
| if (!AddExpectedFailedTest(expected_failure)) { |
| output->assign(output_); |
| return false; |
| } |
| } |
| |
| RunSuiteImpl(); |
| |
| if (*output_dir_.rbegin() != '/') { |
| output_dir_.push_back('/'); |
| } |
| |
| bool ok = true; |
| if (!CheckSetEmpty( |
| unmatched_, "unmatched.txt", |
| absl::StrCat( |
| "These test names were listed in the failure list, but they " |
| "didn't match any actual test name. Remove them from the " |
| "failure list by running from the root of your workspace:\n" |
| " bazel run " |
| "//google/protobuf/conformance:update_failure_list -- ", |
| failure_list_filename_, " --remove ", output_dir_, |
| "unmatched.txt"), |
| output_dir_, &output_)) { |
| ok = false; |
| } |
| |
| if (!CheckSetEmpty( |
| expected_failure_messages_, "expected_failure_messages.txt", |
| absl::StrCat( |
| "These tests (either expanded from wildcard(s) or direct " |
| "matches) were listed in the failure list, but their " |
| "failure messages do not match. Remove their match from the " |
| "failure list by running from the root of your workspace:\n" |
| " bazel run ", |
| "//google/protobuf/conformance:update_failure_list -- ", |
| failure_list_filename_, " --remove ", output_dir_, |
| "expected_failure_messages.txt"), |
| output_dir_, &output_)) { |
| ok = false; |
| } |
| |
| if (!CheckSetEmpty( |
| unexpected_succeeding_tests_, "succeeding_tests.txt", |
| absl::StrCat( |
| "These tests succeeded, even though they were listed in " |
| "the failure list (expanded from wildcard(s) or direct matches). " |
| " Remove their match from the failure list by " |
| "running from the root of your workspace:\n" |
| " bazel run " |
| "//google/protobuf/conformance:update_failure_list -- ", |
| failure_list_filename_, " --remove ", output_dir_, |
| "succeeding_tests.txt"), |
| output_dir_, &output_)) { |
| ok = false; |
| } |
| |
| if (!CheckSetEmpty( |
| exceeded_max_matches_, "exceeded_max_matches.txt", |
| absl::StrFormat( |
| "These failure list entries served as matches to too many test " |
| "names exceeding the max amount of %d. " |
| "Remove them from the failure list by running from the root of " |
| "your workspace:\n" |
| " bazel run " |
| "//google/protobuf/conformance:update_failure_list -- %s " |
| "--remove %sexceeded_max_matches.txt", |
| kMaximumWildcardExpansions, failure_list_filename_, output_dir_), |
| output_dir_, &output_)) { |
| ok = false; |
| } |
| |
| if (!CheckSetEmpty( |
| unexpected_failure_messages_, "unexpected_failure_messages.txt", |
| absl::StrCat( |
| "These tests (expanded from wildcard(s) or direct matches from " |
| "the failure list) failed because their failure messages did " |
| "not match. If they can't be fixed right now, " |
| "you can add them to the failure list so the overall " |
| "suite can succeed. Add them to the failure list by " |
| "running from the root of your workspace:\n" |
| " bazel run " |
| "//google/protobuf/conformance:update_failure_list -- ", |
| failure_list_filename_, " --add ", output_dir_, |
| "unexpected_failure_messages.txt"), |
| output_dir_, &output_)) { |
| ok = false; |
| } |
| |
| if (!CheckSetEmpty( |
| unexpected_failing_tests_, "failing_tests.txt", |
| absl::StrCat( |
| "These tests failed. If they can't be fixed right now, " |
| "you can add them to the failure list so the overall " |
| "suite can succeed. Add them to the failure list by " |
| "running from the root of your workspace:\n" |
| " bazel run " |
| "//google/protobuf/conformance:update_failure_list -- ", |
| failure_list_filename_, " --add ", output_dir_, |
| "failing_tests.txt"), |
| output_dir_, &output_)) { |
| ok = false; |
| } |
| |
| if (verbose_) { |
| CheckSetEmpty(skipped_, "", |
| "These tests were skipped (probably because support for some " |
| "features is not implemented)", |
| output_dir_, &output_); |
| } |
| |
| absl::StrAppendFormat(&output_, |
| "CONFORMANCE SUITE %s: %d successes, %zu skipped, " |
| "%d expected failures, %zu unexpected failures.\n", |
| ok ? "PASSED" : "FAILED", successes_, skipped_.size(), |
| expected_failures_, unexpected_failing_tests_.size()); |
| absl::StrAppendFormat(&output_, "\n"); |
| |
| output->assign(output_); |
| |
| return ok; |
| } |
| |
| } // namespace protobuf |
| } // namespace google |
