conformance/conformance_test.cc - third_party/protobuf - Git at Google

 // Protocol Buffers - Google's data interchange format
 // Copyright 2008 Google Inc.  All rights reserved.
 // https://developers.google.com/protocol-buffers/
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 // copyright notice, this list of conditions and the following disclaimer
 // in the documentation and/or other materials provided with the
 // distribution.
 //     * Neither the name of Google Inc. nor the names of its
 // contributors may be used to endorse or promote products derived from
 // this software without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 #include "conformance_test.h"

 #include <stdarg.h>

 #include <fstream>
 #include <set>
 #include <string>

 #include <google/protobuf/stubs/stringprintf.h>
 #include <google/protobuf/message.h>
 #include <google/protobuf/text_format.h>
 #include <google/protobuf/util/field_comparator.h>
 #include <google/protobuf/util/json_util.h>
 #include <google/protobuf/util/message_differencer.h>
 #include "conformance.pb.h"

 using conformance::ConformanceRequest;
 using conformance::ConformanceResponse;
 using conformance::WireFormat;
 using google::protobuf::TextFormat;
 using google::protobuf::util::DefaultFieldComparator;
 using google::protobuf::util::MessageDifferencer;
 using std::string;

 namespace {

 static string ToOctString(const string& binary_string) {
   string oct_string;
   for (size_t i = 0; i < binary_string.size(); i++) {
     uint8_t c = binary_string.at(i);
     uint8_t high = c / 64;
     uint8_t mid = (c % 64) / 8;
     uint8_t low = c % 8;
     oct_string.push_back('\\');
     oct_string.push_back('0' + high);
     oct_string.push_back('0' + mid);
     oct_string.push_back('0' + low);
   }
   return oct_string;
 }

 }  // namespace

 namespace google {
 namespace protobuf {

 ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
     ConformanceLevel level,
     conformance::WireFormat input_format,
     conformance::WireFormat output_format,
     conformance::TestCategory test_category,
     const Message& prototype_message,
     const string& test_name, const string& input)
     : level_(level),
       input_format_(input_format),
       output_format_(output_format),
       prototype_message_(prototype_message),
       prototype_message_for_compare_(prototype_message.New()),
       test_name_(test_name) {
   switch (input_format) {
     case conformance::PROTOBUF: {
       request_.set_protobuf_payload(input);
       break;
     }

     case conformance::JSON: {
       request_.set_json_payload(input);
       break;
     }

     case conformance::JSPB: {
       request_.set_jspb_payload(input);
       break;
     }

     case conformance::TEXT_FORMAT: {
       request_.set_text_payload(input);
       break;
     }

     default:
       GOOGLE_LOG(FATAL) << "Unspecified input format";
   }

   request_.set_test_category(test_category);

   request_.set_message_type(prototype_message.GetDescriptor()->full_name());
   request_.set_requested_output_format(output_format);
 }

 std::unique_ptr<Message>
 ConformanceTestSuite::ConformanceRequestSetting::NewTestMessage() const {
   return std::unique_ptr<Message>(prototype_message_for_compare_->New());
 }

 string ConformanceTestSuite::ConformanceRequestSetting::
     GetTestName() const {
   string rname =
       prototype_message_.GetDescriptor()->file()->syntax() ==
         FileDescriptor::SYNTAX_PROTO3 ? "Proto3" : "Proto2";

   return StrCat(ConformanceLevelToString(level_), ".", rname, ".",
                       InputFormatString(input_format_), ".", test_name_, ".",
                       OutputFormatString(output_format_));
 }

 string ConformanceTestSuite::ConformanceRequestSetting::
     ConformanceLevelToString(
         ConformanceLevel level) const {
   switch (level) {
     case REQUIRED: return "Required";
     case RECOMMENDED: return "Recommended";
   }
   GOOGLE_LOG(FATAL) << "Unknown value: " << level;
   return "";
 }

 string ConformanceTestSuite::ConformanceRequestSetting::
     InputFormatString(conformance::WireFormat format) const {
   switch (format) {
     case conformance::PROTOBUF:
       return "ProtobufInput";
     case conformance::JSON:
       return "JsonInput";
     case conformance::TEXT_FORMAT:
       return "TextFormatInput";
     default:
       GOOGLE_LOG(FATAL) << "Unspecified output format";
   }
   return "";
 }

 string ConformanceTestSuite::ConformanceRequestSetting::
     OutputFormatString(conformance::WireFormat format) const {
   switch (format) {
     case conformance::PROTOBUF:
       return "ProtobufOutput";
     case conformance::JSON:
       return "JsonOutput";
     case conformance::TEXT_FORMAT:
       return "TextFormatOutput";
     default:
       GOOGLE_LOG(FATAL) << "Unspecified output format";
   }
   return "";
 }

 void ConformanceTestSuite::ReportSuccess(const string& test_name) {
   if (expected_to_fail_.erase(test_name) != 0) {
     StringAppendF(&output_,
                   "ERROR: test %s is in the failure list, but test succeeded.  "
                   "Remove it from the failure list.\n",
                   test_name.c_str());
     unexpected_succeeding_tests_.insert(test_name);
   }
   successes_++;
 }

 void ConformanceTestSuite::ReportFailure(const string& test_name,
                                          ConformanceLevel level,
                                          const ConformanceRequest& request,
                                          const ConformanceResponse& response,
                                          const char* fmt, ...) {
   if (expected_to_fail_.erase(test_name) == 1) {
     expected_failures_++;
     if (!verbose_)
       return;
   } else if (level == RECOMMENDED && !enforce_recommended_) {
     StringAppendF(&output_, "WARNING, test=%s: ", test_name.c_str());
   } else {
     StringAppendF(&output_, "ERROR, test=%s: ", test_name.c_str());
     unexpected_failing_tests_.insert(test_name);
   }
   va_list args;
   va_start(args, fmt);
   StringAppendV(&output_, fmt, args);
   va_end(args);
   StringAppendF(&output_, " request=%s, response=%s\n",
                 request.ShortDebugString().c_str(),
                 response.ShortDebugString().c_str());
 }

 void ConformanceTestSuite::ReportSkip(const string& test_name,
                                       const ConformanceRequest& request,
                                       const ConformanceResponse& response) {
   if (verbose_) {
     StringAppendF(&output_, "SKIPPED, test=%s request=%s, response=%s\n",
                   test_name.c_str(), request.ShortDebugString().c_str(),
                   response.ShortDebugString().c_str());
   }
   skipped_.insert(test_name);
 }

 void ConformanceTestSuite::RunValidInputTest(
     const ConformanceRequestSetting& setting,
     const string& equivalent_text_format) {
   std::unique_ptr<Message> reference_message(setting.NewTestMessage());
   GOOGLE_CHECK(TextFormat::ParseFromString(equivalent_text_format,
                                     reference_message.get()))
       << "Failed to parse data for test case: " << setting.GetTestName()
       << ", data: " << equivalent_text_format;
   const string equivalent_wire_format = reference_message->SerializeAsString();
   RunValidBinaryInputTest(setting, equivalent_wire_format);
 }

 void ConformanceTestSuite::RunValidBinaryInputTest(
     const ConformanceRequestSetting& setting,
     const string& equivalent_wire_format, bool require_same_wire_format) {
   const ConformanceRequest& request = setting.GetRequest();
   ConformanceResponse response;
   RunTest(setting.GetTestName(), request, &response);
   VerifyResponse(setting, equivalent_wire_format, response, true,
                  require_same_wire_format);
 }

 void ConformanceTestSuite::VerifyResponse(
     const ConformanceRequestSetting& setting,
     const string& equivalent_wire_format, const ConformanceResponse& response,
     bool need_report_success, bool require_same_wire_format) {
   std::unique_ptr<Message> test_message(setting.NewTestMessage());
   const ConformanceRequest& request = setting.GetRequest();
   const string& test_name = setting.GetTestName();
   ConformanceLevel level = setting.GetLevel();
   std::unique_ptr<Message> reference_message = setting.NewTestMessage();

   GOOGLE_CHECK(reference_message->ParseFromString(equivalent_wire_format))
       << "Failed to parse wire data for test case: " << test_name;

   switch (response.result_case()) {
     case ConformanceResponse::RESULT_NOT_SET:
       ReportFailure(test_name, level, request, response,
                     "Response didn't have any field in the Response.");
       return;

     case ConformanceResponse::kParseError:
     case ConformanceResponse::kRuntimeError:
     case ConformanceResponse::kSerializeError:
       ReportFailure(test_name, level, request, response,
                     "Failed to parse input or produce output.");
       return;

     case ConformanceResponse::kSkipped:
       ReportSkip(test_name, request, response);
       return;

     default:
       if (!ParseResponse(response, setting, test_message.get())) return;
   }

   MessageDifferencer differencer;
   DefaultFieldComparator field_comparator;
   field_comparator.set_treat_nan_as_equal(true);
   differencer.set_field_comparator(&field_comparator);
   string differences;
   differencer.ReportDifferencesToString(&differences);

   bool check = false;

   if (require_same_wire_format) {
     GOOGLE_DCHECK_EQ(response.result_case(), ConformanceResponse::kProtobufPayload);
     const string& protobuf_payload = response.protobuf_payload();
     check = equivalent_wire_format == protobuf_payload;
     differences = StrCat("Expect: ", ToOctString(equivalent_wire_format),
                                ", but got: ", ToOctString(protobuf_payload));
   } else {
     check = differencer.Compare(*reference_message, *test_message);
   }

   if (check) {
     if (need_report_success) {
       ReportSuccess(test_name);
     }
   } else {
     ReportFailure(test_name, level, request, response,
                   "Output was not equivalent to reference message: %s.",
                   differences.c_str());
   }
 }

 void ConformanceTestSuite::RunTest(const string& test_name,
                                    const ConformanceRequest& request,
                                    ConformanceResponse* response) {
   if (test_names_.insert(test_name).second == false) {
     GOOGLE_LOG(FATAL) << "Duplicated test name: " << test_name;
   }

   string serialized_request;
   string serialized_response;
   request.SerializeToString(&serialized_request);

   runner_->RunTest(test_name, serialized_request, &serialized_response);

   if (!response->ParseFromString(serialized_response)) {
     response->Clear();
     response->set_runtime_error("response proto could not be parsed.");
   }

   if (verbose_) {
     StringAppendF(&output_,
                   "conformance test: name=%s, request=%s, response=%s\n",
                   test_name.c_str(),
                   request.ShortDebugString().c_str(),
                   response->ShortDebugString().c_str());
   }
 }

 bool ConformanceTestSuite::CheckSetEmpty(
     const std::set<string>& set_to_check,
     const std::string& write_to_file,
     const std::string& msg) {
   if (set_to_check.empty()) {
     return true;
   } else {
     StringAppendF(&output_, "\n");
     StringAppendF(&output_, "%s\n\n", msg.c_str());
     for (std::set<string>::const_iterator iter = set_to_check.begin();
          iter != set_to_check.end(); ++iter) {
       StringAppendF(&output_, "  %s\n", iter->c_str());
     }
     StringAppendF(&output_, "\n");

     if (!write_to_file.empty()) {
       std::ofstream os(write_to_file);
       if (os) {
         for (std::set<string>::const_iterator iter = set_to_check.begin();
              iter != set_to_check.end(); ++iter) {
           os << *iter << "\n";
         }
       } else {
         StringAppendF(&output_, "Failed to open file: %s\n",
                       write_to_file.c_str());
       }
     }

     return false;
   }
 }

 string ConformanceTestSuite::WireFormatToString(
     WireFormat wire_format) {
   switch (wire_format) {
     case conformance::PROTOBUF:
       return "PROTOBUF";
     case conformance::JSON:
       return "JSON";
     case conformance::JSPB:
       return "JSPB";
     case conformance::TEXT_FORMAT:
       return "TEXT_FORMAT";
     case conformance::UNSPECIFIED:
       return "UNSPECIFIED";
     default:
       GOOGLE_LOG(FATAL) << "unknown wire type: " << wire_format;
   }
   return "";
 }

 void ConformanceTestSuite::AddExpectedFailedTest(const std::string& test_name) {
   expected_to_fail_.insert(test_name);
 }

 bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
                                     std::string* output, const string& filename,
                                     conformance::FailureSet* failure_list) {
   runner_ = runner;
   successes_ = 0;
   expected_failures_ = 0;
   skipped_.clear();
   test_names_.clear();
   unexpected_failing_tests_.clear();
   unexpected_succeeding_tests_.clear();

   output_ = "\nCONFORMANCE TEST BEGIN ====================================\n\n";

   failure_list_filename_ = filename;
   expected_to_fail_.clear();
   for (const string& failure : failure_list->failure()) {
     AddExpectedFailedTest(failure);
   }
   RunSuiteImpl();

   bool ok = true;
   if (!CheckSetEmpty(expected_to_fail_, "nonexistent_tests.txt",
                      "These tests were listed in the failure list, but they "
                      "don't exist.  Remove them from the failure list by "
                      "running:\n"
                      "  ./update_failure_list.py " + failure_list_filename_ +
                      " --remove nonexistent_tests.txt")) {
     ok = false;
   }
   if (!CheckSetEmpty(unexpected_failing_tests_, "failing_tests.txt",
                      "These tests failed.  If they can't be fixed right now, "
                      "you can add them to the failure list so the overall "
                      "suite can succeed.  Add them to the failure list by "
                      "running:\n"
                      "  ./update_failure_list.py " + failure_list_filename_ +
                      " --add failing_tests.txt")) {
     ok = false;
   }
   if (!CheckSetEmpty(unexpected_succeeding_tests_, "succeeding_tests.txt",
                      "These tests succeeded, even though they were listed in "
                      "the failure list.  Remove them from the failure list "
                      "by running:\n"
                      "  ./update_failure_list.py " + failure_list_filename_ +
                      " --remove succeeding_tests.txt")) {
     ok = false;
   }

   if (verbose_) {
     CheckSetEmpty(skipped_, "",
                   "These tests were skipped (probably because support for some "
                   "features is not implemented)");
   }

   StringAppendF(&output_,
                 "CONFORMANCE SUITE %s: %d successes, %zu skipped, "
                 "%d expected failures, %zu unexpected failures.\n",
                 ok ? "PASSED" : "FAILED", successes_, skipped_.size(),
                 expected_failures_, unexpected_failing_tests_.size());
   StringAppendF(&output_, "\n");

   output->assign(output_);

   return ok;
 }

 }  // namespace protobuf
 }  // namespace google
	// Protocol Buffers - Google's data interchange format
	// Copyright 2008 Google Inc. All rights reserved.
	// https://developers.google.com/protocol-buffers/
	//
	// Redistribution and use in source and binary forms, with or without
	// modification, are permitted provided that the following conditions are
	// met:
	//
	// * Redistributions of source code must retain the above copyright
	// notice, this list of conditions and the following disclaimer.
	// * Redistributions in binary form must reproduce the above
	// copyright notice, this list of conditions and the following disclaimer
	// in the documentation and/or other materials provided with the
	// distribution.
	// * Neither the name of Google Inc. nor the names of its
	// contributors may be used to endorse or promote products derived from
	// this software without specific prior written permission.
	//
	// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	#include "conformance_test.h"

	#include <stdarg.h>

	#include <fstream>
	#include <set>
	#include <string>

	#include <google/protobuf/stubs/stringprintf.h>
	#include <google/protobuf/message.h>
	#include <google/protobuf/text_format.h>
	#include <google/protobuf/util/field_comparator.h>
	#include <google/protobuf/util/json_util.h>
	#include <google/protobuf/util/message_differencer.h>
	#include "conformance.pb.h"

	using conformance::ConformanceRequest;
	using conformance::ConformanceResponse;
	using conformance::WireFormat;
	using google::protobuf::TextFormat;
	using google::protobuf::util::DefaultFieldComparator;
	using google::protobuf::util::MessageDifferencer;
	using std::string;

	namespace {

	static string ToOctString(const string& binary_string) {
	string oct_string;
	for (size_t i = 0; i < binary_string.size(); i++) {
	uint8_t c = binary_string.at(i);
	uint8_t high = c / 64;
	uint8_t mid = (c % 64) / 8;
	uint8_t low = c % 8;
	oct_string.push_back('\\');
	oct_string.push_back('0' + high);
	oct_string.push_back('0' + mid);
	oct_string.push_back('0' + low);
	}
	return oct_string;
	}

	} // namespace

	namespace google {
	namespace protobuf {

	ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
	ConformanceLevel level,
	conformance::WireFormat input_format,
	conformance::WireFormat output_format,
	conformance::TestCategory test_category,
	const Message& prototype_message,
	const string& test_name, const string& input)
	: level_(level),
	input_format_(input_format),
	output_format_(output_format),
	prototype_message_(prototype_message),
	prototype_message_for_compare_(prototype_message.New()),
	test_name_(test_name) {
	switch (input_format) {
	case conformance::PROTOBUF: {
	request_.set_protobuf_payload(input);
	break;
	}

	case conformance::JSON: {
	request_.set_json_payload(input);
	break;
	}

	case conformance::JSPB: {
	request_.set_jspb_payload(input);
	break;
	}

	case conformance::TEXT_FORMAT: {
	request_.set_text_payload(input);
	break;
	}

	default:
	GOOGLE_LOG(FATAL) << "Unspecified input format";
	}

	request_.set_test_category(test_category);

	request_.set_message_type(prototype_message.GetDescriptor()->full_name());
	request_.set_requested_output_format(output_format);
	}

	std::unique_ptr<Message>
	ConformanceTestSuite::ConformanceRequestSetting::NewTestMessage() const {
	return std::unique_ptr<Message>(prototype_message_for_compare_->New());
	}

	string ConformanceTestSuite::ConformanceRequestSetting::
	GetTestName() const {
	string rname =
	prototype_message_.GetDescriptor()->file()->syntax() ==
	FileDescriptor::SYNTAX_PROTO3 ? "Proto3" : "Proto2";

	return StrCat(ConformanceLevelToString(level_), ".", rname, ".",
	InputFormatString(input_format_), ".", test_name_, ".",
	OutputFormatString(output_format_));
	}

	string ConformanceTestSuite::ConformanceRequestSetting::
	ConformanceLevelToString(
	ConformanceLevel level) const {
	switch (level) {
	case REQUIRED: return "Required";
	case RECOMMENDED: return "Recommended";
	}
	GOOGLE_LOG(FATAL) << "Unknown value: " << level;
	return "";
	}

	string ConformanceTestSuite::ConformanceRequestSetting::
	InputFormatString(conformance::WireFormat format) const {
	switch (format) {
	case conformance::PROTOBUF:
	return "ProtobufInput";
	case conformance::JSON:
	return "JsonInput";
	case conformance::TEXT_FORMAT:
	return "TextFormatInput";
	default:
	GOOGLE_LOG(FATAL) << "Unspecified output format";
	}
	return "";
	}

	string ConformanceTestSuite::ConformanceRequestSetting::
	OutputFormatString(conformance::WireFormat format) const {
	switch (format) {
	case conformance::PROTOBUF:
	return "ProtobufOutput";
	case conformance::JSON:
	return "JsonOutput";
	case conformance::TEXT_FORMAT:
	return "TextFormatOutput";
	default:
	GOOGLE_LOG(FATAL) << "Unspecified output format";
	}
	return "";
	}

	void ConformanceTestSuite::ReportSuccess(const string& test_name) {
	if (expected_to_fail_.erase(test_name) != 0) {
	StringAppendF(&output_,
	"ERROR: test %s is in the failure list, but test succeeded. "
	"Remove it from the failure list.\n",
	test_name.c_str());
	unexpected_succeeding_tests_.insert(test_name);
	}
	successes_++;
	}

	void ConformanceTestSuite::ReportFailure(const string& test_name,
	ConformanceLevel level,
	const ConformanceRequest& request,
	const ConformanceResponse& response,
	const char* fmt, ...) {
	if (expected_to_fail_.erase(test_name) == 1) {
	expected_failures_++;
	if (!verbose_)
	return;
	} else if (level == RECOMMENDED && !enforce_recommended_) {
	StringAppendF(&output_, "WARNING, test=%s: ", test_name.c_str());
	} else {
	StringAppendF(&output_, "ERROR, test=%s: ", test_name.c_str());
	unexpected_failing_tests_.insert(test_name);
	}
	va_list args;
	va_start(args, fmt);
	StringAppendV(&output_, fmt, args);
	va_end(args);
	StringAppendF(&output_, " request=%s, response=%s\n",
	request.ShortDebugString().c_str(),
	response.ShortDebugString().c_str());
	}

	void ConformanceTestSuite::ReportSkip(const string& test_name,
	const ConformanceRequest& request,
	const ConformanceResponse& response) {
	if (verbose_) {
	StringAppendF(&output_, "SKIPPED, test=%s request=%s, response=%s\n",
	test_name.c_str(), request.ShortDebugString().c_str(),
	response.ShortDebugString().c_str());
	}
	skipped_.insert(test_name);
	}

	void ConformanceTestSuite::RunValidInputTest(
	const ConformanceRequestSetting& setting,
	const string& equivalent_text_format) {
	std::unique_ptr<Message> reference_message(setting.NewTestMessage());
	GOOGLE_CHECK(TextFormat::ParseFromString(equivalent_text_format,
	reference_message.get()))
	<< "Failed to parse data for test case: " << setting.GetTestName()
	<< ", data: " << equivalent_text_format;
	const string equivalent_wire_format = reference_message->SerializeAsString();
	RunValidBinaryInputTest(setting, equivalent_wire_format);
	}

	void ConformanceTestSuite::RunValidBinaryInputTest(
	const ConformanceRequestSetting& setting,
	const string& equivalent_wire_format, bool require_same_wire_format) {
	const ConformanceRequest& request = setting.GetRequest();
	ConformanceResponse response;
	RunTest(setting.GetTestName(), request, &response);
	VerifyResponse(setting, equivalent_wire_format, response, true,
	require_same_wire_format);
	}

	void ConformanceTestSuite::VerifyResponse(
	const ConformanceRequestSetting& setting,
	const string& equivalent_wire_format, const ConformanceResponse& response,
	bool need_report_success, bool require_same_wire_format) {
	std::unique_ptr<Message> test_message(setting.NewTestMessage());
	const ConformanceRequest& request = setting.GetRequest();
	const string& test_name = setting.GetTestName();
	ConformanceLevel level = setting.GetLevel();
	std::unique_ptr<Message> reference_message = setting.NewTestMessage();

	GOOGLE_CHECK(reference_message->ParseFromString(equivalent_wire_format))
	<< "Failed to parse wire data for test case: " << test_name;

	switch (response.result_case()) {
	case ConformanceResponse::RESULT_NOT_SET:
	ReportFailure(test_name, level, request, response,
	"Response didn't have any field in the Response.");
	return;

	case ConformanceResponse::kParseError:
	case ConformanceResponse::kRuntimeError:
	case ConformanceResponse::kSerializeError:
	ReportFailure(test_name, level, request, response,
	"Failed to parse input or produce output.");
	return;

	case ConformanceResponse::kSkipped:
	ReportSkip(test_name, request, response);
	return;

	default:
	if (!ParseResponse(response, setting, test_message.get())) return;
	}

	MessageDifferencer differencer;
	DefaultFieldComparator field_comparator;
	field_comparator.set_treat_nan_as_equal(true);
	differencer.set_field_comparator(&field_comparator);
	string differences;
	differencer.ReportDifferencesToString(&differences);

	bool check = false;

	if (require_same_wire_format) {
	GOOGLE_DCHECK_EQ(response.result_case(), ConformanceResponse::kProtobufPayload);
	const string& protobuf_payload = response.protobuf_payload();
	check = equivalent_wire_format == protobuf_payload;
	differences = StrCat("Expect: ", ToOctString(equivalent_wire_format),
	", but got: ", ToOctString(protobuf_payload));
	} else {
	check = differencer.Compare(reference_message, test_message);
	}

	if (check) {
	if (need_report_success) {
	ReportSuccess(test_name);
	}
	} else {
	ReportFailure(test_name, level, request, response,
	"Output was not equivalent to reference message: %s.",
	differences.c_str());
	}
	}

	void ConformanceTestSuite::RunTest(const string& test_name,
	const ConformanceRequest& request,
	ConformanceResponse* response) {
	if (test_names_.insert(test_name).second == false) {
	GOOGLE_LOG(FATAL) << "Duplicated test name: " << test_name;
	}

	string serialized_request;
	string serialized_response;
	request.SerializeToString(&serialized_request);

	runner_->RunTest(test_name, serialized_request, &serialized_response);

	if (!response->ParseFromString(serialized_response)) {
	response->Clear();
	response->set_runtime_error("response proto could not be parsed.");
	}

	if (verbose_) {
	StringAppendF(&output_,
	"conformance test: name=%s, request=%s, response=%s\n",
	test_name.c_str(),
	request.ShortDebugString().c_str(),
	response->ShortDebugString().c_str());
	}
	}

	bool ConformanceTestSuite::CheckSetEmpty(
	const std::set<string>& set_to_check,
	const std::string& write_to_file,
	const std::string& msg) {
	if (set_to_check.empty()) {
	return true;
	} else {
	StringAppendF(&output_, "\n");
	StringAppendF(&output_, "%s\n\n", msg.c_str());
	for (std::set<string>::const_iterator iter = set_to_check.begin();
	iter != set_to_check.end(); ++iter) {
	StringAppendF(&output_, " %s\n", iter->c_str());
	}
	StringAppendF(&output_, "\n");

	if (!write_to_file.empty()) {
	std::ofstream os(write_to_file);
	if (os) {
	for (std::set<string>::const_iterator iter = set_to_check.begin();
	iter != set_to_check.end(); ++iter) {
	os << *iter << "\n";
	}
	} else {
	StringAppendF(&output_, "Failed to open file: %s\n",
	write_to_file.c_str());
	}
	}

	return false;
	}
	}

	string ConformanceTestSuite::WireFormatToString(
	WireFormat wire_format) {
	switch (wire_format) {
	case conformance::PROTOBUF:
	return "PROTOBUF";
	case conformance::JSON:
	return "JSON";
	case conformance::JSPB:
	return "JSPB";
	case conformance::TEXT_FORMAT:
	return "TEXT_FORMAT";
	case conformance::UNSPECIFIED:
	return "UNSPECIFIED";
	default:
	GOOGLE_LOG(FATAL) << "unknown wire type: " << wire_format;
	}
	return "";
	}

	void ConformanceTestSuite::AddExpectedFailedTest(const std::string& test_name) {
	expected_to_fail_.insert(test_name);
	}

	bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
	std::string* output, const string& filename,
	conformance::FailureSet* failure_list) {
	runner_ = runner;
	successes_ = 0;
	expected_failures_ = 0;
	skipped_.clear();
	test_names_.clear();
	unexpected_failing_tests_.clear();
	unexpected_succeeding_tests_.clear();

	output_ = "\nCONFORMANCE TEST BEGIN ====================================\n\n";

	failure_list_filename_ = filename;
	expected_to_fail_.clear();
	for (const string& failure : failure_list->failure()) {
	AddExpectedFailedTest(failure);
	}
	RunSuiteImpl();

	bool ok = true;
	if (!CheckSetEmpty(expected_to_fail_, "nonexistent_tests.txt",
	"These tests were listed in the failure list, but they "
	"don't exist. Remove them from the failure list by "
	"running:\n"
	" ./update_failure_list.py " + failure_list_filename_ +
	" --remove nonexistent_tests.txt")) {
	ok = false;
	}
	if (!CheckSetEmpty(unexpected_failing_tests_, "failing_tests.txt",
	"These tests failed. If they can't be fixed right now, "
	"you can add them to the failure list so the overall "
	"suite can succeed. Add them to the failure list by "
	"running:\n"
	" ./update_failure_list.py " + failure_list_filename_ +
	" --add failing_tests.txt")) {
	ok = false;
	}
	if (!CheckSetEmpty(unexpected_succeeding_tests_, "succeeding_tests.txt",
	"These tests succeeded, even though they were listed in "
	"the failure list. Remove them from the failure list "
	"by running:\n"
	" ./update_failure_list.py " + failure_list_filename_ +
	" --remove succeeding_tests.txt")) {
	ok = false;
	}

	if (verbose_) {
	CheckSetEmpty(skipped_, "",
	"These tests were skipped (probably because support for some "
	"features is not implemented)");
	}

	StringAppendF(&output_,
	"CONFORMANCE SUITE %s: %d successes, %zu skipped, "
	"%d expected failures, %zu unexpected failures.\n",
	ok ? "PASSED" : "FAILED", successes_, skipped_.size(),
	expected_failures_, unexpected_failing_tests_.size());
	StringAppendF(&output_, "\n");

	output->assign(output_);

	return ok;
	}

	} // namespace protobuf
	} // namespace google