conformance/test_environment_test.cc

// Protocol Buffers - Google's data interchange format
// Copyright 2026 Google LLC.  All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd

#include "conformance/test_environment.h"

#ifdef _WIN32
#include <direct.h>  // for _chdir, _getcwd
#else
#include <unistd.h>  // for chdir, getcwd
#endif

#include <cstdlib>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "google/protobuf/testing/file.h"
#include "google/protobuf/testing/file.h"
#include "google/protobuf/testing/file.h"
#include "google/protobuf/descriptor.pb.h"
#include <gmock/gmock.h>
#include "google/protobuf/testing/googletest.h"
#include <gtest/gtest-spi.h>
#include <gtest/gtest.h>
#include "absl/log/absl_check.h"
#include "absl/status/status_matchers.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "conformance/binary_wireformat.h"
#include "conformance/conformance.pb.h"
#include "conformance/global_test_environment.h"
#include "conformance/matchers.h"
#include "conformance/matchers_testing.h"
#include "conformance/mock_test_runner.h"
#include "conformance/result_ledger.h"
#include "conformance/result_listener.h"
#include "conformance/result_record.h"
#include "conformance/test_environment_testing.h"
#include "conformance/testee.h"
#include "google/protobuf/test_messages_proto3.pb.h"
#include "google/protobuf/text_format.h"

namespace google {
namespace protobuf {
namespace conformance {
namespace {

using ::absl_testing::IsOk;
using ::google::protobuf::conformance::internal::ConformanceEnvironment;
using ::google::protobuf::conformance::internal::ConformanceEnvironmentOptions;
using ::google::protobuf::conformance::internal::ResultRecord;
using ::google::protobuf::conformance::internal::ResultRecordMessage;
using ::protobuf_test_messages::proto3::TestAllTypesProto3;
using ::testing::_;
using ::testing::Contains;
using ::testing::ElementsAre;
using ::testing::FieldsAre;
using ::testing::HasSubstr;
using ::testing::IsEmpty;
using ::testing::Key;
using ::testing::NiceMock;
using ::testing::Not;
using ::testing::Pair;
using ::testing::Property;
using ::testing::Return;

// The name the testee sees for a kP0 test named "Foo" that parses
// TestAllTypesProto3 from binary and serializes it back to binary.
constexpr absl::string_view kFooTestName =
    "Required.Proto3.ProtobufInput.Foo.ProtobufOutput";
constexpr absl::string_view kBarTestName =
    "Required.Proto3.ProtobufInput.Bar.ProtobufOutput";
constexpr absl::string_view kSetUpTestSuiteTestName =
    "Required.Proto3.ProtobufInput.SetUpTestSuite.ProtobufOutput";

std::string SerializedResponse(absl::string_view textproto) {
  ::conformance::ConformanceResponse response;
  ABSL_CHECK(TextFormat::ParseFromString(textproto, &response));
  return response.SerializeAsString();
}

// A testee that successfully round-trips `optional_int32: 99` for every test
// (by default: being a mock, a test can still set expectations on it).
class FakeTestRunner : public NiceMock<MockTestRunner> {
 public:
  FakeTestRunner() {
    ON_CALL(*this, RunTest)
        .WillByDefault(
            Return(SerializedResponse(R"pb(protobuf_payload: "\010c")pb")));
  }
};

// A FakeTestRunner that reports its destruction.
class FlaggingTestRunner : public FakeTestRunner {
 public:
  explicit FlaggingTestRunner(bool* destroyed) : destroyed_(destroyed) {}
  ~FlaggingTestRunner() override { *destroyed_ = true; }

 private:
  bool* destroyed_;
};

// Sets an environment variable for the lifetime of this object.
class ScopedEnvironmentVariable {
 public:
  ScopedEnvironmentVariable(absl::string_view name,
                            const absl::optional<std::string>& value)
      : name_(name) {
    if (const char* old = std::getenv(name_.c_str()); old != nullptr) {
      old_value_ = old;
    }
    Set(value);
  }
  ~ScopedEnvironmentVariable() { Set(old_value_); }

 private:
  void Set(const absl::optional<std::string>& value) {
#ifdef _WIN32
    // The MSVC CRT has no setenv()/unsetenv(); _putenv_s() with an empty
    // value removes the variable, so std::getenv() then returns null again.
    ABSL_CHECK_EQ(
        _putenv_s(name_.c_str(), value.has_value() ? value->c_str() : ""), 0);
#else
    if (value.has_value()) {
      ABSL_CHECK_EQ(setenv(name_.c_str(), value->c_str(), /*overwrite=*/1), 0);
    } else {
      ABSL_CHECK_EQ(unsetenv(name_.c_str()), 0);
    }
#endif
  }

  std::string name_;
  absl::optional<std::string> old_value_;
};

// Changes the working directory for the lifetime of this object.
class ScopedWorkingDirectory {
 public:
  explicit ScopedWorkingDirectory(const std::string& directory) {
    char* cwd = CurrentDirectory();
    ABSL_CHECK(cwd != nullptr);
    old_directory_ = cwd;
    std::free(cwd);
    ABSL_CHECK_EQ(ChangeDirectory(directory), 0) << directory;
  }
  ~ScopedWorkingDirectory() {
    ABSL_CHECK_EQ(ChangeDirectory(old_directory_), 0) << old_directory_;
  }

 private:
  // Both glibc/macOS (as an extension) and the MSVC CRT malloc() a buffer of
  // the right size when given a null one.
  static char* CurrentDirectory() {
#ifdef _WIN32
    return _getcwd(nullptr, 0);
#else
    return getcwd(nullptr, 0);
#endif
  }
  static int ChangeDirectory(const std::string& directory) {
#ifdef _WIN32
    return _chdir(directory.c_str());
#else
    return chdir(directory.c_str());
#endif
  }

  std::string old_directory_;
};

// The messages of the failures in `failures`.
std::vector<std::string> Messages(
    const testing::TestPartResultArray& failures) {
  std::vector<std::string> messages;
  for (int i = 0; i < failures.size(); ++i) {
    messages.push_back(failures.GetTestPartResult(i).message());
  }
  return messages;
}

// The properties of `result`, as (key, value) pairs.
std::vector<std::pair<std::string, std::string>> Properties(
    const testing::TestResult& result) {
  std::vector<std::pair<std::string, std::string>> properties;
  for (int i = 0; i < result.test_property_count(); ++i) {
    const testing::TestProperty& property = result.GetTestProperty(i);
    properties.emplace_back(property.key(), property.value());
  }
  return properties;
}

// Runs ReportUncheckedResults() on `env` and returns the messages of the
// failures it reports.
std::vector<std::string> UncheckedResultFailures(ConformanceEnvironment& env) {
  testing::TestPartResultArray failures;
  {
    testing::ScopedFakeTestPartResultReporter reporter(
        testing::ScopedFakeTestPartResultReporter::
            INTERCEPT_ONLY_CURRENT_THREAD,
        &failures);
    env.ReportUncheckedResults();
  }
  return Messages(failures);
}

// Creates a kP0 test named `name` against the global testee, as the fixture's
// Testee() would.
internal::Test CreateP0Test(absl::string_view name) {
  return ConformanceEnvironment::Get().testee().CreateTest(name, kP0);
}

// Tears down `env` in a full run, expecting it to report unseen expected
// failures with a message containing `expected_message`.
void TearDownExpectingUnseenFailures(ConformanceEnvironment& env,
                                     absl::string_view expected_message) {
  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(env.TearDown(), std::string(expected_message));
}

class ConformanceEnvironmentTest : public ::testing::Test {
 protected:
  ConformanceEnvironmentTest()
      : tmp_dir_(absl::StrCat(
            TestTempDir(), "/test_environment_test/",
            testing::UnitTest::GetInstance()->current_test_info()->name())) {
    ABSL_CHECK_OK(
        File::RecursivelyCreateDir(tmp_dir_, 0777));
  }

  ~ConformanceEnvironmentTest() override {
    File::DeleteRecursively(tmp_dir_, NULL, NULL);
  }

  // Writes `content` to a file named `name` in this test's temp directory and
  // returns its path.
  std::string WriteFile(absl::string_view name, absl::string_view content) {
    std::string path = TmpPath(name);
    ABSL_CHECK_OK(File::SetContents(path, content, true));
    return path;
  }

  // `path` is a std::string (not a string_view) for the open source
  // File::GetContents() shim.
  std::string ReadFile(const std::string& path) {
    std::string content;
    ABSL_CHECK_OK(File::GetContents(path, &content, true));
    return content;
  }

  std::string TmpPath(absl::string_view name) {
    return absl::StrCat(tmp_dir_, "/", name);
  }

  std::string CreateTmpDir(absl::string_view name) {
    std::string path = TmpPath(name);
    ABSL_CHECK_OK(File::RecursivelyCreateDir(path, 0777));
    return path;
  }

  // Runs a kP0 test named `name` against the global testee.
  internal::TestResult RunP0(absl::string_view name) {
    return ConformanceEnvironment::Get()
        .testee()
        .CreateTest(name, kP0)
        .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99))
        .SerializeBinary();
  }

  internal::TestResult RunP1(absl::string_view name) {
    return ConformanceEnvironment::Get()
        .testee()
        .CreateTest(name, kP1)
        .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99))
        .SerializeBinary();
  }

  FakeTestRunner fake_runner_;

 private:
  std::string tmp_dir_;
};

TEST_F(ConformanceEnvironmentTest, ScopedEnvironmentIsTheGlobalOne) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  EXPECT_EQ(&internal::GetGlobalFailureList(), &env->ledger().failure_list());
}

TEST_F(ConformanceEnvironmentTest, EnforcementLevelIsPassedToTheFailureList) {
  {
    // Every priority is enforced by default.
    internal::ScopedGlobalConformanceEnvironment env(
        {/*runner=*/&fake_runner_});
    EXPECT_THAT(
        env->ledger().failure_list().VerdictOnFailure("P1.x", kP1, "boom"),
        Not(IsOk()));
  }
  {
    ConformanceEnvironmentOptions options;
    options.runner = &fake_runner_;
    options.enforcement_level = kP0;
    internal::ScopedGlobalConformanceEnvironment env(std::move(options));
    EXPECT_THAT(
        env->ledger().failure_list().VerdictOnFailure("P1.x", kP1, "boom"),
        IsOk());
  }
}

TEST_F(ConformanceEnvironmentTest, TesteeRoutesToRunner) {
  MockTestRunner mock;
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&mock});
  env->SetUp();

  EXPECT_CALL(mock, RunTest(kFooTestName, _))
      .WillOnce(Return(SerializedResponse(R"pb(protobuf_payload: "\010c")pb")));
  EXPECT_THAT(RunP0("Foo"),
              Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb"))));
  EXPECT_EQ(env->ledger().expected_successes(), 1);
  EXPECT_EQ(env->ledger().unexpected_failures(), 0);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, UnexpectedFailureFailsTheTest) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  env->SetUp();

  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())),
                        "Should have failed to parse, but didn't.");

  EXPECT_EQ(env->ledger().unexpected_failures(), 1);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, ResultsThatAreNeverCheckedFailTheTest) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  env->SetUp();

  // A result that is stored but never checked and one matched without
  // Yields() both leave the ResultLedger in the dark; only what Yields()
  // records reaches it.  (Dropping a result or applying a leaf matcher to it
  // without Yields() does not compile; see YieldedResult in matchers.h.)
  internal::TestResult unchecked = RunP0("Foo");
  EXPECT_THAT(RunP0("Bar"),
              Property(&internal::TestResult::name, HasSubstr("Bar")));
  EXPECT_THAT(RunP0("Baz"),
              Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb"))));

  EXPECT_THAT(
      UncheckedResultFailures(env.environment()),
      ElementsAre(HasSubstr(absl::StrCat(
                      "TestResult for ", kFooTestName,
                      " was never checked; wrap the matcher in Yields()")),
                  HasSubstr(absl::StrCat(
                      "TestResult for ", kBarTestName,
                      " was never checked; wrap the matcher in Yields()"))));
  // Each result is reported once.
  EXPECT_THAT(UncheckedResultFailures(env.environment()), IsEmpty());
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, TearDownReportsResultsThatWereNeverChecked) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  env->SetUp();
  internal::TestResult unchecked = RunP0("Foo");
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(),
      absl::StrCat("TestResult for ", kFooTestName,
                   " was never checked; wrap the matcher in Yields()"));
}

TEST_F(ConformanceEnvironmentTest, ReportUncheckedResultsIsANoOpAfterTearDown) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  env->SetUp();
  env->TearDown();
  EXPECT_THAT(UncheckedResultFailures(env.environment()), IsEmpty());
}

TEST_F(ConformanceEnvironmentTest, TalliesRecordedResultsAsTheyArrive) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  env->SetUp();

  // What Yields() records (see ResultRecordMessage() in result_record.h), by
  // hand, next to a success that isn't a record.
  SUCCEED() << ResultRecordMessage(kFooTestName,
                                   {kP0, ResultRecord::Status::kFail, "boom"});
  SUCCEED() << "P0 FAIL: not a record";
  EXPECT_THAT(env->ledger().UnexpectedFailures(),
              ElementsAre(FieldsAre(kFooTestName, "boom", absl::nullopt)));
  EXPECT_EQ(env->ledger().expected_successes(), 0);

  // The first outcome recorded for a test is the one that counts.
  SUCCEED() << ResultRecordMessage(kFooTestName,
                                   {kP0, ResultRecord::Status::kPass, ""});
  EXPECT_EQ(env->ledger().unexpected_failures(), 1);
  EXPECT_EQ(env->ledger().expected_successes(), 0);
  env->TearDown();
}

// The environment's ResultListener reports the results Yields() records and
// the statistics when a gtest test or test suite ends (result_listener_test.cc
// drives it by hand), so this only shows through an environment that outlives
// the test body, like the one a test binary installs.  This suite keeps one
// alive across its tests, and until the suite itself has ended, and inspects
// what the listener did after each body has returned.  The failure the
// environment adds for an unchecked result is a real one, so that is checked
// end to end in test_environment_integration_test.cc instead.
class LongLivedEnvironmentTest : public testing::Test {
 protected:
  struct SuiteState {
    FakeTestRunner runner;
    internal::ScopedGlobalConformanceEnvironment env{{/*runner=*/&runner}};
  };

  // Inspects the suite's properties once the suite has ended, then destroys
  // the environment.  gtest tells its listeners about a suite ending in
  // reverse order of registration, so this runs after the environment's
  // listener as long as it is registered first.
  class SuiteEndChecker : public testing::EmptyTestEventListener {
   public:
    explicit SuiteEndChecker(const testing::TestSuite* suite)
        : suite_under_test_(suite) {}

    void OnTestSuiteEnd(const testing::TestSuite& suite) override {
      if (&suite != suite_under_test_ || suite_ == nullptr) return;
      // The suite's own properties hold what SetUpTestSuite() checked and the
      // statistics of the whole suite.
      std::vector<std::pair<std::string, std::string>> properties =
          Properties(suite.ad_hoc_test_result());
      EXPECT_THAT(properties,
                  Contains(Pair(kSetUpTestSuiteTestName, "P0 PASS")));
      EXPECT_THAT(properties, Contains(Pair("expected_successes", "2")));
      EXPECT_THAT(properties, Contains(Pair("unexpected_failures", "0")));
      delete suite_;
      suite_ = nullptr;
    }

   private:
    const testing::TestSuite* const suite_under_test_;
  };

  static void SetUpTestSuite() {
    testing::UnitTest& unit_test = *testing::UnitTest::GetInstance();
    // gtest owns the checker and deletes it when the program ends.
    unit_test.listeners().Append(
        new SuiteEndChecker(unit_test.current_test_suite()));
    suite_ = new SuiteState();
    // A result checked outside any test is the suite's.  It is tallied at
    // once, before the first test starts, so it is neither counted toward that
    // test's statistics nor reported as unchecked when it ends, and it is
    // reported as a property of the suite, not of the test.
    EXPECT_THAT(
        ConformanceEnvironment::Get()
            .testee()
            .CreateTest("SetUpTestSuite", kP0)
            .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99))
            .SerializeBinary(),
        Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb"))));
  }

  static void TearDownTestSuite() {
    // The test's properties hold what Yields() recorded for Bar and the
    // statistics of the test alone.
    const testing::TestSuite& suite =
        *testing::UnitTest::GetInstance()->current_test_suite();
    ASSERT_EQ(suite.total_test_count(), 1);
    std::vector<std::pair<std::string, std::string>> properties =
        Properties(*suite.GetTestInfo(0)->result());
    EXPECT_THAT(properties, Contains(Pair(kBarTestName, "P0 PASS")));
    EXPECT_THAT(properties, Contains(Pair("expected_successes", "1")));
    EXPECT_THAT(properties, Contains(Pair("unexpected_failures", "0")));
    EXPECT_THAT(properties, Not(Contains(Key(kSetUpTestSuiteTestName))));
  }

  static SuiteState* suite_;
};

LongLivedEnvironmentTest::SuiteState* LongLivedEnvironmentTest::suite_ =
    nullptr;

TEST_F(LongLivedEnvironmentTest, ReportsResultsWhenTheTestEnds) {
  EXPECT_THAT(
      CreateP0Test("Bar")
          .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99))
          .SerializeBinary(),
      Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb"))));
}

TEST_F(ConformanceEnvironmentTest, ExpectedFailureFromFailureListPasses) {
  std::string failure_list =
      WriteFile("failures.txt",
                absl::StrCat(kFooTestName, " # Should have failed to parse\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_THAT(RunP0("Foo"), Yields(IsParseError()));

  EXPECT_EQ(env->ledger().expected_failures(), 1);
  EXPECT_EQ(env->ledger().unexpected_failures(), 0);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, LoadsMultipleFailureLists) {
  std::string first = WriteFile("first.txt", absl::StrCat(kFooTestName, "\n"));
  std::string second =
      WriteFile("second.txt", absl::StrCat(kBarTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {first, second};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_THAT(RunP0("Foo"), Yields(IsParseError()));
  EXPECT_THAT(RunP0("Bar"), Yields(IsParseError()));

  EXPECT_EQ(env->ledger().expected_failures(), 2);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, OverlappingFailureListsAreFatalInSetUp) {
  std::string first = WriteFile("first.txt", absl::StrCat(kFooTestName, "\n"));
  std::string second =
      WriteFile("second.txt", absl::StrCat(kFooTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {first, second};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));

  testing::TestPartResultArray failures;
  {
    testing::ScopedFakeTestPartResultReporter reporter(
        testing::ScopedFakeTestPartResultReporter::
            INTERCEPT_ONLY_CURRENT_THREAD,
        &failures);
    env->SetUp();
  }
  ASSERT_EQ(failures.size(), 1);
  EXPECT_TRUE(failures.GetTestPartResult(0).fatally_failed());
  EXPECT_THAT(failures.GetTestPartResult(0).message(),
              HasSubstr(absl::StrCat("Failed to load failure list ", second)));
  EXPECT_THAT(failures.GetTestPartResult(0).message(),
              HasSubstr("already exists"));
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, P1FailureIsToleratedWhenNotEnforced) {
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.enforcement_level = kP0;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_THAT(RunP1("Foo"), Yields(IsParseError()));

  EXPECT_EQ(env->ledger().tolerated_failures(), 1);
  EXPECT_EQ(env->ledger().skipped(), 0);
  EXPECT_EQ(env->ledger().unexpected_failures(), 0);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest,
       ListedP1FailureIsExpectedEvenWhenNotEnforced) {
  std::string failure_list = WriteFile(
      "failures.txt",
      "Recommended.Proto3.ProtobufInput.Foo.ProtobufOutput # Should have "
      "failed to parse\n");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.enforcement_level = kP0;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_THAT(RunP1("Foo"), Yields(IsParseError()));

  EXPECT_EQ(env->ledger().expected_failures(), 1);
  EXPECT_EQ(env->ledger().tolerated_failures(), 0);
  // ...and so it isn't unseen either.
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, MissingFailureListIsFatalInSetUp) {
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {TmpPath("does_not_exist.txt")};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));

  testing::TestPartResultArray failures;
  {
    testing::ScopedFakeTestPartResultReporter reporter(
        testing::ScopedFakeTestPartResultReporter::
            INTERCEPT_ONLY_CURRENT_THREAD,
        &failures);
    env->SetUp();
  }
  ASSERT_EQ(failures.size(), 1);
  EXPECT_TRUE(failures.GetTestPartResult(0).fatally_failed());
  EXPECT_THAT(failures.GetTestPartResult(0).message(),
              HasSubstr("Failed to load failure list"));

  // TearDown must not pile on more failures after a failed SetUp.
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, TearDownFailsOnUnseenExpectedFailures) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  TearDownExpectingUnseenFailures(
      env.environment(),
      absl::StrCat("expected failures were not seen: ", kFooTestName,
                   "\nRemove them from the failure list, or rerun with --fix"));
}

TEST_F(ConformanceEnvironmentTest, TearDownFailsForAnOverexpandedWildcard) {
  std::string entry = "Required.Proto3.ProtobufInput.*.ProtobufOutput";
  std::string failure_list = WriteFile(
      "failures.txt", absl::StrCat(entry, " # Should have failed to parse\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  // Each test is an expected failure; only their number is the problem.
  for (int i = 0; i <= internal::kMaximumWildcardExpansions; ++i) {
    EXPECT_THAT(RunP0(absl::StrCat("Test", i)), Yields(IsParseError()));
  }

  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(),
      absl::StrCat("The failure list entry ", entry, " matched more than ",
                   internal::kMaximumWildcardExpansions, " tests"));
}

TEST_F(ConformanceEnvironmentTest, UnseenExpectedFailuresAreFineInPartialRun) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride partial_run(true);
  env->TearDown();
  EXPECT_THAT(env->ledger().UnmatchedExpectedFailures(),
              ElementsAre(kFooTestName));
}

TEST_F(ConformanceEnvironmentTest, UnseenCheckCanBeDisabled) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.check_unseen_expected_failures = false;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
  EXPECT_THAT(env->ledger().UnmatchedExpectedFailures(),
              ElementsAre(kFooTestName));
}

TEST_F(ConformanceEnvironmentTest, FixWritesToTheOutputFile) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # stale\n"));
  std::string fixed = TmpPath("fixed.txt");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.fix = true;
  options.fix_output_file = fixed;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  // Foo (listed) is never run; Bar (unlisted) fails.
  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                        "Should have failed to parse, but didn't.");
  // The unseen entry is still reported, but the message points at the
  // rewritten list instead of suggesting --fix.
  TearDownExpectingUnseenFailures(
      env.environment(),
      absl::StrCat("expected failures were not seen: ", kFooTestName,
                   "\nThey have been dropped from the updated failure list "
                   "written to ",
                   fixed));

  std::string content = ReadFile(fixed);
  EXPECT_THAT(content, HasSubstr(absl::StrCat(
                           kBarTestName,
                           " # Should have failed to parse, but didn't.")));
  EXPECT_THAT(content, Not(HasSubstr(kFooTestName)));
  // The input is left alone when an explicit output file is given.
  EXPECT_THAT(ReadFile(failure_list), HasSubstr("stale"));
}

TEST_F(ConformanceEnvironmentTest, FixDefaultsToTheSingleAbsoluteFailureList) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # stale\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                        "Should have failed to parse, but didn't.");
  TearDownExpectingUnseenFailures(
      env.environment(),
      absl::StrCat("updated failure list written to ", failure_list));

  std::string content = ReadFile(failure_list);
  EXPECT_THAT(content, HasSubstr(kBarTestName));
  EXPECT_THAT(content, Not(HasSubstr(kFooTestName)));
}

TEST_F(ConformanceEnvironmentTest,
       FixResolvesRelativeFailureListUnderBuildWorkspaceDirectory) {
  // The relative path is loaded from the working directory (the runfiles, under
  // bazel) but rewritten under $BUILD_WORKSPACE_DIRECTORY (the source tree).
  std::string runfiles = CreateTmpDir("runfiles");
  std::string workspace = CreateTmpDir("workspace");
  std::string stale = absl::StrCat(kFooTestName, " # stale\n");
  WriteFile("runfiles/failures.txt", stale);
  WriteFile("workspace/failures.txt", stale);
  ScopedWorkingDirectory cwd(runfiles);
  ScopedEnvironmentVariable build_workspace_directory(
      "BUILD_WORKSPACE_DIRECTORY", workspace);

  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {"failures.txt"};
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                        "Should have failed to parse, but didn't.");
  TearDownExpectingUnseenFailures(
      env.environment(), absl::StrCat("updated failure list written to ",
                                      workspace, "/failures.txt"));

  EXPECT_THAT(ReadFile(TmpPath("workspace/failures.txt")),
              HasSubstr(kBarTestName));
  EXPECT_EQ(ReadFile(TmpPath("runfiles/failures.txt")), stale);
}

TEST_F(ConformanceEnvironmentTest,
       FixWithRelativeFailureListNeedsBuildWorkspaceDirectory) {
  std::string runfiles = CreateTmpDir("runfiles");
  WriteFile("runfiles/failures.txt", "");
  ScopedWorkingDirectory cwd(runfiles);
  ScopedEnvironmentVariable build_workspace_directory(
      "BUILD_WORKSPACE_DIRECTORY", absl::nullopt);

  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {"failures.txt"};
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(),
      "--fix needs either --fix_output_file or $BUILD_WORKSPACE_DIRECTORY");
  EXPECT_EQ(ReadFile(TmpPath("runfiles/failures.txt")), "");
}

TEST_F(ConformanceEnvironmentTest,
       FixResolvesRelativeOutputFileUnderBuildWorkspaceDirectory) {
  // conformance_test() passes a relative --fix_output_file (and no
  // --failure_list) when the failure list doesn't exist yet; --fix must then
  // create it in the source tree, not in the runfiles.
  std::string runfiles = CreateTmpDir("runfiles");
  std::string workspace = CreateTmpDir("workspace");
  ScopedWorkingDirectory cwd(runfiles);
  ScopedEnvironmentVariable build_workspace_directory(
      "BUILD_WORKSPACE_DIRECTORY", workspace);

  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.fix = true;
  options.fix_output_file = "failures.txt";
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                        "Should have failed to parse, but didn't.");
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();

  EXPECT_THAT(ReadFile(TmpPath("workspace/failures.txt")),
              HasSubstr(kBarTestName));
  EXPECT_FALSE(
      File::Exists(TmpPath("runfiles/failures.txt")));
}

TEST_F(ConformanceEnvironmentTest,
       FixWithRelativeOutputFileWritesToTheWorkingDirectoryOutsideBlazeRun) {
  std::string runfiles = CreateTmpDir("runfiles");
  ScopedWorkingDirectory cwd(runfiles);
  ScopedEnvironmentVariable build_workspace_directory(
      "BUILD_WORKSPACE_DIRECTORY", absl::nullopt);

  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.fix = true;
  options.fix_output_file = "failures.txt";
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                        "Should have failed to parse, but didn't.");
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();

  EXPECT_THAT(ReadFile(TmpPath("runfiles/failures.txt")),
              HasSubstr(kBarTestName));
}

TEST_F(ConformanceEnvironmentTest, FixWithMultipleListsNeedsOutputFile) {
  std::string first = WriteFile("first.txt", "");
  std::string second = WriteFile("second.txt", "");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {first, second};
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(),
      "--fix needs --fix_output_file when more than one --failure_list");
}

TEST_F(ConformanceEnvironmentTest, FixWithNoListsNeedsOutputFile) {
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(), "--fix needs --fix_output_file when no --failure_list");
}

TEST_F(ConformanceEnvironmentTest, FixIsRefusedInPartialRun) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # stale\n"));
  std::string fixed = TmpPath("fixed.txt");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.fix = true;
  options.fix_output_file = fixed;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                        "Should have failed to parse, but didn't.");

  // Rewriting would drop Foo, whose test simply wasn't selected.  The unseen
  // check itself stays quiet in a partial run, so this is the only failure.
  internal::ScopedPartialRunOverride partial_run(true);
  EXPECT_NONFATAL_FAILURE(env->TearDown(), "--fix ignored");
  EXPECT_FALSE(File::Exists(fixed));
  EXPECT_THAT(ReadFile(failure_list), HasSubstr("stale"));
}

TEST_F(ConformanceEnvironmentTest, FixReportsWriteFailures) {
  std::string failure_list = WriteFile("failures.txt", "");
  std::string not_a_file = CreateTmpDir("directory");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.fix = true;
  options.fix_output_file = not_a_file;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                        "Should have failed to parse, but didn't.");

  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(),
      absl::StrCat("Failed to write updated failure list to ", not_a_file));
}

TEST_F(ConformanceEnvironmentTest, FixDropsListedSkips) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # listed\n"));
  std::string fixed = TmpPath("fixed.txt");
  MockTestRunner mock;
  ConformanceEnvironmentOptions options;
  options.runner = &mock;
  options.failure_list_files = {failure_list};
  options.fix = true;
  options.fix_output_file = fixed;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  // Foo (listed) is skipped by the testee, which fails it (see matchers.h);
  // the rewritten list no longer has its entry, so the fixed run passes.
  EXPECT_CALL(mock, RunTest(kFooTestName, _))
      .WillOnce(Return(SerializedResponse(R"pb(skipped: "not supported")pb")));
  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())),
                        "is in the failure list but was skipped by the testee");
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();

  EXPECT_THAT(ReadFile(fixed), Not(HasSubstr(kFooTestName)));
}

TEST_F(ConformanceEnvironmentTest, StatisticsSnapshotAndDelta) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  env->SetUp();
  internal::Statistics before = internal::Statistics::From(env->ledger());

  EXPECT_THAT(RunP0("Foo"),
              Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb"))));
  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                        "Should have failed to parse");

  internal::Statistics delta =
      internal::Statistics::From(env->ledger()) - before;
  EXPECT_EQ(delta.expected_successes, 1);
  EXPECT_EQ(delta.unexpected_failures, 1);
  EXPECT_EQ(delta.expected_failures, 0);
  EXPECT_EQ(delta.unexpected_successes, 0);
  EXPECT_EQ(delta.skipped_tests, 0);
  EXPECT_EQ(delta.listed_skips, 0);
  EXPECT_EQ(delta.tolerated_failures, 0);

  internal::Statistics total;
  total += delta;
  total += delta;
  EXPECT_EQ(total.expected_successes, 2);
  EXPECT_EQ(total.unexpected_failures, 2);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, StatisticsCountListedSkips) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # listed\n"));
  MockTestRunner mock;
  ConformanceEnvironmentOptions options;
  options.runner = &mock;
  options.failure_list_files = {failure_list};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();
  internal::Statistics before = internal::Statistics::From(env->ledger());

  // A listed test the testee skips fails (see matchers.h) and is both a skip
  // and a listed skip, but neither an expected nor an unexpected anything.
  EXPECT_CALL(mock, RunTest(kFooTestName, _))
      .WillOnce(Return(SerializedResponse(R"pb(skipped: "not supported")pb")));
  EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())),
                        "is in the failure list but was skipped by the testee");

  internal::Statistics delta =
      internal::Statistics::From(env->ledger()) - before;
  EXPECT_EQ(delta.skipped_tests, 1);
  EXPECT_EQ(delta.listed_skips, 1);
  EXPECT_EQ(delta.expected_failures, 0);
  EXPECT_EQ(delta.unexpected_failures, 0);
  EXPECT_EQ(delta.expected_successes, 0);
  EXPECT_EQ(delta.unexpected_successes, 0);

  internal::Statistics total;
  total += delta;
  total += delta;
  EXPECT_EQ(total.skipped_tests, 2);
  EXPECT_EQ(total.listed_skips, 2);
  // The skip counts as having seen the entry, so this doesn't report it.
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, MaximumEditionIsClampedToProto3) {
  // proto2 and proto3 tests always run, so anything older than proto3
  // (including the legacy runner's EDITION_UNKNOWN default) behaves like
  // proto3 rather than disabling everything.
  for (Edition edition : {EDITION_UNKNOWN, EDITION_LEGACY, EDITION_PROTO2}) {
    ConformanceEnvironmentOptions options;
    options.runner = &fake_runner_;
    options.maximum_edition = edition;
    internal::ScopedGlobalConformanceEnvironment env(std::move(options));
    EXPECT_EQ(env->options().maximum_edition, EDITION_PROTO3);
  }
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.maximum_edition = EDITION_2023;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  EXPECT_EQ(env->options().maximum_edition, EDITION_2023);
}

TEST_F(ConformanceEnvironmentTest, SpawnsForkPipeRunnerLazily) {
  // The testee binary doesn't exist, but ForkPipeRunner only spawns it on
  // first use, so constructing the environment must succeed.
  ConformanceEnvironmentOptions options;
  options.testee_binary = TmpPath("no_such_testee");
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, TearDownShutsDownTheOwnedRunner) {
  bool destroyed = false;
  internal::ScopedGlobalConformanceEnvironment env(
      {/*runner=*/nullptr,
       /*owned_runner=*/std::make_unique<FlaggingTestRunner>(&destroyed)});
  env->SetUp();
  EXPECT_THAT(RunP0("Foo"),
              Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb"))));
  EXPECT_FALSE(destroyed);

  env->TearDown();
  EXPECT_TRUE(destroyed);
  // The statistics survive the testee.
  EXPECT_EQ(env->ledger().expected_successes(), 1);
}

TEST_F(ConformanceEnvironmentTest, TearDownWithoutSetUpOnlyReleasesTheTestee) {
  // gtest skips an environment's SetUp() and TearDown() when it selects no
  // test to run; the merged runner (conformance_test_main.cc) then calls
  // TearDown() itself so that the environment lets go of its runner.  Nothing
  // else may happen: no --fix rewrite and no unseen-entry failure, as the
  // failure list was never loaded.
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # never seen\n"));
  std::string fixed = TmpPath("fixed.txt");
  bool destroyed = false;
  ConformanceEnvironmentOptions options;
  options.owned_runner = std::make_unique<FlaggingTestRunner>(&destroyed);
  options.failure_list_files = {failure_list};
  options.fix = true;
  options.fix_output_file = fixed;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
  EXPECT_TRUE(destroyed);
  EXPECT_FALSE(File::Exists(fixed));
  EXPECT_THAT(ReadFile(failure_list), HasSubstr("never seen"));
  EXPECT_EQ(env->ledger().expected_failures(), 0);
}

TEST_F(ConformanceEnvironmentTest, OwnedRunnerDiesWithTheEnvironment) {
  bool destroyed = false;
  {
    internal::ScopedGlobalConformanceEnvironment env(
        {/*runner=*/nullptr,
         /*owned_runner=*/std::make_unique<FlaggingTestRunner>(&destroyed)});
    EXPECT_FALSE(destroyed);
  }
  EXPECT_TRUE(destroyed);
}

TEST(ConformanceEnvironmentDeathTest, GetWithoutInstall) {
  EXPECT_DEATH(ConformanceEnvironment::Get(), "Install");
}

TEST(ConformanceEnvironmentDeathTest,
     UncheckedResultsAreNotReportedAfterAFatalFailure) {
  // An ASSERT_* that doesn't hold returns from the test before it can check
  // its results.  A real (non-intercepted) fatal failure would fail this test,
  // so the scenario runs in a death test child that exits with the number of
  // stray failures.
  FakeTestRunner runner;
  EXPECT_EXIT(
      {
        internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&runner});
        internal::TestResult unchecked =
            CreateP0Test("Foo")
                .ParseBinary(TestAllTypesProto3::descriptor(),
                             VarintField(1, 99))
                .SerializeBinary();
        // What ASSERT_* records before returning.
        [] { FAIL() << "fatal failure"; }();
        ABSL_CHECK(testing::Test::HasFatalFailure());
        // std::exit() runs no destructors, so the mock is leaked on purpose.
        testing::Mock::AllowLeak(&runner);
        std::exit(static_cast<int>(
            UncheckedResultFailures(env.environment()).size()));
      },
      testing::ExitedWithCode(0), "");
}

TEST(ConformanceEnvironmentDeathTest, RequiresExactlyOneTestee) {
  FakeTestRunner runner;
  EXPECT_DEATH(internal::ScopedGlobalConformanceEnvironment env({}),
               "Exactly one");
  EXPECT_DEATH(internal::ScopedGlobalConformanceEnvironment env(
                   {/*runner=*/&runner, /*owned_runner=*/nullptr,
                    /*testee_binary=*/"/bin/true"}),
               "Exactly one");
  EXPECT_DEATH(internal::ScopedGlobalConformanceEnvironment env(
                   {/*runner=*/&runner,
                    /*owned_runner=*/std::make_unique<FakeTestRunner>()}),
               "Exactly one");
}

TEST(ConformanceEnvironmentDeathTest, OnlyOneGlobalEnvironmentAtATime) {
  FakeTestRunner runner;
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&runner});
  EXPECT_DEATH(
      internal::ScopedGlobalConformanceEnvironment nested({/*runner=*/&runner}),
      "already set");
  EXPECT_DEATH(ConformanceEnvironment::Install({/*runner=*/&runner}),
               "only be called once");
}

TEST(ConformanceEnvironmentDeathTest, OnlyOnePartialRunOverrideAtATime) {
  FakeTestRunner runner;
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&runner});
  internal::ScopedPartialRunOverride partial_run(true);
  EXPECT_DEATH(internal::ScopedPartialRunOverride nested(false),
               "already active");
}

TEST(ConformanceEnvironmentDeathTest, TesteeIsGoneAfterTearDown) {
  FakeTestRunner runner;
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&runner});
  env->SetUp();
  env->TearDown();
  EXPECT_DEATH(env->testee(), "already been shut down");
}

}  // namespace
}  // namespace conformance
}  // namespace protobuf
}  // namespace google