conformance/conformance_test_runner.cc

// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// This file contains a program for running the test suite in a separate
// process.  The other alternative is to run the suite in-process.  See
// conformance.proto for pros/cons of these two options.
//
// This program will fork the process under test and communicate with it over
// its stdin/stdout:
//
//     +--------+   pipe   +----------+
//     | tester | <------> | testee   |
//     |        |          |          |
//     |  C++   |          | any lang |
//     +--------+          +----------+
//
// The tester contains all of the test cases and their expected output.
// The testee is a simple program written in the target language that reads
// each test case and attempts to produce acceptable output for it.
//
// Every test consists of a ConformanceRequest/ConformanceResponse
// request/reply pair.  The protocol on the pipe is simply:
//
//   1. tester sends 4-byte length N (little endian)
//   2. tester sends N bytes representing a ConformanceRequest proto
//   3. testee sends 4-byte length M (little endian)
//   4. testee sends M bytes representing a ConformanceResponse proto

#include <errno.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>

#include <algorithm>
#include <cstdlib>
#include <fstream>
#include <future>
#include <vector>

#include "absl/strings/str_format.h"
#include "conformance/conformance.pb.h"
#include "conformance_test.h"

using conformance::ConformanceResponse;
using google::protobuf::ConformanceTestSuite;
using std::string;
using std::vector;

#define STRINGIFY(x) #x
#define TOSTRING(x) STRINGIFY(x)
#define GOOGLE_CHECK_SYSCALL(call)                            \
  if (call < 0) {                                      \
    perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
    exit(1);                                           \
  }

namespace google {
namespace protobuf {

void ParseFailureList(const char *filename,
                      conformance::FailureSet *failure_list) {
  std::ifstream infile(filename);

  if (!infile.is_open()) {
    fprintf(stderr, "Couldn't open failure list file: %s\n", filename);
    exit(1);
  }

  for (string line; getline(infile, line);) {
    // Remove whitespace.
    line.erase(std::remove_if(line.begin(), line.end(), ::isspace), line.end());

    // Remove comments.
    line = line.substr(0, line.find("#"));

    if (!line.empty()) {
      failure_list->add_failure(line);
    }
  }
}

void UsageError() {
  fprintf(stderr, "Usage: conformance-test-runner [options] <test-program>\n");
  fprintf(stderr, "\n");
  fprintf(stderr, "Options:\n");
  fprintf(stderr,
          "  --failure_list <filename>   Use to specify list of tests\n");
  fprintf(stderr,
          "                              that are expected to fail.  File\n");
  fprintf(stderr,
          "                              should contain one test name per\n");
  fprintf(stderr,
          "                              line.  Use '#' for comments.\n");
  fprintf(stderr,
          "  --text_format_failure_list <filename>   Use to specify list \n");
  fprintf(stderr,
          "                              of tests that are expected to \n");
  fprintf(stderr, "                              fail in the \n");
  fprintf(stderr,
          "                              text_format_conformance_suite.  \n");
  fprintf(stderr,
          "                              File should contain one test name \n");
  fprintf(stderr,
          "                              per line.  Use '#' for comments.\n");

  fprintf(stderr,
          "  --enforce_recommended       Enforce that recommended test\n");
  fprintf(stderr,
          "                              cases are also passing. Specify\n");
  fprintf(stderr,
          "                              this flag if you want to be\n");
  fprintf(stderr,
          "                              strictly conforming to protobuf\n");
  fprintf(stderr, "                              spec.\n");
  fprintf(stderr,
          "  --output_dir                <dirname> Directory to write\n"
          "                              output files.\n");
  exit(1);
}

void ForkPipeRunner::RunTest(const std::string &test_name,
                             const std::string &request,
                             std::string *response) {
  if (child_pid_ < 0) {
    SpawnTestProgram();
  }
  current_test_name_ = test_name;

  uint32_t len = request.size();
  CheckedWrite(write_fd_, &len, sizeof(uint32_t));
  CheckedWrite(write_fd_, request.c_str(), request.size());

  if (!TryRead(read_fd_, &len, sizeof(uint32_t))) {
    // We failed to read from the child, assume a crash and try to reap.
    GOOGLE_LOG(INFO) << "Trying to reap child, pid=" << child_pid_;

    int status = 0;
    waitpid(child_pid_, &status, WEXITED);

    string error_msg;
    conformance::ConformanceResponse response_obj;
    if (WIFEXITED(status)) {
      if (WEXITSTATUS(status) == 0) {
        absl::StrAppendFormat(&error_msg,
                              "child timed out, killed by signal %d",
                              WTERMSIG(status));
        response_obj.set_timeout_error(error_msg);
      } else {
        absl::StrAppendFormat(&error_msg, "child exited, status=%d",
                              WEXITSTATUS(status));
        response_obj.set_runtime_error(error_msg);
      }
    } else if (WIFSIGNALED(status)) {
      absl::StrAppendFormat(&error_msg, "child killed by signal %d",
                            WTERMSIG(status));
    }
    GOOGLE_LOG(INFO) << error_msg;
    child_pid_ = -1;

    response_obj.SerializeToString(response);
    return;
  }

  response->resize(len);
  CheckedRead(read_fd_, (void *)response->c_str(), len);
}

int ForkPipeRunner::Run(int argc, char *argv[],
                        const std::vector<ConformanceTestSuite *> &suites) {
  if (suites.empty()) {
    fprintf(stderr, "No test suites found.\n");
    return EXIT_FAILURE;
  }
  bool all_ok = true;
  for (ConformanceTestSuite *suite : suites) {
    string program;
    std::vector<string> program_args;
    string failure_list_filename;
    conformance::FailureSet failure_list;

    bool performance = false;
    for (int arg = 1; arg < argc; ++arg) {
      if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
        if (++arg == argc) UsageError();
        failure_list_filename = argv[arg];
        ParseFailureList(argv[arg], &failure_list);
      } else if (strcmp(argv[arg], "--performance") == 0) {
        performance = true;
        suite->SetPerformance(true);
      } else if (strcmp(argv[arg], "--verbose") == 0) {
        suite->SetVerbose(true);
      } else if (strcmp(argv[arg], "--enforce_recommended") == 0) {
        suite->SetEnforceRecommended(true);
      } else if (strcmp(argv[arg], "--output_dir") == 0) {
        if (++arg == argc) UsageError();
        suite->SetOutputDir(argv[arg]);
      } else if (argv[arg][0] == '-') {
        bool recognized_flag = false;
        for (ConformanceTestSuite *suite : suites) {
          if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
            if (++arg == argc) UsageError();
            recognized_flag = true;
          }
        }
        if (!recognized_flag) {
          fprintf(stderr, "Unknown option: %s\n", argv[arg]);
          UsageError();
        }
      } else {
        program += argv[arg++];
        while (arg < argc) {
          program_args.push_back(argv[arg]);
          arg++;
        }
      }
    }

    ForkPipeRunner runner(program, program_args, performance);

    std::string output;
    all_ok = all_ok && suite->RunSuite(&runner, &output, failure_list_filename,
                                       &failure_list);

    fwrite(output.c_str(), 1, output.size(), stderr);
  }
  return all_ok ? EXIT_SUCCESS : EXIT_FAILURE;
}

// TODO(haberman): make this work on Windows, instead of using these
// UNIX-specific APIs.
//
// There is a platform-agnostic API in
//    src/google/protobuf/compiler/subprocess.h
//
// However that API only supports sending a single message to the subprocess.
// We really want to be able to send messages and receive responses one at a
// time:
//
// 1. Spawning a new process for each test would take way too long for thousands
//    of tests and subprocesses like java that can take 100ms or more to start
//    up.
//
// 2. Sending all the tests in one big message and receiving all results in one
//    big message would take away our visibility about which test(s) caused a
//    crash or other fatal error.  It would also give us only a single failure
//    instead of all of them.
void ForkPipeRunner::SpawnTestProgram() {
  int toproc_pipe_fd[2];
  int fromproc_pipe_fd[2];
  if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
    perror("pipe");
    exit(1);
  }

  pid_t pid = fork();
  if (pid < 0) {
    perror("fork");
    exit(1);
  }

  if (pid) {
    // Parent.
    GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[0]));
    GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
    write_fd_ = toproc_pipe_fd[1];
    read_fd_ = fromproc_pipe_fd[0];
    child_pid_ = pid;
  } else {
    // Child.
    GOOGLE_CHECK_SYSCALL(close(STDIN_FILENO));
    GOOGLE_CHECK_SYSCALL(close(STDOUT_FILENO));
    GOOGLE_CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
    GOOGLE_CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));

    GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[0]));
    GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
    GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[1]));
    GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[0]));

    std::unique_ptr<char[]> executable(new char[executable_.size() + 1]);
    memcpy(executable.get(), executable_.c_str(), executable_.size());
    executable[executable_.size()] = '\0';

    std::vector<const char *> argv;
    argv.push_back(executable.get());
    GOOGLE_LOG(INFO) << argv[0];
    for (size_t i = 0; i < executable_args_.size(); ++i) {
      argv.push_back(executable_args_[i].c_str());
      GOOGLE_LOG(INFO) << executable_args_[i];
    }
    argv.push_back(nullptr);
    // Never returns.
    GOOGLE_CHECK_SYSCALL(execv(executable.get(), const_cast<char **>(argv.data())));
  }
}

void ForkPipeRunner::CheckedWrite(int fd, const void *buf, size_t len) {
  if (static_cast<size_t>(write(fd, buf, len)) != len) {
    GOOGLE_LOG(FATAL) << current_test_name_
               << ": error writing to test program: " << strerror(errno);
  }
}

bool ForkPipeRunner::TryRead(int fd, void *buf, size_t len) {
  size_t ofs = 0;
  while (len > 0) {
    std::future<ssize_t> future = std::async(
        std::launch::async,
        [](int fd, void *buf, size_t ofs, size_t len) {
          return read(fd, (char *)buf + ofs, len);
        },
        fd, buf, ofs, len);
    std::future_status status;
    if (performance_) {
      status = future.wait_for(std::chrono::seconds(5));
      if (status == std::future_status::timeout) {
        GOOGLE_LOG(ERROR) << current_test_name_ << ": timeout from test program";
        kill(child_pid_, SIGQUIT);
        // TODO(sandyzhang): Only log in flag-guarded mode, since reading output
        // from SIGQUIT is slow and verbose.
        std::vector<char> err;
        err.resize(5000);
        ssize_t err_bytes_read;
        size_t err_ofs = 0;
        do {
          err_bytes_read =
              read(fd, (void *)&err[err_ofs], err.size() - err_ofs);
          err_ofs += err_bytes_read;
        } while (err_bytes_read > 0 && err_ofs < err.size());
        GOOGLE_LOG(ERROR) << "child_pid_=" << child_pid_ << " SIGQUIT: \n" << &err[0];
        return false;
      }
    } else {
      future.wait();
    }
    ssize_t bytes_read = future.get();
    if (bytes_read == 0) {
      GOOGLE_LOG(ERROR) << current_test_name_ << ": unexpected EOF from test program";
      return false;
    } else if (bytes_read < 0) {
      GOOGLE_LOG(ERROR) << current_test_name_
                 << ": error reading from test program: " << strerror(errno);
      return false;
    }

    len -= bytes_read;
    ofs += bytes_read;
  }

  return true;
}

void ForkPipeRunner::CheckedRead(int fd, void *buf, size_t len) {
  if (!TryRead(fd, buf, len)) {
    GOOGLE_LOG(FATAL) << current_test_name_
               << ": error reading from test program: " << strerror(errno);
  }
}

}  // namespace protobuf
}  // namespace google