| #ifndef __HAYAI_MAIN |
| #define __HAYAI_MAIN |
| #include <algorithm> |
| #include <cstdlib> |
| #include <cstring> |
| #include <ctime> |
| #include <errno.h> |
| #include <fstream> |
| #include <set> |
| #include <vector> |
| |
| #include "hayai.hpp" |
| |
| |
| #if defined(_WIN32) |
| # define PATH_SEPARATOR '\\' |
| #else |
| # define PATH_SEPARATOR '/' |
| #endif |
| |
| |
| #define HAYAI_MAIN_FORMAT_FLAG(_desc) \ |
| ::hayai::Console::TextGreen << _desc << ::hayai::Console::TextDefault |
| #define HAYAI_MAIN_FORMAT_ARGUMENT(_desc) \ |
| ::hayai::Console::TextYellow << _desc << ::hayai::Console::TextDefault |
| #define HAYAI_MAIN_FORMAT_ERROR(_desc) \ |
| ::hayai::Console::TextRed << "Error:" << \ |
| ::hayai::Console::TextDefault << " " << _desc |
| #define HAYAI_MAIN_USAGE_ERROR(_desc) \ |
| { \ |
| std::cerr << HAYAI_MAIN_FORMAT_ERROR(_desc) << std::endl \ |
| << std::endl; \ |
| ShowUsage(argv[0]); \ |
| return EXIT_FAILURE; \ |
| } |
| |
| |
| namespace hayai |
| { |
| /// Execution mode. |
| enum MainExecutionMode |
| { |
| /// Run benchmarks. |
| MainRunBenchmarks, |
| |
| |
| /// List benchmarks but do not execute them. |
| MainListBenchmarks |
| }; |
| |
| |
| /// File outputter. |
| class FileOutputter |
| { |
| public: |
| /// File outputter. |
| |
| /// @param path Output path. Expected to be available during the life |
| /// time of the outputter. |
| FileOutputter(const char* path) |
| : _path(path), |
| _outputter(NULL) |
| { |
| |
| } |
| |
| |
| virtual ~FileOutputter() |
| { |
| if (_outputter) |
| delete _outputter; |
| |
| _stream.close(); |
| } |
| |
| |
| /// Set up. |
| |
| /// Opens the output file for writing and initializes the outputter. |
| virtual void SetUp() |
| { |
| _stream.open(_path, |
| std::ios_base::out | |
| std::ios_base::trunc | |
| std::ios_base::binary); |
| if (_stream.bad()) |
| { |
| std::stringstream error; |
| error << "failed to open " << _path << " for writing: " |
| << strerror(errno); |
| throw std::runtime_error(error.str()); |
| } |
| |
| _outputter = CreateOutputter(_stream); |
| } |
| |
| |
| /// Outputter. |
| virtual ::hayai::Outputter& Outputter() |
| { |
| if (!_outputter) |
| throw std::runtime_error("outputter has not been set up"); |
| |
| return *_outputter; |
| } |
| protected: |
| /// Create outputter from output stream. |
| |
| /// @param stream Output stream for the outputter. |
| /// @returns the outputter for the given format. |
| virtual ::hayai::Outputter* CreateOutputter(std::ostream& stream) = 0; |
| private: |
| const char* _path; |
| std::ofstream _stream; |
| ::hayai::Outputter* _outputter; |
| }; |
| |
| |
| #define FILE_OUTPUTTER_IMPLEMENTATION(_prefix) \ |
| class _prefix ## FileOutputter \ |
| : public FileOutputter \ |
| { \ |
| public: \ |
| _prefix ## FileOutputter(const char* path) \ |
| : FileOutputter(path) \ |
| {} \ |
| protected: \ |
| virtual ::hayai::Outputter* CreateOutputter(std::ostream& stream) \ |
| { \ |
| return new ::hayai::_prefix ## Outputter(stream); \ |
| } \ |
| } |
| |
| |
| FILE_OUTPUTTER_IMPLEMENTATION(Json); |
| FILE_OUTPUTTER_IMPLEMENTATION(Console); |
| FILE_OUTPUTTER_IMPLEMENTATION(JUnitXml); |
| |
| #undef FILE_OUTPUTTER_IMPLEMENTATION |
| |
| |
| /// Default main executable runner for Hayai. |
| class MainRunner |
| { |
| public: |
| MainRunner() |
| : ExecutionMode(MainRunBenchmarks), |
| ShuffleBenchmarks(false), |
| StdoutOutputter(NULL) |
| { |
| |
| } |
| |
| |
| ~MainRunner() |
| { |
| // Clean up the outputters. |
| for (std::vector<FileOutputter*>::iterator it = |
| FileOutputters.begin(); |
| it != FileOutputters.end(); |
| ++it) |
| delete *it; |
| |
| if (StdoutOutputter) |
| delete StdoutOutputter; |
| } |
| |
| |
| /// Execution mode. |
| MainExecutionMode ExecutionMode; |
| |
| |
| /// Shuffle benchmarks. |
| bool ShuffleBenchmarks; |
| |
| |
| /// File outputters. |
| /// |
| /// Outputter will be freed by the class on destruction. |
| std::vector<FileOutputter*> FileOutputters; |
| |
| |
| /// Standard output outputter. |
| /// |
| /// Will be freed by the class on destruction. |
| Outputter* StdoutOutputter; |
| |
| |
| /// Parse arguments. |
| |
| /// @param argc Argument count including the executable name. |
| /// @param argv Arguments. |
| /// @param residualArgs Pointer to vector to hold residual arguments |
| /// after parsing. If not NULL, the parser will not fail upon |
| /// encounting an unknown argument but will instead add it to the list |
| /// of residual arguments and return a success code. Note: the parser |
| /// will still fail if an invalid value is provided to a known |
| /// argument. |
| /// @returns 0 on success, otherwise the exit status code to be |
| /// returned from the executable. |
| int ParseArgs(int argc, |
| char** argv, |
| std::vector<char*>* residualArgs = NULL) |
| { |
| int argI = 1; |
| while (argI < argc) |
| { |
| char* arg = argv[argI++]; |
| bool argLast = (argI == argc); |
| std::size_t argLen = strlen(arg); |
| |
| if (argLen == 0) |
| continue; |
| |
| // List flag. |
| if ((!strcmp(arg, "-l")) || (!strcmp(arg, "--list"))) |
| ExecutionMode = ::hayai::MainListBenchmarks; |
| // Shuffle flag. |
| else if ((!strcmp(arg, "-s")) || (!strcmp(arg, "--shuffle"))) |
| ShuffleBenchmarks = true; |
| // Filter flag. |
| else if ((!strcmp(arg, "-f")) || (!strcmp(arg, "--filter"))) |
| { |
| if ((argLast) || (*argv[argI] == 0)) |
| HAYAI_MAIN_USAGE_ERROR(HAYAI_MAIN_FORMAT_FLAG(arg) << |
| " requires a pattern to be specified"); |
| char* pattern = argv[argI++]; |
| |
| ::hayai::Benchmarker::ApplyPatternFilter(pattern); |
| } |
| // Output flag. |
| else if ((!strcmp(arg, "-o")) || (!strcmp(arg, "--output"))) |
| { |
| if (argLast) |
| HAYAI_MAIN_USAGE_ERROR(HAYAI_MAIN_FORMAT_FLAG(arg) << |
| " requires a format to be specified"); |
| char* formatSpecifier = argv[argI++]; |
| |
| char* format = formatSpecifier; |
| char* path = strchr(formatSpecifier, ':'); |
| if (path) |
| { |
| *(path++) = 0; |
| if (!strlen(path)) |
| path = NULL; |
| } |
| |
| #define ADD_OUTPUTTER(_prefix) \ |
| { \ |
| if (path) \ |
| FileOutputters.push_back( \ |
| new ::hayai::_prefix ## FileOutputter(path) \ |
| ); \ |
| else \ |
| { \ |
| if (StdoutOutputter) \ |
| delete StdoutOutputter; \ |
| StdoutOutputter = \ |
| new ::hayai::_prefix ## Outputter(std::cout); \ |
| } \ |
| } |
| |
| if (!strcmp(format, "console")) |
| ADD_OUTPUTTER(Console) |
| else if (!strcmp(format, "json")) |
| ADD_OUTPUTTER(Json) |
| else if (!strcmp(format, "junit")) |
| ADD_OUTPUTTER(JUnitXml) |
| else |
| HAYAI_MAIN_USAGE_ERROR("invalid format: " << format); |
| |
| #undef ADD_OUTPUTTER |
| } |
| // Console coloring flag. |
| else if ((!strcmp(arg, "-c")) || (!strcmp(arg, "--color"))) |
| { |
| if (argLast) |
| HAYAI_MAIN_USAGE_ERROR( |
| HAYAI_MAIN_FORMAT_FLAG(arg) << |
| " requires an argument " << |
| "of either " << HAYAI_MAIN_FORMAT_FLAG("yes") << |
| " or " << HAYAI_MAIN_FORMAT_FLAG("no") |
| ); |
| |
| char* choice = argv[argI++]; |
| bool enabled; |
| |
| if ((!strcmp(choice, "yes")) || |
| (!strcmp(choice, "true")) || |
| (!strcmp(choice, "on")) || |
| (!strcmp(choice, "1"))) |
| enabled = true; |
| else if ((!strcmp(choice, "no")) || |
| (!strcmp(choice, "false")) || |
| (!strcmp(choice, "off")) || |
| (!strcmp(choice, "0"))) |
| enabled = false; |
| else |
| HAYAI_MAIN_USAGE_ERROR( |
| "invalid argument to " << |
| HAYAI_MAIN_FORMAT_FLAG(arg) << |
| ": " << choice |
| ); |
| |
| ::hayai::Console::SetFormattingEnabled(enabled); |
| } |
| // Help. |
| else if ((!strcmp(arg, "-?")) || |
| (!strcmp(arg, "-h")) || |
| (!strcmp(arg, "--help"))) |
| { |
| ShowUsage(argv[0]); |
| return EXIT_FAILURE; |
| } |
| else if (residualArgs) |
| { |
| residualArgs->push_back(arg); |
| } |
| else |
| { |
| HAYAI_MAIN_USAGE_ERROR("unknown option: " << arg); |
| } |
| } |
| |
| return EXIT_SUCCESS; |
| } |
| |
| |
| /// Run the selected execution mode. |
| |
| /// @returns the exit status code to be returned from the executable. |
| int Run() |
| { |
| // Execute based on the selected mode. |
| switch (ExecutionMode) |
| { |
| case ::hayai::MainRunBenchmarks: |
| return RunBenchmarks(); |
| |
| case ::hayai::MainListBenchmarks: |
| return ListBenchmarks(); |
| |
| default: |
| std::cerr << HAYAI_MAIN_FORMAT_ERROR( |
| "invalid execution mode: " << ExecutionMode |
| ) << std::endl; |
| return EXIT_FAILURE; |
| } |
| } |
| private: |
| /// Run benchmarks. |
| |
| /// @returns the exit status code to be returned from the executable. |
| int RunBenchmarks() |
| { |
| // Hook up the outputs. |
| if (StdoutOutputter) |
| ::hayai::Benchmarker::AddOutputter(*StdoutOutputter); |
| |
| for (std::vector< ::hayai::FileOutputter*>::iterator it = |
| FileOutputters.begin(); |
| it < FileOutputters.end(); |
| ++it) |
| { |
| ::hayai::FileOutputter& fileOutputter = **it; |
| |
| try |
| { |
| fileOutputter.SetUp(); |
| } |
| catch (std::exception& e) |
| { |
| std::cerr << HAYAI_MAIN_FORMAT_ERROR(e.what()) << std::endl; |
| return EXIT_FAILURE; |
| } |
| |
| ::hayai::Benchmarker::AddOutputter(fileOutputter.Outputter()); |
| } |
| |
| // Run the benchmarks. |
| if (ShuffleBenchmarks) |
| { |
| std::srand(static_cast<unsigned>(std::time(0))); |
| ::hayai::Benchmarker::ShuffleTests(); |
| } |
| |
| ::hayai::Benchmarker::RunAllTests(); |
| |
| return EXIT_SUCCESS; |
| } |
| |
| |
| /// List benchmarks. |
| |
| /// @returns the exit status code to be returned from the executable. |
| int ListBenchmarks() |
| { |
| // List out the unique benchmark names. |
| std::vector<const ::hayai::TestDescriptor*> tests = |
| ::hayai::Benchmarker::ListTests(); |
| std::vector<std::string> testNames; |
| std::set<std::string> uniqueTestNames; |
| |
| for (std::vector<const ::hayai::TestDescriptor*>::iterator it = |
| tests.begin(); |
| it < tests.end(); |
| ++it) |
| { |
| if (uniqueTestNames.find((*it)->CanonicalName) != |
| uniqueTestNames.end()) |
| continue; |
| |
| testNames.push_back((*it)->CanonicalName); |
| uniqueTestNames.insert((*it)->CanonicalName); |
| } |
| |
| // Sort the benchmark names. |
| std::sort(testNames.begin(), testNames.end()); |
| |
| // Dump the list. |
| for (std::vector<std::string>::iterator it = testNames.begin(); |
| it < testNames.end(); |
| ++it) |
| std::cout << *it << std::endl; |
| |
| return EXIT_SUCCESS; |
| } |
| |
| |
| /// Show usage. |
| |
| /// @param execName Executable name. |
| void ShowUsage(const char* execName) |
| { |
| const char* baseName = strrchr(execName, PATH_SEPARATOR); |
| |
| std::cerr << "Usage: " << (baseName ? baseName + 1 : execName) |
| << " " << HAYAI_MAIN_FORMAT_FLAG("[OPTIONS]") << std::endl |
| << std::endl |
| |
| << " Runs the benchmarks for this project." << std::endl |
| << std::endl |
| |
| << "Benchmark selection options:" << std::endl |
| << " " << HAYAI_MAIN_FORMAT_FLAG("-l") << ", " |
| << HAYAI_MAIN_FORMAT_FLAG("--list") |
| << std::endl |
| << " List the names of all benchmarks instead of " |
| << "running them." << std::endl |
| << " " << HAYAI_MAIN_FORMAT_FLAG("-f") << ", " |
| << HAYAI_MAIN_FORMAT_FLAG("--filter") |
| << " <" << HAYAI_MAIN_FORMAT_ARGUMENT("pattern") << ">" |
| << std::endl |
| << " Run only the tests whose name matches one of the " |
| << "positive patterns but" << std::endl |
| << " none of the negative patterns. '?' matches any " |
| << "single character; '*'" << std::endl |
| << " matches any substring; ':' separates two " |
| << "patterns." |
| << std::endl |
| |
| << "Benchmark execution options:" << std::endl |
| << " " << HAYAI_MAIN_FORMAT_FLAG("-s") << ", " |
| << HAYAI_MAIN_FORMAT_FLAG("--shuffle") |
| << std::endl |
| << " Randomize benchmark execution order." |
| << std::endl |
| << std::endl |
| |
| << "Benchmark output options:" << std::endl |
| << " " << HAYAI_MAIN_FORMAT_FLAG("-o") << ", " |
| << HAYAI_MAIN_FORMAT_FLAG("--output") |
| << " <" << HAYAI_MAIN_FORMAT_ARGUMENT("format") << ">[:" |
| << HAYAI_MAIN_FORMAT_ARGUMENT("<path>") << "]" |
| << std::endl |
| << " Output results in a specific format. If no " |
| << "path is specified, the output" << std::endl |
| << " will be presented on stdout. Can be specified " |
| << "multiple times to get output" << std::endl |
| << " in different formats. The supported formats are:" |
| << std::endl |
| << std::endl |
| << " " << HAYAI_MAIN_FORMAT_ARGUMENT("console") |
| << std::endl |
| << " Standard console output." << std::endl |
| << " " << HAYAI_MAIN_FORMAT_ARGUMENT("json") |
| << std::endl |
| << " JSON." << std::endl |
| << " " << HAYAI_MAIN_FORMAT_ARGUMENT("junit") |
| << std::endl |
| << " JUnit-compatible XML (very restrictive.)" |
| << std::endl |
| << std::endl |
| << " If multiple output formats are provided without " |
| << "a path, only the last" << std::endl |
| << " provided format will be output to stdout." |
| << std::endl |
| << " " << HAYAI_MAIN_FORMAT_FLAG("--c") << ", " |
| << HAYAI_MAIN_FORMAT_FLAG("--color") << " (" |
| << ::hayai::Console::TextGreen << "yes" |
| << ::hayai::Console::TextDefault << "|" |
| << ::hayai::Console::TextGreen << "no" |
| << ::hayai::Console::TextDefault << ")" << std::endl |
| << " Enable colored output when available. Default " |
| << ::hayai::Console::TextGreen << "yes" |
| << ::hayai::Console::TextDefault << "." << std::endl |
| << std::endl |
| |
| << "Miscellaneous options:" << std::endl |
| << " " << HAYAI_MAIN_FORMAT_FLAG("-?") << ", " |
| << HAYAI_MAIN_FORMAT_FLAG("-h") << ", " |
| << HAYAI_MAIN_FORMAT_FLAG("--help") << std::endl |
| << " Show this help information." << std::endl |
| << std::endl |
| |
| << "hayai version: " << HAYAI_VERSION << std::endl |
| << "Clock implementation: " |
| << ::hayai::Clock::Description() |
| << std::endl; |
| } |
| }; |
| } |
| |
| |
| #undef HAYAI_MAIN_FORMAT_FLAG |
| #undef HAYAI_MAIN_FORMAT_ARGUMENT |
| #undef HAYAI_MAIN_FORMAT_ERROR |
| #undef HAYAI_MAIN_USAGE_ERROR |
| |
| #endif |
