feat(test): add regex filtering for suites and cases
This commit is contained in:
parent
723ade84ea
commit
a46f36aefd
GPG key ID:
8C30176798F1A6BA
2 changed files with 250 additions and 56 deletions
|
|
@ -3,22 +3,48 @@
|
|||
using namespace ::lt::test;
|
||||
using namespace ::lt::test::details;
|
||||
|
||||
void parse_option(std::string_view option, Registry::Options &options)
|
||||
void parse_option(std::string_view argument, Registry::Options &options)
|
||||
{
|
||||
if (option == "--stop-on-fail")
|
||||
constexpr auto case_str = std::string_view { "--case=" };
|
||||
constexpr auto suite_str = std::string_view { "--suite=" };
|
||||
|
||||
if (argument == "--stop-on-fail")
|
||||
{
|
||||
options.stop_on_fail = true;
|
||||
return;
|
||||
}
|
||||
|
||||
throw std::invalid_argument { std::format("Invalid argument: {}", option) };
|
||||
if (argument.starts_with("--mode=") && argument.substr(7ul) == "stats")
|
||||
{
|
||||
options.execution_policy = Registry::ExecutionPolicy::stats;
|
||||
return;
|
||||
}
|
||||
|
||||
if (argument.starts_with(suite_str) && argument.length() > suite_str.size())
|
||||
{
|
||||
options.suite_regex = argument.substr(suite_str.length());
|
||||
std::println("SUITE REGEX: {}", options.suite_regex);
|
||||
return;
|
||||
}
|
||||
|
||||
if (argument.starts_with(case_str) && argument.length() > case_str.size())
|
||||
{
|
||||
options.case_regex = argument.substr(case_str.length());
|
||||
std::println("CASE REGEX: {}", options.case_regex);
|
||||
return;
|
||||
}
|
||||
|
||||
throw std::invalid_argument { std::format("Invalid argument: {}", argument) };
|
||||
}
|
||||
|
||||
void print_help()
|
||||
{
|
||||
std::println("Options: ");
|
||||
std::println("--stop-on-fail --> Stops executing the remaining tests on first failure");
|
||||
std::println("--stats --> Print statistics about the tests without running any");
|
||||
std::println("--suite --> Regex for running specific suite(s)");
|
||||
std::println("--case --> Regex for running specific test(s)");
|
||||
std::println("--mode=stats --> Executes tests with an alternative policy");
|
||||
std::println("\t---> stats: Print statistics about the tests without running any");
|
||||
std::println("--help | -h --> ~You just used it! :D");
|
||||
}
|
||||
|
||||
|
|
@ -48,12 +74,24 @@ try
|
|||
{
|
||||
parse_option(argument, options);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw std::invalid_argument { std::format("Invalid argument: {}", argument) };
|
||||
}
|
||||
}
|
||||
|
||||
return Registry::run_all(options);
|
||||
return static_cast<int32_t>(Registry::run_all(options));
|
||||
}
|
||||
catch (const std::exception &exp)
|
||||
{
|
||||
std::cout << "Terminated after uncaught exception:\n";
|
||||
std::cout << "exception.what: " << exp.what();
|
||||
std::println("Terminated due to uncaught exception:");
|
||||
std::println("\twhat: {}", exp.what());
|
||||
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::println("Terminated due to uncaught non-std exception!");
|
||||
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#pragma once
|
||||
|
||||
#include <concepts>
|
||||
#include <format>
|
||||
#include <regex>
|
||||
#include <test/expects.hpp>
|
||||
|
||||
namespace lt::test {
|
||||
|
|
@ -10,9 +12,21 @@ namespace details {
|
|||
class Registry
|
||||
{
|
||||
public:
|
||||
enum class ExecutionPolicy : uint8_t
|
||||
{
|
||||
normal,
|
||||
stats,
|
||||
};
|
||||
|
||||
struct Options
|
||||
{
|
||||
bool stop_on_fail = false;
|
||||
|
||||
ExecutionPolicy execution_policy = ExecutionPolicy::normal;
|
||||
|
||||
std::string suite_regex;
|
||||
|
||||
std::string case_regex;
|
||||
};
|
||||
|
||||
using FuzzFunction = int32_t (*)(const uint8_t *, size_t);
|
||||
|
|
@ -20,7 +34,7 @@ public:
|
|||
|
||||
static void register_suite(SuiteFunction suite)
|
||||
{
|
||||
instance().m_suites.emplace_back(suite);
|
||||
instance().m_suites.emplace_back(suite, "");
|
||||
}
|
||||
|
||||
static void register_fuzz_harness(FuzzFunction suite)
|
||||
|
|
@ -35,37 +49,10 @@ public:
|
|||
instance().m_fuzz_harness = suite;
|
||||
}
|
||||
|
||||
static auto run_all(Options options) -> int32_t
|
||||
static auto run_all(Options options) -> uint32_t
|
||||
{
|
||||
instance().m_options = options;
|
||||
instance().print_options();
|
||||
|
||||
for (auto &test : instance().m_suites)
|
||||
{
|
||||
try
|
||||
{
|
||||
test();
|
||||
}
|
||||
catch (const std::exception &exp)
|
||||
{
|
||||
if (options.stop_on_fail)
|
||||
{
|
||||
std::println("Quitting due to options.stop_on_fail == true");
|
||||
break;
|
||||
}
|
||||
|
||||
std::println("Uncaught exception when running suite:");
|
||||
std::println("\twhat: {}", exp.what());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "Ran " << instance().m_failed_count + instance().m_pasesed_count << " tests:\n"
|
||||
<< "\tpassed: " << instance().m_pasesed_count << '\n'
|
||||
<< "\tfailed: " << instance().m_failed_count << '\n';
|
||||
std::cout << "________________________________________________________________\n\n\n";
|
||||
|
||||
return instance().m_failed_count;
|
||||
instance().m_options = std::move(options);
|
||||
return instance().run_all_impl();
|
||||
}
|
||||
|
||||
static auto process_fuzz_input(const uint8_t *data, size_t size) -> int32_t
|
||||
|
|
@ -80,25 +67,139 @@ public:
|
|||
return instance().m_fuzz_harness(data, size);
|
||||
}
|
||||
|
||||
static void increment_passed_count()
|
||||
static void set_last_suite_name(const char *name)
|
||||
{
|
||||
++instance().m_pasesed_count;
|
||||
instance().m_suites.back().second = name;
|
||||
}
|
||||
|
||||
static void increment_failed_count()
|
||||
static void increment_total_suite_count()
|
||||
{
|
||||
++instance().m_failed_count;
|
||||
++instance().m_total_suite_count;
|
||||
}
|
||||
|
||||
static auto should_return_on_failure() -> bool
|
||||
static void increment_matched_suite_count()
|
||||
{
|
||||
++instance().m_matched_suite_count;
|
||||
}
|
||||
|
||||
static void increment_skipped_suite_count()
|
||||
{
|
||||
++instance().m_skipped_suite_count;
|
||||
}
|
||||
|
||||
static void increment_passed_suite_count()
|
||||
{
|
||||
++instance().m_passed_suite_count;
|
||||
}
|
||||
|
||||
static void increment_failed_suite_count()
|
||||
{
|
||||
++instance().m_failed_suite_count;
|
||||
}
|
||||
|
||||
static void increment_total_case_count()
|
||||
{
|
||||
++instance().m_total_case_count;
|
||||
}
|
||||
|
||||
static void increment_passed_case_count()
|
||||
{
|
||||
++instance().m_passed_case_count;
|
||||
}
|
||||
|
||||
static void increment_failed_case_count()
|
||||
{
|
||||
++instance().m_failed_case_count;
|
||||
}
|
||||
|
||||
[[nodiscard]] static auto should_return_on_failure() -> bool
|
||||
{
|
||||
return instance().m_options.stop_on_fail;
|
||||
}
|
||||
|
||||
[[nodiscard]] static auto get_options() -> const Options &
|
||||
{
|
||||
return instance().m_options;
|
||||
}
|
||||
|
||||
[[nodiscard]] static auto get_case_regex() -> const std::regex &
|
||||
{
|
||||
return instance().m_case_regex;
|
||||
}
|
||||
|
||||
private:
|
||||
auto run_all_impl() -> uint32_t
|
||||
{
|
||||
print_options();
|
||||
m_case_regex = std::regex(m_options.case_regex);
|
||||
|
||||
const auto regex = std::regex(m_options.suite_regex);
|
||||
for (auto &[suite, name] : m_suites)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (std::regex_search(name, regex))
|
||||
{
|
||||
suite();
|
||||
increment_matched_suite_count();
|
||||
}
|
||||
else
|
||||
{
|
||||
increment_skipped_suite_count();
|
||||
}
|
||||
|
||||
increment_total_suite_count();
|
||||
}
|
||||
catch (const std::exception &exp)
|
||||
{
|
||||
if (m_options.stop_on_fail)
|
||||
{
|
||||
std::println("Quitting due to options.stop_on_fail == true");
|
||||
break;
|
||||
}
|
||||
|
||||
std::println("Uncaught exception when running suite:");
|
||||
std::println("\twhat: {}", exp.what());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
switch (m_options.execution_policy)
|
||||
{
|
||||
case ExecutionPolicy::normal:
|
||||
{
|
||||
std::cout << "[-------STATS------]\n"
|
||||
<< "suites:\n"
|
||||
<< "\ttotal: " << m_total_suite_count << '\n'
|
||||
<< "\tpassed: " << m_passed_suite_count << '\n'
|
||||
<< "\tfailed: " << m_failed_suite_count << '\n'
|
||||
<< "\tmatched: " << m_matched_suite_count << '\n'
|
||||
<< "\tskipped: " << m_skipped_suite_count << '\n'
|
||||
<< "tests:\n"
|
||||
<< "\ttotal: " << m_total_case_count << '\n'
|
||||
<< "\tpassed: " << m_passed_case_count << '\n'
|
||||
<< "\tfailed: " << m_failed_case_count << '\n'
|
||||
<< "\tmatched: " << m_matched_case_count << '\n'
|
||||
<< "\tskipped: " << m_skipped_case_count << '\n';
|
||||
|
||||
std::cout << "________________________________________________________________\n\n\n";
|
||||
|
||||
return m_failed_case_count;
|
||||
}
|
||||
case ExecutionPolicy::stats:
|
||||
{
|
||||
std::println("[-------STATS------]");
|
||||
std::println("Total suite count: {}", m_total_suite_count);
|
||||
std::println("Total test count: {}", m_total_case_count);
|
||||
std::println("________________________________________________________________");
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void print_options()
|
||||
{
|
||||
std::println("stop-on-failure: {}", m_options.stop_on_fail);
|
||||
std::println("stop-on-failure: {}", m_options.stop_on_fail);
|
||||
}
|
||||
|
||||
Registry()
|
||||
|
|
@ -114,25 +215,69 @@ private:
|
|||
|
||||
Options m_options {};
|
||||
|
||||
std::vector<SuiteFunction> m_suites;
|
||||
std::vector<std::pair<SuiteFunction, const char *>> m_suites;
|
||||
|
||||
FuzzFunction m_fuzz_harness {};
|
||||
|
||||
int32_t m_pasesed_count {};
|
||||
int32_t m_failed_count {};
|
||||
uint32_t m_total_case_count {};
|
||||
|
||||
uint32_t m_passed_case_count {};
|
||||
|
||||
uint32_t m_failed_case_count {};
|
||||
|
||||
uint32_t m_matched_case_count {};
|
||||
|
||||
uint32_t m_skipped_case_count {};
|
||||
|
||||
uint32_t m_total_suite_count {};
|
||||
|
||||
uint32_t m_passed_suite_count {};
|
||||
|
||||
uint32_t m_failed_suite_count {};
|
||||
|
||||
uint32_t m_matched_suite_count {};
|
||||
|
||||
uint32_t m_skipped_suite_count {};
|
||||
|
||||
std::regex m_case_regex;
|
||||
};
|
||||
|
||||
|
||||
} // namespace details
|
||||
|
||||
struct Case
|
||||
class Case
|
||||
{
|
||||
public:
|
||||
Case(std::string_view name): m_name(name)
|
||||
{
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(misc-unconventional-assign-operator)
|
||||
auto operator=(std::invocable auto test) -> void
|
||||
{
|
||||
std::cout << "[Running-----------] --> ";
|
||||
std::cout << name << '\n';
|
||||
using details::Registry;
|
||||
using enum Registry::ExecutionPolicy;
|
||||
|
||||
switch (Registry::get_options().execution_policy)
|
||||
{
|
||||
case normal: run_normal(std::move(test)); break;
|
||||
case stats: Registry::increment_total_case_count(); break;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void run_normal(std::invocable auto test)
|
||||
{
|
||||
using details::Registry;
|
||||
|
||||
// NOLINTNEXTLINE
|
||||
if (!std::regex_search(m_name.data(), Registry::get_case_regex()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
std::cout << "[Running-----------] --> ";
|
||||
std::cout << m_name << '\n';
|
||||
try
|
||||
{
|
||||
test();
|
||||
|
|
@ -141,30 +286,34 @@ struct Case
|
|||
{
|
||||
std::cout << exp.what() << "\n";
|
||||
std::cout << "[-----------FAIL !!]" << "\n\n";
|
||||
details::Registry::increment_failed_count();
|
||||
Registry::increment_failed_case_count();
|
||||
|
||||
if (details::Registry::should_return_on_failure())
|
||||
if (Registry::should_return_on_failure())
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
details::Registry::increment_passed_count();
|
||||
Registry::increment_passed_case_count();
|
||||
Registry::increment_total_case_count();
|
||||
std::cout << "[--------SUCCESS :D]" << "\n\n";
|
||||
}
|
||||
|
||||
std::string_view name;
|
||||
std::string_view m_name;
|
||||
};
|
||||
|
||||
struct TestSuite
|
||||
{
|
||||
template<class TSuite>
|
||||
constexpr TestSuite(TSuite body)
|
||||
template<typename TSuite>
|
||||
TestSuite(TSuite body)
|
||||
{
|
||||
std::println("CONSTRUCTOR");
|
||||
#ifndef LIGHT_SKIP_TESTS
|
||||
details::Registry::register_suite(+body);
|
||||
#endif
|
||||
}
|
||||
|
||||
TestSuite() = default;
|
||||
};
|
||||
|
||||
struct TestFuzzHarness
|
||||
|
|
@ -179,6 +328,13 @@ struct TestFuzzHarness
|
|||
};
|
||||
|
||||
using Suite = const TestSuite;
|
||||
|
||||
using FuzzHarness = const TestFuzzHarness;
|
||||
|
||||
} // namespace lt::test
|
||||
|
||||
inline auto operator""_suite(const char *name, size_t size) -> lt::test::TestSuite
|
||||
{
|
||||
lt::test::details::Registry::set_last_suite_name(name);
|
||||
return {};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue