From fc0aa02bc9b5f7a2e8047a48d5fa3cab57ab0987 Mon Sep 17 00:00:00 2001 From: tastytea Date: Wed, 26 May 2021 17:23:53 +0200 Subject: [PATCH] Use threads if more than one input file is searched. Use 75% of the available threads (rounded up). Closes: https://schlomp.space/tastytea/epubgrep/issues/4 --- CMakeLists.txt | 1 + src/CMakeLists.txt | 4 +- src/main.cpp | 199 ++++++++++++++++++++++++++++++--------------- 3 files changed, 137 insertions(+), 67 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a594b34..6f6bed9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,7 @@ if(NOT termcolor_FOUND) message(FATAL_ERROR "Termcolor was not found.") endif() endif() +find_package(Threads REQUIRED) add_subdirectory(src) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 668fcf1..df6a810 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -22,7 +22,9 @@ target_link_libraries(${PROJECT_NAME}_lib Boost::regex std::filesystem fmt::fmt - termcolor::termcolor) + termcolor::termcolor + Threads::Threads + m) if(${CMAKE_VERSION} VERSION_LESS 3.17) target_link_libraries(${PROJECT_NAME}_lib diff --git a/src/main.cpp b/src/main.cpp index 91f7f40..8a6e8cd 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -21,16 +21,21 @@ #include #include #include +#include +#include // For compatibility with fmt 4. #include #include +#include #include #include #include +#include #include #include #include -#include +#include +#include #include int main(int argc, char *argv[]) @@ -39,8 +44,11 @@ int main(int argc, char *argv[]) using namespace epubgrep; using boost::locale::translate; + using fmt::format; using std::cerr; using std::cout; + using std::string; + using std::vector; // locale_generator("").name.c_str() returns "*" instead of "". That's why // the global C locale isn't changed. So we have to set it additionally. @@ -74,84 +82,143 @@ int main(int argc, char *argv[]) { cout << "NO INPUT FILE\n"; // TODO: Read data from stdin. + return EXIT_FAILURE; } - else - { - search::options opts; - if (vm.count("basic-regexp") > 0) - { - opts.regex = search::regex_kind::basic; - } - if (vm.count("extended-regexp") > 0) - { - opts.regex = search::regex_kind::extended; - } - if (vm.count("perl-regexp") > 0) - { - opts.regex = search::regex_kind::perl; - } - if (vm.count("grep") > 0) - { - opts.grep_like = true; - } - if (vm.count("ignore-case") > 0) - { - opts.ignore_case = true; - } - if (vm.count("raw") > 0) - { - opts.raw = true; - } - opts.context = vm["context"].as(); - for (const auto &filepath : - vm["input-file"].as>()) + int return_code{EXIT_SUCCESS}; + + search::options opts; + if (vm.count("basic-regexp") > 0) + { + opts.regex = search::regex_kind::basic; + } + if (vm.count("extended-regexp") > 0) + { + opts.regex = search::regex_kind::extended; + } + if (vm.count("perl-regexp") > 0) + { + opts.regex = search::regex_kind::perl; + } + if (vm.count("grep") > 0) + { + opts.grep_like = true; + } + if (vm.count("ignore-case") > 0) + { + opts.ignore_case = true; + } + if (vm.count("raw") > 0) + { + opts.raw = true; + } + opts.context = vm["context"].as(); + + vector> matches_all; + vector> futurepool; + + auto search_file{ + [&vm, &matches_all, &opts](std::string_view filepath) { - for (const auto ®ex : - vm["regexp"].as>()) + for (const auto ®ex : vm["regexp"].as>()) { try { - for (const auto &match : - search::search(filepath, regex, opts)) - { - if (vm.count("no-filename") == 0) - { - cout << match.filepath; - } - if (!match.headline.empty()) - { - if (vm.count("no-filename") == 0) - { - cout << ", "; - } - cout << match.headline; - } - if (!match.page.empty()) - { - cout << ", page " << match.page; - } - cout << ": " << match.context.first; - if (vm.count("nocolor") == 0) - { - cout << termcolor::bright_magenta << match.text - << termcolor::reset; - } - else - { - cout << match.text; - } - cout << match.context.second << '\n'; - } + matches_all.emplace_back( + search::search(filepath, regex, opts)); } catch (const std::exception &e) { // Unknown errors. cerr << '\n' << translate("ERROR: ") << e.what() << '\n'; - cerr << translate("Error while searching.") << '\n'; - // NOTE: Maybe we should continue with the next regex/file? + cerr << format(translate("Error while searching {0:s}.") + .str() + .data(), + filepath) + << '\n'; return EXIT_FAILURE; } } + + return EXIT_SUCCESS; + }}; + + auto futures_cleanup{ + [&futurepool, &return_code](const bool wait = false) + { + using namespace std::chrono_literals; + + for (auto it{futurepool.begin()}; it != futurepool.end();) + { + if (!wait && it->wait_for(100ms) != std::future_status::ready) + { + ++it; + continue; + } + + if (int ret{}; (ret = it->get()) != EXIT_SUCCESS) + { + cerr << "ERROR\n"; + return_code = ret; + } + futurepool.erase(it); + cerr << "ERASED\n"; + } + + return EXIT_SUCCESS; + }}; + + const auto max_threads{ + [] + { + auto n{static_cast(std::thread::hardware_concurrency())}; + return static_cast(std::ceil(n / 2 + n / 4)); + }()}; + + for (const auto &filepath : vm["input-file"].as>()) + { + if (futurepool.size() >= max_threads) + { + futures_cleanup(); + } + futurepool.emplace_back( + std::async(std::launch::async, search_file, filepath)); + cerr << "EMPLACED\n"; + } + futures_cleanup(true); + + for (const auto &matches_file : matches_all) + { + for (const auto &match : matches_file) + { + if (vm.count("no-filename") == 0) + { + cout << match.filepath; + } + if (!match.headline.empty()) + { + if (vm.count("no-filename") == 0) + { + cout << ", "; + } + cout << match.headline; + } + if (!match.page.empty()) + { + cout << ", page " << match.page; + } + cout << ": " << match.context.first; + if (vm.count("nocolor") == 0) + { + cout << termcolor::bright_magenta << match.text + << termcolor::reset; + } + else + { + cout << match.text; + } + cout << match.context.second << '\n'; } } + + return return_code; }