Moved threading into library.
continuous-integration/drone/push Build is passing Details

This commit is contained in:
tastytea 2019-08-09 00:14:26 +02:00
parent ebb1061e99
commit aa622e7cc9
Signed by: tastytea
GPG Key ID: CFC39497F1B26E07
5 changed files with 101 additions and 73 deletions

View File

@ -61,19 +61,36 @@ namespace remwharead
const bool is_re) const;
/*!
* @brief %Search in full text of database entries.
*
* Searches in tags, title, description and full text.
*
* @param expression %Search expression.
* @param is_re Is it a regular expression?
*
* @return List of matching Database::entry.
*
* @since 0.7.0
*/
const list<Database::entry> search_all(string expression,
const bool is_re) const;
* @brief %Search in full text of database entries.
*
* Searches in tags, title, description and full text.
*
* @param expression %Search expression.
* @param is_re Is it a regular expression?
*
* @return List of matching Database::entry.
*
* @since 0.7.0
*/
const list<Database::entry> search_all(string expression,
const bool is_re) const;
/*!
* @brief Spawn threads of search_all(), if it seems sensible.
*
* Figure out if threads could be useful and spawn a sensible amount of
* them.
*
* @param expression %Search expression.
* @param is_re Is it a regular expression?
*
* @return List of matching Database::entry.
*
* @since 0.7.2
*/
// TODO: Think of something more elegant.
const list<Database::entry> search_all_threaded(string expression,
const bool is_re) const;
private:
const list<Database::entry> _entries;

View File

@ -13,7 +13,7 @@ target_include_directories(${PROJECT_NAME}-cli
PRIVATE "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries(${PROJECT_NAME}-cli
PRIVATE ${PROJECT_NAME} pthread)
PRIVATE ${PROJECT_NAME})
# If no Poco*Config.cmake recipes are found, look for headers in standard dirs.
if(PocoUtil_FOUND)

View File

@ -19,9 +19,6 @@
#include <chrono>
#include <fstream>
#include <locale>
#include <thread>
#include <algorithm>
#include <iterator>
#include <list>
#include <cerrno>
#include "sqlite.hpp"
@ -40,8 +37,6 @@ using std::endl;
using std::string;
using std::chrono::system_clock;
using std::ofstream;
using std::thread;
using std::move;
using std::list;
int App::main(const std::vector<std::string> &args)
@ -125,59 +120,8 @@ int App::main(const std::vector<std::string> &args)
}
else if (!_search_all.empty())
{
const size_t len = entries.size();
constexpr size_t min_len = 100;
constexpr size_t min_per_thread = 50;
const size_t n_threads = thread::hardware_concurrency() / 3 + 1;
size_t cut_at = len;
if (len > min_len)
{ // If there are over `min_len` entries, use `n_threads` threads.
cut_at = len / n_threads;
// But don't use less than `min_per_thread` entries per thread.
if (cut_at < min_per_thread)
{
cut_at = min_per_thread;
}
}
list<list<Database::entry>> segments;
// Use threads if list is big.
while (entries.size() > cut_at)
{
list<Database::entry> segment;
auto it = entries.begin();
std::advance(it, cut_at);
// Move the first `cut_at` entries into `segments`.
segment.splice(segment.begin(), entries, entries.begin(), it);
segments.push_back(move(segment));
}
// Move rest of `entries` into `segments`.
segments.push_back(move(entries));
list<thread> threads;
for (auto &segment : segments)
{
thread t(
[&]
{
Search search(segment);
// Replace `segment` with `result`.
segment = search.search_all(_search_all, _regex);
});
threads.push_back(move(t));
}
for (thread &t : threads)
{
t.join();
// Move each of `segments` into `entries`.
entries.splice(entries.end(), segments.front());
segments.pop_front();
}
Search search(entries);
entries = search.search_all_threaded(_search_all, _regex);
}
switch (_format)

View File

@ -22,7 +22,7 @@ target_include_directories(${PROJECT_NAME}
"$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>")
target_link_libraries(${PROJECT_NAME}
PRIVATE PkgConfig::libxdg-basedir
PRIVATE PkgConfig::libxdg-basedir pthread
PUBLIC stdc++fs)
# If no Poco*Config.cmake recipes are found, look for headers in standard dirs.

View File

@ -18,6 +18,9 @@
#include <algorithm>
#include <locale>
#include <list>
#include <thread>
#include <utility>
#include <iterator>
#include <Poco/UTF8String.h>
#include "search.hpp"
@ -29,6 +32,8 @@ namespace remwharead
using std::smatch;
using std::find;
using std::find_if;
using std::thread;
using std::move;
Search::Search(const list<Database::entry> &entries)
:_entries(entries)
@ -195,4 +200,66 @@ namespace remwharead
return result;
}
const list<Database::entry> Search::search_all_threaded(
string expression, const bool is_re) const
{
list<Database::entry> entries = _entries;
const size_t len = entries.size();
constexpr size_t min_len = 100;
constexpr size_t min_per_thread = 50;
const size_t n_threads = thread::hardware_concurrency() / 3 + 1;
size_t cut_at = len;
if (len > min_len)
{ // If there are over `min_len` entries, use `n_threads` threads.
cut_at = len / n_threads;
// But don't use less than `min_per_thread` entries per thread.
if (cut_at < min_per_thread)
{
cut_at = min_per_thread;
}
}
list<list<Database::entry>> segments;
// Use threads if list is big.
while (entries.size() > cut_at)
{
list<Database::entry> segment;
auto it = entries.begin();
std::advance(it, cut_at);
// Move the first `cut_at` entries into `segments`.
segment.splice(segment.begin(), entries, entries.begin(), it);
segments.push_back(move(segment));
}
// Move rest of `entries` into `segments`.
segments.push_back(move(entries));
list<thread> threads;
for (auto &segment : segments)
{
thread t(
[&]
{
Search search(segment);
// Replace `segment` with `result`.
segment = search.search_all(expression, is_re);
});
threads.push_back(move(t));
}
for (thread &t : threads)
{
t.join();
// Move each of `segments` into `entries`.
entries.splice(entries.end(), segments.front());
segments.pop_front();
}
return entries;
}
}