Moved threading into library.
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/push Build is passing
Details
This commit is contained in:
parent
ebb1061e99
commit
aa622e7cc9
|
@ -61,19 +61,36 @@ namespace remwharead
|
||||||
const bool is_re) const;
|
const bool is_re) const;
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
* @brief %Search in full text of database entries.
|
* @brief %Search in full text of database entries.
|
||||||
*
|
*
|
||||||
* Searches in tags, title, description and full text.
|
* Searches in tags, title, description and full text.
|
||||||
*
|
*
|
||||||
* @param expression %Search expression.
|
* @param expression %Search expression.
|
||||||
* @param is_re Is it a regular expression?
|
* @param is_re Is it a regular expression?
|
||||||
*
|
*
|
||||||
* @return List of matching Database::entry.
|
* @return List of matching Database::entry.
|
||||||
*
|
*
|
||||||
* @since 0.7.0
|
* @since 0.7.0
|
||||||
*/
|
*/
|
||||||
const list<Database::entry> search_all(string expression,
|
const list<Database::entry> search_all(string expression,
|
||||||
const bool is_re) const;
|
const bool is_re) const;
|
||||||
|
|
||||||
|
/*!
|
||||||
|
* @brief Spawn threads of search_all(), if it seems sensible.
|
||||||
|
*
|
||||||
|
* Figure out if threads could be useful and spawn a sensible amount of
|
||||||
|
* them.
|
||||||
|
*
|
||||||
|
* @param expression %Search expression.
|
||||||
|
* @param is_re Is it a regular expression?
|
||||||
|
*
|
||||||
|
* @return List of matching Database::entry.
|
||||||
|
*
|
||||||
|
* @since 0.7.2
|
||||||
|
*/
|
||||||
|
// TODO: Think of something more elegant.
|
||||||
|
const list<Database::entry> search_all_threaded(string expression,
|
||||||
|
const bool is_re) const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
const list<Database::entry> _entries;
|
const list<Database::entry> _entries;
|
||||||
|
|
|
@ -13,7 +13,7 @@ target_include_directories(${PROJECT_NAME}-cli
|
||||||
PRIVATE "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}")
|
PRIVATE "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}")
|
||||||
|
|
||||||
target_link_libraries(${PROJECT_NAME}-cli
|
target_link_libraries(${PROJECT_NAME}-cli
|
||||||
PRIVATE ${PROJECT_NAME} pthread)
|
PRIVATE ${PROJECT_NAME})
|
||||||
|
|
||||||
# If no Poco*Config.cmake recipes are found, look for headers in standard dirs.
|
# If no Poco*Config.cmake recipes are found, look for headers in standard dirs.
|
||||||
if(PocoUtil_FOUND)
|
if(PocoUtil_FOUND)
|
||||||
|
|
|
@ -19,9 +19,6 @@
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
#include <locale>
|
#include <locale>
|
||||||
#include <thread>
|
|
||||||
#include <algorithm>
|
|
||||||
#include <iterator>
|
|
||||||
#include <list>
|
#include <list>
|
||||||
#include <cerrno>
|
#include <cerrno>
|
||||||
#include "sqlite.hpp"
|
#include "sqlite.hpp"
|
||||||
|
@ -40,8 +37,6 @@ using std::endl;
|
||||||
using std::string;
|
using std::string;
|
||||||
using std::chrono::system_clock;
|
using std::chrono::system_clock;
|
||||||
using std::ofstream;
|
using std::ofstream;
|
||||||
using std::thread;
|
|
||||||
using std::move;
|
|
||||||
using std::list;
|
using std::list;
|
||||||
|
|
||||||
int App::main(const std::vector<std::string> &args)
|
int App::main(const std::vector<std::string> &args)
|
||||||
|
@ -125,59 +120,8 @@ int App::main(const std::vector<std::string> &args)
|
||||||
}
|
}
|
||||||
else if (!_search_all.empty())
|
else if (!_search_all.empty())
|
||||||
{
|
{
|
||||||
const size_t len = entries.size();
|
Search search(entries);
|
||||||
constexpr size_t min_len = 100;
|
entries = search.search_all_threaded(_search_all, _regex);
|
||||||
constexpr size_t min_per_thread = 50;
|
|
||||||
const size_t n_threads = thread::hardware_concurrency() / 3 + 1;
|
|
||||||
size_t cut_at = len;
|
|
||||||
if (len > min_len)
|
|
||||||
{ // If there are over `min_len` entries, use `n_threads` threads.
|
|
||||||
cut_at = len / n_threads;
|
|
||||||
|
|
||||||
// But don't use less than `min_per_thread` entries per thread.
|
|
||||||
if (cut_at < min_per_thread)
|
|
||||||
{
|
|
||||||
cut_at = min_per_thread;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
list<list<Database::entry>> segments;
|
|
||||||
|
|
||||||
// Use threads if list is big.
|
|
||||||
while (entries.size() > cut_at)
|
|
||||||
{
|
|
||||||
list<Database::entry> segment;
|
|
||||||
|
|
||||||
auto it = entries.begin();
|
|
||||||
std::advance(it, cut_at);
|
|
||||||
|
|
||||||
// Move the first `cut_at` entries into `segments`.
|
|
||||||
segment.splice(segment.begin(), entries, entries.begin(), it);
|
|
||||||
segments.push_back(move(segment));
|
|
||||||
}
|
|
||||||
// Move rest of `entries` into `segments`.
|
|
||||||
segments.push_back(move(entries));
|
|
||||||
|
|
||||||
list<thread> threads;
|
|
||||||
for (auto &segment : segments)
|
|
||||||
{
|
|
||||||
thread t(
|
|
||||||
[&]
|
|
||||||
{
|
|
||||||
Search search(segment);
|
|
||||||
// Replace `segment` with `result`.
|
|
||||||
segment = search.search_all(_search_all, _regex);
|
|
||||||
});
|
|
||||||
threads.push_back(move(t));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (thread &t : threads)
|
|
||||||
{
|
|
||||||
t.join();
|
|
||||||
// Move each of `segments` into `entries`.
|
|
||||||
entries.splice(entries.end(), segments.front());
|
|
||||||
segments.pop_front();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
switch (_format)
|
switch (_format)
|
||||||
|
|
|
@ -22,7 +22,7 @@ target_include_directories(${PROJECT_NAME}
|
||||||
"$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>")
|
"$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>")
|
||||||
|
|
||||||
target_link_libraries(${PROJECT_NAME}
|
target_link_libraries(${PROJECT_NAME}
|
||||||
PRIVATE PkgConfig::libxdg-basedir
|
PRIVATE PkgConfig::libxdg-basedir pthread
|
||||||
PUBLIC stdc++fs)
|
PUBLIC stdc++fs)
|
||||||
|
|
||||||
# If no Poco*Config.cmake recipes are found, look for headers in standard dirs.
|
# If no Poco*Config.cmake recipes are found, look for headers in standard dirs.
|
||||||
|
|
|
@ -18,6 +18,9 @@
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <locale>
|
#include <locale>
|
||||||
#include <list>
|
#include <list>
|
||||||
|
#include <thread>
|
||||||
|
#include <utility>
|
||||||
|
#include <iterator>
|
||||||
#include <Poco/UTF8String.h>
|
#include <Poco/UTF8String.h>
|
||||||
#include "search.hpp"
|
#include "search.hpp"
|
||||||
|
|
||||||
|
@ -29,6 +32,8 @@ namespace remwharead
|
||||||
using std::smatch;
|
using std::smatch;
|
||||||
using std::find;
|
using std::find;
|
||||||
using std::find_if;
|
using std::find_if;
|
||||||
|
using std::thread;
|
||||||
|
using std::move;
|
||||||
|
|
||||||
Search::Search(const list<Database::entry> &entries)
|
Search::Search(const list<Database::entry> &entries)
|
||||||
:_entries(entries)
|
:_entries(entries)
|
||||||
|
@ -195,4 +200,66 @@ namespace remwharead
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const list<Database::entry> Search::search_all_threaded(
|
||||||
|
string expression, const bool is_re) const
|
||||||
|
{
|
||||||
|
list<Database::entry> entries = _entries;
|
||||||
|
|
||||||
|
const size_t len = entries.size();
|
||||||
|
constexpr size_t min_len = 100;
|
||||||
|
constexpr size_t min_per_thread = 50;
|
||||||
|
const size_t n_threads = thread::hardware_concurrency() / 3 + 1;
|
||||||
|
size_t cut_at = len;
|
||||||
|
if (len > min_len)
|
||||||
|
{ // If there are over `min_len` entries, use `n_threads` threads.
|
||||||
|
cut_at = len / n_threads;
|
||||||
|
|
||||||
|
// But don't use less than `min_per_thread` entries per thread.
|
||||||
|
if (cut_at < min_per_thread)
|
||||||
|
{
|
||||||
|
cut_at = min_per_thread;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
list<list<Database::entry>> segments;
|
||||||
|
|
||||||
|
// Use threads if list is big.
|
||||||
|
while (entries.size() > cut_at)
|
||||||
|
{
|
||||||
|
list<Database::entry> segment;
|
||||||
|
|
||||||
|
auto it = entries.begin();
|
||||||
|
std::advance(it, cut_at);
|
||||||
|
|
||||||
|
// Move the first `cut_at` entries into `segments`.
|
||||||
|
segment.splice(segment.begin(), entries, entries.begin(), it);
|
||||||
|
segments.push_back(move(segment));
|
||||||
|
}
|
||||||
|
// Move rest of `entries` into `segments`.
|
||||||
|
segments.push_back(move(entries));
|
||||||
|
|
||||||
|
list<thread> threads;
|
||||||
|
for (auto &segment : segments)
|
||||||
|
{
|
||||||
|
thread t(
|
||||||
|
[&]
|
||||||
|
{
|
||||||
|
Search search(segment);
|
||||||
|
// Replace `segment` with `result`.
|
||||||
|
segment = search.search_all(expression, is_re);
|
||||||
|
});
|
||||||
|
threads.push_back(move(t));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (thread &t : threads)
|
||||||
|
{
|
||||||
|
t.join();
|
||||||
|
// Move each of `segments` into `entries`.
|
||||||
|
entries.splice(entries.end(), segments.front());
|
||||||
|
segments.pop_front();
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue