tastytea
/
remwharead
1
0
Fork 0
Code Issues 1 Pull Requests Releases 37 Activity

Do the archiving parallel to the fetching.
continuous-integration/drone/push Build is passing Details 

Potentially halves the time needed.
This commit is contained in:
tastytea 2019-10-27 20:43:32 +01:00
parent 0a1396fc25
commit b23347f61a
Signed by: tastytea
GPG Key ID: CFC39497F1B26E07
1 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/cli/main.cpp
View File

@ -31,6 +31,7 @@
#include <list> #include <list>
#include <locale> #include <locale>
#include <string> #include <string>
#include <thread>
using namespace remwharead; using namespace remwharead;
using namespace remwharead_cli; using namespace remwharead_cli;
@ -76,6 +77,17 @@ int App::main(const std::vector<std::string> &args)
if (!_uri.empty()) if (!_uri.empty())
{ {
URI uri(_uri); URI uri(_uri);
archive_answer archive_data;
std::thread thread_archive;
if (_archive)
{
thread_archive = std::thread([&archive_data, &uri]
{
archive_data = uri.archive();
});
}
html_extract page = uri.get(); html_extract page = uri.get();
if (!page) if (!page)
{ {
@ -84,17 +96,17 @@ int App::main(const std::vector<std::string> &args)
return 3; return 3;
} }
archive_answer archive;
if (_archive) if (_archive)
{ {
archive = uri.archive(); thread_archive.join();
if (!archive) if (!archive_data)
{ {
cerr << "Error archiving URL: " << archive.error << endl; cerr << "Error archiving URL: " << archive_data.error << endl;
} }
} }
db.store({_uri, archive.uri, system_clock::now(), _tags, page.title,
page.description, page.fulltext}); db.store({_uri, archive_data.uri, system_clock::now(), _tags,
page.title, page.description, page.fulltext});
} }
ofstream file; ofstream file;