Replaced curlpp with POCO. Only HTTPS support fo far.
continuous-integration/drone/push Build is failing
Details
continuous-integration/drone/push Build is failing
Details
curlpp seems dead. 🙁
This commit is contained in:
parent
3895f0531a
commit
9d0dd589a7
27
.drone.yml
27
.drone.yml
|
@ -29,12 +29,9 @@ steps:
|
|||
commands:
|
||||
- rm /etc/apt/apt.conf.d/docker-clean
|
||||
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
|
||||
- echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release
|
||||
- echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
|
||||
- apt-get update -q
|
||||
- apt-get install -qy g++-6 cmake pkg-config
|
||||
- apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- apt-get install -qy -t sid libcurlpp-dev
|
||||
- apt-get install -qy libpoco-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8
|
||||
- rm -rf build && mkdir -p build && cd build
|
||||
- cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_TESTS=YES -DWITH_MOZILLA=YES ..
|
||||
|
@ -57,7 +54,6 @@ steps:
|
|||
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
|
||||
- apt-get update -q
|
||||
- echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release
|
||||
- echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
|
||||
- echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >> /etc/apt/sources.list.d/ubuntu-toolchain-r.list
|
||||
- apt-get install -qy gnupg
|
||||
- gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 0x60c317803a41ba51845e371a1e9377a2ba9ef27f
|
||||
|
@ -65,8 +61,7 @@ steps:
|
|||
- apt-get update -q
|
||||
- apt-get install -qy -t xenial g++-5
|
||||
- apt-get install -qy cmake pkg-config
|
||||
- apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- apt-get install -qy -t sid libcurlpp-dev
|
||||
- apt-get install -qy libpoco-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- rm -rf build && mkdir -p build && cd build
|
||||
- cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES ..
|
||||
- make VERBOSE=1
|
||||
|
@ -86,7 +81,6 @@ steps:
|
|||
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
|
||||
- apt-get update -q
|
||||
- echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release
|
||||
- echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
|
||||
- echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >> /etc/apt/sources.list.d/ubuntu-toolchain-r.list
|
||||
- apt-get install -qy gnupg
|
||||
- gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 0x60c317803a41ba51845e371a1e9377a2ba9ef27f
|
||||
|
@ -94,8 +88,7 @@ steps:
|
|||
- apt-get update -q
|
||||
- apt-get install -qy -t xenial g++-9
|
||||
- apt-get install -qy cmake pkg-config
|
||||
- apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- apt-get install -qy -t sid libcurlpp-dev
|
||||
- apt-get install -qy libpoco-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- rm -rf build && mkdir -p build && cd build
|
||||
- cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES ..
|
||||
- make VERBOSE=1
|
||||
|
@ -113,12 +106,9 @@ steps:
|
|||
commands:
|
||||
- rm /etc/apt/apt.conf.d/docker-clean
|
||||
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
|
||||
- echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release
|
||||
- echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
|
||||
- apt-get update -q
|
||||
- apt-get install -qy clang cmake pkg-config
|
||||
- apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- apt-get install -qy -t sid libcurlpp-dev
|
||||
- apt-get install -qy libpoco-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- rm -rf build && mkdir -p build && cd build
|
||||
- cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES ..
|
||||
- make VERBOSE=1
|
||||
|
@ -137,13 +127,11 @@ steps:
|
|||
- rm /etc/apt/apt.conf.d/docker-clean
|
||||
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
|
||||
- echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release
|
||||
- echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
|
||||
- echo "deb http://deb.debian.org/debian stretch-backports main" >> /etc/apt/sources.list.d/stretch.list
|
||||
- apt-get update -q
|
||||
- apt-get install -qy -t stretch-backports clang-6.0
|
||||
- apt-get install -qy cmake pkg-config
|
||||
- apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- apt-get install -qy -t sid libcurlpp-dev
|
||||
- apt-get install -qy libpoco-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- rm -rf build && mkdir -p build && cd build
|
||||
- cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES ..
|
||||
- make VERBOSE=1
|
||||
|
@ -198,12 +186,9 @@ steps:
|
|||
commands:
|
||||
- rm /etc/apt/apt.conf.d/docker-clean
|
||||
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
|
||||
- echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release
|
||||
- echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
|
||||
- apt-get update -q
|
||||
- apt-get install -qy g++-6 cmake pkg-config
|
||||
- apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- apt-get install -qy -t sid libcurlpp-dev
|
||||
- apt-get install -qy libpoco-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
|
||||
- apt-get install -qy build-essential file
|
||||
- rm -rf build && mkdir -p build && cd build
|
||||
- cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES -DMOZILLA_NMH_DIR="lib/mozilla/native-messaging-hosts" -DWITH_DEB=YES ..
|
||||
|
|
|
@ -60,7 +60,7 @@ only.
|
|||
* https://pkgconfig.freedesktop.org/wiki/[pkgconfig] (tested: 0.29)
|
||||
* https://github.com/badaix/popl[popl] (tested: 1.2)
|
||||
* http://repo.or.cz/w/libxdg-basedir.git[libxdg-basedir] (tested: 1.2)
|
||||
* http://www.curlpp.org/[curlpp] (tested: 0.8 / 0.7)
|
||||
* https://pocoproject.org/[POCO] (tested: 1.9 / 1.7)
|
||||
* http://vsqlite.virtuosic-bytes.com/[vsqlite++] (tested: 0.3)
|
||||
* http://www.icu-project.org/[ICU] (tested: 64.2 / 57.1)
|
||||
* Optional:
|
||||
|
@ -73,12 +73,9 @@ only.
|
|||
====
|
||||
[source,zsh]
|
||||
----
|
||||
echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release
|
||||
echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
|
||||
apt-get update
|
||||
apt-get install g++-6 cmake pkg-config libcurl4-openssl-dev libxdg-basedir-dev \
|
||||
apt-get install g++-6 cmake pkg-config libpoco-dev libxdg-basedir-dev \
|
||||
libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc
|
||||
apt-get install -t sid libcurlpp-dev
|
||||
# Inside the source directory:
|
||||
wget -O src/cli/popl.hpp https://raw.githubusercontent.com/badaix/popl/v1.2.0/include/popl.hpp
|
||||
export CXX="g++-6"
|
||||
|
|
|
@ -3,7 +3,7 @@ include(GNUInstallDirs)
|
|||
|
||||
find_dependency(PkgConfig REQUIRED)
|
||||
pkg_check_modules(libconfig++ REQUIRED IMPORTED_TARGET libconfig++)
|
||||
pkg_check_modules(curlpp REQUIRED IMPORTED_TARGET curlpp)
|
||||
find_depencency(Poco COMPONENTS Net NetSSL Util CONFIG REQUIRED)
|
||||
|
||||
find_file(vsqlitepp NAMES "sqlite/connection.hpp"
|
||||
PATHS ${CMAKE_INSTALL_INCLUDEDIR})
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
#define REMWHAREAD_URI_HPP
|
||||
|
||||
#include <string>
|
||||
#include <curlpp/Easy.hpp>
|
||||
|
||||
namespace remwharead
|
||||
{
|
||||
|
@ -38,6 +37,7 @@ namespace remwharead
|
|||
public:
|
||||
//! Construct object and set URL.
|
||||
explicit URI(const string &uri);
|
||||
~URI();
|
||||
|
||||
//! Download %URI and extract title, description and full text.
|
||||
const html_extract get();
|
||||
|
@ -48,8 +48,8 @@ namespace remwharead
|
|||
protected:
|
||||
string _uri;
|
||||
|
||||
//! Sets common curlpp options.
|
||||
void set_curlpp_options(curlpp::Easy &request);
|
||||
//! Make a HTTPS request.
|
||||
const string https_request(const string &uri) const;
|
||||
|
||||
//! Extract the title from an HTML page.
|
||||
const string extract_title(const string &html);
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
:doctype: manpage
|
||||
:Author: tastytea
|
||||
:Email: tastytea@tastytea.de
|
||||
:Date: 2019-07-25
|
||||
:Date: 2019-08-04
|
||||
:Revision: 0.0.0
|
||||
:man source: remwharead
|
||||
:man manual: General Commands Manual
|
||||
|
@ -141,16 +141,16 @@ finds all things that have either Mountain and Big, or Vegetable in them. You ca
|
|||
use _||_ instead of _OR_ and _&&_ instead of _AND_. Note that *--search-tags*
|
||||
only matches whole tags, Pill does not match Pillow.
|
||||
|
||||
== PROTOCOL SUPPORT
|
||||
// == PROTOCOL SUPPORT
|
||||
|
||||
Since *remwharead* is built on libcurl, it supports the same protocols. See
|
||||
*curl*(1), section _DESCRIPTION_. Titles and descriptions are currently only
|
||||
extracted when using HTTP(S).
|
||||
// Since *remwharead* is built on libcurl, it supports the same protocols. See
|
||||
// *curl*(1), section _DESCRIPTION_. Titles and descriptions are currently only
|
||||
// extracted when using HTTP(S).
|
||||
|
||||
== ENVIRONMENT
|
||||
// == ENVIRONMENT
|
||||
|
||||
You can use proxy servers with the same environment variables as curl. See
|
||||
*curl*(1), section _ENVIRONMENT_.
|
||||
// You can use proxy servers with the same environment variables as curl. See
|
||||
// *curl*(1), section _ENVIRONMENT_.
|
||||
|
||||
== FILES
|
||||
|
||||
|
@ -171,7 +171,7 @@ You can use proxy servers with the same environment variables as curl. See
|
|||
|
||||
== SEE ALSO
|
||||
|
||||
*curl*(1), *crontab*(1), *crontab*(5)
|
||||
*crontab*(1), *crontab*(5)
|
||||
|
||||
== REPORTING BUGS
|
||||
|
||||
|
|
|
@ -45,7 +45,7 @@ if (WITH_RPM)
|
|||
set(CPACK_RPM_PACKAGE_LICENSE "GPL-3")
|
||||
set(CPACK_RPM_PACKAGE_URL "https://schlomp.space/tastytea/${PROJECT_NAME}")
|
||||
set(CPACK_RPM_PACKAGE_REQUIRES
|
||||
"curlpp >= 0.7.3, libxdg-basedir, vsqlite++ >= 0.3.13, libicu-devel")
|
||||
"poco-netssl >= 1.6.1, libxdg-basedir, vsqlite++ >= 0.3.13, libicu-devel")
|
||||
set(CPACK_PACKAGE_FILE_NAME
|
||||
"${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-0.${CPACK_PACKAGE_ARCHITECTURE}")
|
||||
set(CPACK_SOURCE_PACKAGE_FILE_NAME
|
||||
|
|
|
@ -8,6 +8,6 @@ Name: ${name}
|
|||
Description: @PROJECT_DESCRIPTION@
|
||||
Version: @PROJECT_VERSION@
|
||||
Cflags: -I${includedir}
|
||||
Requires: curlpp
|
||||
Libs: -L${libdir} -l${name} -lvsqlitepp -lstdc++fs
|
||||
Requires.private: libxdg-basedir, icu-uc, icu-i18n
|
||||
Libs.private: -lPocoNet -lPocoNetSSL -lPocoUtil
|
||||
|
|
|
@ -2,8 +2,8 @@ include(GNUInstallDirs)
|
|||
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(libxdg-basedir REQUIRED IMPORTED_TARGET libxdg-basedir)
|
||||
pkg_check_modules(curlpp REQUIRED IMPORTED_TARGET curlpp)
|
||||
find_package(ICU COMPONENTS uc i18n REQUIRED)
|
||||
find_package(Poco COMPONENTS Net NetSSL Util CONFIG REQUIRED)
|
||||
|
||||
file(GLOB_RECURSE sources_lib *.cpp)
|
||||
file(GLOB_RECURSE headers_lib ../../include/*.hpp)
|
||||
|
@ -22,8 +22,9 @@ target_include_directories(${PROJECT_NAME}
|
|||
"$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>")
|
||||
|
||||
target_link_libraries(${PROJECT_NAME}
|
||||
PRIVATE PkgConfig::libxdg-basedir ICU::uc ICU::i18n
|
||||
PUBLIC PkgConfig::curlpp vsqlitepp stdc++fs)
|
||||
PRIVATE
|
||||
PkgConfig::libxdg-basedir ICU::uc ICU::i18n Poco::Net Poco::NetSSL Poco::Util
|
||||
PUBLIC vsqlitepp stdc++fs)
|
||||
|
||||
install(TARGETS ${PROJECT_NAME}
|
||||
EXPORT "${PROJECT_NAME}Targets"
|
||||
|
|
|
@ -20,7 +20,6 @@
|
|||
#include <algorithm>
|
||||
#include <utility>
|
||||
#include <locale>
|
||||
#include <curlpp/cURLpp.hpp>
|
||||
#include "version.hpp"
|
||||
#include "time.hpp"
|
||||
#include "export/adoc.hpp"
|
||||
|
|
|
@ -20,16 +20,17 @@
|
|||
#include <regex>
|
||||
#include <locale>
|
||||
#include <codecvt>
|
||||
#include <curlpp/cURLpp.hpp>
|
||||
#include <curlpp/Options.hpp>
|
||||
#include <curlpp/Exception.hpp>
|
||||
#include <curlpp/Infos.hpp>
|
||||
#include <version.hpp>
|
||||
#include "Poco/Net/HTTPSClientSession.h"
|
||||
#include "Poco/Net/HTTPRequest.h"
|
||||
#include "Poco/Net/HTTPResponse.h"
|
||||
#include "Poco/StreamCopier.h"
|
||||
#include "Poco/Path.h"
|
||||
#include "Poco/URI.h"
|
||||
#include "version.hpp"
|
||||
#include "uri.hpp"
|
||||
|
||||
namespace remwharead
|
||||
{
|
||||
namespace curlopts = curlpp::options;
|
||||
using std::uint64_t;
|
||||
using std::cerr;
|
||||
using std::endl;
|
||||
|
@ -39,10 +40,23 @@ namespace remwharead
|
|||
using std::smatch;
|
||||
using std::regex_constants::icase;
|
||||
using std::array;
|
||||
using std::istream;
|
||||
using Poco::Net::HTTPSClientSession;
|
||||
using Poco::Net::HTTPRequest;
|
||||
using Poco::Net::HTTPResponse;
|
||||
using Poco::Net::HTTPMessage;
|
||||
using Poco::StreamCopier;
|
||||
using Poco::Path;
|
||||
|
||||
URI::URI(const string &uri)
|
||||
:_uri(uri)
|
||||
{
|
||||
Poco::Net::initializeSSL();
|
||||
}
|
||||
|
||||
URI::~URI()
|
||||
{
|
||||
Poco::Net::uninitializeSSL();
|
||||
}
|
||||
|
||||
const html_extract URI::get()
|
||||
|
@ -50,17 +64,11 @@ namespace remwharead
|
|||
try
|
||||
{
|
||||
std::ostringstream oss;
|
||||
curlpp::Easy request;
|
||||
set_curlpp_options(request);
|
||||
request.setOpt<curlopts::Url>(_uri);
|
||||
request.setOpt<curlopts::WriteStream>(&oss);
|
||||
request.perform();
|
||||
|
||||
const string answer = oss.str();
|
||||
const string answer = https_request(_uri);
|
||||
if (answer.empty())
|
||||
{
|
||||
cerr << "Error: Could not download page. Response code: "
|
||||
<< curlpp::infos::ResponseCode::get(request) << endl;
|
||||
cerr << "Error: Could not download page.\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -74,18 +82,40 @@ namespace remwharead
|
|||
}
|
||||
catch (const std::exception &e)
|
||||
{
|
||||
// Prints something like "SSL Exception", nothing specific.
|
||||
cerr << "Error in " << __func__ << ": " << e.what() << endl;
|
||||
}
|
||||
|
||||
return { "", "", "" };
|
||||
}
|
||||
|
||||
void URI::set_curlpp_options(curlpp::Easy &request)
|
||||
const string URI::https_request(const string &uri) const
|
||||
{
|
||||
request.setOpt<curlopts::UserAgent>(string("remwharead/")
|
||||
+ global::version);
|
||||
request.setOpt<curlopts::HttpHeader>({ "Connection: close" });
|
||||
request.setOpt<curlopts::FollowLocation>(true);
|
||||
Poco::URI poco_uri(uri);
|
||||
string path = poco_uri.getPathAndQuery();
|
||||
if (path.empty())
|
||||
{
|
||||
path = "/";
|
||||
}
|
||||
HTTPSClientSession session(poco_uri.getHost(), poco_uri.getPort());
|
||||
HTTPRequest request(HTTPRequest::HTTP_GET, path,
|
||||
HTTPMessage::HTTP_1_1);
|
||||
HTTPResponse response;
|
||||
request.set("User-Agent", string("remwharead/") + global::version);
|
||||
|
||||
session.sendRequest(request);
|
||||
istream &rs = session.receiveResponse(response);
|
||||
if (response.getStatus() == HTTPResponse::HTTP_OK)
|
||||
{
|
||||
string answer;
|
||||
StreamCopier::copyToString(rs, answer);
|
||||
return answer;
|
||||
}
|
||||
else
|
||||
{
|
||||
cerr << response.getStatus() << " " << response.getReason() << endl;
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
const string URI::extract_title(const string &html)
|
||||
|
@ -481,31 +511,22 @@ namespace remwharead
|
|||
|
||||
try
|
||||
{
|
||||
std::ostringstream oss;
|
||||
curlpp::Easy request;
|
||||
set_curlpp_options(request);
|
||||
request.setOpt<curlopts::Url>("https://web.archive.org/save/"
|
||||
const string answer = https_request("https://web.archive.org/save/"
|
||||
+ _uri);
|
||||
request.setOpt<curlopts::WriteStream>(&oss);
|
||||
request.setOpt<curlopts::NoBody>(true); // Make HEAD request.
|
||||
request.setOpt<curlpp::options::Header>(true); // Keep headers.
|
||||
request.perform();
|
||||
|
||||
smatch match;
|
||||
const string answer = oss.str();
|
||||
if (regex_search(answer, match, regex("Content-Location: (.+)\r")))
|
||||
{
|
||||
return "https://web.archive.org" + match[1].str();
|
||||
}
|
||||
else
|
||||
{
|
||||
cerr << "Error: Could not archive page. HTTP status: "
|
||||
<< curlpp::infos::ResponseCode::get(request) << endl;
|
||||
cerr << "Error: Could not archive page.\n";
|
||||
}
|
||||
}
|
||||
catch (const std::exception &e)
|
||||
{
|
||||
cerr << "Error in " << __func__ << ": " << e.what() << endl;
|
||||
cerr << "Error in " << __func__ << ": " << e.what() << ".\n";
|
||||
}
|
||||
|
||||
return "";
|
||||
|
|
Loading…
Reference in New Issue