2019-07-27 22:46:58 +02:00
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
< html xmlns = "http://www.w3.org/1999/xhtml" >
< head >
< meta http-equiv = "Content-Type" content = "text/xhtml;charset=UTF-8" / >
< meta http-equiv = "X-UA-Compatible" content = "IE=9" / >
< meta name = "generator" content = "Doxygen 1.8.14" / >
< meta name = "viewport" content = "width=device-width, initial-scale=1" / >
< title > remwharead: remwharead::URI Class Reference< / title >
< link href = "tabs.css" rel = "stylesheet" type = "text/css" / >
< script type = "text/javascript" src = "jquery.js" > < / script >
< script type = "text/javascript" src = "dynsections.js" > < / script >
< link href = "search/search.css" rel = "stylesheet" type = "text/css" / >
< script type = "text/javascript" src = "search/searchdata.js" > < / script >
< script type = "text/javascript" src = "search/search.js" > < / script >
< link href = "doxygen.css" rel = "stylesheet" type = "text/css" / >
< / head >
< body >
< div id = "top" > <!-- do not remove this div, it is closed by doxygen! -->
< div id = "titlearea" >
< table cellspacing = "0" cellpadding = "0" >
< tbody >
< tr style = "height: 56px;" >
< td id = "projectalign" style = "padding-left: 0.5em;" >
< div id = "projectname" > remwharead
  < span id = "projectnumber" > 0.6.3< / span >
< / div >
< / td >
< / tr >
< / tbody >
< / table >
< / div >
<!-- end header part -->
<!-- Generated by Doxygen 1.8.14 -->
< script type = "text/javascript" >
/* @license magnet:?xt=urn:btih:cf05388f2679ee054f2beb29a391d25f4e673ac3& dn=gpl-2.0.txt GPL-v2 */
var searchBox = new SearchBox("searchBox", "search",false,'Search');
/* @license-end */
< / script >
< script type = "text/javascript" src = "menudata.js" > < / script >
< script type = "text/javascript" src = "menu.js" > < / script >
< script type = "text/javascript" >
/* @license magnet:?xt=urn:btih:cf05388f2679ee054f2beb29a391d25f4e673ac3& dn=gpl-2.0.txt GPL-v2 */
$(function() {
initMenu('',true,false,'search.php','Search');
$(document).ready(function() { init_search(); });
});
/* @license-end */< / script >
< div id = "main-nav" > < / div >
<!-- window showing the filter options -->
< div id = "MSearchSelectWindow"
onmouseover="return searchBox.OnSearchSelectShow()"
onmouseout="return searchBox.OnSearchSelectHide()"
onkeydown="return searchBox.OnSearchSelectKey(event)">
< / div >
<!-- iframe showing the search results (closed by default) -->
< div id = "MSearchResultsWindow" >
< iframe src = "javascript:void(0)" frameborder = "0"
name="MSearchResults" id="MSearchResults">
< / iframe >
< / div >
< div id = "nav-path" class = "navpath" >
< ul >
< li class = "navelem" > < b > remwharead< / b > < / li > < li class = "navelem" > < a class = "el" href = "classremwharead_1_1URI.html" > URI< / a > < / li > < / ul >
< / div >
< / div > <!-- top -->
< div class = "header" >
< div class = "summary" >
< a href = "#pub-methods" > Public Member Functions< / a > |
< a href = "#pro-methods" > Protected Member Functions< / a > |
< a href = "#pro-attribs" > Protected Attributes< / a > |
< a href = "classremwharead_1_1URI-members.html" > List of all members< / a > < / div >
< div class = "headertitle" >
< div class = "title" > remwharead::URI Class Reference< / div > < / div >
< / div > <!-- header -->
< div class = "contents" >
2019-07-28 02:30:30 +02:00
< p > Download, archive and process an URI.
< a href = "classremwharead_1_1URI.html#details" > More...< / a > < / p >
2019-08-05 23:28:43 +02:00
< p > < code > #include < < a class = "el" href = "uri_8hpp_source.html" > remwharead/uri.hpp< / a > > < / code > < / p >
2019-07-27 22:46:58 +02:00
< table class = "memberdecls" >
< tr class = "heading" > < td colspan = "2" > < h2 class = "groupheader" > < a name = "pub-methods" > < / a >
Public Member Functions< / h2 > < / td > < / tr >
2019-07-28 02:30:30 +02:00
< tr class = "memitem:acda508768b1fd3b4df81ea66dd4fab87" > < td class = "memItemLeft" align = "right" valign = "top" >   < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#acda508768b1fd3b4df81ea66dd4fab87" > URI< / a > (const string & uri)< / td > < / tr >
< tr class = "memdesc:acda508768b1fd3b4df81ea66dd4fab87" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Construct object and set URL. < a href = "#acda508768b1fd3b4df81ea66dd4fab87" > More...< / a > < br / > < / td > < / tr >
2019-07-27 22:46:58 +02:00
< tr class = "separator:acda508768b1fd3b4df81ea66dd4fab87" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
2019-07-28 02:30:30 +02:00
< tr class = "memitem:a8d6ac084a823749ed38c12e7bf8f3461" > < td class = "memItemLeft" align = "right" valign = "top" > const < a class = "el" href = "structremwharead_1_1html__extract.html" > html_extract< / a >   < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#a8d6ac084a823749ed38c12e7bf8f3461" > get< / a > ()< / td > < / tr >
< tr class = "memdesc:a8d6ac084a823749ed38c12e7bf8f3461" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Download URI and extract title, description and full text. < a href = "#a8d6ac084a823749ed38c12e7bf8f3461" > More...< / a > < br / > < / td > < / tr >
2019-07-27 22:46:58 +02:00
< tr class = "separator:a8d6ac084a823749ed38c12e7bf8f3461" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
2019-07-28 02:30:30 +02:00
< tr class = "memitem:a074b3aa3ec7658b184869599c9ab6445" > < td class = "memItemLeft" align = "right" valign = "top" > const string  < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#a074b3aa3ec7658b184869599c9ab6445" > archive< / a > ()< / td > < / tr >
< tr class = "memdesc:a074b3aa3ec7658b184869599c9ab6445" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Save URI in archive and return archive-URI. < a href = "#a074b3aa3ec7658b184869599c9ab6445" > More...< / a > < br / > < / td > < / tr >
2019-07-27 22:46:58 +02:00
< tr class = "separator:a074b3aa3ec7658b184869599c9ab6445" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
< / table > < table class = "memberdecls" >
< tr class = "heading" > < td colspan = "2" > < h2 class = "groupheader" > < a name = "pro-methods" > < / a >
Protected Member Functions< / h2 > < / td > < / tr >
2019-08-05 22:01:08 +02:00
< tr class = "memitem:a2eadd0b24e6a37c8c42d710707bbfedf" > < td class = "memItemLeft" align = "right" valign = "top" > const string  < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#a2eadd0b24e6a37c8c42d710707bbfedf" > make_request< / a > (const string & uri) const< / td > < / tr >
< tr class = "memdesc:a2eadd0b24e6a37c8c42d710707bbfedf" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Make a HTTP(S) request. < a href = "#a2eadd0b24e6a37c8c42d710707bbfedf" > More...< / a > < br / > < / td > < / tr >
< tr class = "separator:a2eadd0b24e6a37c8c42d710707bbfedf" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
2019-07-28 02:30:30 +02:00
< tr class = "memitem:a37f93c46371d9b3753ae04bd2ef2c362" > < td class = "memItemLeft" align = "right" valign = "top" > const string  < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#a37f93c46371d9b3753ae04bd2ef2c362" > extract_title< / a > (const string & html)< / td > < / tr >
< tr class = "memdesc:a37f93c46371d9b3753ae04bd2ef2c362" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Extract the title from an HTML page. < a href = "#a37f93c46371d9b3753ae04bd2ef2c362" > More...< / a > < br / > < / td > < / tr >
2019-07-27 22:46:58 +02:00
< tr class = "separator:a37f93c46371d9b3753ae04bd2ef2c362" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
2019-07-28 02:30:30 +02:00
< tr class = "memitem:ae6ff7a41b9529eb8f4c7f2ace7260dc7" > < td class = "memItemLeft" align = "right" valign = "top" > const string  < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#ae6ff7a41b9529eb8f4c7f2ace7260dc7" > extract_description< / a > (const string & html)< / td > < / tr >
< tr class = "memdesc:ae6ff7a41b9529eb8f4c7f2ace7260dc7" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Extract the description from an HTML page. < a href = "#ae6ff7a41b9529eb8f4c7f2ace7260dc7" > More...< / a > < br / > < / td > < / tr >
2019-07-27 22:46:58 +02:00
< tr class = "separator:ae6ff7a41b9529eb8f4c7f2ace7260dc7" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
2019-07-28 02:30:30 +02:00
< tr class = "memitem:ad6ad5351ecf2983e01f9f4a51c2057a5" > < td class = "memItemLeft" align = "right" valign = "top" > const string  < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#ad6ad5351ecf2983e01f9f4a51c2057a5" > strip_html< / a > (const string & html)< / td > < / tr >
< tr class = "memdesc:ad6ad5351ecf2983e01f9f4a51c2057a5" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Removes HTML tags and superflous spaces from an HTML page. < a href = "#ad6ad5351ecf2983e01f9f4a51c2057a5" > More...< / a > < br / > < / td > < / tr >
2019-07-27 22:46:58 +02:00
< tr class = "separator:ad6ad5351ecf2983e01f9f4a51c2057a5" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
2019-07-28 02:30:30 +02:00
< tr class = "memitem:a8b340b13ccf0bc3ae9059872ce48e06a" > < td class = "memItemLeft" align = "right" valign = "top" > const string  < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#a8b340b13ccf0bc3ae9059872ce48e06a" > remove_html_tags< / a > (const string & html, const string & tag=" " )< / td > < / tr >
< tr class = "memdesc:a8b340b13ccf0bc3ae9059872ce48e06a" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Remove HTML tags. < a href = "#a8b340b13ccf0bc3ae9059872ce48e06a" > More...< / a > < br / > < / td > < / tr >
2019-07-27 22:46:58 +02:00
< tr class = "separator:a8b340b13ccf0bc3ae9059872ce48e06a" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
2019-07-28 02:30:30 +02:00
< tr class = "memitem:a37c36dbf8ccc03c8cc132be29e49f4ec" > < td class = "memItemLeft" align = "right" valign = "top" > const string  < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#a37c36dbf8ccc03c8cc132be29e49f4ec" > unescape_html< / a > (const string & html)< / td > < / tr >
< tr class = "memdesc:a37c36dbf8ccc03c8cc132be29e49f4ec" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Convert HTML entities to UTF-8. < a href = "#a37c36dbf8ccc03c8cc132be29e49f4ec" > More...< / a > < br / > < / td > < / tr >
2019-07-27 22:46:58 +02:00
< tr class = "separator:a37c36dbf8ccc03c8cc132be29e49f4ec" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
2019-07-28 02:30:30 +02:00
< tr class = "memitem:a9373cb28de198ae2db624980273ece4a" > < td class = "memItemLeft" align = "right" valign = "top" > const string  < / td > < td class = "memItemRight" valign = "bottom" > < a class = "el" href = "classremwharead_1_1URI.html#a9373cb28de198ae2db624980273ece4a" > remove_newlines< / a > (string text)< / td > < / tr >
< tr class = "memdesc:a9373cb28de198ae2db624980273ece4a" > < td class = "mdescLeft" >   < / td > < td class = "mdescRight" > Replace newlines with spaces. < a href = "#a9373cb28de198ae2db624980273ece4a" > More...< / a > < br / > < / td > < / tr >
2019-07-27 22:46:58 +02:00
< tr class = "separator:a9373cb28de198ae2db624980273ece4a" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
< / table > < table class = "memberdecls" >
< tr class = "heading" > < td colspan = "2" > < h2 class = "groupheader" > < a name = "pro-attribs" > < / a >
Protected Attributes< / h2 > < / td > < / tr >
< tr class = "memitem:a6d76848066779348084046a63bdaedc0" > < td class = "memItemLeft" align = "right" valign = "top" > < a id = "a6d76848066779348084046a63bdaedc0" > < / a >
string  < / td > < td class = "memItemRight" valign = "bottom" > < b > _uri< / b > < / td > < / tr >
< tr class = "separator:a6d76848066779348084046a63bdaedc0" > < td class = "memSeparator" colspan = "2" >   < / td > < / tr >
< / table >
2019-07-28 02:30:30 +02:00
< a name = "details" id = "details" > < / a > < h2 class = "groupheader" > Detailed Description< / h2 >
< div class = "textblock" > < p > Download, archive and process an URI. < / p >
2019-08-05 23:28:43 +02:00
< dl class = "section since" > < dt > Since< / dt > < dd > 0.6.0 < / dd > < / dl >
2019-07-28 02:30:30 +02:00
< / div > < h2 class = "groupheader" > Constructor & Destructor Documentation< / h2 >
< a id = "acda508768b1fd3b4df81ea66dd4fab87" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#acda508768b1fd3b4df81ea66dd4fab87" > ◆ < / a > < / span > URI()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "mlabels" >
< tr >
< td class = "mlabels-left" >
< table class = "memname" >
< tr >
< td class = "memname" > remwharead::URI::URI < / td >
< td > (< / td >
< td class = "paramtype" > const string &   < / td >
< td class = "paramname" > < em > uri< / em > < / td > < td > )< / td >
< td > < / td >
< / tr >
< / table >
< / td >
< td class = "mlabels-right" >
< span class = "mlabels" > < span class = "mlabel" > explicit< / span > < / span > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Construct object and set URL. < / p >
2019-08-05 22:01:08 +02:00
< div class = "fragment" > < div class = "line" > < a name = "l00057" > < / a > < span class = "lineno" > 57< / span >   :_uri(uri)< / div > < div class = "line" > < a name = "l00058" > < / a > < span class = "lineno" > 58< / span >   {< / div > < div class = "line" > < a name = "l00059" > < / a > < span class = "lineno" > 59< / span >   Poco::Net::initializeSSL();< / div > < div class = "line" > < a name = "l00060" > < / a > < span class = "lineno" > 60< / span >   < / div > < div class = "line" > < a name = "l00061" > < / a > < span class = "lineno" > 61< / span >   < span class = "keywordflow" > try< / span > < / div > < div class = "line" > < a name = "l00062" > < / a > < span class = "lineno" > 62< / span >   {< / div > < div class = "line" > < a name = "l00063" > < / a > < span class = "lineno" > 63< / span >   HTTPClientSession::ProxyConfig proxy;< / div > < div class = "line" > < a name = "l00064" > < / a > < span class = "lineno" > 64< / span >   < span class = "keywordtype" > string< / span > proxy_env = Environment::get(< span class = "stringliteral" > " http_proxy" < / span > );< / div > < div class = "line" > < a name = "l00065" > < / a > < span class = "lineno" > 65< / span >   < span class = "keywordtype" > size_t< / span > pos;< / div > < div class = "line" > < a name = "l00066" > < / a > < span class = "lineno" > 66< / span >   < / div > < div class = "line" > < a name = "l00067" > < / a > < span class = "lineno" > 67< / span >   < span class = "comment" > // Only keep text between // and /.< / span > < / div > < div class = "line" > < a name = "l00068" > < / a > < span class = "lineno" > 68< / span >   < span class = "keywordflow" > if< / span > ((pos = proxy_env.find(< span class = "stringliteral" > " //" < / span > )) != string::npos)< / div > < div class = "line" > < a name = "l00069" > < / a > < span class = "lineno" > 69< / span >   {< / div > < div class = "line" > < a name = "l00070" > < / a > < span class = "lineno" > 70< / span >   proxy_env = proxy_env.substr(pos + 2);< / div > < div class = "line" > < a name = "l00071" > < / a > < span class = "lineno" > 71< / span >   }< / div > < div class = "line" > < a name = "l00072" > < / a > < span class = "lineno" > 72< / span >   < span class = "keywordflow" > if< / span > ((pos = proxy_env.find(< span class = "charliteral" > ' /' < / span > )) != string::npos)< / div > < div class = "line" > < a name = "l00073" > < / a > < span class = "lineno" > 73< / span >   {< / div > < div class = "line" > < a name = "l00074" > < / a > < span class = "lineno" > 74< / span >   proxy_env = proxy_env.substr(0, pos);< / div > < div class = "line" > < a name = "l00075" > < / a > < span class = "lineno" > 75< / span >   }< / div > < div class = "line" > < a name = "l00076" > < / a > < span class = "lineno" > 76< / span >   < span class = "keywordflow" > if< / span > ((pos = proxy_env.find(< span class = "charliteral" > ' :' < / span > )) != string::npos)< / div > < div class = "line" > < a name = "l00077" > < / a > < span class = "lineno" > 77< / span >   {< / div > < div class = "line" > < a name = "l00078" > < / a > < span class = "lineno" > 78< / span >   proxy.host = proxy_env.substr(0, pos);< / div > < div class = "line" > < a name = "l00079" > < / a > < span class = "lineno" > 79< / span >   proxy.port = std::stoi(proxy_env.substr(pos + 1));< / div > < div class = "line" > < a name = "l00080" > < / a > < span class = "lineno" > 80< / span >   }< / div > < div class = "line" > < a name = "l00081" > < / a > < span class = "lineno" > 81< / span >   < span class = "keywordflow" > else< / span > < / div > < div class = "line" > < a name = "l00082" > < / a > < span class = "lineno" > 82< / span >   {< / div > < div class = "line" > < a name = "l00083" > < / a > < span class = "lineno" > 83< / span >   proxy.host = proxy_env;< / div > < div class = "line" > < a name = "l00084" > < / a > < span class = "lineno" > 84< / span >   }< / div > < div class = "line" > < a name = "l00085" > < / a > < span class = "lineno" > 85< / span >   < / div > < div class = "line" > < a name = "l00086" > < / a > < span class = "lineno" > 86< / span >   HTTPClientSession::setGlobalProxyConfig(proxy);< / div > < div class = "line" > < a name = "l00087" > < / a > < span class = "lineno" > 87< / span >   }< / div > < div class = "line" > < a name = "l00088" > < / a > < span class = "lineno" > 88< / span >   < span class = "keywordflow" > catch< / span > (< span class = "keyw
2019-07-28 02:30:30 +02:00
< / div >
< / div >
< h2 class = "groupheader" > Member Function Documentation< / h2 >
< a id = "a074b3aa3ec7658b184869599c9ab6445" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#a074b3aa3ec7658b184869599c9ab6445" > ◆ < / a > < / span > archive()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "memname" >
< tr >
< td class = "memname" > const string remwharead::URI::archive < / td >
< td > (< / td >
< td class = "paramname" > < / td > < td > )< / td >
< td > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Save URI in archive and return archive-URI. < / p >
2019-08-05 22:01:08 +02:00
< div class = "fragment" > < div class = "line" > < a name = "l00569" > < / a > < span class = "lineno" > 569< / span >   {< / div > < div class = "line" > < a name = "l00570" > < / a > < span class = "lineno" > 570< / span >   < span class = "keywordflow" > if< / span > (_uri.substr(0, 4) != < span class = "stringliteral" > " http" < / span > )< / div > < div class = "line" > < a name = "l00571" > < / a > < span class = "lineno" > 571< / span >   {< / div > < div class = "line" > < a name = "l00572" > < / a > < span class = "lineno" > 572< / span >   < span class = "keywordflow" > return< / span > < span class = "stringliteral" > " " < / span > ;< / div > < div class = "line" > < a name = "l00573" > < / a > < span class = "lineno" > 573< / span >   }< / div > < div class = "line" > < a name = "l00574" > < / a > < span class = "lineno" > 574< / span >   < / div > < div class = "line" > < a name = "l00575" > < / a > < span class = "lineno" > 575< / span >   < span class = "keywordflow" > try< / span > < / div > < div class = "line" > < a name = "l00576" > < / a > < span class = "lineno" > 576< / span >   {< / div > < div class = "line" > < a name = "l00577" > < / a > < span class = "lineno" > 577< / span >   < span class = "keyword" > const< / span > < span class = "keywordtype" > string< / span > answer = < a class = "code" href = "classremwharead_1_1URI.html#a2eadd0b24e6a37c8c42d710707bbfedf" > make_request< / a > (< span class = "stringliteral" > " https://web.archive.org/save/" < / span > < / div > < div class = "line" > < a name = "l00578" > < / a > < span class = "lineno" > 578< / span >   + _uri);< / div > < div class = "line" > < a name = "l00579" > < / a > < span class = "lineno" > 579< / span >   < / div > < div class = "line" > < a name = "l00580" > < / a > < span class = "lineno" > 580< / span >   smatch match;< / div > < div class = "line" > < a name = "l00581" > < / a > < span class = "lineno" > 581< / span >   < span class = "keywordflow" > if< / span > (regex_search(answer, match, regex(< span class = "stringliteral" > " Content-Location: (.+)\r" < / span > )))< / div > < div class = "line" > < a name = "l00582" > < / a > < span class = "lineno" > 582< / span >   {< / div > < div class = "line" > < a name = "l00583" > < / a > < span class = "lineno" > 583< / span >   < span class = "keywordflow" > return< / span > < span class = "stringliteral" > " https://web.archive.org" < / span > + match[1].str();< / div > < div class = "line" > < a name = "l00584" > < / a > < span class = "lineno" > 584< / span >   }< / div > < div class = "line" > < a name = "l00585" > < / a > < span class = "lineno" > 585< / span >   < span class = "keywordflow" > else< / span > < / div > < div class = "line" > < a name = "l00586" > < / a > < span class = "lineno" > 586< / span >   {< / div > < div class = "line" > < a name = "l00587" > < / a > < span class = "lineno" > 587< / span >   cerr < < < span class = "stringliteral" > " Error: Could not archive page.\n" < / span > ;< / div > < div class = "line" > < a name = "l00588" > < / a > < span class = "lineno" > 588< / span >   }< / div > < div class = "line" > < a name = "l00589" > < / a > < span class = "lineno" > 589< / span >   }< / div > < div class = "line" > < a name = "l00590" > < / a > < span class = "lineno" > 590< / span >   < span class = "keywordflow" > catch< / span > (< span class = "keyword" > const< / span > std::exception & e)< / div > < div class = "line" > < a name = "l00591" > < / a > < span class = "lineno" > 591< / span >   {< / div > < div class = "line" > < a name = "l00592" > < / a > < span class = "lineno" > 592< / span >   cerr < < < span class = "stringliteral" > " Error in " < / span > < < __func__ < < < span class = "stringliteral" > " : " < / span > < < e.what() < < < span class = "stringliteral" > " .\n" < / span > ;< / div > < div class = "line" > < a name = "l00593" > < / a > < span class = "lineno" > 593< / span >   }< / div > < div class = "line" > < a name = "l00594" > < / a > < span class = "lineno" > 594< / span >   < / div > < div class = "line" > < a name = "l00595" > < / a > < span class = "lineno" > 595< / span >   < span class = "keywordflow" > return< / span > < span class = "stringliteral" > " " < / span > ;< / div > < div class = "line" > < a name = "l00596" > < / a > < span class = "lineno" > 596< / span >   }< / div > < div class = "ttc" id = "classremwharead_1_1URI_html_a2eadd0b24e6a37c8c42d710707bbfedf" > < div
2019-07-28 02:30:30 +02:00
< / div > <!-- fragment -->
< / div >
< / div >
< a id = "ae6ff7a41b9529eb8f4c7f2ace7260dc7" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#ae6ff7a41b9529eb8f4c7f2ace7260dc7" > ◆ < / a > < / span > extract_description()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "mlabels" >
< tr >
< td class = "mlabels-left" >
< table class = "memname" >
< tr >
< td class = "memname" > const string remwharead::URI::extract_description < / td >
< td > (< / td >
< td class = "paramtype" > const string &   < / td >
< td class = "paramname" > < em > html< / em > < / td > < td > )< / td >
< td > < / td >
< / tr >
< / table >
< / td >
< td class = "mlabels-right" >
< span class = "mlabels" > < span class = "mlabel" > protected< / span > < / span > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Extract the description from an HTML page. < / p >
2019-08-05 22:01:08 +02:00
< div class = "fragment" > < div class = "line" > < a name = "l00198" > < / a > < span class = "lineno" > 198< / span >   {< / div > < div class = "line" > < a name = "l00199" > < / a > < span class = "lineno" > 199< / span >   < span class = "keyword" > const< / span > regex re_htmlfile(< span class = "stringliteral" > " \\.(.?html?|xml|rss)$" < / span > );< / div > < div class = "line" > < a name = "l00200" > < / a > < span class = "lineno" > 200< / span >   < span class = "keywordflow" > if< / span > (_uri.substr(0, 4) == < span class = "stringliteral" > " http" < / span > || regex_search(_uri, re_htmlfile))< / div > < div class = "line" > < a name = "l00201" > < / a > < span class = "lineno" > 201< / span >   {< / div > < div class = "line" > < a name = "l00202" > < / a > < span class = "lineno" > 202< / span >   smatch match;< / div > < div class = "line" > < a name = "l00203" > < / a > < span class = "lineno" > 203< / span >   < span class = "keyword" > const< / span > regex re(< span class = "stringliteral" > " description\" [^> ]+content=\" ([^\" ]+)" < / span > , icase);< / div > < div class = "line" > < a name = "l00204" > < / a > < span class = "lineno" > 204< / span >   regex_search(html, match, re);< / div > < div class = "line" > < a name = "l00205" > < / a > < span class = "lineno" > 205< / span >   < span class = "keywordflow" > return< / span > < a class = "code" href = "classremwharead_1_1URI.html#a9373cb28de198ae2db624980273ece4a" > remove_newlines< / a > (< a class = "code" href = "classremwharead_1_1URI.html#ad6ad5351ecf2983e01f9f4a51c2057a5" > strip_html< / a > (match[1].str()));< / div > < div class = "line" > < a name = "l00206" > < / a > < span class = "lineno" > 206< / span >   }< / div > < div class = "line" > < a name = "l00207" > < / a > < span class = "lineno" > 207< / span >   < / div > < div class = "line" > < a name = "l00208" > < / a > < span class = "lineno" > 208< / span >   < span class = "keywordflow" > return< / span > < span class = "stringliteral" > " " < / span > ;< / div > < div class = "line" > < a name = "l00209" > < / a > < span class = "lineno" > 209< / span >   }< / div > < div class = "ttc" id = "classremwharead_1_1URI_html_ad6ad5351ecf2983e01f9f4a51c2057a5" > < div class = "ttname" > < a href = "classremwharead_1_1URI.html#ad6ad5351ecf2983e01f9f4a51c2057a5" > remwharead::URI::strip_html< / a > < / div > < div class = "ttdeci" > const string strip_html(const string & html)< / div > < div class = "ttdoc" > Removes HTML tags and superflous spaces from an HTML page. < / div > < div class = "ttdef" > < b > Definition:< / b > uri.cpp:211< / div > < / div >
< div class = "ttc" id = "classremwharead_1_1URI_html_a9373cb28de198ae2db624980273ece4a" > < div class = "ttname" > < a href = "classremwharead_1_1URI.html#a9373cb28de198ae2db624980273ece4a" > remwharead::URI::remove_newlines< / a > < / div > < div class = "ttdeci" > const string remove_newlines(string text)< / div > < div class = "ttdoc" > Replace newlines with spaces. < / div > < div class = "ttdef" > < b > Definition:< / b > uri.cpp:598< / div > < / div >
2019-07-28 02:30:30 +02:00
< / div > <!-- fragment -->
< / div >
< / div >
< a id = "a37f93c46371d9b3753ae04bd2ef2c362" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#a37f93c46371d9b3753ae04bd2ef2c362" > ◆ < / a > < / span > extract_title()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "mlabels" >
< tr >
< td class = "mlabels-left" >
< table class = "memname" >
< tr >
< td class = "memname" > const string remwharead::URI::extract_title < / td >
< td > (< / td >
< td class = "paramtype" > const string &   < / td >
< td class = "paramname" > < em > html< / em > < / td > < td > )< / td >
< td > < / td >
< / tr >
< / table >
< / td >
< td class = "mlabels-right" >
< span class = "mlabels" > < span class = "mlabel" > protected< / span > < / span > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Extract the title from an HTML page. < / p >
2019-08-05 22:01:08 +02:00
< div class = "fragment" > < div class = "line" > < a name = "l00185" > < / a > < span class = "lineno" > 185< / span >   {< / div > < div class = "line" > < a name = "l00186" > < / a > < span class = "lineno" > 186< / span >   < span class = "keyword" > const< / span > regex re_htmlfile(< span class = "stringliteral" > " \\.(.?html?|xml|rss)$" < / span > );< / div > < div class = "line" > < a name = "l00187" > < / a > < span class = "lineno" > 187< / span >   < span class = "keywordflow" > if< / span > (_uri.substr(0, 4) == < span class = "stringliteral" > " http" < / span > || regex_search(_uri, re_htmlfile))< / div > < div class = "line" > < a name = "l00188" > < / a > < span class = "lineno" > 188< / span >   {< / div > < div class = "line" > < a name = "l00189" > < / a > < span class = "lineno" > 189< / span >   smatch match;< / div > < div class = "line" > < a name = "l00190" > < / a > < span class = "lineno" > 190< / span >   regex_search(html, match, regex(< span class = "stringliteral" > " < title> ([^< ]+)" < / span > , icase));< / div > < div class = "line" > < a name = "l00191" > < / a > < span class = "lineno" > 191< / span >   < span class = "keywordflow" > return< / span > < a class = "code" href = "classremwharead_1_1URI.html#a9373cb28de198ae2db624980273ece4a" > remove_newlines< / a > (< a class = "code" href = "classremwharead_1_1URI.html#a37c36dbf8ccc03c8cc132be29e49f4ec" > unescape_html< / a > (match[1].str()));< / div > < div class = "line" > < a name = "l00192" > < / a > < span class = "lineno" > 192< / span >   }< / div > < div class = "line" > < a name = "l00193" > < / a > < span class = "lineno" > 193< / span >   < / div > < div class = "line" > < a name = "l00194" > < / a > < span class = "lineno" > 194< / span >   < span class = "keywordflow" > return< / span > < span class = "stringliteral" > " " < / span > ;< / div > < div class = "line" > < a name = "l00195" > < / a > < span class = "lineno" > 195< / span >   }< / div > < div class = "ttc" id = "classremwharead_1_1URI_html_a9373cb28de198ae2db624980273ece4a" > < div class = "ttname" > < a href = "classremwharead_1_1URI.html#a9373cb28de198ae2db624980273ece4a" > remwharead::URI::remove_newlines< / a > < / div > < div class = "ttdeci" > const string remove_newlines(string text)< / div > < div class = "ttdoc" > Replace newlines with spaces. < / div > < div class = "ttdef" > < b > Definition:< / b > uri.cpp:598< / div > < / div >
< div class = "ttc" id = "classremwharead_1_1URI_html_a37c36dbf8ccc03c8cc132be29e49f4ec" > < div class = "ttname" > < a href = "classremwharead_1_1URI.html#a37c36dbf8ccc03c8cc132be29e49f4ec" > remwharead::URI::unescape_html< / a > < / div > < div class = "ttdeci" > const string unescape_html(const string & html)< / div > < div class = "ttdoc" > Convert HTML entities to UTF-8. < / div > < div class = "ttdef" > < b > Definition:< / b > uri.cpp:268< / div > < / div >
2019-07-28 02:30:30 +02:00
< / div > <!-- fragment -->
< / div >
< / div >
< a id = "a8d6ac084a823749ed38c12e7bf8f3461" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#a8d6ac084a823749ed38c12e7bf8f3461" > ◆ < / a > < / span > get()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "memname" >
< tr >
< td class = "memname" > const < a class = "el" href = "structremwharead_1_1html__extract.html" > html_extract< / a > remwharead::URI::get < / td >
< td > (< / td >
< td class = "paramname" > < / td > < td > )< / td >
< td > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Download URI and extract title, description and full text. < / p >
2019-08-05 22:01:08 +02:00
< div class = "fragment" > < div class = "line" > < a name = "l00100" > < / a > < span class = "lineno" > 100< / span >   {< / div > < div class = "line" > < a name = "l00101" > < / a > < span class = "lineno" > 101< / span >   < span class = "keywordflow" > try< / span > < / div > < div class = "line" > < a name = "l00102" > < / a > < span class = "lineno" > 102< / span >   {< / div > < div class = "line" > < a name = "l00103" > < / a > < span class = "lineno" > 103< / span >   std::ostringstream oss;< / div > < div class = "line" > < a name = "l00104" > < / a > < span class = "lineno" > 104< / span >   < / div > < div class = "line" > < a name = "l00105" > < / a > < span class = "lineno" > 105< / span >   < span class = "keyword" > const< / span > < span class = "keywordtype" > string< / span > answer = < a class = "code" href = "classremwharead_1_1URI.html#a2eadd0b24e6a37c8c42d710707bbfedf" > make_request< / a > (_uri);< / div > < div class = "line" > < a name = "l00106" > < / a > < span class = "lineno" > 106< / span >   < span class = "keywordflow" > if< / span > (answer.empty())< / div > < div class = "line" > < a name = "l00107" > < / a > < span class = "lineno" > 107< / span >   {< / div > < div class = "line" > < a name = "l00108" > < / a > < span class = "lineno" > 108< / span >   cerr < < < span class = "stringliteral" > " Error: Could not download page.\n" < / span > ;< / div > < div class = "line" > < a name = "l00109" > < / a > < span class = "lineno" > 109< / span >   }< / div > < div class = "line" > < a name = "l00110" > < / a > < span class = "lineno" > 110< / span >   < span class = "keywordflow" > else< / span > < / div > < div class = "line" > < a name = "l00111" > < / a > < span class = "lineno" > 111< / span >   {< / div > < div class = "line" > < a name = "l00112" > < / a > < span class = "lineno" > 112< / span >   < span class = "keywordflow" > return< / span > < / div > < div class = "line" > < a name = "l00113" > < / a > < span class = "lineno" > 113< / span >   {< / div > < div class = "line" > < a name = "l00114" > < / a > < span class = "lineno" > 114< / span >   < a class = "code" href = "classremwharead_1_1URI.html#a37f93c46371d9b3753ae04bd2ef2c362" > extract_title< / a > (answer),< / div > < div class = "line" > < a name = "l00115" > < / a > < span class = "lineno" > 115< / span >   < a class = "code" href = "classremwharead_1_1URI.html#ae6ff7a41b9529eb8f4c7f2ace7260dc7" > extract_description< / a > (answer),< / div > < div class = "line" > < a name = "l00116" > < / a > < span class = "lineno" > 116< / span >   < a class = "code" href = "classremwharead_1_1URI.html#ad6ad5351ecf2983e01f9f4a51c2057a5" > strip_html< / a > (answer)< / div > < div class = "line" > < a name = "l00117" > < / a > < span class = "lineno" > 117< / span >   };< / div > < div class = "line" > < a name = "l00118" > < / a > < span class = "lineno" > 118< / span >   }< / div > < div class = "line" > < a name = "l00119" > < / a > < span class = "lineno" > 119< / span >   }< / div > < div class = "line" > < a name = "l00120" > < / a > < span class = "lineno" > 120< / span >   < span class = "keywordflow" > catch< / span > (< span class = "keyword" > const< / span > std::exception & e)< / div > < div class = "line" > < a name = "l00121" > < / a > < span class = "lineno" > 121< / span >   {< / div > < div class = "line" > < a name = "l00122" > < / a > < span class = "lineno" > 122< / span >   < span class = "comment" > // Prints something like " SSL Exception" , nothing specific.< / span > < / div > < div class = "line" > < a name = "l00123" > < / a > < span class = "lineno" > 123< / span >   cerr < < < span class = "stringliteral" > " Error in " < / span > < < __func__ < < < span class = "stringliteral" > " : " < / span > < < e.what() < < endl;< / div > < div class = "line" > < a name = "l00124" > < / a > < span class = "lineno" > 124< / span >   }< / div > < div class = "line" > < a name = "l00125" > < / a > < span class = "lineno" > 125< / span >   < / div > < div class = "line" > < a name = "l00126" > < / a > < span class = "lineno" > 126< / span >   < span class = "keywordflow" > return< / span > { < span class = "stringliteral" > " " < / span > , < span class = "stringliteral" > " " < / span > , < span class = "stringliteral" > " " < / span > };< / div > < div class = "line" > < a name = "l00127" > < / a > < span class = "lineno" > 127< / span >   }< / div > < div c
< div class = "ttc" id = "classremwharead_1_1URI_html_a37f93c46371d9b3753ae04bd2ef2c362" > < div class = "ttname" > < a href = "classremwharead_1_1URI.html#a37f93c46371d9b3753ae04bd2ef2c362" > remwharead::URI::extract_title< / a > < / div > < div class = "ttdeci" > const string extract_title(const string & html)< / div > < div class = "ttdoc" > Extract the title from an HTML page. < / div > < div class = "ttdef" > < b > Definition:< / b > uri.cpp:184< / div > < / div >
< div class = "ttc" id = "classremwharead_1_1URI_html_ae6ff7a41b9529eb8f4c7f2ace7260dc7" > < div class = "ttname" > < a href = "classremwharead_1_1URI.html#ae6ff7a41b9529eb8f4c7f2ace7260dc7" > remwharead::URI::extract_description< / a > < / div > < div class = "ttdeci" > const string extract_description(const string & html)< / div > < div class = "ttdoc" > Extract the description from an HTML page. < / div > < div class = "ttdef" > < b > Definition:< / b > uri.cpp:197< / div > < / div >
< div class = "ttc" id = "classremwharead_1_1URI_html_a2eadd0b24e6a37c8c42d710707bbfedf" > < div class = "ttname" > < a href = "classremwharead_1_1URI.html#a2eadd0b24e6a37c8c42d710707bbfedf" > remwharead::URI::make_request< / a > < / div > < div class = "ttdeci" > const string make_request(const string & uri) const< / div > < div class = "ttdoc" > Make a HTTP(S) request. < / div > < div class = "ttdef" > < b > Definition:< / b > uri.cpp:129< / div > < / div >
< / div > <!-- fragment -->
< / div >
< / div >
< a id = "a2eadd0b24e6a37c8c42d710707bbfedf" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#a2eadd0b24e6a37c8c42d710707bbfedf" > ◆ < / a > < / span > make_request()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "mlabels" >
< tr >
< td class = "mlabels-left" >
< table class = "memname" >
< tr >
< td class = "memname" > const string remwharead::URI::make_request < / td >
< td > (< / td >
< td class = "paramtype" > const string &   < / td >
< td class = "paramname" > < em > uri< / em > < / td > < td > )< / td >
< td > const< / td >
< / tr >
< / table >
< / td >
< td class = "mlabels-right" >
< span class = "mlabels" > < span class = "mlabel" > protected< / span > < / span > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Make a HTTP(S) request. < / p >
< div class = "fragment" > < div class = "line" > < a name = "l00130" > < / a > < span class = "lineno" > 130< / span >   {< / div > < div class = "line" > < a name = "l00131" > < / a > < span class = "lineno" > 131< / span >   Poco::URI poco_uri(uri);< / div > < div class = "line" > < a name = "l00132" > < / a > < span class = "lineno" > 132< / span >   < span class = "keywordtype" > string< / span > path = poco_uri.getPathAndQuery();< / div > < div class = "line" > < a name = "l00133" > < / a > < span class = "lineno" > 133< / span >   < span class = "keywordflow" > if< / span > (path.empty())< / div > < div class = "line" > < a name = "l00134" > < / a > < span class = "lineno" > 134< / span >   {< / div > < div class = "line" > < a name = "l00135" > < / a > < span class = "lineno" > 135< / span >   path = < span class = "stringliteral" > " /" < / span > ;< / div > < div class = "line" > < a name = "l00136" > < / a > < span class = "lineno" > 136< / span >   }< / div > < div class = "line" > < a name = "l00137" > < / a > < span class = "lineno" > 137< / span >   < / div > < div class = "line" > < a name = "l00138" > < / a > < span class = "lineno" > 138< / span >   unique_ptr< HTTPClientSession> session;< / div > < div class = "line" > < a name = "l00139" > < / a > < span class = "lineno" > 139< / span >   < span class = "keywordflow" > if< / span > (poco_uri.getScheme() == < span class = "stringliteral" > " https" < / span > )< / div > < div class = "line" > < a name = "l00140" > < / a > < span class = "lineno" > 140< / span >   {< / div > < div class = "line" > < a name = "l00141" > < / a > < span class = "lineno" > 141< / span >   session = make_unique< HTTPSClientSession> (poco_uri.getHost(),< / div > < div class = "line" > < a name = "l00142" > < / a > < span class = "lineno" > 142< / span >   poco_uri.getPort());< / div > < div class = "line" > < a name = "l00143" > < / a > < span class = "lineno" > 143< / span >   }< / div > < div class = "line" > < a name = "l00144" > < / a > < span class = "lineno" > 144< / span >   < span class = "keywordflow" > else< / span > < span class = "keywordflow" > if< / span > (poco_uri.getScheme() == < span class = "stringliteral" > " http" < / span > )< / div > < div class = "line" > < a name = "l00145" > < / a > < span class = "lineno" > 145< / span >   {< / div > < div class = "line" > < a name = "l00146" > < / a > < span class = "lineno" > 146< / span >   session = make_unique< HTTPClientSession> (poco_uri.getHost(),< / div > < div class = "line" > < a name = "l00147" > < / a > < span class = "lineno" > 147< / span >   poco_uri.getPort());< / div > < div class = "line" > < a name = "l00148" > < / a > < span class = "lineno" > 148< / span >   }< / div > < div class = "line" > < a name = "l00149" > < / a > < span class = "lineno" > 149< / span >   < / div > < div class = "line" > < a name = "l00150" > < / a > < span class = "lineno" > 150< / span >   HTTPRequest request(HTTPRequest::HTTP_GET, path,< / div > < div class = "line" > < a name = "l00151" > < / a > < span class = "lineno" > 151< / span >   HTTPMessage::HTTP_1_1);< / div > < div class = "line" > < a name = "l00152" > < / a > < span class = "lineno" > 152< / span >   request.set(< span class = "stringliteral" > " User-Agent" < / span > , < span class = "keywordtype" > string< / span > (< span class = "stringliteral" > " remwharead/" < / span > ) + global::version);< / div > < div class = "line" > < a name = "l00153" > < / a > < span class = "lineno" > 153< / span >   < / div > < div class = "line" > < a name = "l00154" > < / a > < span class = "lineno" > 154< / span >   HTTPResponse response;< / div > < div class = "line" > < a name = "l00155" > < / a > < span class = "lineno" > 155< / span >   < / div > < div class = "line" > < a name = "l00156" > < / a > < span class = "lineno" > 156< / span >   session-> sendRequest(request);< / div > < div class = "line" > < a name = "l00157" > < / a > < span class = "lineno" > 157< / span >   istream & rs = session-> receiveResponse(response);< / div > < div class = "line" > < a name = "l00158" > < / a > < span class = "lineno" > 158< / span >   < / div > < div class = "line" > < a name = "l00159" > < / a > < span class = "lineno" > 159< / span >   < span class = "comment" > // Not using the constants because some are too new for Debian stretch.< / span > < / div > < div class = "line" > < a name = "l00160" > < / a > < span class = "lineno" > 160< /sp
2019-07-28 02:30:30 +02:00
< / div > <!-- fragment -->
< / div >
< / div >
< a id = "a8b340b13ccf0bc3ae9059872ce48e06a" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#a8b340b13ccf0bc3ae9059872ce48e06a" > ◆ < / a > < / span > remove_html_tags()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "mlabels" >
< tr >
< td class = "mlabels-left" >
< table class = "memname" >
< tr >
< td class = "memname" > const string remwharead::URI::remove_html_tags < / td >
< td > (< / td >
< td class = "paramtype" > const string &   < / td >
< td class = "paramname" > < em > html< / em > , < / td >
< / tr >
< tr >
< td class = "paramkey" > < / td >
< td > < / td >
< td class = "paramtype" > const string &   < / td >
< td class = "paramname" > < em > tag< / em > = < code > " " < / code >   < / td >
< / tr >
< tr >
< td > < / td >
< td > )< / td >
< td > < / td > < td > < / td >
< / tr >
< / table >
< / td >
< td class = "mlabels-right" >
< span class = "mlabels" > < span class = "mlabel" > protected< / span > < / span > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Remove HTML tags. < / p >
< dl class = "params" > < dt > Parameters< / dt > < dd >
< table class = "params" >
< tr > < td class = "paramname" > html< / td > < td > HTML page. < / td > < / tr >
< tr > < td class = "paramname" > tag< / td > < td > If set, only remove this tag. < / td > < / tr >
< / table >
< / dd >
< / dl >
2019-08-05 22:01:08 +02:00
< div class = "fragment" > < div class = "line" > < a name = "l00231" > < / a > < span class = "lineno" > 231< / span >   {< / div > < div class = "line" > < a name = "l00232" > < / a > < span class = "lineno" > 232< / span >   < span class = "comment" > // NOTE: I did this with regex_replace before, but libstdc++ segfaulted.< / span > < / div > < div class = "line" > < a name = "l00233" > < / a > < span class = "lineno" > 233< / span >   < span class = "keywordtype" > string< / span > out;< / div > < div class = "line" > < a name = "l00234" > < / a > < span class = "lineno" > 234< / span >   < span class = "keywordflow" > if< / span > (tag.empty())< / div > < div class = "line" > < a name = "l00235" > < / a > < span class = "lineno" > 235< / span >   {< / div > < div class = "line" > < a name = "l00236" > < / a > < span class = "lineno" > 236< / span >   < span class = "keywordtype" > size_t< / span > pos = 0;< / div > < div class = "line" > < a name = "l00237" > < / a > < span class = "lineno" > 237< / span >   < span class = "keywordflow" > while< / span > (pos != std::string::npos)< / div > < div class = "line" > < a name = "l00238" > < / a > < span class = "lineno" > 238< / span >   {< / div > < div class = "line" > < a name = "l00239" > < / a > < span class = "lineno" > 239< / span >   < span class = "keywordtype" > size_t< / span > startpos = html.find(< span class = "charliteral" > ' < ' < / span > , pos);< / div > < div class = "line" > < a name = "l00240" > < / a > < span class = "lineno" > 240< / span >   < span class = "keywordtype" > size_t< / span > endpos = html.find(< span class = "charliteral" > ' > ' < / span > , startpos);< / div > < div class = "line" > < a name = "l00241" > < / a > < span class = "lineno" > 241< / span >   out += html.substr(pos, startpos - pos);< / div > < div class = "line" > < a name = "l00242" > < / a > < span class = "lineno" > 242< / span >   pos = endpos;< / div > < div class = "line" > < a name = "l00243" > < / a > < span class = "lineno" > 243< / span >   < span class = "keywordflow" > if< / span > (pos != std::string::npos)< / div > < div class = "line" > < a name = "l00244" > < / a > < span class = "lineno" > 244< / span >   {< / div > < div class = "line" > < a name = "l00245" > < / a > < span class = "lineno" > 245< / span >   ++pos;< / div > < div class = "line" > < a name = "l00246" > < / a > < span class = "lineno" > 246< / span >   }< / div > < div class = "line" > < a name = "l00247" > < / a > < span class = "lineno" > 247< / span >   }< / div > < div class = "line" > < a name = "l00248" > < / a > < span class = "lineno" > 248< / span >   }< / div > < div class = "line" > < a name = "l00249" > < / a > < span class = "lineno" > 249< / span >   < span class = "keywordflow" > else< / span > < / div > < div class = "line" > < a name = "l00250" > < / a > < span class = "lineno" > 250< / span >   {< / div > < div class = "line" > < a name = "l00251" > < / a > < span class = "lineno" > 251< / span >   < span class = "keywordtype" > size_t< / span > pos = 0;< / div > < div class = "line" > < a name = "l00252" > < / a > < span class = "lineno" > 252< / span >   out = html;< / div > < div class = "line" > < a name = "l00253" > < / a > < span class = "lineno" > 253< / span >   < span class = "keywordflow" > while< / span > ((pos = out.find(< span class = "stringliteral" > " < " < / span > + tag)) != std::string::npos)< / div > < div class = "line" > < a name = "l00254" > < / a > < span class = "lineno" > 254< / span >   {< / div > < div class = "line" > < a name = "l00255" > < / a > < span class = "lineno" > 255< / span >   < span class = "keywordtype" > size_t< / span > endpos = out.find(< span class = "stringliteral" > " < /" < / span > + tag, pos);< / div > < div class = "line" > < a name = "l00256" > < / a > < span class = "lineno" > 256< / span >   < span class = "keywordflow" > if< / span > (endpos == std::string::npos)< / div > < div class = "line" > < a name = "l00257" > < / a > < span class = "lineno" > 257< / span >   {< / div > < div class = "line" > < a name = "l00258" > < / a > < span class = "lineno" > 258< / span >   < span class = "keywordflow" > break< / span > ;< / div > < div class = "line" > < a name = "l00259" > < / a > < span class = "lineno" > 259< / span >   }< / div > < div class = "line" > < a name = "l00260" > < / a > < span class = "lineno" > 260< / span >   endpos += 3 + tag.length(); < span class
2019-07-28 02:30:30 +02:00
< / div >
< / div >
< a id = "a9373cb28de198ae2db624980273ece4a" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#a9373cb28de198ae2db624980273ece4a" > ◆ < / a > < / span > remove_newlines()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "mlabels" >
< tr >
< td class = "mlabels-left" >
< table class = "memname" >
< tr >
< td class = "memname" > const string remwharead::URI::remove_newlines < / td >
< td > (< / td >
< td class = "paramtype" > string  < / td >
< td class = "paramname" > < em > text< / em > < / td > < td > )< / td >
< td > < / td >
< / tr >
< / table >
< / td >
< td class = "mlabels-right" >
< span class = "mlabels" > < span class = "mlabel" > protected< / span > < / span > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Replace newlines with spaces. < / p >
2019-08-05 22:01:08 +02:00
< div class = "fragment" > < div class = "line" > < a name = "l00599" > < / a > < span class = "lineno" > 599< / span >   {< / div > < div class = "line" > < a name = "l00600" > < / a > < span class = "lineno" > 600< / span >   < span class = "keywordtype" > size_t< / span > posn = 0;< / div > < div class = "line" > < a name = "l00601" > < / a > < span class = "lineno" > 601< / span >   < span class = "keywordflow" > while< / span > ((posn = text.find(< span class = "charliteral" > ' \n' < / span > , posn)) != std::string::npos)< / div > < div class = "line" > < a name = "l00602" > < / a > < span class = "lineno" > 602< / span >   {< / div > < div class = "line" > < a name = "l00603" > < / a > < span class = "lineno" > 603< / span >   text.replace(posn, 1, < span class = "stringliteral" > " " < / span > );< / div > < div class = "line" > < a name = "l00604" > < / a > < span class = "lineno" > 604< / span >   < / div > < div class = "line" > < a name = "l00605" > < / a > < span class = "lineno" > 605< / span >   < span class = "keywordtype" > size_t< / span > posr = posn - 1;< / div > < div class = "line" > < a name = "l00606" > < / a > < span class = "lineno" > 606< / span >   < span class = "keywordflow" > if< / span > (text[posr] == < span class = "charliteral" > ' \r' < / span > )< / div > < div class = "line" > < a name = "l00607" > < / a > < span class = "lineno" > 607< / span >   {< / div > < div class = "line" > < a name = "l00608" > < / a > < span class = "lineno" > 608< / span >   text.replace(posr, 1, < span class = "stringliteral" > " " < / span > );< / div > < div class = "line" > < a name = "l00609" > < / a > < span class = "lineno" > 609< / span >   }< / div > < div class = "line" > < a name = "l00610" > < / a > < span class = "lineno" > 610< / span >   ++posn;< / div > < div class = "line" > < a name = "l00611" > < / a > < span class = "lineno" > 611< / span >   }< / div > < div class = "line" > < a name = "l00612" > < / a > < span class = "lineno" > 612< / span >   < / div > < div class = "line" > < a name = "l00613" > < / a > < span class = "lineno" > 613< / span >   < span class = "keywordflow" > return< / span > text;< / div > < div class = "line" > < a name = "l00614" > < / a > < span class = "lineno" > 614< / span >   }< / div > < / div > <!-- fragment -->
2019-07-28 02:30:30 +02:00
< / div >
< / div >
< a id = "ad6ad5351ecf2983e01f9f4a51c2057a5" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#ad6ad5351ecf2983e01f9f4a51c2057a5" > ◆ < / a > < / span > strip_html()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "mlabels" >
< tr >
< td class = "mlabels-left" >
< table class = "memname" >
< tr >
< td class = "memname" > const string remwharead::URI::strip_html < / td >
< td > (< / td >
< td class = "paramtype" > const string &   < / td >
< td class = "paramname" > < em > html< / em > < / td > < td > )< / td >
< td > < / td >
< / tr >
< / table >
< / td >
< td class = "mlabels-right" >
< span class = "mlabels" > < span class = "mlabel" > protected< / span > < / span > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Removes HTML tags and superflous spaces from an HTML page. < / p >
2019-08-05 22:01:08 +02:00
< div class = "fragment" > < div class = "line" > < a name = "l00212" > < / a > < span class = "lineno" > 212< / span >   {< / div > < div class = "line" > < a name = "l00213" > < / a > < span class = "lineno" > 213< / span >   < span class = "keywordtype" > string< / span > out;< / div > < div class = "line" > < a name = "l00214" > < / a > < span class = "lineno" > 214< / span >   < / div > < div class = "line" > < a name = "l00215" > < / a > < span class = "lineno" > 215< / span >   out = < a class = "code" href = "classremwharead_1_1URI.html#a8b340b13ccf0bc3ae9059872ce48e06a" > remove_html_tags< / a > (html, < span class = "stringliteral" > " script" < / span > ); < span class = "comment" > // Remove JavaScript.< / span > < / div > < div class = "line" > < a name = "l00216" > < / a > < span class = "lineno" > 216< / span >   out = < a class = "code" href = "classremwharead_1_1URI.html#a8b340b13ccf0bc3ae9059872ce48e06a" > remove_html_tags< / a > (out, < span class = "stringliteral" > " style" < / span > ); < span class = "comment" > // Remove CSS.< / span > < / div > < div class = "line" > < a name = "l00217" > < / a > < span class = "lineno" > 217< / span >   out = < a class = "code" href = "classremwharead_1_1URI.html#a8b340b13ccf0bc3ae9059872ce48e06a" > remove_html_tags< / a > (out); < span class = "comment" > // Remove tags.< / span > < / div > < div class = "line" > < a name = "l00218" > < / a > < span class = "lineno" > 218< / span >   < / div > < div class = "line" > < a name = "l00219" > < / a > < span class = "lineno" > 219< / span >   < span class = "keywordtype" > size_t< / span > pos = 0;< / div > < div class = "line" > < a name = "l00220" > < / a > < span class = "lineno" > 220< / span >   < span class = "keywordflow" > while< / span > ((pos = out.find(< span class = "stringliteral" > " \r" < / span > , pos)) != std::string::npos) < span class = "comment" > // Remove CR.< / span > < / div > < div class = "line" > < a name = "l00221" > < / a > < span class = "lineno" > 221< / span >   {< / div > < div class = "line" > < a name = "l00222" > < / a > < span class = "lineno" > 222< / span >   out.replace(pos, 1, < span class = "stringliteral" > " " < / span > );< / div > < div class = "line" > < a name = "l00223" > < / a > < span class = "lineno" > 223< / span >   }< / div > < div class = "line" > < a name = "l00224" > < / a > < span class = "lineno" > 224< / span >   < / div > < div class = "line" > < a name = "l00225" > < / a > < span class = "lineno" > 225< / span >   out = regex_replace(out, regex(< span class = "stringliteral" > " \\s+\n" < / span > ), < span class = "stringliteral" > " \n" < / span > ); < span class = "comment" > // Remove space at eol.< / span > < / div > < div class = "line" > < a name = "l00226" > < / a > < span class = "lineno" > 226< / span >   out = regex_replace(out, regex(< span class = "stringliteral" > " \n{2,}" < / span > ), < span class = "stringliteral" > " \n" < / span > ); < span class = "comment" > // Reduce newlines.< / span > < / div > < div class = "line" > < a name = "l00227" > < / a > < span class = "lineno" > 227< / span >   < / div > < div class = "line" > < a name = "l00228" > < / a > < span class = "lineno" > 228< / span >   < span class = "keywordflow" > return< / span > < a class = "code" href = "classremwharead_1_1URI.html#a37c36dbf8ccc03c8cc132be29e49f4ec" > unescape_html< / a > (out);< / div > < div class = "line" > < a name = "l00229" > < / a > < span class = "lineno" > 229< / span >   }< / div > < div class = "ttc" id = "classremwharead_1_1URI_html_a8b340b13ccf0bc3ae9059872ce48e06a" > < div class = "ttname" > < a href = "classremwharead_1_1URI.html#a8b340b13ccf0bc3ae9059872ce48e06a" > remwharead::URI::remove_html_tags< / a > < / div > < div class = "ttdeci" > const string remove_html_tags(const string & html, const string & tag=" " )< / div > < div class = "ttdoc" > Remove HTML tags. < / div > < div class = "ttdef" > < b > Definition:< / b > uri.cpp:230< / div > < / div >
< div class = "ttc" id = "classremwharead_1_1URI_html_a37c36dbf8ccc03c8cc132be29e49f4ec" > < div class = "ttname" > < a href = "classremwharead_1_1URI.html#a37c36dbf8ccc03c8cc132be29e49f4ec" > remwharead::URI::unescape_html< / a > < / div > < div class = "ttdeci" > const string unescape_html(const string & html)< / div > < div class = "ttdoc" > Convert HTML entities to UTF-8. < / div > < div class = "ttdef" > < b > Definition:< / b > uri.cpp:268< / div > < / div >
2019-07-28 02:30:30 +02:00
< / div > <!-- fragment -->
< / div >
< / div >
< a id = "a37c36dbf8ccc03c8cc132be29e49f4ec" > < / a >
< h2 class = "memtitle" > < span class = "permalink" > < a href = "#a37c36dbf8ccc03c8cc132be29e49f4ec" > ◆ < / a > < / span > unescape_html()< / h2 >
< div class = "memitem" >
< div class = "memproto" >
< table class = "mlabels" >
< tr >
< td class = "mlabels-left" >
< table class = "memname" >
< tr >
< td class = "memname" > const string remwharead::URI::unescape_html < / td >
< td > (< / td >
< td class = "paramtype" > const string &   < / td >
< td class = "paramname" > < em > html< / em > < / td > < td > )< / td >
< td > < / td >
< / tr >
< / table >
< / td >
< td class = "mlabels-right" >
< span class = "mlabels" > < span class = "mlabel" > protected< / span > < / span > < / td >
< / tr >
< / table >
< / div > < div class = "memdoc" >
< p > Convert HTML entities to UTF-8. < / p >
2019-08-05 22:01:08 +02:00
< div class = "fragment" > < div class = "line" > < a name = "l00269" > < / a > < span class = "lineno" > 269< / span >   {< / div > < div class = "line" > < a name = "l00270" > < / a > < span class = "lineno" > 270< / span >   < span class = "keywordtype" > string< / span > buffer = html;< / div > < div class = "line" > < a name = "l00271" > < / a > < span class = "lineno" > 271< / span >   < span class = "keywordtype" > string< / span > output;< / div > < div class = "line" > < a name = "l00272" > < / a > < span class = "lineno" > 272< / span >   < / div > < div class = "line" > < a name = "l00273" > < / a > < span class = "lineno" > 273< / span >   < span class = "comment" > // Used to convert int to utf-8 char.< / span > < / div > < div class = "line" > < a name = "l00274" > < / a > < span class = "lineno" > 274< / span >   std::wstring_convert< std::codecvt_utf8< char32_t> , char32_t> u8c;< / div > < div class = "line" > < a name = "l00275" > < / a > < span class = "lineno" > 275< / span >   regex re_entity(< span class = "stringliteral" > " & #(x)?([[:alnum:]]{1,8});" < / span > );< / div > < div class = "line" > < a name = "l00276" > < / a > < span class = "lineno" > 276< / span >   smatch match;< / div > < div class = "line" > < a name = "l00277" > < / a > < span class = "lineno" > 277< / span >   < / div > < div class = "line" > < a name = "l00278" > < / a > < span class = "lineno" > 278< / span >   < span class = "keywordflow" > while< / span > (regex_search(buffer, match, re_entity))< / div > < div class = "line" > < a name = "l00279" > < / a > < span class = "lineno" > 279< / span >   {< / div > < div class = "line" > < a name = "l00280" > < / a > < span class = "lineno" > 280< / span >   char32_t codepoint = 0;< / div > < div class = "line" > < a name = "l00281" > < / a > < span class = "lineno" > 281< / span >   < span class = "comment" > // ' x' in front of the number means it' s hexadecimal, else decimal.< / span > < / div > < div class = "line" > < a name = "l00282" > < / a > < span class = "lineno" > 282< / span >   < span class = "keywordflow" > if< / span > (match[1].length() == 1)< / div > < div class = "line" > < a name = "l00283" > < / a > < span class = "lineno" > 283< / span >   {< / div > < div class = "line" > < a name = "l00284" > < / a > < span class = "lineno" > 284< / span >   codepoint = std::stoi(match[2].str(), < span class = "keyword" > nullptr< / span > , 16);< / div > < div class = "line" > < a name = "l00285" > < / a > < span class = "lineno" > 285< / span >   }< / div > < div class = "line" > < a name = "l00286" > < / a > < span class = "lineno" > 286< / span >   < span class = "keywordflow" > else< / span > < / div > < div class = "line" > < a name = "l00287" > < / a > < span class = "lineno" > 287< / span >   {< / div > < div class = "line" > < a name = "l00288" > < / a > < span class = "lineno" > 288< / span >   codepoint = std::stoi(match[2].str(), < span class = "keyword" > nullptr< / span > , 10);< / div > < div class = "line" > < a name = "l00289" > < / a > < span class = "lineno" > 289< / span >   }< / div > < div class = "line" > < a name = "l00290" > < / a > < span class = "lineno" > 290< / span >   output += match.prefix().str() + u8c.to_bytes(codepoint);< / div > < div class = "line" > < a name = "l00291" > < / a > < span class = "lineno" > 291< / span >   buffer = match.suffix().str();< / div > < div class = "line" > < a name = "l00292" > < / a > < span class = "lineno" > 292< / span >   }< / div > < div class = "line" > < a name = "l00293" > < / a > < span class = "lineno" > 293< / span >   output += buffer;< / div > < div class = "line" > < a name = "l00294" > < / a > < span class = "lineno" > 294< / span >   < / div > < div class = "line" > < a name = "l00295" > < / a > < span class = "lineno" > 295< / span >   < span class = "comment" > // Source: https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_< / span > < / div > < div class = "line" > < a name = "l00296" > < / a > < span class = "lineno" > 296< / span >   < span class = "comment" > // entity_references#Character_entity_references_in_HTML< / span > < / div > < div class = "line" > < a name = "l00297" > < / a > < span class = "lineno" > 297< / span >   < span class = "keyword" > const< / span > array< const std::pair< const string, const char32_t> , 258> names =< / div > < div class = "line" > < a name = "l00298" > < / a > < span class = "lineno" > 298< / span >   {{< / div > < div class = "line" > < a name = "l00299" > < / a > < span
2019-07-28 02:30:30 +02:00
< / div >
< / div >
2019-07-27 22:46:58 +02:00
< hr / > The documentation for this class was generated from the following files:< ul >
2019-08-05 22:01:08 +02:00
< li > include/< a class = "el" href = "uri_8hpp_source.html" > uri.hpp< / a > < / li >
2019-07-27 22:46:58 +02:00
< li > src/lib/uri.cpp< / li >
< / ul >
< / div > <!-- contents -->
<!-- start footer part -->
< hr class = "footer" / > < address class = "footer" > < small >
Generated by   < a href = "http://www.doxygen.org/index.html" >
< img class = "footer" src = "doxygen.png" alt = "doxygen" / >
< / a > 1.8.14
< / small > < / address >
< / body >
< / html >
