remwharead  0.8.3
Public Member Functions | Protected Member Functions | Protected Attributes | List of all members
remwharead::URI Class Reference

Download, archive and process an URI. More...

#include <remwharead/uri.hpp>

Public Member Functions

 URI (const string &uri)
 Construct object and set URL. More...
 
const html_extract get ()
 Download URI and extract title, description and full text. More...
 
const archive_answer archive ()
 Save URI in archive and return archive-URI. More...
 

Protected Member Functions

const string make_request (const string &uri, bool archive=false) const
 Make a HTTP(S) request. More...
 
const string extract_title (const string &html)
 Extract the title from an HTML page. More...
 
const string extract_description (const string &html)
 Extract the description from an HTML page. More...
 
const string strip_html (const string &html)
 Removes HTML tags and superflous spaces from an HTML page. More...
 
const string remove_html_tags (const string &html, const string &tag="")
 Remove HTML tags. More...
 
const string unescape_html (string html)
 Convert HTML entities to UTF-8. More...
 
const string remove_newlines (string text)
 Replace newlines with spaces. More...
 

Protected Attributes

string _uri
 

Detailed Description

Download, archive and process an URI.

Since
0.6.0

Constructor & Destructor Documentation

◆ URI()

remwharead::URI::URI ( const string &  uri)
explicit

Construct object and set URL.

Initializes TLS and sets proxy from the environment variable http_proxy, if possible.

Since
0.6.0
65  :_uri(uri)
66  {
67  Poco::Net::initializeSSL();
68 
69  try
70  {
71  HTTPClientSession::ProxyConfig proxy;
72  const string env_proxy = Environment::get("http_proxy");
73  const RegEx re_proxy("^(?:https?://)?(?:([^:]+):?([^@]*)@)?"
74  "([^:/]+)(?::([\\d]{1,5}))?/?$");
75  vector<string> matches;
76 
77  if (re_proxy.split(env_proxy, matches) >= 4)
78  {
79  proxy.username = matches[1];
80  proxy.password = matches[2];
81  proxy.host = matches[3];
82  if (!matches[4].empty())
83  {
84  const std::uint32_t &port = std::stoul(matches[4]);
85  if (port > 65535)
86  {
87  throw std::invalid_argument("Port number out of range");
88  }
89  proxy.port = port;
90  }
91  }
92  HTTPClientSession::setGlobalProxyConfig(proxy);
93  }
94  catch (const Poco::RegularExpressionException &e)
95  {
96  cerr << "Error: Proxy could not be set ("
97  << e.displayText() << ")\n";
98  }
99  catch (const std::invalid_argument &e)
100  {
101  cerr << "Error: " << e.what() << endl;
102  }
103  catch (const std::exception &)
104  {
105  // No proxy found, no problem.
106  }
107  }

Member Function Documentation

◆ archive()

const archive_answer remwharead::URI::archive ( )

Save URI in archive and return archive-URI.

Since
0.6.0
607  {
608  if (_uri.substr(0, 4) != "http")
609  {
610  return { false, "Only HTTP(S) is archivable.", "" };
611  }
612 
613  try
614  {
615  const string answer = make_request("https://web.archive.org/save/"
616  + _uri, true);
617 
618  if (!answer.empty())
619  {
620  return { true, "", "https://web.archive.org" + answer };
621  }
622  }
623  catch (const Poco::Exception &e)
624  {
625  return { false, e.displayText(), "" };
626  }
627 
628  return { false, "Unknown error.", "" };
629  }
remwharead::URI::make_request
const string make_request(const string &uri, bool archive=false) const
Make a HTTP(S) request.
Definition: uri.cpp:139

◆ extract_description()

const string remwharead::URI::extract_description ( const string &  html)
protected

Extract the description from an HTML page.

Since
0.6.0
230  {
231  const RegEx re_htmlfile(".*\\.(.?html?|xml|rss)$", RegEx::RE_CASELESS);
232  if (_uri.substr(0, 4) == "http" || re_htmlfile.match(_uri))
233  {
234  const RegEx re_desc("description\"[^>]+content=\"([^\"]+)",
235  RegEx::RE_CASELESS);
236  vector<string> matches;
237  re_desc.split(html, matches);
238  if (matches.size() >= 2)
239  {
240  return remove_newlines(unescape_html(matches[1]));
241  }
242  }
243 
244  return "";
245  }
remwharead::URI::unescape_html
const string unescape_html(string html)
Convert HTML entities to UTF-8.
Definition: uri.cpp:306
remwharead::URI::remove_newlines
const string remove_newlines(string text)
Replace newlines with spaces.
Definition: uri.cpp:631

◆ extract_title()

const string remwharead::URI::extract_title ( const string &  html)
protected

Extract the title from an HTML page.

Since
0.6.0
213  {
214  const RegEx re_htmlfile(".*\\.(.?html?|xml|rss)$", RegEx::RE_CASELESS);
215  if (_uri.substr(0, 4) == "http" || re_htmlfile.match(_uri))
216  {
217  const RegEx re_title("<title>([^<]+)", RegEx::RE_CASELESS);
218  vector<string> matches;
219  re_title.split(html, matches);
220  if (matches.size() >= 2)
221  {
222  return remove_newlines(unescape_html(matches[1]));
223  }
224  }
225 
226  return "";
227  }
remwharead::URI::unescape_html
const string unescape_html(string html)
Convert HTML entities to UTF-8.
Definition: uri.cpp:306
remwharead::URI::remove_newlines
const string remove_newlines(string text)
Replace newlines with spaces.
Definition: uri.cpp:631

◆ get()

const html_extract remwharead::URI::get ( )

Download URI and extract title, description and full text.

Since
0.6.0
115  {
116  try
117  {
118  const string answer = make_request(_uri);
119  if (!answer.empty())
120  {
121  return
122  {
123  true,
124  "",
125  extract_title(answer),
126  extract_description(answer),
127  strip_html(answer)
128  };
129  }
130  }
131  catch (const Poco::Exception &e)
132  {
133  return { false, e.displayText(), "", "", "" };
134  }
135 
136  return { false, "Unknown error.", "", "", "" };
137  }
remwharead::URI::strip_html
const string strip_html(const string &html)
Removes HTML tags and superflous spaces from an HTML page.
Definition: uri.cpp:247
remwharead::URI::extract_title
const string extract_title(const string &html)
Extract the title from an HTML page.
Definition: uri.cpp:212
remwharead::URI::make_request
const string make_request(const string &uri, bool archive=false) const
Make a HTTP(S) request.
Definition: uri.cpp:139
remwharead::URI::extract_description
const string extract_description(const string &html)
Extract the description from an HTML page.
Definition: uri.cpp:229

◆ make_request()

const string remwharead::URI::make_request ( const string &  uri,
bool  archive = false 
) const
protected

Make a HTTP(S) request.

Since
0.6.0
140  {
141  Poco::URI poco_uri(uri);
142  string method =
143  archive ? HTTPRequest::HTTP_HEAD : HTTPRequest::HTTP_GET;
144  string path = poco_uri.getPathAndQuery();
145  if (path.empty())
146  {
147  path = "/";
148  }
149 
150  unique_ptr<HTTPClientSession> session;
151  if (poco_uri.getScheme() == "https")
152  {
153  session = make_unique<HTTPSClientSession>(poco_uri.getHost(),
154  poco_uri.getPort());
155  }
156  else if (poco_uri.getScheme() == "http")
157  {
158  session = make_unique<HTTPClientSession>(poco_uri.getHost(),
159  poco_uri.getPort());
160  }
161  else
162  {
163  throw Poco::Exception("Protocol not supported.");
164  }
165 
166  HTTPRequest request(method, path, HTTPMessage::HTTP_1_1);
167  request.set("User-Agent", string("remwharead/") + global::version);
168 
169  HTTPResponse response;
170 
171  session->sendRequest(request);
172  istream &rs = session->receiveResponse(response);
173 
174  // Not using the constants because some are too new for Debian stretch.
175  switch (response.getStatus())
176  {
177  case 301: // HTTPResponse::HTTP_MOVED_PERMANENTLY
178  case 308: // HTTPResponse::HTTP_PERMANENT_REDIRECT
179  case 302: // HTTPResponse::HTTP_FOUND
180  case 303: // HTTPResponse::HTTP_SEE_OTHER
181  case 307: // HTTPResponse::HTTP_TEMPORARY_REDIRECT
182  {
183  string location = response.get("Location");
184  if (location.substr(0, 4) != "http")
185  {
186  location = poco_uri.getScheme() + "://" + poco_uri.getHost()
187  + location;
188  }
189  return make_request(location);
190  }
191  case HTTPResponse::HTTP_OK:
192  {
193  string answer;
194  if (archive)
195  {
196  answer = response.get("Content-Location");
197  }
198  else
199  {
200  StreamCopier::copyToString(rs, answer);
201  }
202  return answer;
203  }
204  default:
205  {
206  throw Poco::Exception(response.getReason());
207  return "";
208  }
209  }
210  }
remwharead::URI::make_request
const string make_request(const string &uri, bool archive=false) const
Make a HTTP(S) request.
Definition: uri.cpp:139
remwharead::URI::archive
const archive_answer archive()
Save URI in archive and return archive-URI.
Definition: uri.cpp:606

◆ remove_html_tags()

const string remwharead::URI::remove_html_tags ( const string &  html,
const string &  tag = "" 
)
protected

Remove HTML tags.

Parameters
htmlHTML page.
tagIf set, only remove this tag.
Since
0.6.0
269  {
270  // NOTE: I did this with regex_replace before, but libstdc++ segfaulted.
271  string out;
272  if (tag.empty())
273  {
274  size_t pos = 0;
275  while (pos != std::string::npos)
276  {
277  size_t startpos = html.find('<', pos);
278  size_t endpos = html.find('>', startpos);
279  out += html.substr(pos, startpos - pos);
280  pos = endpos;
281  if (pos != std::string::npos)
282  {
283  ++pos;
284  }
285  }
286  }
287  else
288  {
289  size_t pos = 0;
290  out = html;
291  while ((pos = out.find("<" + tag)) != std::string::npos)
292  {
293  size_t endpos = out.find("</" + tag, pos);
294  if (endpos == std::string::npos)
295  {
296  break;
297  }
298  endpos += 3 + tag.length(); // tag + </ + >
299  out.replace(pos, endpos - pos, "");
300  }
301  }
302 
303  return out;
304  }

◆ remove_newlines()

const string remwharead::URI::remove_newlines ( string  text)
protected

Replace newlines with spaces.

Since
0.6.0
632  {
633  size_t posn = 0;
634  while ((posn = text.find('\n', posn)) != std::string::npos)
635  {
636  text.replace(posn, 1, " ");
637 
638  size_t posr = posn - 1;
639  if (text[posr] == '\r')
640  {
641  text.replace(posr, 1, " ");
642  }
643  ++posn;
644  }
645 
646  return text;
647  }

◆ strip_html()

const string remwharead::URI::strip_html ( const string &  html)
protected

Removes HTML tags and superflous spaces from an HTML page.

Since
0.6.0
248  {
249  string out;
250 
251  out = remove_html_tags(html, "script"); // Remove JavaScript.
252  out = remove_html_tags(out, "style"); // Remove CSS.
253  out = remove_html_tags(out); // Remove tags.
254 
255  size_t pos = 0;
256  while ((pos = out.find("\r", pos)) != std::string::npos) // Remove CR.
257  {
258  out.replace(pos, 1, "");
259  }
260 
261  // Remove whitespace at eol.
262  RegEx("\\s+\n").subst(out, "\n", RegEx::RE_GLOBAL);
263  RegEx("\n{2,}").subst(out, "\n", RegEx::RE_GLOBAL); // Reduce newlines.
264 
265  return unescape_html(out);
266  }
remwharead::URI::unescape_html
const string unescape_html(string html)
Convert HTML entities to UTF-8.
Definition: uri.cpp:306
remwharead::URI::remove_html_tags
const string remove_html_tags(const string &html, const string &tag="")
Remove HTML tags.
Definition: uri.cpp:268

◆ unescape_html()

const string remwharead::URI::unescape_html ( string  html)
protected

Convert HTML entities to UTF-8.

Since
0.6.0
307  {
308  // Used to convert int to utf-8 char.
309  std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> u8c;
310  const RegEx re_entity("&#(x)?([[:alnum:]]{1,8});");
311  RegEx::MatchVec matches;
312  string::size_type pos = 0;
313 
314  while (re_entity.match(html, pos, matches) != 0)
315  {
316  char32_t codepoint = 0;
317  const string number = html.substr(matches[2].offset,
318  matches[2].length);
319  // 'x' in front of the number means it's hexadecimal, else decimal.
320  if (matches[1].length != 0)
321  {
322  codepoint = std::stoi(number, nullptr, 16);
323  }
324  else
325  {
326  codepoint = std::stoi(number, nullptr, 10);
327  }
328  const string unicode = u8c.to_bytes(codepoint);
329  html.replace(matches[0].offset, matches[0].length, unicode);
330  pos = matches[0].offset + unicode.length();
331  }
332 
333  // Source: https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_
334  // entity_references#Character_entity_references_in_HTML
335  const array<const std::pair<const string, const char32_t>, 258> names =
336  {{
337  { "exclamation", 0x0021 },
338  { "quot", 0x0022 },
339  { "percent", 0x0025 },
340  { "amp", 0x0026 },
341  { "apos", 0x0027 },
342  { "add", 0x002B },
343  { "lt", 0x003C },
344  { "equal", 0x003D },
345  { "gt", 0x003E },
346  { "nbsp", 0x00A0 },
347  { "iexcl", 0x00A1 },
348  { "cent", 0x00A2 },
349  { "pound", 0x00A3 },
350  { "curren", 0x00A4 },
351  { "yen", 0x00A5 },
352  { "brvbar", 0x00A6 },
353  { "sect", 0x00A7 },
354  { "uml", 0x00A8 },
355  { "copy", 0x00A9 },
356  { "ordf", 0x00AA },
357  { "laquo", 0x00AB },
358  { "not", 0x00AC },
359  { "shy", 0x00AD },
360  { "reg", 0x00AE },
361  { "macr", 0x00AF },
362  { "deg", 0x00B0 },
363  { "plusmn", 0x00B1 },
364  { "sup2", 0x00B2 },
365  { "sup3", 0x00B3 },
366  { "acute", 0x00B4 },
367  { "micro", 0x00B5 },
368  { "para", 0x00B6 },
369  { "middot", 0x00B7 },
370  { "cedil", 0x00B8 },
371  { "sup1", 0x00B9 },
372  { "ordm", 0x00BA },
373  { "raquo", 0x00BB },
374  { "frac14", 0x00BC },
375  { "frac12", 0x00BD },
376  { "frac34", 0x00BE },
377  { "iquest", 0x00BF },
378  { "Agrave", 0x00C0 },
379  { "Aacute", 0x00C1 },
380  { "Acirc", 0x00C2 },
381  { "Atilde", 0x00C3 },
382  { "Auml", 0x00C4 },
383  { "Aring", 0x00C5 },
384  { "AElig", 0x00C6 },
385  { "Ccedil", 0x00C7 },
386  { "Egrave", 0x00C8 },
387  { "Eacute", 0x00C9 },
388  { "Ecirc", 0x00CA },
389  { "Euml", 0x00CB },
390  { "Igrave", 0x00CC },
391  { "Iacute", 0x00CD },
392  { "Icirc", 0x00CE },
393  { "Iuml", 0x00CF },
394  { "ETH", 0x00D0 },
395  { "Ntilde", 0x00D1 },
396  { "Ograve", 0x00D2 },
397  { "Oacute", 0x00D3 },
398  { "Ocirc", 0x00D4 },
399  { "Otilde", 0x00D5 },
400  { "Ouml", 0x00D6 },
401  { "times", 0x00D7 },
402  { "Oslash", 0x00D8 },
403  { "Ugrave", 0x00D9 },
404  { "Uacute", 0x00DA },
405  { "Ucirc", 0x00DB },
406  { "Uuml", 0x00DC },
407  { "Yacute", 0x00DD },
408  { "THORN", 0x00DE },
409  { "szlig", 0x00DF },
410  { "agrave", 0x00E0 },
411  { "aacute", 0x00E1 },
412  { "acirc", 0x00E2 },
413  { "atilde", 0x00E3 },
414  { "auml", 0x00E4 },
415  { "aring", 0x00E5 },
416  { "aelig", 0x00E6 },
417  { "ccedil", 0x00E7 },
418  { "egrave", 0x00E8 },
419  { "eacute", 0x00E9 },
420  { "ecirc", 0x00EA },
421  { "euml", 0x00EB },
422  { "igrave", 0x00EC },
423  { "iacute", 0x00ED },
424  { "icirc", 0x00EE },
425  { "iuml", 0x00EF },
426  { "eth", 0x00F0 },
427  { "ntilde", 0x00F1 },
428  { "ograve", 0x00F2 },
429  { "oacute", 0x00F3 },
430  { "ocirc", 0x00F4 },
431  { "otilde", 0x00F5 },
432  { "ouml", 0x00F6 },
433  { "divide", 0x00F7 },
434  { "oslash", 0x00F8 },
435  { "ugrave", 0x00F9 },
436  { "uacute", 0x00FA },
437  { "ucirc", 0x00FB },
438  { "uuml", 0x00FC },
439  { "yacute", 0x00FD },
440  { "thorn", 0x00FE },
441  { "yuml", 0x00FF },
442  { "OElig", 0x0152 },
443  { "oelig", 0x0153 },
444  { "Scaron", 0x0160 },
445  { "scaron", 0x0161 },
446  { "Yuml", 0x0178 },
447  { "fnof", 0x0192 },
448  { "circ", 0x02C6 },
449  { "tilde", 0x02DC },
450  { "Alpha", 0x0391 },
451  { "Beta", 0x0392 },
452  { "Gamma", 0x0393 },
453  { "Delta", 0x0394 },
454  { "Epsilon", 0x0395 },
455  { "Zeta", 0x0396 },
456  { "Eta", 0x0397 },
457  { "Theta", 0x0398 },
458  { "Iota", 0x0399 },
459  { "Kappa", 0x039A },
460  { "Lambda", 0x039B },
461  { "Mu", 0x039C },
462  { "Nu", 0x039D },
463  { "Xi", 0x039E },
464  { "Omicron", 0x039F },
465  { "Pi", 0x03A0 },
466  { "Rho", 0x03A1 },
467  { "Sigma", 0x03A3 },
468  { "Tau", 0x03A4 },
469  { "Upsilon", 0x03A5 },
470  { "Phi", 0x03A6 },
471  { "Chi", 0x03A7 },
472  { "Psi", 0x03A8 },
473  { "Omega", 0x03A9 },
474  { "alpha", 0x03B1 },
475  { "beta", 0x03B2 },
476  { "gamma", 0x03B3 },
477  { "delta", 0x03B4 },
478  { "epsilon", 0x03B5 },
479  { "zeta", 0x03B6 },
480  { "eta", 0x03B7 },
481  { "theta", 0x03B8 },
482  { "iota", 0x03B9 },
483  { "kappa", 0x03BA },
484  { "lambda", 0x03BB },
485  { "mu", 0x03BC },
486  { "nu", 0x03BD },
487  { "xi", 0x03BE },
488  { "omicron", 0x03BF },
489  { "pi", 0x03C0 },
490  { "rho", 0x03C1 },
491  { "sigmaf", 0x03C2 },
492  { "sigma", 0x03C3 },
493  { "tau", 0x03C4 },
494  { "upsilon", 0x03C5 },
495  { "phi", 0x03C6 },
496  { "chi", 0x03C7 },
497  { "psi", 0x03C8 },
498  { "omega", 0x03C9 },
499  { "thetasym", 0x03D1 },
500  { "upsih", 0x03D2 },
501  { "piv", 0x03D6 },
502  { "ensp", 0x2002 },
503  { "emsp", 0x2003 },
504  { "thinsp", 0x2009 },
505  { "zwnj", 0x200C },
506  { "zwj", 0x200D },
507  { "lrm", 0x200E },
508  { "rlm", 0x200F },
509  { "ndash", 0x2013 },
510  { "mdash", 0x2014 },
511  { "horbar", 0x2015 },
512  { "lsquo", 0x2018 },
513  { "rsquo", 0x2019 },
514  { "sbquo", 0x201A },
515  { "ldquo", 0x201C },
516  { "rdquo", 0x201D },
517  { "bdquo", 0x201E },
518  { "dagger", 0x2020 },
519  { "Dagger", 0x2021 },
520  { "bull", 0x2022 },
521  { "hellip", 0x2026 },
522  { "permil", 0x2030 },
523  { "prime", 0x2032 },
524  { "Prime", 0x2033 },
525  { "lsaquo", 0x2039 },
526  { "rsaquo", 0x203A },
527  { "oline", 0x203E },
528  { "frasl", 0x2044 },
529  { "euro", 0x20AC },
530  { "image", 0x2111 },
531  { "weierp", 0x2118 },
532  { "real", 0x211C },
533  { "trade", 0x2122 },
534  { "alefsym", 0x2135 },
535  { "larr", 0x2190 },
536  { "uarr", 0x2191 },
537  { "rarr", 0x2192 },
538  { "darr", 0x2193 },
539  { "harr", 0x2194 },
540  { "crarr", 0x21B5 },
541  { "lArr", 0x21D0 },
542  { "uArr", 0x21D1 },
543  { "rArr", 0x21D2 },
544  { "dArr", 0x21D3 },
545  { "hArr", 0x21D4 },
546  { "forall", 0x2200 },
547  { "part", 0x2202 },
548  { "exist", 0x2203 },
549  { "empty", 0x2205 },
550  { "nabla", 0x2207 },
551  { "isin", 0x2208 },
552  { "notin", 0x2209 },
553  { "ni", 0x220B },
554  { "prod", 0x220F },
555  { "sum", 0x2211 },
556  { "minus", 0x2212 },
557  { "lowast", 0x2217 },
558  { "radic", 0x221A },
559  { "prop", 0x221D },
560  { "infin", 0x221E },
561  { "ang", 0x2220 },
562  { "and", 0x2227 },
563  { "or", 0x2228 },
564  { "cap", 0x2229 },
565  { "cup", 0x222A },
566  { "int", 0x222B },
567  { "there4", 0x2234 },
568  { "sim", 0x223C },
569  { "cong", 0x2245 },
570  { "asymp", 0x2248 },
571  { "ne", 0x2260 },
572  { "equiv", 0x2261 },
573  { "le", 0x2264 },
574  { "ge", 0x2265 },
575  { "sub", 0x2282 },
576  { "sup", 0x2283 },
577  { "nsub", 0x2284 },
578  { "sube", 0x2286 },
579  { "supe", 0x2287 },
580  { "oplus", 0x2295 },
581  { "otimes", 0x2297 },
582  { "perp", 0x22A5 },
583  { "sdot", 0x22C5 },
584  { "lceil", 0x2308 },
585  { "rceil", 0x2309 },
586  { "lfloor", 0x230A },
587  { "rfloor", 0x230B },
588  { "lang", 0x2329 },
589  { "rang", 0x232A },
590  { "loz", 0x25CA },
591  { "spades", 0x2660 },
592  { "clubs", 0x2663 },
593  { "hearts", 0x2665 },
594  { "diams", 0x2666 }
595  }};
596 
597  for (auto &pair : names)
598  {
599  const RegEx re('&' + pair.first + ';');
600  re.subst(html, u8c.to_bytes(pair.second), RegEx::RE_GLOBAL);
601  }
602 
603  return html;
604  }
The documentation for this class was generated from the following files:
Generated by   doxygen 1.8.15