Compare commits

...

155 Commits
0.8.2 ... main

Author SHA1 Message Date
tastytea 6123c7fbb3
Use sub-project curl_wrapper instead of custom implementation.
continuous-integration/drone/push Build is passing Details
Via git subtree from <https://schlomp.space/tastytea/curl_wrapper>.
2020-11-13 22:41:56 +01:00
tastytea 5b56ad00b3
Merge commit 'abd97989f27429dd88e26110aa4feae3e9e58b0e' into main 2020-11-13 22:08:57 +01:00
tastytea abd97989f2 Squashed 'src/curl_wrapper/' changes from 5af456a..e3b5476
e3b5476 Build position independent code (-fPIC).

git-subtree-dir: src/curl_wrapper
git-subtree-split: e3b5476d7e5fd031a775f2bc6f66fbd534fb8602
2020-11-13 22:08:57 +01:00
tastytea dead7481d8 Merge commit '54afa6babfaee9123f882fd2f2bb12b8824b4e5a' as 'src/curl_wrapper' 2020-11-13 21:27:54 +01:00
tastytea 54afa6babf Squashed 'src/curl_wrapper/' content from commit 5af456a
git-subtree-dir: src/curl_wrapper
git-subtree-split: 5af456a18b84805e0b65c187512d6810ec7961e9
2020-11-13 21:27:54 +01:00
tastytea 79bf24942d
Add curl dependency to readme.
continuous-integration/drone/push Build is passing Details
2020-10-31 19:04:26 +01:00
tastytea 26f52d23a6
Add libcurl dependency to CI and RPM.
continuous-integration/drone/push Build was killed Details
2020-10-31 18:35:23 +01:00
tastytea 6426305366
Use libcurl to download URIs.
continuous-integration/drone/push Build is failing Details
2020-10-31 18:26:05 +01:00
tastytea 4dc9faf1df
Make reference in ranged for const.
continuous-integration/drone/push Build is passing Details
2020-10-31 13:30:40 +01:00
tastytea a198f1323a
Reformat remwharead_wrapper.cpp. 2020-10-31 13:30:04 +01:00
tastytea 4e9e750e35
Initialize variables in native-wrapper. 2020-10-31 13:28:45 +01:00
tastytea 5b82bb0ef0
Reformat uri.hpp. 2020-10-31 13:25:45 +01:00
tastytea b7d5b21899
Make operator bool() in html_extract and archive_answer const. 2020-10-31 13:25:02 +01:00
tastytea eab10e28ae
Reformat uri.cpp. 2020-10-31 13:21:40 +01:00
tastytea 6d45af58a2
Fix Debian dependencies example. 2020-10-31 13:09:52 +01:00
tastytea 8b953ff817
Remove Hunter support. 2020-10-31 13:04:19 +01:00
tastytea 0b32de2846
Add .clang-format. 2020-10-31 13:01:53 +01:00
tastytea b40f9910d0
Make use of CMake up to 3.17. 2020-10-31 13:00:47 +01:00
tastytea 4f4f4dacad
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2020-02-07 11:58:32 +01:00
tastytea 4f78da24b5
Allow answer of archiving service to be an HTTP redirection.
continuous-integration/drone/push Build is passing Details
2020-02-07 11:56:05 +01:00
tastytea 6802c7563d
Version bump 0.10.0. 2020-01-31 05:57:47 +01:00
tastytea d75342e71f
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2020-01-31 05:08:25 +01:00
tastytea 7e147012fe
Rewrite wrapper to be more readable.
continuous-integration/drone/push Build is passing Details
2020-01-28 02:48:58 +01:00
tastytea 2b8e443da7
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2020-01-27 10:49:02 +01:00
tastytea 5e9937c9a0
WebExtension: Use \u001f as separator.
continuous-integration/drone/push Build is passing Details
2020-01-27 10:35:41 +01:00
tastytea 1c2828369d
Unescape the escaped string we get from the WebExtension. 2020-01-27 10:34:46 +01:00
tastytea 53230aedaa
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2020-01-27 07:41:28 +01:00
tastytea b79dacf742
Remove debug statement.
continuous-integration/drone/push Build is passing Details
2020-01-27 07:26:39 +01:00
tastytea ee4debb005
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2020-01-27 06:55:05 +01:00
tastytea 8a4f518cb9
Fix clang-tidy warnings.
continuous-integration/drone/push Build is passing Details
2020-01-27 06:32:28 +01:00
tastytea 3a7f4ba8e2
Update .clang-tidy. 2020-01-27 06:18:58 +01:00
tastytea bb5d9cecad
Change namespace of version. 2020-01-27 04:50:33 +01:00
tastytea 8c1c18ed54
Add support for more robust message parsing to native-wrapper.
continuous-integration/drone/push Build is passing Details
Step 1 for #7.
2020-01-27 04:27:45 +01:00
tastytea a28d674ac0
Add .clang-tidy. 2020-01-27 03:24:12 +01:00
tastytea 4b6d1c19e3
Add support for clang-tidy to CMake recipe. 2020-01-27 03:23:47 +01:00
tastytea 70cc91e0fc
Remove unnecessary -g from debug flags. 2020-01-27 03:22:08 +01:00
tastytea 2ce2530ac3
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-12-31 11:23:44 +01:00
tastytea 62fd8a8abb
Use Poco::URI::encode to percent-encode URIs in AsciiDoc export.
continuous-integration/drone/push Build was killed Details
2019-12-31 11:22:57 +01:00
tastytea 489e1fba99
Fix regex for extracting charset.
continuous-integration/drone/push Build is passing Details
2019-12-19 21:49:53 +01:00
tastytea 92db2241b0
Add Boost extras to global locale.
continuous-integration/drone/push Build is passing Details
Needed for boost::locale::to_lower() to work.
2019-12-17 13:37:06 +01:00
tastytea f9101a3aa1
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-12-15 14:52:28 +01:00
tastytea c3307dddf0
Prevent interpretation of AsciiDoc syntax in descriptions.
continuous-integration/drone/push Build was killed Details
2019-12-15 14:50:06 +01:00
tastytea 56dc4083ce
Do not attempt to convert encoding if it is already utf-8. 2019-12-11 23:42:45 +01:00
tastytea d4d7cd4efd
Simplify is_html(). 2019-12-11 23:42:45 +01:00
tastytea ab94a9e6b0
Add Boost to dependencies.
continuous-integration/drone/push Build is passing Details
2019-12-11 23:42:12 +01:00
tastytea dc29da15ac
Version bump 0.9.2.
continuous-integration/drone/push Build is passing Details
2019-12-11 14:33:36 +01:00
tastytea c68f77262f
Merge branch 'develop' into main 2019-12-11 14:33:20 +01:00
tastytea c5dc9d4098
Add Boost to Hunter config.
continuous-integration/drone/push Build is passing Details
2019-12-11 14:23:04 +01:00
tastytea 3889a1b915
Add Boost to drone recipe. 2019-12-11 13:58:58 +01:00
tastytea 6fa611cf42
Detect file encoding of web page and convert to UTF-8.
continuous-integration/drone/push Build is failing Details
Fixes #6.
2019-12-11 13:01:44 +01:00
tastytea 7c7d28b7bc
Store document in class variable. 2019-12-11 13:00:43 +01:00
tastytea 0431b4a8ca
Version bump 0.9.1.
continuous-integration/drone/push Build is passing Details
2019-12-04 07:20:37 +01:00
tastytea 9706327c37
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-12-02 16:10:17 +01:00
tastytea b6e7449a6a
Allow <title> to have arguments.
continuous-integration/drone/push Build is passing Details
Fixes title extraction on medium.com posts.
2019-12-02 15:57:56 +01:00
tastytea d264ead7f9
Version bump 0.9.0.
continuous-integration/drone/push Build is passing Details
2019-11-30 03:51:22 +01:00
tastytea 8b95b43c45
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-11-28 09:52:21 +01:00
tastytea 970fb1486a
Add rofi export to documentation.
continuous-integration/drone/push Build is passing Details
2019-11-28 09:45:23 +01:00
tastytea cce04a60d9
Fix path in rofi script. 2019-11-28 09:33:27 +01:00
tastytea 4ac61852ed
Add test for rofi export.
continuous-integration/drone/push Build is passing Details
2019-11-28 09:22:03 +01:00
tastytea bce12ae53b
Add remwharead-rofi. 2019-11-28 09:04:07 +01:00
tastytea ad676bbc13
Add rofi export. 2019-11-28 09:03:52 +01:00
tastytea f8fd9b8c6d
Add text_to_string(). 2019-11-28 09:03:38 +01:00
tastytea 138170975a
Add list.hpp to remwharead.hpp. 2019-11-28 05:08:39 +01:00
tastytea 6c5328595f
Add example for mass-deletion.
continuous-integration/drone/push Build is passing Details
2019-11-28 02:26:02 +01:00
tastytea 21db0f3a62
Turn examples into codeblocks. 2019-11-28 02:26:02 +01:00
tastytea bf735b7654
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-11-28 01:27:36 +01:00
tastytea 3fe1e78af4
Add `--delete`, to delete URIs from the database.
continuous-integration/drone/push Build is passing Details
Closes #5.
2019-11-28 00:56:20 +01:00
tastytea 3cb491ccc9
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-11-27 23:31:50 +01:00
tastytea 859154a3e9
Add test for link export.
continuous-integration/drone/push Build is passing Details
2019-11-27 21:58:13 +01:00
tastytea 45668b6c9f
Add export: link.
A plain list of links, separated by newline. Closes #2.
2019-11-27 21:53:48 +01:00
tastytea e544ccb031
Version bump 0.8.6.
continuous-integration/drone/push Build is passing Details
2019-11-27 09:29:39 +01:00
tastytea 17f0398b1f
Merge branch 'develop' into main 2019-11-27 09:19:51 +01:00
tastytea e1ad1a3588
Replace deprecated timelocal() with mktime().
continuous-integration/drone/push Build is passing Details
2019-11-27 02:11:30 +01:00
tastytea b585a543fb
Cosmetic: Move const up. 2019-11-09 22:37:09 +01:00
tastytea 91fc599ec5
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-11-09 12:54:36 +01:00
tastytea 8564e5496f
Explain Hunter in greater detail.
continuous-integration/drone/push Build is passing Details
2019-11-09 12:25:08 +01:00
tastytea 4ea24f0101
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-11-06 12:59:42 +01:00
tastytea 693580a44a
Add support for Hunter package manager.
continuous-integration/drone/push Build is passing Details
2019-11-06 12:58:26 +01:00
tastytea 37f6da6710
[[nodiscard]] all the functions.
continuous-integration/drone/push Build is passing Details
2019-10-30 08:51:07 +01:00
tastytea 32cf647131
Nest namespaces. 2019-10-30 08:06:12 +01:00
tastytea 382c35856e
Switch to C++17, but support g++-6 / clang++-6.
continuous-integration/drone/push Build is passing Details
2019-10-30 07:49:12 +01:00
tastytea 47d89ce01a
Enable -Wmissing-declarations. 2019-10-30 07:48:50 +01:00
tastytea a5ec7609f9
Fix/supress clang-tidy warnings in Mozilla-wrapper. 2019-10-30 07:47:39 +01:00
tastytea e8f0156296
Do not terminate before all threads are joined. 2019-10-30 06:54:40 +01:00
tastytea 6b5a6c4b85
Move debug compiler flags into CMake module.
continuous-integration/drone/push Build is passing Details
2019-10-30 05:26:48 +01:00
tastytea cada434f2b
Default to Release build. 2019-10-30 05:04:00 +01:00
tastytea 6a9d96c208
Remove spaces betweeen `if` and `(` in root CMake file. 2019-10-30 04:58:59 +01:00
tastytea 3a30cedfb6
Fix indentation of root CMake recipe. 2019-10-30 04:50:11 +01:00
tastytea 6ff101370e
Require at least CMake 3.9. 2019-10-30 04:48:12 +01:00
tastytea bd60a4970d
Specify different indentation for CMake files in EditorConfig. 2019-10-30 04:46:15 +01:00
tastytea e40140fb03
Remove unnecessary const in function declaration.
continuous-integration/drone/push Build is passing Details
2019-10-28 06:46:58 +01:00
tastytea ea585751fc
Fix typo in NOLINT-comment. 2019-10-28 06:46:34 +01:00
tastytea eb77d8fc75
Implement all special member functions for ExportBase. 2019-10-28 06:46:08 +01:00
tastytea 46c7ab4a82
Reduce unnecessary indentation.
continuous-integration/drone/push Build is passing Details
2019-10-28 03:04:25 +01:00
tastytea e63a1c65c9
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-10-27 23:49:22 +01:00
tastytea b23347f61a
Do the archiving parallel to the fetching.
continuous-integration/drone/push Build is passing Details
Potentially halves the time needed.
2019-10-27 20:43:32 +01:00
tastytea 0a1396fc25
Mark `URI::archive()` const. 2019-10-27 20:38:58 +01:00
tastytea ff7b6338a0
Replace `Poco::Path::dataHome()` with `get_data_home()`.
continuous-integration/drone/push Build is passing Details
`Poco::Path::dataHome()` does not follow the XDG Base Directory Specification.
2019-10-27 07:21:18 +01:00
tastytea 253047c0b5
CI: Prepare rpm build.
continuous-integration/drone/push Build is passing Details
I couldn't find poco anywhere in the repos or EPEL.
2019-10-27 07:11:08 +01:00
tastytea 9937ecf5b0
CI: Compile with g++-7 instead of 8.
continuous-integration/drone/push Build is passing Details
2019-10-27 05:06:12 +01:00
tastytea e1ee5d7b37
Remove dependency on libxdg-basedir and pkg-config. 2019-10-27 05:05:51 +01:00
tastytea cae1d99f9a
Add missing header.
continuous-integration/drone/push Build is passing Details
2019-10-25 06:01:14 +02:00
tastytea d821a9310f
Add virtual destructor to ExportBase. 2019-10-25 06:00:56 +02:00
tastytea f8040f4803
Make type conversions explicit. 2019-10-25 06:00:36 +02:00
tastytea 7c2e33949b
Add more warnings to compiler flags. 2019-10-25 05:59:13 +02:00
tastytea d19dc71e3a
Version bump 0.8.5.
continuous-integration/drone/push Build is passing Details
2019-10-15 16:01:18 +02:00
tastytea bee3001901
Put = between options and arguments in manpage. 2019-10-15 15:59:26 +02:00
tastytea 869fb59be6
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-10-15 15:34:26 +02:00
tastytea d8102b017e
Cut descriptions at 500 characters.
continuous-integration/drone/push Build is passing Details
2019-10-15 15:10:39 +02:00
tastytea 2b9768a5a9
Provide help for specific arguments (--help=option).
continuous-integration/drone/push Build is passing Details
2019-10-10 14:36:59 +02:00
tastytea 43d9e3947a
Disable clang-tidy warnings I can't do anything about.
continuous-integration/drone/push Build is passing Details
2019-10-08 18:03:27 +02:00
tastytea 61aa6b4653
Removed alias DB = Database.
continuous-integration/drone/push Build is passing Details
2019-09-30 13:24:00 +02:00
tastytea 2c4fb47f63
Changed namespace-indentation and header order. 2019-09-30 13:23:45 +02:00
tastytea 714cad1a53
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-09-29 07:04:35 +02:00
tastytea e486be3a2a
Add test for RSS export.
continuous-integration/drone/push Build is passing Details
2019-09-29 06:23:51 +02:00
tastytea 019085cb73
Add test for JSON export. 2019-09-29 06:20:52 +02:00
tastytea b1ce2437c0
Simplified structs.
continuous-integration/drone/push Build is passing Details
using X = struct X → struct X.
2019-09-29 01:06:27 +02:00
tastytea 5c0e08a9b8
Fixed required CMake version in readme. 2019-09-29 01:05:33 +02:00
tastytea cdb8589d3a
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-09-27 03:47:59 +02:00
tastytea a5c21f4109
Fix contact-variable in COC.
continuous-integration/drone/push Build is passing Details
2019-09-27 03:30:22 +02:00
tastytea 5e5ce2e31e
use more variables in contribution guidelines. 2019-09-27 03:30:15 +02:00
tastytea 8d2cb99e0c
Include contributions guidelines in readme. 2019-09-27 03:19:31 +02:00
tastytea bb67d09f03
Make more use of variables in readme. 2019-09-27 03:19:04 +02:00
tastytea 47538c0350
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-09-26 08:12:06 +02:00
tastytea 5383bc59eb
Fix hyperlink to manpage.
continuous-integration/drone/push Build was killed Details
2019-09-26 08:10:35 +02:00
tastytea 4fb7067efc
Add hyperlink to blogpost.
continuous-integration/drone/push Build was killed Details
2019-09-26 08:08:52 +02:00
tastytea 1a4261f308
Replace inline-hyperlinks with variables in readme. 2019-09-26 08:08:30 +02:00
tastytea 432bac8c67
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-09-25 06:52:28 +02:00
tastytea f48b07f323
Refactored for better readability.
continuous-integration/drone/push Build is passing Details
Ran clang-tidy over the code, took most of the advice.
2019-09-25 03:58:29 +02:00
tastytea 31de8ff620
Replaced raw array with std::array. 2019-09-23 22:53:11 +02:00
tastytea a414b998f0
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-09-23 21:56:40 +02:00
tastytea da88eaf703
Moved tagpair-lambda up for better readability.
continuous-integration/drone/push Build is passing Details
2019-09-23 21:52:10 +02:00
tastytea b4fb62b15a
Add readability-newline.
continuous-integration/drone/push Build is passing Details
2019-09-23 19:28:19 +02:00
tastytea 8584ac2dfe
Add “patch via email”-paragraph to contributing guidelines. 2019-09-23 19:25:52 +02:00
tastytea c484800f6d
Improved short help for --export. 2019-09-23 03:16:58 +02:00
tastytea 4e58279767
Fixed hyperlink to bookmarks spec in manpage. 2019-09-23 03:16:32 +02:00
tastytea 68d590c82d
Version bump 0.8.4.
continuous-integration/drone/push Build is passing Details
2019-09-22 23:46:31 +02:00
tastytea c8598fca42
Merge branch 'develop' into main 2019-09-22 23:45:41 +02:00
tastytea e374405863
Retain archive-flag on HTTP redirects.
continuous-integration/drone/push Build is passing Details
2019-09-22 23:43:46 +02:00
tastytea ab3085023b
Mark numbers in wrapper const if possible.
continuous-integration/drone/push Build is passing Details
2019-09-22 22:05:44 +02:00
tastytea d5b07b51c7
Remove filesystem from wrapper.
It was a leftover from an obsolete feature.
2019-09-22 22:04:35 +02:00
tastytea 7ac7bd2edb
Make reading input in wrapper more robust.
We read the size of the message now instead of getting everything between
quotes.
2019-09-22 21:59:46 +02:00
tastytea d746b8c266
Remove obsolete code from wrapper.
continuous-integration/drone/push Build is passing Details
2019-09-22 20:58:59 +02:00
tastytea 8db8f97bde
Add tab_with to EditorConfig. 2019-09-22 20:58:51 +02:00
tastytea 5e4dc6edc1
Version bump 0.8.3.
continuous-integration/drone/push Build is passing Details
2019-09-22 03:05:10 +02:00
tastytea 86cc616eb2
Merge branch 'develop' into main 2019-09-22 03:04:26 +02:00
tastytea d521f21f20
Fix header installation path.
continuous-integration/drone/push Build is passing Details
2019-09-22 03:03:42 +02:00
tastytea 95498ded36
Add link to CONTRIBUTUNG.adoc to readme.
continuous-integration/drone/push Build is passing Details
2019-09-22 02:20:49 +02:00
tastytea f6658b0179
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-09-22 01:59:39 +02:00
tastytea e45d0d780d
Add Code of Conduct.
continuous-integration/drone/push Build was killed Details
2019-09-22 01:56:49 +02:00
tastytea 89229a7bf9
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-09-22 00:57:45 +02:00
tastytea 0392e8f0e3
Added EditorConfig.
continuous-integration/drone/push Build was killed Details
2019-09-22 00:56:18 +02:00
tastytea 5584a3f722
Added contributing guidelines. 2019-09-22 00:32:27 +02:00
tastytea 630cfba48b
Merge branch 'develop' into main
continuous-integration/drone/push Build is passing Details
2019-09-21 22:45:37 +02:00
tastytea d69b8b06fa
Fix `date` call in examples in manpage.
The default precision of `date -I` is day, implying the time "00:00:00". The
current day would be left out.
2019-09-21 22:41:04 +02:00
77 changed files with 4889 additions and 2244 deletions

131
.clang-format Normal file
View File

@ -0,0 +1,131 @@
# -*- mode: yaml -*-
# Written for clang-format 10.
# https://releases.llvm.org/10.0.0/tools/clang/docs/ClangFormatStyleOptions.html
---
DisableFormat: false
Language: Cpp
AccessModifierOffset: -4
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
# AlignConsecutiveBitFields: false # clang-format 11
AlignConsecutiveDeclarations: false
AlignConsecutiveMacros: false
AlignEscapedNewlines: DontAlign
AlignOperands: true # clang-format 11: Align
AlignTrailingComments: true
AllowAllArgumentsOnNextLine: false
AllowAllConstructorInitializersOnNextLine: false
AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: Empty
AllowShortCaseLabelsOnASingleLine: false
# AllowShortEnumsOnASingleLine: false # clang-format 11
AllowShortFunctionsOnASingleLine: Empty
AllowShortIfStatementsOnASingleLine: Never
AllowShortLambdasOnASingleLine: Inline
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: Yes
BinPackArguments: true
BinPackParameters: true
BraceWrapping: # If BreakBeforeBraces is set to Custom.
AfterCaseLabel: true
AfterClass: true
AfterControlStatement: Always
AfterEnum: true
AfterFunction: true
AfterNamespace: true
AfterStruct: true
AfterUnion: true
AfterExternBlock: true
BeforeCatch: true
BeforeElse: true
# BeforeLambdaBody: true # clang-format 11
# BeforeWhile: true # clang-format 11
IndentBraces: false
SplitEmptyFunction: false
SplitEmptyRecord: false
SplitEmptyNamespace: false
BreakBeforeBinaryOperators: NonAssignment
BreakBeforeBraces: Custom
BreakBeforeTernaryOperators: true
BreakConstructorInitializers: BeforeComma
BreakInheritanceList: BeforeComma
BreakStringLiterals: true
ColumnLimit: 80
# CommentPragmas: 'regex'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: false
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DeriveLineEnding: true
DerivePointerAlignment: false
FixNamespaceComments: true
ForEachMacros:
- FOREACH
- RANGES_FOR
- Q_FOREACH
- BOOST_FOREACH
IncludeBlocks: Regroup
IncludeCategories: # stdlib headers into own group.
- Regex: '^[^\.]+$'
Priority: 4
# IndentCaseBlocks: false # clang-format 11
IndentCaseLabels: false
# IndentExternBlock: NoIndent # clang-format 11
IndentGotoLabels: false
IndentPPDirectives: AfterHash
IndentWidth: 4
IndentWrappedFunctionNames: false
KeepEmptyLinesAtTheStartOfBlocks: true
# MacroBlockBegin: 'string'
# MacroBlockEnd: 'string'
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
# NamespaceMacros: 'string'
PenaltyBreakAssignment: 250
PenaltyBreakBeforeFirstCallParameter: 300
# PenaltyBreakComment: 300
# PenaltyBreakFirstLessLess: 120
# PenaltyBreakString: 1000
# PenaltyBreakTemplateDeclaration: 10
# PenaltyExcessCharacter: 1000000
# PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Right
# RawStringFormats: # <YAML>
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: false
SpaceAfterLogicalNot: false
SpaceAfterTemplateKeyword: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeCpp11BracedList: false
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatements
SpaceBeforeRangeBasedForLoopColon: true
SpaceBeforeSquareBrackets: false
SpaceInEmptyBlock: false
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInCStyleCastParentheses: false
SpacesInConditionalStatement: false
SpacesInContainerLiterals: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Auto
# StatementMacros:
# - Q_UNUSED
# - QT_REQUIRE_VERSION
TabWidth: 4
# TypenameMacros:
# - STACK_OF
# - LIST
UseCRLF: false
UseTab: Never
# WhitespaceSensitiveMacros: ['string', 'string'] # clang-format 11
...

43
.clang-tidy Normal file
View File

@ -0,0 +1,43 @@
# -*- mode: conf; fill-column: 100; -*-
# Written for clang-tidy 9.
---
Checks: '*,
-cppcoreguidelines-non-private-member-variables-in-classes,
-fuchsia-default-arguments-calls,
-fuchsia-default-arguments-declarations,
-fuchsia-default-arguments,
-llvm-include-order,
-llvm-header-guard,
-misc-non-private-member-variables-in-classes,
-fuchsia-overloaded-operator,
-cppcoreguidelines-avoid-magic-numbers,
-readability-magic-numbers,
-cppcoreguidelines-pro-bounds-array-to-pointer-decay,
-hicpp-no-array-decay,
-modernize-avoid-c-arrays,
-cppcoreguidelines-avoid-c-arrays,
-hicpp-avoid-c-arrays,
-google-build-using-namespace,
-readability-named-parameter,
-google-runtime-references,
-hicpp-avoid-goto,
-hicpp-vararg,
-fuchsia-statically-constructed-objects,
-google-readability-todo,
-modernize-use-trailing-return-type'
CheckOptions: - { key: readability-identifier-naming.ClassCase, value: CamelCase }
# Clashes with constant private member prefix. (const int _var;)
# - { key: readability-identifier-naming.ConstantCase, value: lower_case }
- { key: readability-identifier-naming.EnumCase, value: lower_case }
- { key: readability-identifier-naming.FunctionCase, value: lower_case }
- { key: readability-identifier-naming.MemberCase, value: lower_case }
- { key: readability-identifier-naming.ParameterCase, value: lower_case }
- { key: readability-identifier-naming.PrivateMemberCase, value: lower_case }
- { key: readability-identifier-naming.PrivateMemberPrefix, value: _ }
- { key: readability-identifier-naming.ProtextedMemberCase, value: lower_case }
- { key: readability-identifier-naming.ProtectedMemberPrefix, value: _ }
- { key: readability-identifier-naming.StructCase, value: lower_case }
# Clashes with static private member prefix. (static int _var;)
# - { key: readability-identifier-naming.VariableCase, value: lower_case }
...

View File

@ -13,18 +13,18 @@ trigger:
- tag
steps:
- name: gcc8
- name: gcc7
image: debian:buster-slim
pull: always
environment:
CXX: g++-8
CXX: g++-7
CXXFLAGS: -pipe -O2
LANG: en_US.utf-8
commands:
- rm /etc/apt/apt.conf.d/docker-clean
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
- apt-get update -q
- apt-get install -qy g++ cmake pkg-config libpoco-dev libxdg-basedir-dev asciidoc catch
- apt-get install -qy g++-7 cmake libpoco-dev libboost-locale-dev libcurl4-openssl-dev asciidoc catch
- rm -rf build && mkdir -p build && cd build
- cmake -DWITH_MOZILLA=YES -DWITH_TESTS=YES ..
- make VERBOSE=1
@ -52,7 +52,7 @@ steps:
- gpg --armor --export 0x60c317803a41ba51845e371a1e9377a2ba9ef27f | apt-key add -
- apt-get update -q
- apt-get install -qy -t bionic g++-9
- apt-get install -qy cmake pkg-config libpoco-dev libxdg-basedir-dev asciidoc catch
- apt-get install -qy cmake libpoco-dev libboost-locale-dev libcurl4-openssl-dev asciidoc catch
- rm -rf build && mkdir -p build && cd build
- cmake -DWITH_MOZILLA=YES ..
- make VERBOSE=1
@ -71,7 +71,7 @@ steps:
- rm /etc/apt/apt.conf.d/docker-clean
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
- apt-get update -q
- apt-get install -qy clang-6.0 cmake pkg-config libpoco-dev libxdg-basedir-dev asciidoc catch
- apt-get install -qy clang-6.0 cmake libpoco-dev libboost-locale-dev libcurl4-openssl-dev asciidoc catch
- rm -rf build && mkdir -p build && cd build
- cmake -DWITH_MOZILLA=YES ..
- make VERBOSE=1
@ -90,7 +90,7 @@ steps:
- rm /etc/apt/apt.conf.d/docker-clean
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
- apt-get update -q
- apt-get install -qy clang cmake pkg-config libpoco-dev libxdg-basedir-dev asciidoc catch
- apt-get install -qy clang cmake libpoco-dev libboost-locale-dev libcurl4-openssl-dev asciidoc catch
- rm -rf build && mkdir -p build && cd build
- cmake -DWITH_MOZILLA=YES ..
- make VERBOSE=1
@ -140,7 +140,7 @@ steps:
- rm /etc/apt/apt.conf.d/docker-clean
- alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get'
- apt-get update -q
- apt-get install -qy g++ cmake pkg-config libpoco-dev libxdg-basedir-dev asciidoc catch
- apt-get install -qy g++ cmake libpoco-dev libboost-locale-dev libcurl4-openssl-dev asciidoc catch
- apt-get install -qy build-essential file zip
- rm -rf build && mkdir -p build && cd build
- cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES -DMOZILLA_NMH_DIR="lib/mozilla/native-messaging-hosts" -DWITH_DEB=YES ..
@ -153,6 +153,26 @@ steps:
- name: debian-package-cache
path: /var/cache/apt/archives
# - name: rpm
# image: centos:8
# pull: always
# environment:
# CXX: g++
# CXXFLAGS: -pipe -O2
# commands:
# - sed -i 's/keepcache=0/keepcache=1/' /etc/yum.conf
# - yum install -qy epel-release
# - yum install --enablerepo=PowerTools --enablerepo=epel -qy gcc-c++ cmake poco-devel openssl-devel doxygen asciidoc rpm-build
# - rm -rf build && mkdir -p build && cd build
# - cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_RPM=YES ..
# - make package
# - cmake -DWITH_RPM=YES -DWITH_DOC=NO ..
# - make package
# - cp -v remwharead-${DRONE_TAG}-0.x86_64.rpm ..
# volumes:
# - name: centos-package-cache
# path: /var/cache/yum
- name: release
image: plugins/gitea-release
pull: always

20
.editorconfig Normal file
View File

@ -0,0 +1,20 @@
# Configuration file for EditorConfig.
# More information is available under <https://editorconfig.org/>.
root = true
[*]
indent_style = space
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
max_line_length = 80
[*.?pp]
indent_size = 4
tab_width = 4
[{CMakeLists.txt,*.cmake}]
indent_size = 2
tab_width = 2

View File

@ -1,65 +1,56 @@
# Support version 3.6 and above, but use policy settings up to 3.14.
# 3.6 is needed because of IMPORTED_TARGET in pkg_check_modules().
cmake_minimum_required(VERSION 3.6...3.14)
# Support version 3.9 and above, but use policy settings up to 3.17.
# 3.9 is needed for project description.
cmake_minimum_required(VERSION 3.9...3.17)
# Ranges are supported from 3.12, set policy to current for < 3.12.
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
project(remwharead
VERSION 0.8.2
LANGUAGES CXX)
# DESCRIPTION was introduced in version 3.9.
if(NOT (${CMAKE_VERSION} VERSION_LESS 3.9))
set(PROJECT_DESCRIPTION
"Saves URIs of things you want to remember in a database.")
endif()
# Global build options.
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "The type of build.")
option(BUILD_SHARED_LIBS "Build shared libraries." YES)
project(remwharead
VERSION 0.10.0
DESCRIPTION "Saves URIs of things you want to remember in a database.")
set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
include(GNUInstallDirs)
# All custom build switches.
# Project build options.
option(WITH_MAN "Compile and install manpage." YES)
option(WITH_TESTS "Compile tests." NO)
option(WITH_MOZILLA "Build and install wrapper for Mozilla browsers." YES)
option(BUILD_SHARED_LIBS "Build shared libraries." YES)
set(MOZILLA_NMH_DIR "${CMAKE_INSTALL_LIBDIR}/mozilla/native-messaging-hosts"
CACHE STRING "Directory for the Mozilla extension wrapper.")
option(WITH_CLANG-TIDY "Check sourcecode with clang-tidy while compiling." NO)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
set(DEBUG_CXXFLAGS
"-Wall"
"-Wextra"
"-Wpedantic"
"-ftrapv"
"-fsanitize=undefined"
"-g"
"-Og"
"-fno-omit-frame-pointer")
set(DEBUG_LDFLAGS
"-fsanitize=undefined")
add_compile_options("$<$<CONFIG:Debug>:${DEBUG_CXXFLAGS}>")
# add_link_options was introduced in version 3.13.
if(${CMAKE_VERSION} VERSION_LESS 3.13)
set(CMAKE_SHARED_LINKER_FLAGS_DEBUG "${DEBUG_LDFLAGS}")
else()
add_link_options("$<$<CONFIG:Debug>:${DEBUG_LDFLAGS}>")
include(debug_flags)
if(WITH_CLANG-TIDY)
set(CMAKE_CXX_CLANG_TIDY
"clang-tidy"
"-header-filter=${PROJECT_SOURCE_DIR}"
"-quiet")
endif()
add_subdirectory(src)
add_subdirectory(src/curl_wrapper)
add_subdirectory(src/lib)
add_subdirectory(include)
add_subdirectory(src/cli)
add_subdirectory(pkg-config)
add_subdirectory(cmake)
if (WITH_MAN)
if(WITH_MAN)
add_subdirectory(man)
endif()
if (WITH_MOZILLA)
if(WITH_MOZILLA)
add_subdirectory(browser-plugins/webextension/native-wrapper)
endif()

49
CODE_OF_CONDUCT.adoc Normal file
View File

@ -0,0 +1,49 @@
:contact-coc: tastytea@tastytea.de
== Code of Conduct
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, education, ethnicity, gender identity and expression, level of
experience, nationality, personal appearance, race, religion, sex
characteristics, sexual identity and orientation or socio-economic status.
=== Examples
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language.
* Being respectful of differing viewpoints and experiences.
* Gracefully accepting constructive criticism.
* Focusing on what is best for the community.
* Showing empathy towards other community members.
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances.
* Trolling, insulting/derogatory comments, and personal attacks.
* Public or private harassment.
* Publishing others private information, such as a physical or electronic
address, without explicit permission.
=== Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at {contact-coc}.
All complaints will be reviewed and investigated and will result in a response
that is deemed necessary and appropriate to the circumstances. The project team
is obligated to maintain confidentiality with regard to the reporter of an
incident.
=== Attribution
This Code of Conduct is adapted from the Contributor Covenant, version 1.4,
available at
https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq

29
CONTRIBUTING.adoc Normal file
View File

@ -0,0 +1,29 @@
:project: remwharead
:uri-base: https://schlomp.space/tastytea/{project}
:uri-coc: {uri-base}/src/branch/main/CODE_OF_CONDUCT.adoc
:contact-email: tastytea@tastytea.de
:contact-xmpp: {contact-email}
:contact-fediverse: https://likeable.space/users/tastytea
== How to contribute
Read the link:{uri-coc}[Code of Conduct].
=== Reporting bugs or suggesting enhancements
Before reporting a bug, please
https://schlomp.space/tastytea/{project}/issues[perform a search] to see if the
problem has already been reported. If it has, add a comment to the existing
issue instead of opening a new one. Same for enhancements.
You can also contact me via mailto:{contact-email}[E-Mail],
link:xmpp:{contact-xmpp}[XMPP] or the {contact-fediverse}[Fediverse] if you
don't want to open an account.
=== Pull requests
Please use similar coding conventions as the rest of the project. The basic rule
to remember is to write code in the same style as the existing/surrounding code.
You can also send me your patches via mailto:{contact-email}[E-Mail], ideally
using `git format-patch` or `git send-email`.

View File

@ -1,6 +1,29 @@
= remwharead
:toc: preamble
:project: remwharead
:uri-base: https://schlomp.space/tastytea/{project}
:uri-branch-main: {uri-base}/src/branch/main
:uri-reference: https://doc.schlomp.space/{project}/
:uri-images-base: https://doc.schlomp.space/.{project}
:uri-archive: https://archive.org/
:uri-overlay: https://schlomp.space/tastytea/overlay
:uri-blogpost: https://blog.tastytea.de/posts/keep-track-of-what-you-have-read-online-with-remwharead/
:uri-gcc: https://gcc.gnu.org/
:uri-clang: https://clang.llvm.org/
:uri-cmake: https://cmake.org/
:uri-poco: https://pocoproject.org/
:uri-asciidoc: http://asciidoc.org/
:uri-catch: https://github.com/catchorg/Catch2
:uri-dpkg: https://packages.qa.debian.org/dpkg
:uri-rpm: http://www.rpm.org/
:uri-ff-addon: https://addons.mozilla.org/firefox/addon/remwharead
:uri-papirus: https://github.com/PapirusDevelopmentTeam/papirus-icon-theme
:uri-rofi: https://github.com/davatorium/rofi
:uri-boost: https://www.boost.org/
:uri-clang-tidy: https://clang.llvm.org/extra/clang-tidy/
:uri-curl: https://curl.haxx.se/libcurl/
*remwharead* saves URIs of things you want to remember in a database along with
an URI to the archived version, the current date and time, title, description,
the full text of the page and optional tags.
@ -9,29 +32,38 @@ The database can be filtered by time, tags and full text and exported to CSV,
AsciiDoc, JSON or RSS.
Archiving is done using the Wayback machine from the
https://archive.org/[Internet Archive].
{uri-archive}[Internet Archive].
.AsciiDoc export formatted with Asciidoctor.
====
image::https://doc.schlomp.space/.remwharead/example_dates.png[Dates view, height=250, link="https://doc.schlomp.space/.remwharead/example_dates.png", role=left]
image::https://doc.schlomp.space/.remwharead/example_tags.png[Tags view, height=250, link="https://doc.schlomp.space/.remwharead/example_tags.png"]
[alt="Dates view", height=250, link="{uri-images-base}/example_dates.png", role=left]
image::{uri-images-base}/example_dates.png[]
[alt="Tags view", height=250, link="{uri-images-base}/example_tags.png"]
image::{uri-images-base}/example_tags.png[]
====
== Usage
See
https://schlomp.space/tastytea/remwharead/src/branch/main/man/remwharead.1.adoc[manpage].
See {uri-branch-main}/man/remwharead.1.adoc[manpage] and/or read
{uri-blogpost}[the blogpost].
=== With rofi
The link:{uri-rofi}[rofi] export makes integration with rofi simple. See
link:{uri-branch-main}/scripts/remwharead-rofi[scripts/remwharead-rofi] for an
example.
=== In your programs
The complete functionality is implemented in a C++ library, libremwharead. Take
a look at the https://doc.schlomp.space/remwharead/[reference] for more info.
a look at the {uri-reference}[reference] for more info.
== Install
=== Gentoo
Add my https://schlomp.space/tastytea/overlay[repository] and install it from
Add my {uri-overlay}[repository] and install it from
there.
[source,zsh]
@ -45,7 +77,7 @@ emerge -a www-misc/remwharead
=== Debian and Debian based
Download the `.deb`-package from
https://schlomp.space/tastytea/remwharead/releases[schlomp.space] and install
{uri-base}/releases[schlomp.space] and install
with `apt install ./rewharead_*.deb`. The package works for 64 bit installations
only.
@ -54,23 +86,22 @@ only.
==== Dependencies
* Tested OS: Linux
* C++ compiler (tested: https://gcc.gnu.org/[gcc] 8/9,
https://llvm.org/[clang] 6/7)
* https://cmake.org/[cmake] (at least: 3.2)
* https://pkgconfig.freedesktop.org/wiki/[pkgconfig] (tested: 0.29)
* http://repo.or.cz/w/libxdg-basedir.git[libxdg-basedir] (tested: 1.2)
* https://pocoproject.org/[POCO] (tested: 1.9 / 1.7)
* C++ compiler ({uri-gcc}[gcc] 6+, {uri-clang}[clang] 6+)
* {uri-cmake}[cmake] (at least: 3.9)
* {uri-poco}[POCO] (tested: 1.9 / 1.7)
* {uri-boost}[Boost] (tested: 1.71 / 1.67)
* {uri-curl}[libcurl] (at least: 7.52)
* Optional:
** Manpage: http://asciidoc.org/[asciidoc] (tested: 8.6)
** Tests: https://github.com/catchorg/Catch2[catch] (tested: 2.5 / 1.2)
** DEB package: https://packages.qa.debian.org/dpkg[dpkg] (tested: 1.18)
** RPM package: http://www.rpm.org[rpm-build] (tested: 4.11)
** Manpage: {uri-asciidoc}[asciidoc] (tested: 8.6)
** Tests: {uri-catch}[catch] (tested: 2.5 / 1.2)
** DEB package: {uri-dpkg}[dpkg] (tested: 1.18)
** RPM package: {uri-rpm}[rpm-build] (tested: 4.11)
.Install dependencies in Debian buster.
====
[source,zsh]
----
apt-get install g++ cmake pkg-config libpoco-dev libxdg-basedir-dev asciidoc dpkg
apt-get install g++ cmake libpoco-dev libboost-dev asciidoc dpkg
----
====
@ -78,8 +109,7 @@ apt-get install g++ cmake pkg-config libpoco-dev libxdg-basedir-dev asciidoc dpk
===== Releases
Download the current release at
https://schlomp.space/tastytea/remwharead/releases[schlomp.space].
Download the current release at {uri-base}/releases[schlomp.space].
===== Development version
@ -106,6 +136,8 @@ cmake --build .
* `-DMOZILLA_NMH_DIR` lets you set the directory for the Mozilla
extension wrapper. The complete path is
`${CMAKE_INSTALL_PREFIX}/${MOZILLA_NMH_DIR}`.
* `-DWITH_CLANG-TIDY=YES` to check the sourcecode with
link:{uri-clang-tidy}[clang-tidy] while compiling.
* One of:
** `-DWITH_DEB=YES` if you want to be able to generate a deb-package.
** `-DWITH_RPM=YES` if you want to be able to generate an rpm-package.
@ -117,18 +149,17 @@ generate binary packages with `make package`.
=== WebExtension
The
https://schlomp.space/tastytea/remwharead/src/branch/main/browser-plugins/webextension[WebExtension]
works in Firefox and possibly other browsers with WebExtension support. You
can install it from
https://addons.mozilla.org/en-US/firefox/addon/remwharead/[addons.mozilla.org]
or build it yourself with `build_xpi.sh`.
The {uri-branch-main}/browser-plugins/webextension[WebExtension] works in
Firefox and possibly other browsers with WebExtension support. You can install
it from {uri-ff-addon}/[addons.mozilla.org] or build it yourself with
`build_xpi.sh`.
include::{uri-base}/raw/branch/main/CONTRIBUTING.adoc[]
== Copyright
The icons of the plugins are from the
https://github.com/PapirusDevelopmentTeam/papirus-icon-theme[Papirus icon
theme] with the license GPLv3.
The icons of the plugins are from the {uri-papirus}[Papirus icon theme] with the
license GPLv3.
----
Copyright © 2019 tastytea <tastytea@tastytea.de>.

View File

@ -1,7 +1,7 @@
{
"manifest_version": 2,
"name": "remwharead",
"version": "0.4.1",
"version": "0.5.0",
"description": "Integrates remwharead into your Browser.",
"homepage_url": "https://schlomp.space/tastytea/remwharead",

View File

@ -2,8 +2,6 @@ set(INSTALL_MOZILLA_NMH_DIR "${CMAKE_INSTALL_PREFIX}/${MOZILLA_NMH_DIR}")
add_executable(${PROJECT_NAME}_wrapper ${PROJECT_NAME}_wrapper.cpp)
target_link_libraries(${PROJECT_NAME}_wrapper PRIVATE stdc++fs)
install(TARGETS ${PROJECT_NAME}_wrapper DESTINATION ${MOZILLA_NMH_DIR})
configure_file("${PROJECT_NAME}.json.in"

View File

@ -1,5 +1,5 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
* Copyright © 2019, 2020 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -14,104 +14,146 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <string>
#include <iostream>
#include <experimental/filesystem>
#include <cstdint>
#include <cstdlib>
#include <Poco/Message.h>
#include <sys/wait.h>
using std::string;
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
using std::cin;
using std::cout;
using std::uint32_t;
using std::string;
using std::system;
using std::uint32_t;
namespace fs = std::experimental::filesystem;
const string read_input()
class Message
{
string input;
char c;
public:
//! Read message from stdin and store it in _msg.
Message();
bool start = false;
while (cin.read(&c, 1).good())
//! Decode message and return argument string for launch().
[[nodiscard]] string decode();
private:
string _msg;
//! Replace "\u001f" with the char in _msg.
void unescape();
//! Replace " with \" in field.
static void replace_in_field(string &field);
};
//! Send a message back.
void send_message(const string &message);
//! Launch remwharead with args.
int launch(const string &args);
int main()
{
Message message;
const auto args{message.decode()};
const int ret = launch(args);
if (ret == 0)
{
if (!start)
{
if (c == '"')
{
start = true;
}
continue;
}
if (c != '"')
{
input += c;
}
else
{
break;
}
send_message("Command successful.");
}
else
{
send_message("Command failed with status: " + std::to_string(ret)
+ '.');
}
return ret;
}
Message::Message()
{
// Read message length.
uint32_t length{0};
char buffer[4];
cin.read(buffer, sizeof(length));
std::memcpy(&length, buffer, sizeof(length));
// Ignore quotes.
length -= 2;
cin.ignore(1);
// Read message.
char c{'\0'};
for (; length > 0; --length)
{
cin.read(&c, 1);
_msg += c;
}
}
string Message::decode()
{
unescape();
constexpr char separator{'\u001f'}; // UNIT SEPARATOR.
if (_msg[0] != separator) // Extension uses old method.
{
return _msg;
}
size_t pos{1};
size_t endpos{0};
string newargs;
while ((endpos = _msg.find(separator, pos)) != string::npos)
{
string field{_msg.substr(pos, endpos - pos)};
replace_in_field(field);
newargs += " \"" + field + '"';
pos = endpos + 1;
}
return newargs;
}
void Message::unescape()
{
size_t pos{0};
while ((pos = _msg.find(R"(\u001f)", pos)) != string::npos)
{
_msg.replace(pos, 6, "\u001f");
}
}
void Message::replace_in_field(string &field)
{
size_t pos{0};
while ((pos = field.find('"', pos)) != string::npos)
{
field.replace(pos, 1, R"(\")");
pos += 2;
}
return input;
}
void send_message(const string &message)
{
uint32_t length = message.length() + 2;
cout.write(reinterpret_cast<const char*>(&length), sizeof(length));
const auto length = static_cast<uint32_t>(message.length() + 2);
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
cout.write(reinterpret_cast<const char *>(&length), sizeof(uint32_t));
cout << '"' << message << '"';
}
int launch(const string &args)
{
const string cmd = "remwharead " + args + " 2>/dev/null";
// NOLINTNEXTLINE(cert-env33-c)
int ret = system(cmd.c_str());
if (WIFEXITED(ret))
if (WIFEXITED(ret)) // NOLINT(hicpp-signed-bitwise)
{
ret = WEXITSTATUS(ret);
}
return ret;
}
int main()
{
string args = read_input();
// size_t pos = args.find("TEMPFILE");
// string tmpfile;
// if (pos != string::npos)
// {
// try
// {
// tmpfile = fs::temp_directory_path() / "remwharead.html";
// args.replace(pos, 8, tmpfile);
// }
// catch (const fs::filesystem_error &e)
// {
// send_message("Could not create temporary file.");
// return 3;
// }
// }
int ret = launch(args);
if (ret == 0)
{
// if (!tmpfile.empty())
// {
// send_message("FILE:" + tmpfile);
// }
// else
// {
send_message("Command successful.");
// }
}
else
{
send_message("Command failed with status: " + std::to_string(ret) + '.');
ret = WEXITSTATUS(ret); // NOLINT(hicpp-signed-bitwise)
}
return ret;

View File

@ -7,12 +7,13 @@ const chkarchive = document.getElementById("chkarchive");
const btnadd = document.getElementById("btnadd");
const msgstatus = document.getElementById("msgstatus");
const msgerror = document.getElementById("msgerror");
const separator = '\u001f';
function set_taburl(tabs) // Set taburl to URL of current tab.
{
const tab = tabs[0];
taburl = '\'' + tab.url + '\'';
taburl = separator + tab.url + separator;
}
function get_tags() // get tags from text input.
@ -20,7 +21,7 @@ function get_tags() // get tags from text input.
const tags = txttags.value;
if (tags != "")
{
return "-t '" + tags + "' ";
return separator + "-t " + tags;
}
return "";
}
@ -72,7 +73,7 @@ function add()
let archive = "";
if (chkarchive.checked === false)
{
archive = "--no-archive ";
archive = separator + "--no-archive";
}
const args = get_tags() + archive + taburl;
console.log(args);

57
cmake/debug_flags.cmake Normal file
View File

@ -0,0 +1,57 @@
# Set compiler flags for Debug builds.
# Only has an effect on GCC/Clang >= 5.0.
set(DEBUG_CXXFLAGS "")
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang"
AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5")
list(APPEND DEBUG_CXXFLAGS
"-Wall"
"-Wextra"
"-Wpedantic"
"-Wuninitialized"
"-Wshadow"
"-Wnon-virtual-dtor"
"-Wconversion"
"-Wsign-conversion"
"-Wold-style-cast"
"-Wzero-as-null-pointer-constant"
"-Wmissing-declarations"
"-Wcast-align"
"-Wunused"
"-Woverloaded-virtual"
"-Wdouble-promotion"
"-Wformat=2"
"-ftrapv"
"-fsanitize=undefined"
"-Og"
"-fno-omit-frame-pointer")
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
list(APPEND DEBUG_CXXFLAGS
"-Wlogical-op"
"-Wuseless-cast")
if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "6")
list(APPEND DEBUG_CXXFLAGS
"-Wmisleading-indentation"
"-Wduplicated-cond"
"-Wnull-dereference")
if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7")
list(APPEND DEBUG_CXXFLAGS
"-Wduplicated-branches")
endif()
endif()
endif()
add_compile_options("$<$<CONFIG:Debug>:${DEBUG_CXXFLAGS}>")
set(DEBUG_LDFLAGS
"-fsanitize=undefined")
# add_link_options was introduced in version 3.13.
if(${CMAKE_VERSION} VERSION_LESS 3.13)
set(CMAKE_SHARED_LINKER_FLAGS_DEBUG "${DEBUG_LDFLAGS}")
else()
add_link_options("$<$<CONFIG:Debug>:${DEBUG_LDFLAGS}>")
endif()
else()
message(STATUS
"No additional compiler flags were set, "
"because your compiler was not anticipated.")
endif()

View File

@ -3,8 +3,7 @@ set(CPACK_PACKAGE_VERSION_MAJOR ${${PROJECT_NAME}_VERSION_MAJOR})
set(CPACK_PACKAGE_VERSION_MINOR ${${PROJECT_NAME}_VERSION_MINOR})
set(CPACK_PACKAGE_VERSION_PATCH ${${PROJECT_NAME}_VERSION_PATCH})
set(CPACK_PACKAGE_VERSION ${${PROJECT_NAME}_VERSION})
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
"Saves URIs of things you want to remember in a database. ")
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY ${PROJECT_DESCRIPTION})
set(CPACK_PACKAGE_CONTACT "tastytea <tastytea@tastytea.de>")
set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE")
set(CPACK_RESOURCE_FILE_README "${PROJECT_SOURCE_DIR}/README.adoc")
@ -45,7 +44,7 @@ if (WITH_RPM)
set(CPACK_RPM_PACKAGE_LICENSE "GPL-3")
set(CPACK_RPM_PACKAGE_URL "https://schlomp.space/tastytea/${PROJECT_NAME}")
set(CPACK_RPM_PACKAGE_REQUIRES
"poco-netssl >= 1.6, poco-sqlite >= 1.6, libxdg-basedir")
"poco-net >= 1.6, poco-sqlite >= 1.6, libcurl >= 7.52")
set(CPACK_PACKAGE_FILE_NAME
"${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-0.${CPACK_PACKAGE_ARCHITECTURE}")
set(CPACK_SOURCE_PACKAGE_FILE_NAME

View File

@ -4,7 +4,7 @@ include(GNUInstallDirs)
find_depencency(Poco
COMPONENTS Foundation Net NetSSL Data DataSQLite JSON XML
CONFIG REQUIRED)
find_dependency(PkgConfig REQUIRED)
pkg_check_modules(libxdg-basedir REQUIRED IMPORTED_TARGET libxdg-basedir)
find_dependency(Boost 1.48.0 REQUIRED COMPONENTS locale)
find_dependency(CURL 7.52 REQUIRED)
include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")

View File

@ -1,5 +1,6 @@
include(GNUInstallDirs)
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
# The trailing / is important.
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}"
FILES_MATCHING PATTERN "*.hpp")

View File

@ -14,61 +14,64 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef REMWHAREAD_ADOC_HPP
#define REMWHAREAD_ADOC_HPP
#ifndef REMWHAREAD_EXPORT_ADOC_HPP
#define REMWHAREAD_EXPORT_ADOC_HPP
#include "export.hpp"
#include "sqlite.hpp"
#include <map>
#include <string>
#include <vector>
#include "sqlite.hpp"
#include "export.hpp"
namespace remwharead
namespace remwharead::Export
{
namespace Export
using std::string;
/*!
* @brief Export as %AsciiDoc document.
*
* @since 0.6.0
*
* @headerfile adoc.hpp remwharead/export/adoc.hpp
*/
class AsciiDoc : protected ExportBase
{
using std::string;
public:
using ExportBase::ExportBase;
/*!
* @brief Export as %AsciiDoc document.
*
* @since 0.6.0
*
* @headerfile adoc.hpp remwharead/export/adoc.hpp
*/
class AsciiDoc : protected ExportBase
{
public:
using ExportBase::ExportBase;
void print() const override;
void print() const override;
private:
using tagmap = std::map<string, list<Database::entry>>;
using replacemap = const std::map<const string, const string>;
private:
using tagmap = std::map<string, list<Database::entry>>;
using replacemap = const std::map<const string, const string>;
//! Replace strings in text.
[[nodiscard]]
static string replace(string text, const replacemap &replacements);
//! Replace strings in text.
const string replace(string text, const replacemap &replacements) const;
//! Replaces characters in tags that asciidoctor doesn't like.
[[nodiscard]]
static string replace_in_tag(const string &text);
//! Replaces characters in tags that asciidoctor doesn't like.
const string replace_in_tag(const string &text) const;
//! Replaces characters in title that asciidoctor doesn't like.
[[nodiscard]]
static string replace_in_title(const string &text);
//! Replaces characters in title that asciidoctor doesn't like.
const string replace_in_title(const string &text) const;
//! Replaces characters in URI that asciidoctor doesn't like.
[[nodiscard]]
static string replace_in_uri(const string &text);
//! Replaces characters in URI that asciidoctor doesn't like.
const string replace_in_uri(const string &text) const;
//! Print things sorted by tag.
void print_tags(const tagmap &tags) const;
//! Print things sorted by tag.
void print_tags(const tagmap &tags) const;
//! Get ISO-8601 day from Database::entry.
[[nodiscard]]
static string get_day(const Database::entry &entry);
//! Get ISO-8601 day from Database::entry.
const string get_day(const Database::entry &entry) const;
//! Get ISO-8601 time from Database::entry.
[[nodiscard]]
static string get_time(const Database::entry &entry);
};
} // namespace remwharead::Export
//! Get ISO-8601 time from Database::entry.
const string get_time(const Database::entry &entry) const;
};
}
}
#endif // REMWHAREAD_ADOC_HPP
#endif // REMWHAREAD_EXPORT_ADOC_HPP

View File

@ -14,14 +14,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef REMWHAREAD_BOOKMARKS_HPP
#define REMWHAREAD_BOOKMARKS_HPP
#ifndef REMWHAREAD_EXPORT_BOOKMARKS_HPP
#define REMWHAREAD_EXPORT_BOOKMARKS_HPP
#include "export.hpp"
namespace remwharead
{
namespace Export
namespace remwharead::Export
{
/*!
* @brief Export as Netscape bookmark file.
@ -34,9 +32,9 @@ namespace Export
{
public:
using ExportBase::ExportBase;
virtual void print() const override;
};
}
}
#endif // REMWHAREAD_BOOKMARKS_HPP
void print() const override;
};
} // namespace remwharead::Export
#endif // REMWHAREAD_EXPORT_BOOKMARKS_HPP

View File

@ -14,15 +14,13 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef REMWHAREAD_CSV_HPP
#define REMWHAREAD_CSV_HPP
#ifndef REMWHAREAD_EXPORT_CSV_HPP
#define REMWHAREAD_EXPORT_CSV_HPP
#include <string>
#include "export.hpp"
#include <string>
namespace remwharead
{
namespace Export
namespace remwharead::Export
{
using std::string;
@ -38,13 +36,13 @@ namespace Export
public:
using ExportBase::ExportBase;
virtual void print() const override;
void print() const override;
private:
//! replaces " with "".
const string quote(string field) const;
[[nodiscard]]
static string quote(string field);
};
}
}
} // namespace remwharead::Export
#endif // REMWHAREAD_CSV_HPP
#endif // REMWHAREAD_EXPORT_CSV_HPP

View File

@ -14,60 +14,62 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef REMWHAREAD_EXPORT_HPP
#define REMWHAREAD_EXPORT_HPP
#ifndef REMWHAREAD_EXPORT_EXPORT_HPP
#define REMWHAREAD_EXPORT_EXPORT_HPP
#include <list>
#include <iostream>
#include "sqlite.hpp"
#include <iostream>
#include <list>
namespace remwharead::Export
{
using std::list;
using std::ostream;
using std::cout;
namespace remwharead
{
namespace Export
/*!
* @brief Base class for exports.
*
* @since 0.6.0
*
* @headerfile export.hpp remwharead/export/export.hpp
*/
class ExportBase
{
public:
/*!
* @brief Base class for exports.
* @brief Export list of Database::entry.
*
* @since 0.6.0
*
* @headerfile export.hpp remwharead/export/export.hpp
* @param entries List of Database::entry to export.
* @param out Output stream.
*/
class ExportBase
{
public:
/*!
* @brief Export list of Database::entry.
*
* @param entries List of Database::entry to export.
* @param out Output stream.
*/
explicit ExportBase(const list<Database::entry> &entries,
ostream &out = cout);
explicit ExportBase(const list<Database::entry> &entries,
ostream &out = cout);
virtual ~ExportBase() = default;
ExportBase(const ExportBase &) = delete;
ExportBase &operator=(const ExportBase &) = delete;
ExportBase(ExportBase &&) = delete;
ExportBase &operator=(ExportBase &&) = delete;
/*!
* @brief Print output to std::ostream.
*/
virtual void print() const = 0;
protected:
const list<Database::entry> _entries;
ostream &_out;
/*!
* @brief Print output to std::ostream.
*/
virtual void print() const = 0;
/*!
* @brief Sort entries from newest to oldest and remove duplicates.
*
* @param entries List of Database::entry to sort.
*
* @return Sorted list of Database::entry.
*/
const list<Database::entry>
sort_entries(list<Database::entry> entries) const;
};
}
}
protected:
const list<Database::entry> _entries;
ostream &_out;
#endif // REMWHAREAD_EXPORT_HPP
/*!
* @brief Sort entries from newest to oldest and remove duplicates.
*
* @param entries List of Database::entry to sort.
*
* @return Sorted list of Database::entry.
*/
[[nodiscard]]
static list<Database::entry> sort_entries(list<Database::entry> entries);
};
} // namespace remwharead::Export
#endif // REMWHAREAD_EXPORT_EXPORT_HPP

View File

@ -14,33 +14,30 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef REMWHAREAD_JSON_HPP
#define REMWHAREAD_JSON_HPP
#ifndef REMWHAREAD_EXPORT_JSON_HPP
#define REMWHAREAD_EXPORT_JSON_HPP
#include <string>
#include "export.hpp"
#include <string>
namespace remwharead
namespace remwharead::Export
{
namespace Export
using std::string;
/*!
* @brief Export as %JSON array.
*
* @since 0.8.0
*
* @headerfile json.hpp remwharead/export/json.hpp
*/
class JSON : protected ExportBase
{
using std::string;
public:
using ExportBase::ExportBase;
/*!
* @brief Export as %JSON array.
*
* @since 0.8.0
*
* @headerfile json.hpp remwharead/export/json.hpp
*/
class JSON : protected ExportBase
{
public:
using ExportBase::ExportBase;
void print() const override;
};
} // namespace remwharead::Export
virtual void print() const override;
};
}
}
#endif // REMWHAREAD_JSON_HPP
#endif // REMWHAREAD_EXPORT_JSON_HPP

39
include/export/link.hpp Normal file
View File

@ -0,0 +1,39 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef REMWHAREAD_EXPORT_LINK_HPP
#define REMWHAREAD_EXPORT_LINK_HPP
#include "export.hpp"
namespace remwharead::Export
{
/*!
* @brief Export as list of hyperlinks.
*
* @since 0.9.0
*
* @headerfile link.hpp remwharead/export/link.hpp
*/
class Link : protected ExportBase
{
public:
using ExportBase::ExportBase;
void print() const override;
};
} // namespace remwharead::Export
#endif // REMWHAREAD_EXPORT_LINK_HPP

39
include/export/rofi.hpp Normal file
View File

@ -0,0 +1,39 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef REMWHAREAD_EXPORT_ROFI_HPP
#define REMWHAREAD_EXPORT_ROFI_HPP
#include "export.hpp"
namespace remwharead::Export
{
/*!
* @brief Export title, tags and URL for consumption by rofi.
*
* @since 0.9.0
*
* @headerfile rofi.hpp remwharead/export/rofi.hpp
*/
class Rofi : protected ExportBase
{
public:
using ExportBase::ExportBase;
void print() const override;
};
} // namespace remwharead::Export
#endif // REMWHAREAD_EXPORT_ROFI_HPP

View File

@ -14,33 +14,30 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef REMWHAREAD_RSS_HPP
#define REMWHAREAD_RSS_HPP
#ifndef REMWHAREAD_EXPORT_RSS_HPP
#define REMWHAREAD_EXPORT_RSS_HPP
#include <string>
#include "export.hpp"
namespace remwharead
namespace remwharead::Export
{
namespace Export
using std::string;
/*!
* @brief Export as %RSS feed.
*
* @since 0.8.0
*
* @headerfile rss.hpp remwharead/export/rss.hpp
*/
class RSS : protected ExportBase
{
using std::string;
public:
using ExportBase::ExportBase;
/*!
* @brief Export as %RSS feed.
*
* @since 0.8.0
*
* @headerfile rss.hpp remwharead/export/rss.hpp
*/
class RSS : protected ExportBase
{
public:
using ExportBase::ExportBase;
void print() const override;
};
} // namespace remwharead::Export
virtual void print() const override;
};
}
}
#endif // REMWHAREAD_RSS_HPP
#endif // REMWHAREAD_EXPORT_RSS_HPP

View File

@ -14,29 +14,26 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef REMWHAREAD_SIMPLE_HPP
#define REMWHAREAD_SIMPLE_HPP
#ifndef REMWHAREAD_EXPORT_SIMPLE_HPP
#define REMWHAREAD_EXPORT_SIMPLE_HPP
#include "export.hpp"
namespace remwharead
namespace remwharead::Export
{
namespace Export
/*!
* @brief Export as simple list.
*
* @since 0.6.0
*
* @headerfile simple.hpp remwharead/export/simple.hpp
*/
class Simple : protected ExportBase
{
/*!
* @brief Export as simple list.
*
* @since 0.6.0
*
* @headerfile simple.hpp remwharead/export/simple.hpp
*/
class Simple : protected ExportBase
{
public:
using ExportBase::ExportBase;
virtual void print() const override;
};
}
}
public:
using ExportBase::ExportBase;
void print() const override;
};
} // namespace remwharead::Export
#endif // REMWHAREAD_SIMPLE_HPP
#endif // REMWHAREAD_EXPORT_SIMPLE_HPP

View File

@ -43,9 +43,11 @@
#include "export/bookmarks.hpp"
#include "export/csv.hpp"
#include "export/export.hpp"
#include "export/simple.hpp"
#include "export/json.hpp"
#include "export/rss.hpp"
#include "export/simple.hpp"
#include "export/list.hpp"
#include "export/rofi.hpp"
#include "search.hpp"
#include "sqlite.hpp"
#include "time.hpp"

View File

@ -17,104 +17,109 @@
#ifndef REMWHAREAD_SEARCH_HPP
#define REMWHAREAD_SEARCH_HPP
#include "sqlite.hpp"
#include <list>
#include <string>
#include <vector>
#include "sqlite.hpp"
namespace remwharead
{
using std::list;
using std::string;
using std::vector;
using std::list;
using std::string;
using std::vector;
/*!
* @brief %Search in database entries.
*
* @since 0.7.0
*
* @headerfile search.hpp remwharead/search.hpp
*/
class Search
{
public:
/*!
* @brief %Search in database entries.
* @brief Defines the entries to search.
*
* @since 0.7.0
*
* @headerfile search.hpp remwharead/search.hpp
*/
class Search
{
public:
/*!
* @brief Defines the entries to search.
*
* @since 0.7.0
*/
explicit Search(const list<Database::entry> &entries);
explicit Search(list<Database::entry> entries);
/*!
* @brief %Search in tags of database entries.
*
* Only matches whole tags, *Pill* does not match *Pillow*.
*
* @param expression %Search expression.
* @param is_re Is it a regular expression?
*
* @return List of matching Database::entry.
*
* @since 0.7.0
*/
const list<Database::entry> search_tags(string expression,
const bool is_re) const;
/*!
* @brief %Search in tags of database entries.
*
* Only matches whole tags, *Pill* does not match *Pillow*.
*
* @param expression %Search expression.
* @param is_re Is it a regular expression?
*
* @return List of matching Database::entry.
*
* @since 0.7.0
*/
[[nodiscard]]
list<Database::entry> search_tags(const string &expression, bool is_re)
const;
/*!
* @brief %Search in full text of database entries.
*
* Searches in tags, title, description and full text.
*
* @param expression %Search expression.
* @param is_re Is it a regular expression?
*
* @return List of matching Database::entry.
*
* @since 0.7.0
*/
const list<Database::entry> search_all(string expression,
const bool is_re) const;
/*!
* @brief %Search in full text of database entries.
*
* Searches in tags, title, description and full text.
*
* @param expression %Search expression.
* @param is_re Is it a regular expression?
*
* @return List of matching Database::entry.
*
* @since 0.7.0
*/
[[nodiscard]]
list<Database::entry> search_all(const string &expression, bool is_re)
const;
/*!
* @brief Spawn threads of search_all(), if it seems sensible.
*
* Figure out if threads could be useful and spawn a sensible amount of
* them.
*
* @param expression %Search expression.
* @param is_re Is it a regular expression?
*
* @return List of matching Database::entry.
*
* @since 0.7.2
*/
// TODO: Think of something more elegant.
const list<Database::entry> search_all_threaded(string expression,
const bool is_re) const;
/*!
* @brief Spawn threads of search_all(), if it seems sensible.
*
* Figure out if threads could be useful and spawn a sensible amount of
* them.
*
* @param expression %Search expression.
* @param is_re Is it a regular expression?
*
* @return List of matching Database::entry.
*
* @since 0.7.2
*/
// TODO(tastytea): Think of something more elegant.
[[nodiscard]]
list<Database::entry> search_all_threaded(const string &expression,
bool is_re) const;
private:
const list<Database::entry> _entries;
private:
const list<Database::entry> _entries;
/*!
* @brief Split expression into subexpressions.
*
* First it splits at `OR` or `||`, then it splits the subexpressions
* at `AND` or `&&`. The first vector contains all tags before the
* first `OR`.
*
* @return Vector of `OR`-vectors of `AND`-tags.
*
* @since 0.7.0
*/
const vector<vector<string>> parse_expression(string expression) const;
/*!
* @brief Split expression into subexpressions.
*
* First it splits at `OR` or `||`, then it splits the subexpressions
* at `AND` or `&&`. The first vector contains all tags before the
* first `OR`.
*
* @return Vector of `OR`-vectors of `AND`-tags.
*
* @since 0.7.0
*/
[[nodiscard]]
static vector<vector<string>> parse_expression(const string &expression);
/*!
* @brief Convert str to lowercase. Works with unicode.
*
* @since 0.7.0
*/
inline const string to_lowercase(const string &str) const;
};
}
/*!
* @brief Convert str to lowercase. Works with unicode.
*
* @since 0.7.0
*/
[[nodiscard]]
static inline string to_lowercase(const string &str);
};
} // namespace remwharead
#endif // REMWHAREAD_SEARCH_HPP

View File

@ -17,104 +17,123 @@
#ifndef REMWHAREAD_SQLITE_HPP
#define REMWHAREAD_SQLITE_HPP
#include <Poco/Data/Session.h>
#include <chrono>
#include <experimental/filesystem>
#include <list>
#include <memory>
#include <string>
#include <vector>
#include <chrono>
#include <list>
#include <Poco/Data/Session.h>
namespace remwharead
{
namespace fs = std::experimental::filesystem;
using std::string;
using std::vector;
using std::chrono::system_clock;
using time_point = system_clock::time_point;
using std::list;
using Poco::Data::Session;
namespace fs = std::experimental::filesystem;
using std::string;
using std::vector;
using std::chrono::system_clock;
using time_point = system_clock::time_point;
using std::list;
using Poco::Data::Session;
/*!
* @brief Store and retrieve files from/to SQLite.
*
* @since 0.6.0
*
* @headerfile sqlite.hpp remwharead/sqlite.hpp
*/
class Database
{
public:
/*!
* @brief Store and retrieve files from/to SQLite.
* @brief Describes a database entry.
*
* @since 0.6.0
*
* @headerfile sqlite.hpp remwharead/sqlite.hpp
*/
class Database
struct entry
{
public:
/*!
* @brief Describes a database entry.
*
* @since 0.6.0
*
* @headerfile sqlite.hpp remwharead/sqlite.hpp
*/
typedef struct entry
{
string uri;
string archive_uri;
time_point datetime;
vector<string> tags;
string title;
string description;
string fulltext;
/*!
* @brief Returns true if date and time are equal.
*
* @since 0.6.0
*/
friend bool operator ==(const Database::entry &a,
const Database::entry &b);
/*!
* @brief The full text in one line.
*
* @since 0.6.0
*/
const string fulltext_oneline() const;
} entry;
string uri;
string archive_uri;
time_point datetime;
vector<string> tags;
string title;
string description;
string fulltext;
/*!
* @brief Connects to the database and creates it if necessary.
* @brief Returns true if date and time are equal.
*
* @since 0.6.0
*/
Database();
friend bool operator ==(const Database::entry &a,
const Database::entry &b);
/*!
* @brief Returns true if connected to the database.
* @brief The full text in one line.
*
* @since 0.6.0
*/
operator bool() const;
/*!
* @brief Store a Database::entry in the database.
*
* @since 0.6.0
*/
void store(const entry &data) const;
/*!
* @brief Retrieve a list of Database::entry from the database.
*
* @since 0.6.0
*/
const list<entry> retrieve(
const time_point &start = time_point(),
const time_point &end = system_clock::now()) const;
private:
fs::path _dbpath;
std::unique_ptr<Session> _session;
bool _connected;
[[nodiscard]]
string fulltext_oneline() const;
};
using DB = Database;
}
/*!
* @brief Connects to the database and creates it if necessary.
*
* @since 0.6.0
*/
Database();
/*!
* @brief Returns true if connected to the database.
*
* @since 0.6.0
*/
explicit operator bool() const;
/*!
* @brief Store a Database::entry in the database.
*
* @since 0.6.0
*/
void store(const entry &data) const;
/*!
* @brief Retrieve a list of Database::entry from the database.
*
* @since 0.6.0
*/
[[nodiscard]]
list<entry> retrieve(const time_point &start = time_point(),
const time_point &end = system_clock::now()) const;
/*!
* @brief Remove all entries with this URI from database.
*
* @return Number of removed entries.
*
* @since 0.9.0
*/
size_t remove(const string &uri);
/*!
* @brief Returns tags as comma separated string.
*
* @since 0.9.0
*/
[[nodiscard]]
static string tags_to_string(const vector<string> &tags);
private:
fs::path _dbpath;
std::unique_ptr<Session> _session;
bool _connected;
[[nodiscard]]
static fs::path get_data_home();
};
} // namespace remwharead
#endif // REMWHAREAD_SQLITE_HPP

View File

@ -17,37 +17,38 @@
#ifndef REMWHAREAD_TIME_HPP
#define REMWHAREAD_TIME_HPP
#include <string>
#include <chrono>
#include <string>
//! @file
namespace remwharead
{
using std::string;
using std::chrono::system_clock;
using time_point = system_clock::time_point;
using std::string;
using std::chrono::system_clock;
using time_point = system_clock::time_point;
/*!
* @brief Convert ISO 8601 or SQLite time-string to time_point.
*
* The SQLite format is *YY-MM-DD hh:mm:ss* instead of *YY-MM-DDThh:mm:ss*.
*
* @param strtime Time string in ISO 8601 or SQLite format.
* @param sqlite Is the string in SQLite format?
*/
const time_point string_to_timepoint(const string &strtime,
bool sqlite = false);
/*!
* @brief Convert ISO 8601 or SQLite time-string to time_point.
*
* The SQLite format is *YY-MM-DD hh:mm:ss* instead of *YY-MM-DDThh:mm:ss*.
*
* @param strtime Time string in ISO 8601 or SQLite format.
* @param sqlite Is the string in SQLite format?
*/
[[nodiscard]]
time_point string_to_timepoint(const string &strtime, bool sqlite = false);
/*!
* @brief Convert time_point to ISO 8601 or SQLite time-string.
*
* The SQLite format is *YY-MM-DD hh:mm:ss* instead of *YY-MM-DDThh:mm:ss*.
*
* @param time_point The std::chrono::system_clock::time_point.
* @param sqlite Is the string in SQLite format?
*/
const string timepoint_to_string(const time_point &tp, bool sqlite = false);
}
/*!
* @brief Convert time_point to ISO 8601 or SQLite time-string.
*
* The SQLite format is *YY-MM-DD hh:mm:ss* instead of *YY-MM-DDThh:mm:ss*.
*
* @param time_point The std::chrono::system_clock::time_point.
* @param sqlite Is the string in SQLite format?
*/
[[nodiscard]]
string timepoint_to_string(const time_point &tp, bool sqlite = false);
} // namespace remwharead
#endif // REMWHAREAD_TIME_HPP

View File

@ -21,23 +21,25 @@
namespace remwharead
{
/*!
* @brief Format of the export.
*
* @since 0.6.0
*
* @headerfile types.hpp remwharead/types.hpp
*/
enum class export_format
{
undefined,
csv,
asciidoc,
bookmarks,
simple,
json,
rss
};
}
/*!
* @brief Format of the export.
*
* @since 0.6.0
*
* @headerfile types.hpp remwharead/types.hpp
*/
enum class export_format
{
undefined,
csv,
asciidoc,
bookmarks,
simple,
json,
rss,
link,
rofi
};
} // namespace remwharead
#endif // REMWHAREAD_TYPES_HPP

View File

@ -1,5 +1,5 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
* Copyright © 2019, 2020 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -17,142 +17,180 @@
#ifndef REMWHAREAD_URI_HPP
#define REMWHAREAD_URI_HPP
#include <curl/curl.h>
#include <cstdint>
#include <string>
namespace remwharead
{
using std::string;
using std::string;
using std::uint16_t;
/*!
* @brief A processed HTML page.
*
* @return true if successful, when cast to bool.
*
* @since 0.7.0
*
* @headerfile uri.hpp remwharead/uri.hpp
*/
struct html_extract
{
bool successful = false;
string error;
string title;
string description;
string fulltext;
explicit operator bool() const;
};
/*!
* @brief The result of the call to the archive service.
*
* @return true if successful, when cast to bool.
*
* @since 0.7.0
*
* @headerfile uri.hpp remwharead/uri.hpp
*/
struct archive_answer
{
bool successful = false;
string error;
string uri;
explicit operator bool() const;
};
/*!
* @brief Download, archive and process an %URI.
*
* @since 0.6.0
*
* @headerfile uri.hpp remwharead/uri.hpp
*/
class URI
{
public:
/*!
* @brief A processed HTML page.
* @brief Construct object and set URL.
*
* @return true if successful, when cast to bool.
*
* @since 0.7.0
*
* @headerfile uri.hpp remwharead/uri.hpp
*/
typedef struct html_extract
{
bool successful = false;
string error;
string title;
string description;
string fulltext;
operator bool();
} html_extract;
/*!
* @brief The result of the call to the archive service.
*
* @return true if successful, when cast to bool.
*
* @since 0.7.0
*
* @headerfile uri.hpp remwharead/uri.hpp
*/
typedef struct archive_answer
{
bool successful = false;
string error;
string uri;
operator bool();
} archive_answer;
/*!
* @brief Download, archive and process an %URI.
* Initializes TLS and sets proxy from the environment variable
* `http_proxy`, if possible.
*
* @since 0.6.0
*
* @headerfile uri.hpp remwharead/uri.hpp
*/
class URI
{
public:
/*!
* @brief Construct object and set URL.
*
* Initializes TLS and sets proxy from the environment variable
* `http_proxy`, if possible.
*
* @since 0.6.0
*/
explicit URI(const string &uri);
virtual ~URI();
explicit URI(string uri);
virtual ~URI() = default;
/*!
* @brief Download %URI and extract title, description and full text.
*
* @since 0.6.0
*/
const html_extract get();
URI(const URI &other) = default;
URI &operator=(const URI &other) = default;
URI(URI &&other) = default;
URI &operator=(URI &&other) = default;
/*!
* @brief Save %URI in archive and return archive-URI.
*
* @since 0.6.0
*/
const archive_answer archive();
/*!
* @brief Download %URI and extract title, description and full text.
*
* @since 0.6.0
*/
[[nodiscard]] html_extract get();
protected:
string _uri;
/*!
* @brief Save %URI in archive and return archive-URI.
*
* @since 0.6.0
*/
[[nodiscard]] archive_answer archive() const;
/*!
* @brief Make a HTTP(S) request.
*
* @since 0.6.0
*/
const string make_request(const string &uri,
bool archive = false) const;
protected:
string _uri;
string _encoding;
string _document;
/*!
* @brief Extract the title from an HTML page.
*
* @since 0.6.0
*/
const string extract_title(const string &html);
/*!
* @brief Extract the title from an HTML page.
*
* @since 0.6.0
*/
[[nodiscard]] string extract_title() const;
/*!
* @brief Extract the description from an HTML page.
*
* @since 0.6.0
*/
const string extract_description(const string &html);
/*!
* @brief Extract the description from an HTML page.
*
* @since 0.6.0
*/
[[nodiscard]] string extract_description() const;
/*!
* @brief Removes HTML tags and superflous spaces from an HTML page.
*
* @since 0.6.0
*/
const string strip_html(const string &html);
/*!
* @brief Removes HTML tags and superflous spaces from an HTML page.
*
* @since 0.6.0
*/
[[nodiscard]] string strip_html() const;
/*!
* @brief Remove HTML tags.
*
* @param html HTML page.
* @param tag If set, only remove this tag.
*
* @since 0.6.0
*/
const string remove_html_tags(const string &html,
const string &tag = "");
/*!
* @brief Remove HTML tags.
*
* @param html HTML page.
* @param tag If set, only remove this tag.
*
* @since 0.6.0
*/
[[nodiscard]] static string remove_html_tags(const string &html,
const string &tag = "");
/*!
* @brief Convert HTML entities to UTF-8.
*
* @since 0.6.0
*/
const string unescape_html(string html);
/*!
* @brief Convert HTML entities to UTF-8.
*
* @since 0.6.0
*/
[[nodiscard]] static string unescape_html(string html);
/*!
* @brief Replace newlines with spaces.
*
* @since 0.6.0
*/
const string remove_newlines(string text);
};
}
/*!
* @brief Replace newlines with spaces.
*
* @since 0.6.0
*/
[[nodiscard]] static string remove_newlines(string text);
#endif // REMWHAREAD_URI_HPP
/*!
* @brief Set proxy server.
*
* @since 0.8.5
*/
static void set_proxy();
/*!
* @brief Limits text to N characters, cuts at space.
*
* @since 0.8.5
*/
[[nodiscard]] static string cut_text(const string &text, uint16_t n_chars);
/*!
* @brief Converts string to UTF-8.
*
* @since 0.9.2
*/
[[nodiscard]] inline string to_utf8(const string &str);
/*!
* @brief Try to detect the encoding of the document.
*
* @since 0.9.2
*/
void detect_encoding();
/*!
* @brief Returns true if document is *HTML.
*
* @since 0.9.2
*/
[[nodiscard]] bool is_html() const;
};
} // namespace remwharead
#endif // REMWHAREAD_URI_HPP

View File

@ -2,7 +2,7 @@
:doctype: manpage
:Author: tastytea
:Email: tastytea@tastytea.de
:Date: 2019-09-21
:Date: 2020-10-31
:Revision: 0.0.0
:man source: remwharead
:man manual: General Commands Manual
@ -13,9 +13,11 @@ remwharead - Saves URIs of things you want to remember in a database
== SYNOPSIS
*remwharead* [*-t* _tags_] [*-N*] _URI_
*remwharead* [*-t*=_tags_] [*-N*] _URI_
*remwharead* *-e* _format_ [*-f* _file_] [*-T* _start_,_end_] [[*-s*|*-S*] _expression_] [*-r*]
*remwharead* *-e*=_format_ [*-f*=_file_] [*-T*=_start_,_end_] [[*-s*|*-S*]=_expression_] [*-r*]
*remwharead* [*-d*=_URI_]
== DESCRIPTION
@ -24,34 +26,35 @@ remwharead - Saves URIs of things you want to remember in a database
the full text of the page and optional tags.
The database can be filtered by time, tags and full text and exported to CSV,
AsciiDoc, a bookmarks file, JSON or RSS.
AsciiDoc, a bookmarks file, JSON, RSS, a list of hyperlinks or a rofi-compatible
list.
Archiving is done using the Wayback machine from the
https://archive.org/[Internet Archive].
== OPTIONS
*-t* _tags_, *--tags* _tags_::
*-t*=_tags_, *--tags*=_tags_::
Add tags to _URI_, delimited by commas.
*-e* _format_, *--export* _format_::
*-e*=_format_, *--export*=_format_::
Export to _format_. Possible values are _csv_, _asciidoc_, _bookmarks_,
_simple_, _json_ or _rss_. See _FORMATS_.
_simple_, _json_, _rss_, _link_ or _rofi_. See _FORMATS_.
*-f* _file_, *--file* _file_::
*-f*=_file_, *--file*=_file_::
Save output to _file_. Default is stdout.
*-T* _start_,_end_, *--time-span* _start_,_end_::
*-T*=_start_,_end_, *--time-span*=_start_,_end_::
Only export entries between and including _start_ and _end_. _start_ and _end_
are date and time representations according to ISO 8601
(YYYY-MM-DDThh:mm:ss). Time zones are ignored.
Example: `--time-span 2019-01-01,2019-02-10T12:30`.
Example: `--time-span=2019-01-01,2019-02-10T12:30`.
*-s* _expression_, *--search-tags* _expression_::
*-s*=_expression_, *--search-tags*=_expression_::
Search in tags. Format: _tag1 AND tag2 OR tag3_. See _SEARCH EXPRESSIONS_. Case
insensitive.
*-S* _expression_, *--search-all* _expression_::
*-S*=_expression_, *--search-all*=_expression_::
Search in tags, title, description and full text. See _SEARCH EXPRESSIONS_. Case
insensitive.
@ -62,6 +65,9 @@ every tag is enclosed by _^_ and _$_.
*-N*, *--no-archive*::
Do not archive URI.
*-d*=_URI_, *--delete*=_URI_::
Remove all entries with this URI from the database.
*-h*, *--help*::
Show help message.
@ -72,32 +78,65 @@ Print version, copyright and license.
.Save a thing into the database, with tags.
====
`remwharead -t tag1,tag2 https://example.com/article.html`
[source,shell]
----
remwharead -t=tag1,tag2 https://example.com/article.html
----
====
.Export all things between and including 2019-04-01 and 2019-05-31 to a file.
====
`remwharead -e asciidoc -f out.adoc -T 2019-04-01,2019-05-31`
[source,shell]
----
remwharead -e=asciidoc -f=out.adoc -T=2019-04-01,2019-05-31
----
====
.Export all things to an HTML file.
====
`remwharead -e asciidoc | asciidoctor --backend=html5 --out-file=out.html -`
[source,shell]
----
remwharead -e=asciidoc | asciidoctor --backend=html5 --out-file=out.html -
----
====
.Export all things about GRUB the boot-loader, but nothing about caterpillars.
====
`remwharead -e csv -s "grub AND boot"`
[source,shell]
----
remwharead -e=csv -s="grub AND boot"
----
====
.Output all articles by Jan Müller, consider different spellings.
====
`remwharead -e simple -S 'Jan[\s]+M(ü|ue?)ller' -r`
[source,shell]
----
remwharead -e=simple -S='Jan[\s]+M(ü|ue?)ller' -r
----
====
.Export all things from the last week to an RSS feed.
====
`remwharead -e rss -T $(date -d "-1 week" -I),$(date -I) | sed 's|<link/>|<link>https://example.com/</link>|' > /var/www/feed.rss`
[source,shell]
----
remwharead -e=rss -T=$(date -d "-1 week" -I),$(date -Iminutes) | sed 's|<link/>|<link>https://example.com/</link>|' > /var/www/feed.rss
----
====
.Remove all entries that are tagged with mountain
====
[source,shell]
----
OLDIFS=${IFS}
IFS=$'\n'
for uri in $(remwharead -e link -s mountain); do
remwharead -d "${uri}"
done
IFS=${OLDIFS}
----
====
=== Display database
@ -109,7 +148,7 @@ can periodically generate an HTML file with cron and display it in the browser.
====
[source,crontab]
----
*/30 * * * * remwharead -e asciidoc -T $(date -d "-6 months" -I),$(date -I) | asciidoctor --backend=html5 --out-file=${HOME}/remwharead.html -
*/30 * * * * remwharead -e=asciidoc -T=$(date -d "-6 months" -I),$(date -Iminutes) | asciidoctor --backend=html5 --out-file=${HOME}/remwharead.html -
----
====
@ -129,8 +168,7 @@ HTML, PDF and many other formats.
=== bookmarks
The
https://docs.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/platform-apis/aa753582(v=vs.85)[Netscape
The https://msdn.microsoft.com/en-us/library/aa753582(VS.85).aspx[Netscape
Bookmark file format] is a format for exporting and importing bookmarks that is
understood by most browsers.
@ -150,6 +188,15 @@ Export as http://www.rssboard.org/rss-specification[RSS] feed. Because the URL
of the feed is unknown to *remwharead*, the generated feed is slightly out of
specification (the element _link_ in _channel_ is empty).
=== link
Export as a plain list of links, separated by newlines.
=== rofi
Export title, tags and URL for consumption by rofi. See the `scripts/` directory
on https://schlomp.space/tastytea/remwharead for an example.
== SEARCH EXPRESSIONS
A search expression is either a single term, or several terms separated by _AND_
@ -164,9 +211,8 @@ Currently only HTTP and HTTPS are supported.
== PROXY SUPPORT
*remwharead* supports HTTP proxies set via the environment variable
_http_proxy_. Accepted formats are: _\http://[user[:password]@]host[:port]/_ or
_[user[:password]@]host[:port]_. No SOCKS proxy support yet, sorry.
Since *remwharead* is built on libcurl, it respects the same proxy environment
variables. See *curl*(1), section _ENVIRONMENT_.
Example: http_proxy="http://localhost:3128/"
@ -188,7 +234,7 @@ Example: http_proxy="http://localhost:3128/"
== SEE ALSO
*crontab*(1), *crontab*(5)
*crontab*(1), *crontab*(5), *curl*(1)
== REPORTING BUGS

View File

@ -9,5 +9,5 @@ Description: @PROJECT_DESCRIPTION@
Version: @PROJECT_VERSION@
Cflags: -I${includedir}
Libs: -L${libdir} -l${name} -lPocoData -lstdc++fs
Requires.private: libxdg-basedir, icu-uc, icu-i18n
Requires.private: icu-uc, icu-i18n
Libs.private: -lPocoFoundation -lPocoNet -lPocoNetSSL -lPocoDataSQLite -lPocoJSON -lPocoXML

15
scripts/remwharead-rofi Executable file
View File

@ -0,0 +1,15 @@
#!/bin/sh
# Open the whole database in rofi. Searches in title, tags and URI.
# The selected entry will be opened with the default browser.
if [ -n "${2}" ]; then
uri=$(echo "${*}" | sed -E 's|^.+>([^><]+)</span>$|\1|')
xdg-open "${uri}"
exit 0
fi
if [ "${1}" = "runremwharead" ]; then
remwharead -e rofi
else
rofi -show remwharead -modi remwharead:"${0} runremwharead"
fi

View File

@ -1,3 +1,3 @@
# Write version in header
configure_file("version.hpp.in"
"${PROJECT_BINARY_DIR}/version.hpp")
"${CMAKE_CURRENT_BINARY_DIR}/version.hpp")

View File

@ -10,7 +10,7 @@ set_target_properties(${PROJECT_NAME}-cli
PROPERTIES OUTPUT_NAME ${PROJECT_NAME})
target_include_directories(${PROJECT_NAME}-cli
PRIVATE "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}")
PRIVATE "${PROJECT_BINARY_DIR}/src" "${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries(${PROJECT_NAME}-cli
PRIVATE ${PROJECT_NAME})

View File

@ -14,25 +14,29 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <string>
#include <chrono>
#include <fstream>
#include <locale>
#include <list>
#include "sqlite.hpp"
#include "remwharead_cli.hpp"
#include "uri.hpp"
#include "types.hpp"
#include "export/csv.hpp"
#include "export/adoc.hpp"
#include "export/bookmarks.hpp"
#include "export/simple.hpp"
#include "export/csv.hpp"
#include "export/json.hpp"
#include "export/link.hpp"
#include "export/rofi.hpp"
#include "export/rss.hpp"
#include "export/simple.hpp"
#include "search.hpp"
#include "sqlite.hpp"
#include "types.hpp"
#include "uri.hpp"
#include <chrono>
#include <fstream>
#include <iostream>
#include <list>
#include <locale>
#include <string>
#include <thread>
using namespace remwharead;
using namespace remwharead_cli;
using std::cerr;
using std::endl;
using std::string;
@ -44,29 +48,23 @@ int App::main(const std::vector<std::string> &args)
{
std::locale::global(std::locale("")); // Set locale globally.
if (_version_requested)
if (_exit_requested)
{
print_version();
return 0;
}
else if (_help_requested)
if (_argument_error)
{
print_help();
return 1;
}
else
if (!args.empty())
{
if (_argument_error)
{
return 1;
}
if (args.size() > 0)
{
_uri = args[0];
}
if (_uri.empty() && _format == export_format::undefined)
{
cerr << "Error: You have to specify either an URI or --export.\n";
return 1;
}
_uri = args[0];
}
if (_uri.empty() && _format == export_format::undefined)
{
cerr << "Error: You have to specify either an URI or --export.\n";
return 1;
}
Database db;
@ -79,23 +77,38 @@ int App::main(const std::vector<std::string> &args)
if (!_uri.empty())
{
URI uri(_uri);
archive_answer archive_data;
std::thread thread_archive;
if (_archive)
{
// clang-format off
thread_archive = std::thread([&archive_data, &uri]
{
archive_data = uri.archive();
});
// clang-format on
}
html_extract page = uri.get();
if (_archive)
{
thread_archive.join();
if (!archive_data)
{
cerr << "Error archiving URL: " << archive_data.error << endl;
}
}
if (!page)
{
cerr << "Error: Could not fetch page.\n";
cerr << page.error << endl;
return 3;
}
archive_answer archive;
if (_archive)
{
archive = uri.archive();
if (!archive)
{
cerr << "Error archiving URL: " << archive.error << endl;
}
}
db.store({_uri, archive.uri, system_clock::now(), _tags,
db.store({_uri, archive_data.uri, system_clock::now(), _tags,
page.title, page.description, page.fulltext});
}
@ -205,6 +218,32 @@ int App::main(const std::vector<std::string> &args)
}
break;
}
case export_format::link:
{
if (file.is_open())
{
Export::Link(entries, file).print();
file.close();
}
else
{
Export::Link(entries).print();
}
break;
}
case export_format::rofi:
{
if (file.is_open())
{
Export::Rofi(entries, file).print();
file.close();
}
else
{
Export::Rofi(entries).print();
}
break;
}
default:
{
break;

View File

@ -14,12 +14,16 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <Poco/Util/Option.h>
#include <Poco/Util/HelpFormatter.h>
#include "version.hpp"
#include "remwharead_cli.hpp"
#include "sqlite.hpp"
#include "time.hpp"
#include "version.hpp"
#include <Poco/Util/HelpFormatter.h>
#include <Poco/Util/Option.h>
#include <iostream>
#include <memory>
using namespace remwharead_cli;
using std::cout;
using std::cerr;
using std::endl;
@ -28,31 +32,29 @@ using Poco::Util::OptionCallback;
using Poco::Util::HelpFormatter;
App::App()
: _help_requested(false)
, _version_requested(false)
, _argument_error(false)
, _uri()
, _tags()
, _format(export_format::undefined)
, _timespan({ time_point(), system_clock::now() })
, _archive(true)
, _regex(false)
: _exit_requested{false}
, _argument_error{false}
, _format{export_format::undefined}
, _timespan{{time_point(), system_clock::now()}}
, _archive{true}
, _regex{false}
{}
void App::defineOptions(OptionSet& options)
{
options.addOption(
Option("help", "h", "Show this help message.")
.callback(OptionCallback<App>(this, &App::handle_info)));
.argument("option", false)
.callback(OptionCallback<App>(this, &App::handle_options)));
options.addOption(
Option("version", "V", "Print version, copyright and license.")
.callback(OptionCallback<App>(this, &App::handle_info)));
.callback(OptionCallback<App>(this, &App::handle_options)));
options.addOption(
Option("tags", "t", "Add tags to URI, delimited by commas.")
.argument("tags")
.callback(OptionCallback<App>(this, &App::handle_options)));
options.addOption(
Option("export", "e", "Export to format.")
Option("export", "e", "Export to format. See manpage for a list.")
.argument("format")
.callback(OptionCallback<App>(this, &App::handle_options)));
options.addOption(
@ -80,25 +82,28 @@ void App::defineOptions(OptionSet& options)
options.addOption(
Option("no-archive", "N", "Do not archive URI.")
.callback(OptionCallback<App>(this, &App::handle_options)));
}
void App::handle_info(const std::string &name, const std::string &)
{
if (name == "help")
{
_help_requested = true;
}
else if (name == "version")
{
_version_requested = true;
}
stopOptionsProcessing();
options.addOption(
Option("delete", "d",
"Remove all entries with this URI from database.")
.argument("URI")
.callback(OptionCallback<App>(this, &App::handle_options)));
}
void App::handle_options(const std::string &name, const std::string &value)
{
if (name == "tags")
if (name == "help")
{
_exit_requested = true;
print_help(value);
stopOptionsProcessing();
}
else if (name == "version")
{
_exit_requested = true;
print_version();
stopOptionsProcessing();
}
else if (name == "tags")
{
size_t pos_end = 0;
size_t pos_start = 0;
@ -147,6 +152,14 @@ void App::handle_options(const std::string &name, const std::string &value)
{
_format = export_format::rss;
}
else if (value == "link")
{
_format = export_format::link;
}
else if (value == "rofi")
{
_format = export_format::rofi;
}
else
{
cerr << "Error: Unknown format.\n";
@ -191,21 +204,40 @@ void App::handle_options(const std::string &name, const std::string &value)
{
_regex = true;
}
else if (name == "delete")
{
Database db;
cout << "Deleted " << db.remove(value) << " entries.\n";
_exit_requested = true;
}
}
void App::print_help()
void App::print_help(const string &option)
{
HelpFormatter helpFormatter(options());
helpFormatter.setCommand(commandName());
helpFormatter.setUsage("[-t tags] [-N] URI\n"
"-e format [-f file] [-T start,end] "
"[[-s|-S] expression] [-r]");
helpFormatter.format(cout);
std::unique_ptr<HelpFormatter> helpFormatter;
OptionSet oneoption;
if (option.empty())
{
helpFormatter = std::make_unique<HelpFormatter>(options());
helpFormatter->setCommand(commandName());
helpFormatter->setUsage("[-t tags] [-N] URI\n"
"-e format [-f file] [-T start,end] "
"[[-s|-S] expression] [-r]\n"
"-d URI");
}
else
{
oneoption.addOption(options().getOption(option));
helpFormatter = std::make_unique<HelpFormatter>(oneoption);
}
helpFormatter->format(cout);
}
void App::print_version()
{
cout << "remwharead " << global::version << endl <<
cout << "remwharead " << remwharead::version << endl <<
"Copyright (C) 2019 tastytea <tastytea@tastytea.de>\n"
"License GPLv3: GNU GPL version 3 "
"<https://www.gnu.org/licenses/gpl-3.0.html>.\n"

View File

@ -17,15 +17,16 @@
#ifndef REMWHAREAD_PARSE_OPTIONS_HPP
#define REMWHAREAD_PARSE_OPTIONS_HPP
#include <string>
#include <vector>
#include <array>
#include <chrono>
#include "types.hpp"
#include <Poco/Util/Application.h>
#include <Poco/Util/OptionSet.h>
#include "types.hpp"
#include "time.hpp"
#include <array>
#include <chrono>
#include <string>
#include <vector>
namespace remwharead_cli
{
using namespace remwharead;
using std::string;
using std::vector;
@ -40,16 +41,14 @@ public:
App();
protected:
void defineOptions(OptionSet& options);
void handle_info(const std::string &name, const std::string &);
void handle_options(const std::string &name, const std::string &value);
void print_help();
void print_version();
int main(const std::vector<std::string> &args);
void defineOptions(OptionSet& options) override;
void handle_options(const string &name, const string &value);
void print_help(const string &option);
static void print_version();
int main(const std::vector<string> &args) override;
private:
bool _help_requested;
bool _version_requested;
bool _exit_requested;
bool _argument_error;
string _uri;
vector<string> _tags;
@ -61,5 +60,6 @@ private:
bool _archive;
bool _regex;
};
} // namespace remwharead_cli
#endif // REMWHAREAD_PARSE_OPTIONS_HPP

View File

@ -0,0 +1,131 @@
# -*- mode: yaml -*-
# Written for clang-format 10.
# https://releases.llvm.org/10.0.0/tools/clang/docs/ClangFormatStyleOptions.html
---
DisableFormat: false
Language: Cpp
AccessModifierOffset: -4
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
# AlignConsecutiveBitFields: false # clang-format 11
AlignConsecutiveDeclarations: false
AlignConsecutiveMacros: false
AlignEscapedNewlines: DontAlign
AlignOperands: true # clang-format 11: Align
AlignTrailingComments: true
AllowAllArgumentsOnNextLine: false
AllowAllConstructorInitializersOnNextLine: false
AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: Empty
AllowShortCaseLabelsOnASingleLine: false
# AllowShortEnumsOnASingleLine: false # clang-format 11
AllowShortFunctionsOnASingleLine: Empty
AllowShortIfStatementsOnASingleLine: Never
AllowShortLambdasOnASingleLine: Inline
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: Yes
BinPackArguments: true
BinPackParameters: true
BraceWrapping: # If BreakBeforeBraces is set to Custom.
AfterCaseLabel: true
AfterClass: true
AfterControlStatement: Always
AfterEnum: true
AfterFunction: true
AfterNamespace: true
AfterStruct: true
AfterUnion: true
AfterExternBlock: true
BeforeCatch: true
BeforeElse: true
# BeforeLambdaBody: true # clang-format 11
# BeforeWhile: true # clang-format 11
IndentBraces: false
SplitEmptyFunction: false
SplitEmptyRecord: false
SplitEmptyNamespace: false
BreakBeforeBinaryOperators: NonAssignment
BreakBeforeBraces: Custom
BreakBeforeTernaryOperators: true
BreakConstructorInitializers: BeforeComma
BreakInheritanceList: BeforeComma
BreakStringLiterals: true
ColumnLimit: 80
# CommentPragmas: 'regex'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: false
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DeriveLineEnding: true
DerivePointerAlignment: false
FixNamespaceComments: true
ForEachMacros:
- FOREACH
- RANGES_FOR
- Q_FOREACH
- BOOST_FOREACH
IncludeBlocks: Regroup
IncludeCategories: # stdlib headers into own group.
- Regex: '^[^\.]+$'
Priority: 4
# IndentCaseBlocks: false # clang-format 11
IndentCaseLabels: false
# IndentExternBlock: NoIndent # clang-format 11
IndentGotoLabels: false
IndentPPDirectives: AfterHash
IndentWidth: 4
IndentWrappedFunctionNames: false
KeepEmptyLinesAtTheStartOfBlocks: true
# MacroBlockBegin: 'string'
# MacroBlockEnd: 'string'
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
# NamespaceMacros: 'string'
PenaltyBreakAssignment: 250
PenaltyBreakBeforeFirstCallParameter: 300
# PenaltyBreakComment: 300
# PenaltyBreakFirstLessLess: 120
# PenaltyBreakString: 1000
# PenaltyBreakTemplateDeclaration: 10
# PenaltyExcessCharacter: 1000000
# PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Right
# RawStringFormats: # <YAML>
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: false
SpaceAfterLogicalNot: false
SpaceAfterTemplateKeyword: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeCpp11BracedList: false
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatements
SpaceBeforeRangeBasedForLoopColon: true
SpaceBeforeSquareBrackets: false
SpaceInEmptyBlock: false
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInCStyleCastParentheses: false
SpacesInConditionalStatement: false
SpacesInContainerLiterals: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Auto
# StatementMacros:
# - Q_UNUSED
# - QT_REQUIRE_VERSION
TabWidth: 4
# TypenameMacros:
# - STACK_OF
# - LIST
UseCRLF: false
UseTab: Never
# WhitespaceSensitiveMacros: ['string', 'string'] # clang-format 11
...

View File

@ -0,0 +1,20 @@
# Configuration file for EditorConfig.
# More information is available under <https://editorconfig.org/>.
root = true
[*]
indent_style = space
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
max_line_length = 80
[*.?pp]
indent_size = 4
tab_width = 4
[{CMakeLists.txt,*.cmake}]
indent_size = 2
tab_width = 2

1
src/curl_wrapper/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/build/

View File

@ -0,0 +1,27 @@
# Support version 3.9 and above, but use policy settings up to 3.17.
cmake_minimum_required(VERSION 3.9...3.17)
# Ranges are supported from 3.12, set policy to current for < 3.12.
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
project(curl_wrapper
VERSION 0.1.0
DESCRIPTION "Light libcurl wrapper."
LANGUAGES CXX)
option(WITH_CURL_WRAPPER_TESTS "Compile tests for curl_wrapper." NO)
option(WITH_CURL_WRAPPER_DOC "Compile API reference for curl_wrapper." NO)
find_package(CURL 7.52 REQUIRED)
add_subdirectory("src")
if(WITH_CURL_WRAPPER_TESTS)
add_subdirectory("tests")
endif()
if(WITH_CURL_WRAPPER_DOC)
include("cmake/Doxygen.cmake")
enable_doxygen("src")
endif()

661
src/curl_wrapper/LICENSE Normal file
View File

@ -0,0 +1,661 @@
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU Affero General Public License is a free, copyleft license for
software and other kinds of works, specifically designed to ensure
cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
our General Public Licenses are intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights
with two steps: (1) assert copyright on the software, and (2) offer
you this License which gives you legal permission to copy, distribute
and/or modify the software.
A secondary benefit of defending all users' freedom is that
improvements made in alternate versions of the program, if they
receive widespread use, become available for other developers to
incorporate. Many developers of free software are heartened and
encouraged by the resulting cooperation. However, in the case of
software used on network servers, this result may fail to come about.
The GNU General Public License permits making a modified version and
letting the public access it on a server without ever releasing its
source code to the public.
The GNU Affero General Public License is designed specifically to
ensure that, in such cases, the modified source code becomes available
to the community. It requires the operator of a network server to
provide the source code of the modified version running there to the
users of that server. Therefore, public use of a modified version, on
a publicly accessible server, gives the public access to the source
code of the modified version.
An older license, called the Affero General Public License and
published by Affero, was designed to accomplish similar goals. This is
a different license, not a version of the Affero GPL, but Affero has
released a new version of the Affero GPL which permits relicensing under
this license.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the
Program, your modified version must prominently offer all users
interacting with it remotely through a computer network (if your version
supports such interaction) an opportunity to receive the Corresponding
Source of your version by providing access to the Corresponding Source
from a network server at no charge, through some standard or customary
means of facilitating copying of software. This Corresponding Source
shall include the Corresponding Source for any work covered by version 3
of the GNU General Public License that is incorporated pursuant to the
following paragraph.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the work with which it is combined will remain governed by version
3 of the GNU General Public License.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU Affero General Public License from time to time. Such new versions
will be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU Affero General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU Affero General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU Affero General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer
network, you should also make sure that it provides a way for users to
get its source. For example, if your program is a web application, its
interface could display a "Source" link that leads users to an archive
of the code. There are many ways you could offer source, and different
solutions will be better for different programs; see section 13 for the
specific requirements.
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU AGPL, see
<https://www.gnu.org/licenses/>.

View File

@ -0,0 +1,65 @@
= curl_wrapper
:project: curl_wrapper
:uri-base: https://schlomp.space/tastytea/{project}
:uri-cmake: https://cmake.org/
:uri-libcurl: https://curl.haxx.se/libcurl/
:uri-catch: https://github.com/catchorg/Catch2
:uri-doxygen: http://www.doxygen.nl/
Light libcurl wrapper for when you need to GET a website with minimum effort.
This is _not_ supposed to be a package on its own, but a thing you drop into
your project.
I made this because the curl wrapper I used before is no longer maintained and
the other wrappers are either incomplete or unmaintained as well. _I do not
guarantee anything, use at your own risk._
URL: <{uri-base}>.
== Example program
[source,cpp]
--------------------------------------------------------------------------------
#include "curl_wrapper.hpp"
#include <iostream>
namespace cw = curl_wrapper;
int main()
{
cw::CURLWrapper curl;
const auto answer{curl.make_http_request(cw::http_method::GET,
"http://example.com/")};
if (answer)
{
std::cout << answer;
}
}
--------------------------------------------------------------------------------
== Use with CMake
Drop this project into a subfolder in your project tree. It will be compiled as
a static library.
[source,cmake]
--------------------------------------------------------------------------------
add_subdirectory(curl_wrapper)
add_executable(test)
target_link_libraries(test PRIVATE curl_wrapper)
--------------------------------------------------------------------------------
.CMake options:
* `-DWITH_CURL_WRAPPER_TESTS=YES` Compiles the tests.
* `-DWITH_CURL_WRAPPER_DOC=YES` Generate API reference.
== Dependencies
* C++17
* link:{uri-cmake}[CMake] >= 3.9
* link:{uri-libcurl}[libcurl] >= 7.52
* Optional:
** Tests: link:{uri-catch}[Catch] >= 1.2
** Documentation: link:{uri-doxygen}[Doxygen] >= 1.8

View File

@ -0,0 +1,33 @@
function(enable_doxygen)
find_package(Doxygen REQUIRED dot)
set(DOXYGEN_RECURSIVE YES)
set(DOXYGEN_GENERATE_HTML YES)
set(DOXYGEN_HTML_OUTPUT "doc/html")
set(DOXYGEN_GENERATE_LATEX NO)
set(DOXYGEN_ALLOW_UNICODE_NAMES YES)
set(DOXYGEN_BRIEF_MEMBER_DESC YES)
set(DOXYGEN_REPEAT_BRIEF YES)
set(DOXYGEN_ALWAYS_DETAILED_SEC YES)
set(DOXYGEN_INLINE_INHERITED_MEMB NO)
set(DOXYGEN_INHERIT_DOCS YES)
set(DOXYGEN_SEPARATE_MEMBER_PAGES NO)
set(DOXYGEN_TAB_SIZE 4)
set(DOXYGEN_MARKDOWN_SUPPORT YES)
set(DOXYGEN_AUTOLINK_SUPPORT YES)
set(DOXYGEN_INLINE_SIMPLE_STRUCTS NO)
set(DOXYGEN_QUIET YES)
set(DOXYGEN_WARNINGS YES)
set(DOXYGEN_WARN_IF_UNDOCUMENTED YES)
set(DOXYGEN_BUILTIN_STL_SUPPORT YES)
set(DOXYGEN_VERBATIM_HEADERS YES)
set(DOXYGEN_INLINE_SOURCES YES)
set(DOXYGEN_SEARCHENGINE YES)
set(DOXYGEN_SHOW_FILES YES)
file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/doc")
doxygen_add_docs(${PROJECT_NAME}_doxygen "${ARGV}")
# Make sure doxygen is run with every build.
add_custom_target(${PROJECT_NAME}_docs ALL DEPENDS ${PROJECT_NAME}_doxygen)
endfunction()

View File

@ -0,0 +1,23 @@
file(GLOB sources "*.cpp")
file(GLOB headers "*.hpp")
add_library(${PROJECT_NAME} STATIC ${sources} ${headers})
unset(sources)
unset(headers)
set_target_properties(${PROJECT_NAME}
PROPERTIES
CXX_STANDARD 17
CXX_STANDARD_REQUIRED ON
CXX_EXTENSIONS OFF
POSITION_INDEPENDENT_CODE ON)
# FindCURL provides an IMPORTED target since CMake 3.12.
if(NOT ${CMAKE_VERSION} VERSION_LESS 3.12)
target_link_libraries(${PROJECT_NAME} PUBLIC CURL::libcurl)
else()
target_include_directories(${PROJECT_NAME} PUBLIC ${CURL_INCLUDE_DIRS})
target_link_libraries(${PROJECT_NAME} PUBLIC ${CURL_LIBRARIES})
endif()
target_include_directories(${PROJECT_NAME}
PUBLIC "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>")

View File

@ -0,0 +1,240 @@
/* This file is part of curl_wrapper.
* Copyright © 2020 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "curl_wrapper.hpp"
#include "types.hpp"
#include <curl/curl.h>
#include <atomic>
#include <cstdint>
#include <exception>
#include <stdexcept>
#include <string>
namespace curl_wrapper
{
inline static std::atomic<std::uint64_t> curlwrapper_instances{0};
CURLWrapper::CURLWrapper()
{
if (curlwrapper_instances == 0)
{
// NOLINTNEXTLINE(hicpp-signed-bitwise)
check(curl_global_init(CURL_GLOBAL_ALL));
}
++curlwrapper_instances;
_connection = curl_easy_init();
if (_connection == nullptr)
{
throw std::runtime_error{"Failed to initialize curl."};
}
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
curl_easy_setopt(_connection, CURLOPT_ERRORBUFFER, _buffer_error);
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
curl_easy_setopt(_connection, CURLOPT_WRITEFUNCTION, writer_body_wrapper);
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
curl_easy_setopt(_connection, CURLOPT_WRITEDATA, this);
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
curl_easy_setopt(_connection, CURLOPT_HEADERFUNCTION,
writer_headers_wrapper);
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
curl_easy_setopt(_connection, CURLOPT_HEADERDATA, this);
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
check(curl_easy_setopt(_connection, CURLOPT_FOLLOWLOCATION, 1L));
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
curl_easy_setopt(_connection, CURLOPT_MAXREDIRS, 5L);
}
CURLWrapper::~CURLWrapper() noexcept
{
curl_easy_cleanup(_connection);
--curlwrapper_instances;
if (curlwrapper_instances == 0)
{
curl_global_cleanup();
}
}
string CURLWrapper::escape_url(const string_view url) const
{
char *cbuf{curl_easy_escape(_connection, url.data(),
static_cast<int>(url.size()))};
string sbuf{cbuf};
curl_free(cbuf);
return sbuf;
}
string CURLWrapper::unescape_url(const string_view url) const
{
char *cbuf{curl_easy_unescape(_connection, url.data(),
static_cast<int>(url.size()), nullptr)};
string sbuf{cbuf};
curl_free(cbuf);
return sbuf;
}
void CURLWrapper::set_useragent(const string_view useragent)
{
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
check(curl_easy_setopt(_connection, CURLOPT_USERAGENT, useragent.data()));
}
void CURLWrapper::set_proxy(const string_view proxy)
{
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
check(curl_easy_setopt(_connection, CURLOPT_PROXY, proxy.data()));
}
answer CURLWrapper::make_http_request(http_method method, string_view uri)
{
_buffer_headers.clear();
_buffer_body.clear();
switch (method)
{
case http_method::DELETE:
{
// NOTE: Use CURLOPT_MIMEPOST, then set to DELETE to send data.
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
check(curl_easy_setopt(_connection, CURLOPT_CUSTOMREQUEST, "DELETE"));
break;
}
case http_method::GET:
{
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
curl_easy_setopt(_connection, CURLOPT_HTTPGET, 1L);
break;
}
case http_method::HEAD:
{
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
check(curl_easy_setopt(_connection, CURLOPT_CUSTOMREQUEST, "HEAD"));
break;
}
case http_method::PATCH:
{
// NOTE: Use CURLOPT_MIMEPOST, then set to PATCH to send data.
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
check(curl_easy_setopt(_connection, CURLOPT_CUSTOMREQUEST, "PATCH"));
break;
}
case http_method::POST:
{
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
curl_easy_setopt(_connection, CURLOPT_POST, 1L);
// NOTE: Use CURLOPT_MIMEPOST to send data.
break;
}
case http_method::PUT:
{
// NOTE: Use CURLOPT_MIMEPOST, then set to PUT to send data.
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
check(curl_easy_setopt(_connection, CURLOPT_CUSTOMREQUEST, "PUT"));
break;
}
}
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
check(curl_easy_setopt(_connection, CURLOPT_URL, uri.data()));
try
{
check(curl_easy_perform(_connection));
}
catch (const CURLException &e)
{
// PARTIAL_FILE error seems to be normal for HEAD requests.
if (!(method == http_method::HEAD
&& e.error_code == CURLE_PARTIAL_FILE))
{
std::rethrow_exception(std::current_exception());
}
}
long http_status{0}; // NOLINT(google-runtime-int)
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
check(curl_easy_getinfo(_connection, CURLINFO_RESPONSE_CODE, &http_status));
return {static_cast<std::uint16_t>(http_status), _buffer_headers,
_buffer_body};
}
size_t CURLWrapper::writer_body(char *data, size_t size, size_t nmemb)
{
if (data == nullptr)
{
return 0;
}
_buffer_body.append(data, size * nmemb);
return size * nmemb;
}
size_t CURLWrapper::writer_headers(char *data, size_t size, size_t nmemb)
{
if (data == nullptr)
{
return 0;
}
_buffer_headers.append(data, size * nmemb);
return size * nmemb;
}
void CURLWrapper::check(const CURLcode code)
{
if (code != CURLE_OK)
{
throw CURLException{code, _buffer_error};
}
}
const char *CURLException::what() const noexcept
{
// NOTE: The string has to be static, or it'll vanish before it can be
// used. Couldn't find good documentation on that.
static string error_string;
error_string = _error_message;
if (!error_string.empty())
{
error_string = " " + error_string;
}
error_string = "libcurl error: " + std::to_string(error_code)
+ error_string;
return error_string.c_str();
}
} // namespace curl_wrapper

View File

@ -0,0 +1,244 @@
/* This file is part of curl_wrapper.
* Copyright © 2020 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef CURL_WRAPPER_HPP
#define CURL_WRAPPER_HPP
#include "types.hpp"
#include <curl/curl.h>
#include <exception>
#include <string>
#include <string_view>
namespace curl_wrapper
{
using std::string;
using std::string_view;
/*!
* @brief Light wrapper around libcurl.
*
* @since 0.1.0
*/
class CURLWrapper
{
public:
/*!
* @brief Initializes curl and sets up connection.
*
* The first time an instance of CURLWrapper is created, it calls
* `curl_global_init`, which is not thread-safe. For more information
* consult [curl_global_init(3)]
* (https://curl.haxx.se/libcurl/c/curl_global_init.html).
*
* May throw CURLException or std::runtime_error.
*
* @since 0.1.0
*/
CURLWrapper();
/*!
* @brief Cleans up curl and connection.
*
* Calls `curl_global_cleanup`, which is not thread-safe, when the last
* instance of CURLWrapper is destroyed. For more information consult
* [curl_global_cleanup(3)]
* (https://curl.haxx.se/libcurl/c/curl_global_cleanup.html).
*
* @since 0.1.0
*/
virtual ~CURLWrapper() noexcept;
//! Copy constructor. @since 0.1.0
CURLWrapper(const CURLWrapper &other) = delete;
//! Move constructor @since 0.1.0
CURLWrapper(CURLWrapper &&other) noexcept = delete;
//! Copy assignment operator @since 0.1.0
CURLWrapper &operator=(const CURLWrapper &other) = delete;
//! Move assignment operator @since 0.1.0
CURLWrapper &operator=(CURLWrapper &&other) noexcept = delete;
/*!
* @brief Returns pointer to the CURL easy handle.
*
* You can use this handle to set or modify curl options. For more
* information consult [curl_easy_setopt(3)]
* (https://curl.haxx.se/libcurl/c/curl_easy_setopt.html).
*
* @since 0.1.0
*/
[[nodiscard]] inline CURL *get_curl_easy_handle() const
{
return _connection;
}
/*!
* @brief URL encodes the given string.
*
* For more information consult [curl_easy_escape(3)]
* (https://curl.haxx.se/libcurl/c/curl_easy_escape.html).
*
* @param url String to escape.
*
* @return The escaped string or {} if it failed.
*
* @since 0.1.0
*/
[[nodiscard]] string escape_url(string_view url) const;
/*!
* @brief URL decodes the given string.
*
* For more information consult [curl_easy_unescape(3)]
* (https://curl.haxx.se/libcurl/c/curl_easy_unescape.html).
*
* @param url String to unescape.
*
* @return The unescaped string or {} if it failed.
*
* @since 0.1.0
*/
[[nodiscard]] string unescape_url(string_view url) const;
/*!
* @brief Set the User-Agent.
*
* May throw CURLException.
*
* @since 0.1.0
*/
void set_useragent(string_view useragent);
/*!
* @brief Set a proxy.
*
* For more information consult [CURLOPT_PROXY(3)]
* (https://curl.haxx.se/libcurl/c/CURLOPT_PROXY.html).
*
* May throw CURLException.
*
* @since 0.1.1
*/
void set_proxy(string_view proxy);
/*!
* @brief Make a HTTP request.
*
* May throw CURLException.
*
* @param method The HTTP method.
* @param uri The full URI.
*
* @return The status code, headers and body of the page.
*
* @since 0.1.0
*/
[[nodiscard]] answer make_http_request(http_method method, string_view uri);
private:
CURL *_connection{};
char _buffer_error[CURL_ERROR_SIZE]{};
string _buffer_headers;
string _buffer_body;
/*!
* @brief libcurl write callback function.
*
* @since 0.1.0
*/
size_t writer_body(char *data, size_t size, size_t nmemb);
/*!
* @brief Wrapper for curl, because it can only call static member
* functions.
*
* <https://curl.haxx.se/docs/faq.html#Using_C_non_static_functions_f>
*
* @since 0.1.0
*/
static inline size_t writer_body_wrapper(char *data, size_t sz,
size_t nmemb, void *f)
{
return static_cast<CURLWrapper *>(f)->writer_body(data, sz, nmemb);
}
//! @copydoc writer_body
size_t writer_headers(char *data, size_t size, size_t nmemb);
//! @copydoc writer_body_wrapper
static inline size_t writer_headers_wrapper(char *data, size_t sz,
size_t nmemb, void *f)
{
return static_cast<CURLWrapper *>(f)->writer_headers(data, sz, nmemb);
}
/*!
* @brief Throw CURLException if command doesn't return CURLE_OK.
*
* @since 0.1.0
*/
void check(CURLcode code);
};
/*!
* @brief Exception for libcurl errors.
*
* @since 0.1.0
*/
class CURLException : public std::exception
{
public:
/*!
* @brief Constructor with error code.
*
* @since 0.1.0
*/
explicit CURLException(const CURLcode code)
: error_code{code}
{}
/*!
* @brief Constructor with error code and error buffer.
*
* @since 0.1.0
*/
explicit CURLException(const CURLcode code, string_view error_buffer)
: error_code{code}
, _error_message{error_buffer}
{}
const CURLcode error_code; //!< Error code from libcurl.
/*!
* @brief Error message.
*
* @since 0.1.0
*/
[[nodiscard]] const char *what() const noexcept override;
private:
string _error_message;
};
} // namespace curl_wrapper
#endif // CURL_WRAPPER_HPP

View File

@ -0,0 +1,51 @@
/* This file is part of curl_wrapper.
* Copyright © 2020 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "types.hpp"
#include <algorithm>
#include <cctype>
#include <string>
#include <string_view>
namespace curl_wrapper
{
using std::tolower;
std::string_view answer::get_header(const std::string_view field) const
{
const string searchstring{string(field) += ":"};
// clang-format off
auto it{std::search(headers.begin(), headers.end(), searchstring.begin(),
searchstring.end(),
[](unsigned char a, unsigned char b)
{ return tolower(a) == tolower(b); })};
// clang-format on
if (it != headers.end())
{
auto pos{static_cast<size_t>(it - headers.begin())};
pos = headers.find(':', pos) + 1;
pos = headers.find_first_not_of(' ', pos);
const auto endpos{headers.find_first_of("\r\n", pos)};
return string_view(&headers[pos], endpos - pos);
}
return {};
}
} // namespace curl_wrapper

View File

@ -0,0 +1,106 @@
/* This file is part of curl_wrapper.
* Copyright © 2020 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef CURL_WRAPPER_TYPES_HPP
#define CURL_WRAPPER_TYPES_HPP
#include <cstdint>
#include <ostream>
#include <string>
#include <string_view>
namespace curl_wrapper
{
using std::ostream;
using std::string;
using std::string_view;
/*!
* @brief The HTTP method.
*
* @since 0.1.0
*/
enum class http_method
{
DELETE,
GET,
HEAD,
PATCH,
POST,
PUT
};
/*!
* @brief Return type for network requests.
*
* Currently only HTTP is considered.
*
* @since 0.1.0
*/
struct answer
{
std::uint16_t status{0}; //!< Status code.
string headers; //!< The headers of the response from the server.
string body; //!< The response from the server.
/*!
* @brief Returns true if #status is 200.
*
* @since 0.1.0
*/
[[nodiscard]] inline explicit operator bool() const
{
return (status == 200);
}
/*!
* @brief Returns std::string_view of the #body.
*
* @since 0.1.0
*/
[[nodiscard]] inline explicit operator string_view() const
{
return body;
}
/*!
* @brief Returns #body as std::ostream.
*
* @since 0.1.0
*/
inline friend ostream &operator<<(ostream &out, const answer &answer)
{
out << answer.body;
return out;
}
/*!
* @brief Returns the value of a header field.
*
* @param field Case insensitive, ASCII only.
*
* @return A std::string_view to the value of the header field or {} if not
* found.
*
* @since 0.1.0
*/
[[nodiscard]] string_view get_header(string_view field) const;
};
} // namespace curl_wrapper
#endif // CURL_WRAPPER_TYPES_HPP

View File

@ -0,0 +1,37 @@
include(CTest)
file(GLOB sources_tests "test_*.cpp")
find_package(Catch2 CONFIG)
if(Catch2_FOUND) # Catch 2.x
include(Catch)
add_executable(all_tests main.cpp ${sources_tests})
set_target_properties(all_tests
PROPERTIES
CXX_STANDARD 17
CXX_STANDARD_REQUIRED ON
CXX_EXTENSIONS OFF)
target_link_libraries(all_tests
PRIVATE Catch2::Catch2 ${PROJECT_NAME})
target_include_directories(all_tests PRIVATE "/usr/include/catch2")
catch_discover_tests(all_tests EXTRA_ARGS "${EXTRA_TEST_ARGS}")
else() # Catch 1.x
if(EXISTS "/usr/include/catch.hpp")
message(STATUS "Catch 1.x found.")
foreach(src ${sources_tests})
get_filename_component(bin "${src}" NAME_WE)
add_executable(${bin} "main.cpp" "${src}")
set_target_properties(${bin}
PROPERTIES
CXX_STANDARD 17
CXX_STANDARD_REQUIRED ON
CXX_EXTENSIONS OFF)
target_link_libraries(${bin}
PRIVATE ${PROJECT_NAME})
add_test(${bin} ${bin} "${EXTRA_TEST_ARGS}")
endforeach()
else()
message(FATAL_ERROR
"Neither Catch 2.x nor Catch 1.x could be found.")
endif()
endif()

View File

@ -0,0 +1,3 @@
#define CATCH_CONFIG_MAIN
#include <catch.hpp>

View File

@ -0,0 +1,62 @@
#include "curl_wrapper.hpp"
#include <catch.hpp>
#include <exception>
#include <string>
namespace curl_wrapper
{
using std::string;
SCENARIO("URL encoding / decoding")
{
const string text_raw{"Hüpfburg am rande!"};
const string text_escaped("H%C3%BCpfburg%20am%20rande%21");
bool exception = false;
string answer;
WHEN("Encoding " + text_raw)
{
try
{
CURLWrapper curl;
answer = curl.escape_url(text_raw);
}
catch (const std::exception &e)
{
exception = true;
}
THEN("No exception is thrown")
AND_THEN("The text is successfully encoded")
{
REQUIRE_FALSE(exception);
REQUIRE(answer == text_escaped);
}
}
WHEN("Decoding " + text_escaped)
{
try
{
CURLWrapper curl;
answer = curl.unescape_url(text_escaped);
}
catch (const std::exception &e)
{
exception = true;
}
THEN("No exception is thrown")
AND_THEN("The text is successfully decoded")
{
REQUIRE_FALSE(exception);
REQUIRE(answer == text_raw);
}
}
}
} // namespace curl_wrapper

View File

@ -0,0 +1,41 @@
#include "curl_wrapper.hpp"
#include <catch.hpp>
#include <exception>
#include <string>
namespace curl_wrapper
{
using std::string;
SCENARIO("HTTP GET", "[http]")
{
const string uri{"https://schlomp.space/api/v1/version"};
bool exception = false;
string answer;
WHEN("GETing " + uri)
{
try
{
CURLWrapper curl;
answer = curl.make_http_request(http_method::GET, uri).body;
}
catch (const std::exception &e)
{
exception = true;
}
THEN("No exception is thrown")
AND_THEN("We get the right answer")
{
REQUIRE_FALSE(exception);
REQUIRE(answer.substr(0, 11) == R"({"version":)");
}
}
}
} // namespace curl_wrapper

View File

@ -0,0 +1,75 @@
#include "types.hpp"
#include <catch.hpp>
#include <exception>
#include <iostream>
#include <locale>
#include <string>
namespace curl_wrapper
{
using std::string;
SCENARIO("Extract header")
{
answer ret;
ret.headers = R"(HTTP/1.1 200 OK
Server: nginx/1.18.0
Date: Sat, 07 Nov 2020 22:26:13 GMT
Content-Type: application/rss+xml; charset=utf-8
Connection: keep-alive
Keep-Alive: timeout=20
Expires: Sat, 07 Nov 2020 22:56:13 GMT
Cache-Control: max-age=1800
X-Cache: HIT
X-UmläÜt: 🙂
)";
bool exception = false;
string value;
WHEN("We search for “cache-control”")
{
try
{
value = ret.get_header("cache-control");
}
catch (const std::exception &e)
{
std::cerr << "Exception: " << e.what() << '\n';
exception = true;
}
THEN("No exception is thrown")
AND_THEN("The value is successfully extracted")
{
REQUIRE_FALSE(exception);
REQUIRE(value == "max-age=1800");
}
}
// WHEN("We search for “X-UMLÄÜT”")
// {
// std::locale::global(std::locale("de_DE.UTF-8"));
//
// try
// {
// value = ret.get_header("X-UMLÄÜT");
// }
// catch (const std::exception &e)
// {
// std::cerr << "Exception: " << e.what() << '\n';
// exception = true;
// }
// THEN("No exception is thrown")
// AND_THEN("The value is successfully extracted")
// {
// REQUIRE_FALSE(exception);
// REQUIRE(value == "🙂");
// }
// }
}
} // namespace curl_wrapper

View File

@ -1,11 +1,9 @@
include(GNUInstallDirs)
find_package(PkgConfig REQUIRED)
pkg_check_modules(libxdg-basedir REQUIRED IMPORTED_TARGET libxdg-basedir)
# Some distributions do not contain Poco*Config.cmake recipes.
find_package(Poco
COMPONENTS Foundation Net NetSSL Data DataSQLite JSON XML
CONFIG)
find_package(Poco CONFIG
COMPONENTS Foundation Net Data DataSQLite JSON XML)
find_package(Boost 1.48.0 REQUIRED COMPONENTS locale)
file(GLOB_RECURSE sources_lib *.cpp)
file(GLOB_RECURSE headers_lib ../../include/*.hpp)
@ -18,19 +16,19 @@ set_target_properties(${PROJECT_NAME} PROPERTIES
target_include_directories(${PROJECT_NAME}
PRIVATE
"$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>" # version.hpp
"$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src>" # version.hpp
PUBLIC
"$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>"
"$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>")
target_link_libraries(${PROJECT_NAME}
PRIVATE PkgConfig::libxdg-basedir pthread
PRIVATE pthread Boost::locale curl_wrapper
PUBLIC stdc++fs)
# If no Poco*Config.cmake recipes are found, look for headers in standard dirs.
if(PocoNetSSL_FOUND)
if(Poco_FOUND)
target_link_libraries(${PROJECT_NAME}
PRIVATE Poco::Foundation Poco::Net Poco::NetSSL Poco::DataSQLite
PRIVATE Poco::Foundation Poco::Net Poco::DataSQLite
Poco::JSON Poco::XML
PUBLIC Poco::Data)
else()
@ -45,7 +43,7 @@ else()
"but the files seem to be in the standard directories. "
"Let's hope this works.")
target_link_libraries(${PROJECT_NAME}
PRIVATE PocoFoundation PocoNet PocoNetSSL PocoDataSQLite PocoJSON PocoXML
PRIVATE PocoFoundation PocoNet PocoDataSQLite PocoJSON PocoXML
PUBLIC PocoData)
endif()
endif()
@ -54,4 +52,3 @@ install(TARGETS ${PROJECT_NAME}
EXPORT "${PROJECT_NAME}Targets"
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")

View File

@ -14,238 +14,237 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <string>
#include <algorithm>
#include <utility>
#include <locale>
#include "version.hpp"
#include "time.hpp"
#include "export/adoc.hpp"
#include "time.hpp"
#include "version.hpp"
#include <Poco/URI.h>
#include <algorithm>
#include <iostream>
#include <locale>
#include <string>
#include <utility>
namespace remwharead
{
using std::string;
using std::cerr;
using std::endl;
using tagpair = std::pair<string,list<Database::entry>>;
using std::string;
using std::cerr;
using std::endl;
using tagpair = std::pair<string,list<Database::entry>>;
void Export::AsciiDoc::print() const
void Export::AsciiDoc::print() const
{
try
{
try
_out << "= Visited things\n"
<< ":Author: remwharead " << version << "\n"
<< ":Date: "
<< timepoint_to_string(system_clock::now()) << "\n"
<< ":TOC: right\n"
<< ":TOCLevels: 2\n"
<< ":!webfonts:\n\n";
tagmap alltags;
string day;
for (const Database::entry &entry : _entries)
{
_out << "= Visited things\n"
<< ":Author: remwharead " << global::version << endl
<< ":Date: "
<< timepoint_to_string(system_clock::now()) << endl
<< ":TOC: right\n"
<< ":TOCLevels: 2\n"
<< ":!webfonts:\n\n";
const string newday = get_day(entry);
tagmap alltags;
string day;
for (const Database::entry &entry : _entries)
if (newday != day)
{
const string newday = get_day(entry);
day = newday;
_out << "== " << day << endl << endl;
}
if (newday != day)
_out << "[[dt_" << timepoint_to_string(entry.datetime)
<< "]]\n" << "* link:" << replace_in_uri(entry.uri);
if (!entry.title.empty())
{
_out << '[' << replace_in_title(entry.title) << ']';
}
else
{
_out << "[]";
}
_out << " +" << endl;
_out << '_' << get_time(entry).substr(0, 5) << '_';
if (!entry.archive_uri.empty())
{
_out << " (link:" << replace_in_uri(entry.archive_uri)
<< "[archived version])";
}
bool separator = false;
for (const string &tag : entry.tags)
{
if (tag.empty())
{
day = newday;
_out << "== " << day << endl << endl;
continue;
}
if (!separator)
{
_out << "\n| ";
separator = true;
}
_out << "[[dt_" << timepoint_to_string(entry.datetime)
<< "]]\n" << "* link:" << replace_in_uri(entry.uri);
if (!entry.title.empty())
auto globaltag = alltags.find(tag);
if (globaltag != alltags.end())
{
_out << '[' << replace_in_title(entry.title) << ']';
globaltag->second.push_back(entry);
}
else
{
_out << "[]";
alltags.insert({ tag, { entry } });
}
_out << " +" << endl;
_out << '_' << get_time(entry).substr(0, 5) << '_';
if (!entry.archive_uri.empty())
_out << "xref:t_" << replace_in_tag(tag)
<< "[" << tag << ']';
if (tag != *(entry.tags.rbegin()))
{
_out << " (link:" << replace_in_uri(entry.archive_uri)
<< "[archived version])";
_out << ", ";
}
bool separator = false;
for (const string &tag : entry.tags)
{
if (tag.empty())
{
continue;
}
if (!separator)
{
_out << "\n| ";
separator = true;
}
auto globaltag = alltags.find(tag);
if (globaltag != alltags.end())
{
globaltag->second.push_back(entry);
}
else
{
alltags.insert({ tag, { entry } });
}
_out << "xref:t_" << replace_in_tag(tag)
<< "[" << tag << ']';
if (tag != *(entry.tags.rbegin()))
{
_out << ", ";
}
}
if (!entry.description.empty())
{
_out << " +" << endl << entry.description;
}
_out << endl << endl;
}
if (!alltags.empty())
if (!entry.description.empty())
{
print_tags(alltags);
_out << " +\n+" << entry.description << '+';
}
_out << endl << endl;
}
catch (std::exception &e)
if (!alltags.empty())
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
print_tags(alltags);
}
}
const string Export::AsciiDoc::replace(string text,
const replacemap &replacements) const
catch (std::exception &e)
{
for (const std::pair<const string, const string> &sr : replacements)
cerr << "Error in " << __func__ << ": " << e.what() << endl;
}
}
string Export::AsciiDoc::replace(string text, const replacemap &replacements)
{
for (const std::pair<const string, const string> &sr : replacements)
{
size_t pos = 0;
while ((pos = text.find(sr.first, pos)) != std::string::npos)
{
size_t pos = 0;
while ((pos = text.find(sr.first, pos)) != std::string::npos)
{
text.replace(pos, sr.first.length(), sr.second);
pos += sr.second.length();
}
text.replace(pos, sr.first.length(), sr.second);
pos += sr.second.length();
}
return text;
}
const string Export::AsciiDoc::replace_in_tag(const string &text) const
{
// TODO: Find a better solution.
const replacemap replacements =
{
{ " ", "-" }, { "§", "-" },
{ "$", "-" }, { "%", "-" },
{ "&", "-" }, { "/", "-" },
{ "=", "-" }, { "^", "-" },
{ "!", "-" }, { "?", "-" },
{ "'", "-" }, { "\"", "-" },
{ "´", "-" }, { "`", "-" },
{ "", "-" }, { "#", "-" },
{ "", "0" }, { "", "0" },
{ "", "1" }, { "¹", "1" },
{ "", "2" }, { "²", "2" },
{ "", "3" }, { "³", "3" },
{ "", "4" }, { "", "4" },
{ "", "5" }, { "", "5" },
{ "", "6" }, { "", "6" },
{ "", "7" }, { "", "7" },
{ "", "8" }, { "", "8" },
{ "", "9" }, { "", "9" }
};
return replace(text, replacements);
}
const string Export::AsciiDoc::replace_in_title(const string &text) const
{
// [ is implicitly escaped if the corresponding ] is.
return replace(text, {{ "]", "\\]" }});
}
const string Export::AsciiDoc::replace_in_uri(const string &text) const
{
return replace(text,
{
{ "[", "%5B" }, { "]", "%5D" }
});
}
void Export::AsciiDoc::print_tags(const tagmap &tags) const
{
_out << "== Tags\n\n";
vector<tagpair> sortedtags(tags.size());
std::move(tags.begin(), tags.end(), sortedtags.begin());
std::sort(sortedtags.begin(), sortedtags.end(),
[](const tagpair &a, tagpair &b)
{
if (a.second.size() != b.second.size())
{ // Sort by number of occurrences if they are different.
return a.second.size() > b.second.size();
}
else
{ // Sort by tag names otherwise.
std::locale loc;
const std::collate<char> &coll =
std::use_facet<std::collate<char>>(loc);
return (coll.compare(
a.first.data(), a.first.data()
+ a.first.length(),
b.first.data(), b.first.data()
+ b.first.length()) == -1);
}
});
bool othertags = false; // Have we printed “Less used tags” already?
for (const auto &tag : sortedtags)
return text;
}
string Export::AsciiDoc::replace_in_tag(const string &text)
{
// TODO(tastytea): Find a better solution.
const replacemap replacements =
{
// If we have more than 20 tags, group all tags that occur only 1
// time under the section “Less used tags”.
if (sortedtags.size() > 20 && tag.second.size() == 1)
{
if (!othertags)
{
_out << "=== Less used tags\n\n";
othertags = true;
}
_out << "=";
{ " ", "-" }, { "§", "-" },
{ "$", "-" }, { "%", "-" },
{ "&", "-" }, { "/", "-" },
{ "=", "-" }, { "^", "-" },
{ "!", "-" }, { "?", "-" },
{ "'", "-" }, { "\"", "-" },
{ "´", "-" }, { "`", "-" },
{ "", "-" }, { "#", "-" },
{ "", "0" }, { "", "0" },
{ "", "1" }, { "¹", "1" },
{ "", "2" }, { "²", "2" },
{ "", "3" }, { "³", "3" },
{ "", "4" }, { "", "4" },
{ "", "5" }, { "", "5" },
{ "", "6" }, { "", "6" },
{ "", "7" }, { "", "7" },
{ "", "8" }, { "", "8" },
{ "", "9" }, { "", "9" }
};
return replace(text, replacements);
}
string Export::AsciiDoc::replace_in_title(const string &text)
{
// [ is implicitly escaped if the corresponding ] is.
return replace(text, {{ "]", "\\]" }});
}
string Export::AsciiDoc::replace_in_uri(const string &text)
{
string out;
Poco::URI::encode(text, "+", out);
return out;
}
void Export::AsciiDoc::print_tags(const tagmap &tags) const
{
_out << "== Tags\n\n";
vector<tagpair> sortedtags(tags.size());
std::move(tags.begin(), tags.end(), sortedtags.begin());
const auto compare_tags =
[](const tagpair &a, tagpair &b)
{
if (a.second.size() != b.second.size())
{ // Sort by number of occurrences if they are different.
return a.second.size() > b.second.size();
}
_out << "=== [[t_" << replace_in_tag(tag.first) << "]]"
<< tag.first << endl;
for (const Database::entry &entry : tag.second)
// Sort by tag names otherwise.
const std::locale loc;
const auto &coll = std::use_facet<std::collate<char>>(loc);
return (coll.compare(
// NOLINTNEXTLINE pointer arithmetic
&a.first[0], &a.first[0] + a.first.size(),
// NOLINTNEXTLINE pointer arithmetic
&b.first[0], &b.first[0] + b.first.size()) == -1);
};
std::sort(sortedtags.begin(), sortedtags.end(), compare_tags);
bool othertags = false; // Have we printed “Less used tags” already?
for (const auto &tag : sortedtags)
{
// If we have more than 20 tags, group all tags that occur only 1
// time under the section “Less used tags”.
if (sortedtags.size() > 20 && tag.second.size() == 1)
{
if (!othertags)
{
const string datetime = timepoint_to_string(entry.datetime);
const string date = datetime.substr(0, datetime.find('T'));
string title = replace_in_title(entry.title);
if (title.empty())
{
title = "++" + entry.uri + "++";
}
_out << endl << "* xref:dt_" << datetime
<< '[' << title << "] _(" << date << ")_" << endl;
_out << "=== Less used tags\n\n";
othertags = true;
}
_out << endl;
_out << "=";
}
_out << "=== [[t_" << replace_in_tag(tag.first) << "]]"
<< tag.first << '\n';
for (const Database::entry &entry : tag.second)
{
const string datetime = timepoint_to_string(entry.datetime);
const string date = datetime.substr(0, datetime.find('T'));
string title = replace_in_title(entry.title);
if (title.empty())
{
title = "++" + entry.uri + "++";
}
_out << "\n* xref:dt_" << datetime << '[' << title << "] _("
<< date << ")_" << endl;
}
_out << endl;
}
const string Export::AsciiDoc::get_day(const Database::entry &entry) const
{
const string datetime = timepoint_to_string(entry.datetime);
return datetime.substr(0, datetime.find('T'));
}
const string Export::AsciiDoc::get_time(const Database::entry &entry) const
{
const string datetime = timepoint_to_string(entry.datetime);
return datetime.substr(datetime.find('T') + 1);
}
_out << endl;
}
string Export::AsciiDoc::get_day(const Database::entry &entry)
{
const string datetime = timepoint_to_string(entry.datetime);
return datetime.substr(0, datetime.find('T'));
}
string Export::AsciiDoc::get_time(const Database::entry &entry)
{
const string datetime = timepoint_to_string(entry.datetime);
return datetime.substr(datetime.find('T') + 1);
}
} // namespace remwharead

View File

@ -14,46 +14,46 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "export/bookmarks.hpp"
#include "sqlite.hpp"
#include <chrono>
#include <string>
#include "sqlite.hpp"
#include "export/bookmarks.hpp"
namespace remwharead
{
using std::chrono::system_clock;
using std::chrono::duration_cast;
using std::chrono::seconds;
using std::string;
using std::chrono::system_clock;
using std::chrono::duration_cast;
using std::chrono::seconds;
using std::string;
void Export::Bookmarks::print() const
void Export::Bookmarks::print() const
{
_out << "<!DOCTYPE NETSCAPE-Bookmark-file-1>\n"
"<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; "
"charset=UTF-8\">\n"
"<TITLE>Bookmarks from remwharead</TITLE>\n"
"<H1>Bookmarks from remwharead<H1>\n\n"
"<DL><p>\n"
"<DT><H3>remwharead</H3>\n"
"<DL><p>\n";
for (const Database::entry & entry : _entries)
{
_out << "<!DOCTYPE NETSCAPE-Bookmark-file-1>\n"
"<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; "
"charset=UTF-8\">\n"
"<TITLE>Bookmarks from remwharead</TITLE>\n"
"<H1>Bookmarks from remwharead<H1>\n\n"
"<DL><p>\n"
"<DT><H3>remwharead</H3>\n"
"<DL><p>\n";
for (const Database::entry & entry : _entries)
string title = entry.title;
if (title.empty())
{
string title = entry.title;
if (title.empty())
{
title = entry.uri;
}
system_clock::time_point tp = entry.datetime;
system_clock::duration duration = tp.time_since_epoch();
string time_seconds =
std::to_string(duration_cast<seconds>(duration).count());
_out << "<DT><A HREF=\"" << entry.uri << "\" "
<< "ADD_DATE=\"" << time_seconds << "\">"
<< title << "</A>\n";
title = entry.uri;
}
_out << "</DL><p>\n"
<< "</DL><p>\n";
system_clock::time_point tp = entry.datetime;
system_clock::duration duration = tp.time_since_epoch();
string time_seconds =
std::to_string(duration_cast<seconds>(duration).count());
_out << "<DT><A HREF=\"" << entry.uri << "\" "
<< "ADD_DATE=\"" << time_seconds << "\">"
<< title << "</A>\n";
}
_out << "</DL><p>\n"
<< "</DL><p>\n";
}
} // namespace remwharead

View File

@ -14,53 +14,44 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "time.hpp"
#include "export/csv.hpp"
#include "time.hpp"
namespace remwharead
{
using std::cerr;
using std::endl;
using std::cerr;
using std::endl;
void Export::CSV::print() const
void Export::CSV::print() const
{
try
{
try
_out << R"("URI","Archived URI","Date & time","Tags",)"
<< R"("Title","Description","Full text")" << "\r\n";
for (const Database::entry &entry : _entries)
{
_out << "\"URI\",\"Archived URI\",\"Date & time\",\"Tags\","
<< "\"Title\",\"Description\",\"Full text\"\r\n";
for (const Database::entry &entry : _entries)
{
string strtags;
for (const string &tag : entry.tags)
{
strtags += tag;
if (tag != *(entry.tags.rbegin()))
{
strtags += ",";
}
}
_out << '"' << quote(entry.uri) << "\",\""
<< quote(entry.archive_uri) << "\",\""
<< timepoint_to_string(entry.datetime) << "\",\""
<< quote(strtags) << "\",\""
<< quote(entry.title) << "\",\""
<< quote(entry.description) << "\",\""
<< quote(entry.fulltext_oneline()) << '"'<< "\r\n";
}
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
_out << '"' << quote(entry.uri) << "\",\""
<< quote(entry.archive_uri) << "\",\""
<< timepoint_to_string(entry.datetime) << "\",\""
<< quote(Database::tags_to_string(entry.tags)) << "\",\""
<< quote(entry.title) << "\",\""
<< quote(entry.description) << "\",\""
<< quote(entry.fulltext_oneline()) << '"'<< "\r\n";
}
}
const string Export::CSV::quote(string field) const
catch (std::exception &e)
{
size_t pos = 0;
while ((pos = field.find('"', pos)) != std::string::npos)
{
field.replace(pos, 1, "\"\"");
}
return field;
cerr << "Error in " << __func__ << ": " << e.what() << endl;
}
}
string Export::CSV::quote(string field)
{
size_t pos = 0;
while ((pos = field.find('"', pos)) != std::string::npos)
{
field.replace(pos, 1, "\"\"");
}
return field;
}
} // namespace remwharead

View File

@ -14,28 +14,25 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <algorithm>
#include "export/export.hpp"
#include <algorithm>
namespace remwharead
namespace remwharead::Export
{
namespace Export
ExportBase::ExportBase(const list<Database::entry> &entries, ostream &out)
: _entries(sort_entries(entries))
, _out(out)
{}
list<Database::entry>
ExportBase::sort_entries(list<Database::entry> entries)
{
ExportBase::ExportBase(const list<Database::entry> &entries, ostream &out)
: _entries(sort_entries(entries))
, _out(out)
{}
entries.sort([](const auto &a, const auto &b)
{
return (a.datetime > b.datetime);
});
entries.unique();
const list<Database::entry>
ExportBase::sort_entries(list<Database::entry> entries) const
{
entries.sort([](const auto &a, const auto &b)
{
return (a.datetime > b.datetime);
});
entries.unique();
return entries;
}
}
return entries;
}
} // namespace remwharead::Export

View File

@ -14,48 +14,49 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "export/json.hpp"
#include "time.hpp"
#include <Poco/JSON/Object.h>
#include <Poco/JSON/Stringifier.h>
#include "time.hpp"
#include "export/json.hpp"
namespace remwharead
{
using std::cerr;
using std::endl;
using std::cerr;
using std::endl;
void Export::JSON::print() const
void Export::JSON::print() const
{
try
{
try
{
Poco::JSON::Array root = Poco::JSON::Array();
Poco::JSON::Array root = Poco::JSON::Array();
for (const Database::entry &entry : _entries)
for (const Database::entry &entry : _entries)
{
Poco::JSON::Object json_entry = Poco::JSON::Object();
json_entry.set("uri", entry.uri);
json_entry.set("archive_uri", entry.archive_uri);
json_entry.set("datetime", timepoint_to_string(entry.datetime));
Poco::JSON::Array tags = Poco::JSON::Array();
for (const string &tag : entry.tags)
{
Poco::JSON::Object json_entry = Poco::JSON::Object();
json_entry.set("uri", entry.uri);
json_entry.set("archive_uri", entry.archive_uri);
json_entry.set("datetime", timepoint_to_string(entry.datetime));
Poco::JSON::Array tags = Poco::JSON::Array();
for (const string &tag : entry.tags)
{
tags.add(tag);
}
json_entry.set("tags", tags);
json_entry.set("title", entry.title);
json_entry.set("description", entry.description);
json_entry.set("fulltext", entry.fulltext);
root.add(json_entry);
tags.add(tag);
}
json_entry.set("tags", tags);
json_entry.set("title", entry.title);
json_entry.set("description", entry.description);
json_entry.set("fulltext", entry.fulltext);
root.stringify(_out);
_out << endl;
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
root.add(json_entry);
}
root.stringify(_out);
_out << endl;
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
}
}
} // namespace remwharead

32
src/lib/export/link.cpp Normal file
View File

@ -0,0 +1,32 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "export/link.hpp"
#include "sqlite.hpp"
#include <string>
namespace remwharead
{
using std::string;
void Export::Link::print() const
{
for (const Database::entry & entry : _entries)
{
_out << entry.uri << '\n';
}
}
} // namespace remwharead

39
src/lib/export/rofi.cpp Normal file
View File

@ -0,0 +1,39 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "export/rofi.hpp"
#include "sqlite.hpp"
#include <string>
namespace remwharead
{
using std::string;
void Export::Rofi::print() const
{
_out << static_cast<char>(0x00) << "markup-rows"
<< static_cast<char>(0x1f) << "true\n";
for (const Database::entry & entry : _entries)
{
_out << entry.title
<< R"( <span size="small" weight="light" style="italic">()"
<< Database::tags_to_string(entry.tags) << ")</span> "
<< R"(<span size="xx-small" weight="ultralight">)"
<< entry.uri << "</span>\n";
}
}
} // namespace remwharead

View File

@ -14,142 +14,143 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <ctime>
#include <cstdint>
#include <Poco/XML/XMLWriter.h>
#include <Poco/SAX/AttributesImpl.h>
#include <Poco/DateTime.h>
#include <Poco/DateTimeFormatter.h>
#include <Poco/Timestamp.h>
#include "export/rss.hpp"
#include "time.hpp"
#include "version.hpp"
#include "export/rss.hpp"
#include <Poco/DateTime.h>
#include <Poco/DateTimeFormatter.h>
#include <Poco/SAX/AttributesImpl.h>
#include <Poco/Timestamp.h>
#include <Poco/XML/XMLWriter.h>
#include <cstdint>
#include <ctime>
namespace remwharead
{
using std::cerr;
using std::endl;
using std::time_t;
using Poco::XML::XMLWriter;
using Poco::XML::AttributesImpl;
using Poco::DateTime;
using Poco::DateTimeFormatter;
using Poco::Timestamp;
using std::cerr;
using std::endl;
using std::time_t;
using Poco::XML::XMLWriter;
using Poco::XML::AttributesImpl;
using Poco::DateTime;
using Poco::DateTimeFormatter;
using Poco::Timestamp;
void Export::RSS::print() const
void Export::RSS::print() const
{
try
{
try
XMLWriter writer(_out, XMLWriter::CANONICAL);
AttributesImpl attrs_rss;
AttributesImpl attrs_guid;
constexpr char timefmt_rfc822[] = "%w, %d %b %Y %H:%M:%S %Z";
attrs_rss.addAttribute("", "", "version", "", "2.0");
attrs_rss.addAttribute("", "", "xmlns:atom", "",
"http://www.w3.org/2005/Atom");
attrs_guid.addAttribute("", "", "isPermaLink", "", "false");
writer.startDocument();
writer.startElement("", "", "rss", attrs_rss);
writer.startElement("", "", "channel");
writer.startElement("", "", "title");
writer.characters("Visited things");
writer.endElement("", "", "title");
writer.startElement("", "", "link");
writer.endElement("", "", "link");
writer.startElement("", "", "description");
writer.characters("Export from remwharead.");
writer.endElement("", "", "description");
writer.startElement("", "", "generator");
writer.characters(string("remwharead ") + version);
writer.endElement("", "", "generator");
const string now = DateTimeFormatter::format(DateTime(),
timefmt_rfc822);
writer.startElement("", "", "lastBuildDate");
writer.characters(now);
writer.endElement("", "", "lastBuildDate");
for (const Database::entry &entry : _entries)
{
XMLWriter writer(_out, XMLWriter::CANONICAL);
AttributesImpl attrs_rss, attrs_guid;
constexpr char timefmt_rfc822[] = "%w, %d %b %Y %H:%M:%S %Z";
attrs_rss.addAttribute("", "", "version", "", "2.0");
attrs_rss.addAttribute("", "", "xmlns:atom", "",
"http://www.w3.org/2005/Atom");
attrs_guid.addAttribute("", "", "isPermaLink", "", "false");
writer.startDocument();
writer.startElement("", "", "rss", attrs_rss);
writer.startElement("", "", "channel");
writer.startElement("", "", "item");
writer.startElement("", "", "title");
writer.characters("Visited things");
if (!entry.title.empty())
{
writer.characters(entry.title);
}
else
{
constexpr std::uint8_t maxlen = 100;
string title = entry.description.substr(0, maxlen);
if (entry.description.length() > maxlen)
{
title += " […]";
}
writer.characters(title);
}
writer.endElement("", "", "title");
writer.startElement("", "", "link");
writer.characters(entry.uri);
writer.endElement("", "", "link");
writer.startElement("", "", "guid", attrs_guid);
writer.characters(entry.uri + " at "
+ timepoint_to_string(entry.datetime));
writer.endElement("", "", "guid");
const time_t time = system_clock::to_time_t(entry.datetime);
const string time_visited = DateTimeFormatter::format(
Timestamp::fromEpochTime(time), timefmt_rfc822);
writer.startElement("", "", "pubDate");
writer.characters(time_visited);
writer.endElement("", "", "pubDate");
string description = entry.description;
if (!description.empty())
{
description = "<p>" + description.append("</p>");
}
if (!entry.tags.empty())
{
description += "<p><strong>Tags:</strong> ";
for (const string &tag : entry.tags)
{
description += tag;
if (tag != *(entry.tags.rbegin()))
{
description += ", ";
}
}
description += "</p>";
}
if (!entry.archive_uri.empty())
{
description += "<p><strong>Archived version:</strong> "
"<a href=\"" + entry.archive_uri + "\">"
+ entry.archive_uri + "</a></p>";
}
writer.startElement("", "", "description");
writer.characters("Export from remwharead.");
writer.characters(description);
writer.endElement("", "", "description");
writer.startElement("", "", "generator");
writer.characters(string("remwharead ") + global::version);
writer.endElement("", "", "generator");
const string now = DateTimeFormatter::format(DateTime(),
timefmt_rfc822);
writer.startElement("", "", "lastBuildDate");
writer.characters(now);
writer.endElement("", "", "lastBuildDate");
for (const Database::entry &entry : _entries)
{
writer.startElement("", "", "item");
writer.startElement("", "", "title");
if (!entry.title.empty())
{
writer.characters(entry.title);
}
else
{
constexpr std::uint8_t maxlen = 100;
string title = entry.description.substr(0, maxlen);
if (entry.description.length() > maxlen)
{
title += " […]";
}
writer.characters(title);
}
writer.endElement("", "", "title");
writer.startElement("", "", "link");
writer.characters(entry.uri);
writer.endElement("", "", "link");
writer.startElement("", "", "guid", attrs_guid);
writer.characters(entry.uri + " at " +
timepoint_to_string(entry.datetime));
writer.endElement("", "", "guid");
const time_t time = system_clock::to_time_t(entry.datetime);
const string time_visited = DateTimeFormatter::format(
Timestamp::fromEpochTime(time), timefmt_rfc822);
writer.startElement("", "", "pubDate");
writer.characters(time_visited);
writer.endElement("", "", "pubDate");
string description = entry.description;
if (!description.empty())
{
description = "<p>" + description + "</p>";
}
if (!entry.tags.empty())
{
description += "<p><strong>Tags:</strong> ";
for (const string &tag : entry.tags)
{
description += tag;
if (tag != *(entry.tags.rbegin()))
{
description += ", ";
}
}
description += "</p>";
}
if (!entry.archive_uri.empty())
{
description += "<p><strong>Archived version:</strong> "
"<a href=\"" + entry.archive_uri + "\">"
+ entry.archive_uri + "</a>";
}
writer.startElement("", "", "description");
writer.characters(description);
writer.endElement("", "", "description");
writer.endElement("", "", "item");
}
writer.endElement("", "", "channel");
writer.endElement("", "", "rss");
writer.endDocument();
_out << endl;
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
writer.endElement("", "", "item");
}
writer.endElement("", "", "channel");
writer.endElement("", "", "rss");
writer.endDocument();
_out << endl;
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
}
}
} // namespace remwharead

View File

@ -14,28 +14,28 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <string>
#include "export/simple.hpp"
#include "sqlite.hpp"
#include "time.hpp"
#include "export/simple.hpp"
#include <string>
namespace remwharead
{
using std::string;
using std::string;
void Export::Simple::print() const
void Export::Simple::print() const
{
for (const Database::entry & entry : _entries)
{
for (const Database::entry & entry : _entries)
const string timestring = timepoint_to_string(entry.datetime);
_out << timestring.substr(0, timestring.find('T')) << ": ";
if (!entry.title.empty())
{
const string timestring = timepoint_to_string(entry.datetime);
_out << timestring.substr(0, timestring.find('T')) << ": ";
if (!entry.title.empty())
{
_out << entry.title << '\n';
_out << " ";
}
_out << "<" << entry.uri << ">\n";
_out << entry.title << '\n';
_out << " ";
}
_out << "<" << entry.uri << ">\n";
}
}
} // namespace remwharead

View File

@ -14,257 +14,255 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "search.hpp"
#include <Poco/RegularExpression.h>
#include <Poco/UTF8String.h>
#include <algorithm>
#include <locale>
#include <iterator>
#include <list>
#include <locale>
#include <thread>
#include <utility>
#include <iterator>
#include <Poco/UTF8String.h>
#include <Poco/RegularExpression.h>
#include "search.hpp"
namespace remwharead
{
using std::list;
using std::find;
using std::find_if;
using std::thread;
using std::move;
using RegEx = Poco::RegularExpression;
using std::list;
using std::find;
using std::find_if;
using std::thread;
using std::move;
using RegEx = Poco::RegularExpression;
Search::Search(const list<Database::entry> &entries)
:_entries(entries)
{}
Search::Search(list<Database::entry> entries)
:_entries(move(entries))
{}
vector<vector<string>> Search::parse_expression(const string &expression)
{
vector<vector<string>> searchlist;
const RegEx re_or("(.+?) (OR|\\|\\|) ");
const RegEx re_and("(.+?) (AND|&&) ");
RegEx::MatchVec matches;
string::size_type pos = 0;
vector<string> subexpressions;
{ // Split expression at OR.
while (re_or.match(expression, pos, matches) != 0)
{
const string &subexpr = expression.substr(matches[1].offset,
matches[1].length);
subexpressions.push_back(subexpr);
pos = matches[0].offset + matches[0].length;
}
subexpressions.push_back(expression.substr(pos));
}
const vector<vector<string>> Search::parse_expression(string expression)
const
{
vector<vector<string>> searchlist;
const RegEx re_or("(.+?) (OR|\\|\\|) ");
const RegEx re_and("(.+?) (AND|&&) ");
RegEx::MatchVec matches;
string::size_type pos = 0;
for (const string &sub : subexpressions)
{ // Split each OR-slice at AND.
vector<string> terms;
pos = 0;
vector<string> subexpressions;
{ // Split expression at OR.
while (re_or.match(expression, pos, matches) != 0)
while (re_and.match(sub, pos, matches) != 0)
{
const string &subexpr = expression.substr(matches[1].offset,
matches[1].length);
subexpressions.push_back(subexpr);
const string &term = sub.substr(matches[1].offset,
matches[1].length);
terms.push_back(to_lowercase(term));
pos = matches[0].offset + matches[0].length;
}
subexpressions.push_back(expression.substr(pos));
terms.push_back(to_lowercase(sub.substr(pos)));
searchlist.push_back(terms);
}
{
for (string sub : subexpressions)
{ // Split each OR-slice at AND.
vector<string> terms;
pos = 0;
while (re_and.match(sub, pos, matches) != 0)
{
const string &term = sub.substr(matches[1].offset,
matches[1].length);
terms.push_back(to_lowercase(term));
pos = matches[0].offset + matches[0].length;
}
terms.push_back(to_lowercase(sub.substr(pos)));
searchlist.push_back(terms);
}
}
return searchlist;
}
const string Search::to_lowercase(const string &str) const
{
return Poco::UTF8::toLower(str);
}
const list<DB::entry> Search::search_tags(string expression,
const bool is_re) const
{
vector<vector<string>> searchlist = parse_expression(expression);
list<DB::entry> result;
for (const vector<string> &tags_or : searchlist)
{
for (const DB::entry &entry : _entries)
{ // Add entry to result if all tags in an OR-slice match.
bool matched = true;
for (const string &tag : tags_or)
{
const auto it = find_if(
entry.tags.begin(), entry.tags.end(),
[&, is_re](string s)
{
s = to_lowercase(s);
if (is_re)
{
const RegEx re("^" + tag + "$");
return (re == s);
}
else
{
return (s == tag);
}
});
if (it == entry.tags.end())
{
matched = false;
}
}
if (matched == true)
{
result.push_back(entry);
}
}
}
return result;
}
const list<DB::entry> Search::search_all(string expression,
const bool is_re) const
{
vector<vector<string>> searchlist = parse_expression(expression);
list<DB::entry> result = search_tags(expression, is_re);
for (const vector<string> &terms_or : searchlist)
{
for (const DB::entry &entry : _entries)
{
// Add entry to result if all terms in an OR-slice match title,
// description or full text.
bool matched_title = true;
bool matched_description = true;
bool matched_fulltext = true;
const auto it = find(result.begin(), result.end(), entry);
if (it != result.end())
{ // Skip if already in result list.
continue;
}
for (const string &term : terms_or)
{
const string title = to_lowercase(entry.title);
const string description = to_lowercase(entry.description);
const string fulltext = to_lowercase(entry.fulltext);
// Set matched_* to false if term is not found.
if (is_re)
{
const RegEx re(term);
if (!(re == title))
{
matched_title = false;
}
if (!(re == description))
{
matched_description = false;
}
if (!(re == fulltext))
{
matched_fulltext = false;
}
}
else
{
if (title.find(term) == string::npos)
{
matched_title = false;
}
if (description.find(term) == string::npos)
{
matched_description = false;
}
if (fulltext.find(term) == string::npos)
{
matched_fulltext = false;
}
}
}
if (matched_title == true
|| matched_description == true
|| matched_fulltext == true)
{
result.push_back(entry);
}
}
}
return result;
}
const list<Database::entry> Search::search_all_threaded(
string expression, const bool is_re) const
{
list<Database::entry> entries = _entries;
const size_t len = entries.size();
constexpr size_t min_len = 100;
constexpr size_t min_per_thread = 50;
const size_t n_threads = thread::hardware_concurrency() / 3 + 1;
size_t cut_at = len;
if (len > min_len)
{ // If there are over `min_len` entries, use `n_threads` threads.
cut_at = len / n_threads;
// But don't use less than `min_per_thread` entries per thread.
if (cut_at < min_per_thread)
{
cut_at = min_per_thread;
}
}
list<list<Database::entry>> segments;
// Use threads if list is big.
while (entries.size() > cut_at)
{
list<Database::entry> segment;
auto it = entries.begin();
std::advance(it, cut_at);
// Move the first `cut_at` entries into `segments`.
segment.splice(segment.begin(), entries, entries.begin(), it);
segments.push_back(move(segment));
}
// Move rest of `entries` into `segments`.
segments.push_back(move(entries));
list<thread> threads;
for (auto &segment : segments)
{
thread t(
[&]
{
Search search(segment);
// Replace `segment` with `result`.
segment = search.search_all(expression, is_re);
});
threads.push_back(move(t));
}
for (thread &t : threads)
{
t.join();
// Move each of `segments` into `entries`.
entries.splice(entries.end(), segments.front());
segments.pop_front();
}
return entries;
}
return searchlist;
}
string Search::to_lowercase(const string &str)
{
return Poco::UTF8::toLower(str);
}
list<Database::entry> Search::search_tags(const string &expression,
const bool is_re) const
{
vector<vector<string>> searchlist = parse_expression(expression);
list<Database::entry> result;
for (const vector<string> &tags_or : searchlist)
{
for (const Database::entry &entry : _entries)
{ // Add entry to result if all tags in an OR-slice match.
bool matched = true;
for (const string &tag : tags_or)
{
const auto it = find_if(
entry.tags.begin(), entry.tags.end(),
[&, is_re](string s)
{
s = to_lowercase(s);
if (is_re)
{
const RegEx re("^" + tag + "$");
return (re == s);
}
return (s == tag);
});
if (it == entry.tags.end())
{
matched = false;
}
}
if (matched)
{
result.push_back(entry);
}
}
}
return result;
}
list<Database::entry> Search::search_all(const string &expression,
const bool is_re) const
{
vector<vector<string>> searchlist = parse_expression(expression);
list<Database::entry> result = search_tags(expression, is_re);
for (const vector<string> &terms_or : searchlist)
{
for (const Database::entry &entry : _entries)
{
// Add entry to result if all terms in an OR-slice match title,
// description or full text.
bool matched_title = true;
bool matched_description = true;
bool matched_fulltext = true;
const auto it = find(result.begin(), result.end(), entry);
if (it != result.end())
{ // Skip if already in result list.
continue;
}
for (const string &term : terms_or)
{
const string title = to_lowercase(entry.title);
const string description = to_lowercase(entry.description);
const string fulltext = to_lowercase(entry.fulltext);
// Set matched_* to false if term is not found.
if (is_re)
{
const RegEx re(term);
if (!(re == title))
{
matched_title = false;
}
if (!(re == description))
{
matched_description = false;
}
if (!(re == fulltext))
{
matched_fulltext = false;
}
}
else
{
if (title.find(term) == string::npos)
{
matched_title = false;
}
if (description.find(term) == string::npos)
{
matched_description = false;
}
if (fulltext.find(term) == string::npos)
{
matched_fulltext = false;
}
}
}
if (matched_title || matched_description || matched_fulltext)
{
result.push_back(entry);
}
}
}
return result;
}
list<Database::entry> Search::search_all_threaded(const string &expression,
const bool is_re) const
{
list<Database::entry> entries = _entries;
const size_t len = entries.size();
constexpr size_t min_len = 100;
constexpr size_t min_per_thread = 50;
const size_t n_threads = thread::hardware_concurrency() / 3 + 1;
size_t cut_at = len;
if (len > min_len)
{ // If there are over `min_len` entries, use `n_threads` threads.
cut_at = len / n_threads;
// But don't use less than `min_per_thread` entries per thread.
if (cut_at < min_per_thread)
{
cut_at = min_per_thread;
}
}
list<list<Database::entry>> segments;
// Use threads if list is big.
while (entries.size() > cut_at)
{
list<Database::entry> segment;
auto it = entries.begin();
std::advance(it, cut_at);
// Move the first `cut_at` entries into `segments`.
segment.splice(segment.begin(), entries, entries.begin(), it);
segments.push_back(move(segment));
}
// Move rest of `entries` into `segments`.
list<Database::entry> rest;
rest.splice(rest.begin(), entries);
segments.push_back(move(rest));
list<thread> threads;
for (auto &segment : segments)
{
thread t(
[&]
{
Search search(segment);
// Replace `segment` with `result`.
segment = search.search_all(expression, is_re);
});
threads.push_back(move(t));
}
for (thread &t : threads)
{
t.join();
// Move each of `segments` into `entries`.
entries.splice(entries.end(), segments.front());
segments.pop_front();
}
return entries;
}
} // namespace remwharead

View File

@ -14,156 +14,187 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sqlite.hpp"
#include "time.hpp"
#include <Poco/Data/SQLite/Connector.h>
#include <Poco/Data/Session.h>
#include <Poco/Version.h>
#include <Poco/Environment.h>
#include <algorithm>
#include <exception>
#include <iostream>
#include <algorithm>
#include <basedir.h>
#include <Poco/Data/Session.h>
#include <Poco/Data/SQLite/Connector.h>
#include "time.hpp"
#include "sqlite.hpp"
namespace remwharead
{
using std::cerr;
using std::endl;
using namespace Poco::Data::Keywords;
using Poco::Data::Statement;
using std::cerr;
using std::endl;
using namespace Poco::Data::Keywords;
using Poco::Data::Statement;
using Poco::Environment;
Database::Database()
: _connected(false)
Database::Database()
: _connected{false}
{
try
{
try
_dbpath = get_data_home();
if (!fs::exists(_dbpath))
{
xdgHandle xdg;
xdgInitHandle(&xdg);
_dbpath = xdgDataHome(&xdg) / fs::path("remwharead");
xdgWipeHandle(&xdg);
if (!fs::exists(_dbpath))
{
fs::create_directories(_dbpath);
}
_dbpath /= "database.sqlite";
Poco::Data::SQLite::Connector::registerConnector();
_session = std::make_unique<Session>("SQLite", _dbpath);
*_session << "CREATE TABLE IF NOT EXISTS remwharead("
"uri TEXT, archive_uri TEXT, datetime TEXT, "
"tags TEXT, title TEXT, description TEXT, fulltext TEXT);", now;
_connected = true;
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
fs::create_directories(_dbpath);
}
_dbpath /= "database.sqlite";
Poco::Data::SQLite::Connector::registerConnector();
_session = std::make_unique<Session>("SQLite", _dbpath);
*_session << "CREATE TABLE IF NOT EXISTS remwharead("
"uri TEXT, archive_uri TEXT, datetime TEXT, "
"tags TEXT, title TEXT, description TEXT, fulltext TEXT);", now;
_connected = true;
}
Database::operator bool() const
catch (std::exception &e)
{
return _connected;
}
bool operator ==(const Database::entry &a, const Database::entry &b)
{
return (a.datetime == b.datetime);
}
const string Database::entry::fulltext_oneline() const
{
string oneline = fulltext;
size_t pos = 0;
while ((pos = oneline.find('\n', pos)) != string::npos)
{
oneline.replace(pos, 1, "\\n");
}
return oneline;
}
void Database::store(const Database::entry &data) const
{
try
{
const string strdatetime = timepoint_to_string(data.datetime, true);
string strtags;
Statement insert(*_session);
for (const string &tag : data.tags)
{
strtags += tag;
if (tag != *(data.tags.rbegin()))
{
strtags += ",";
}
}
// useRef() uses the const reference.
insert << "INSERT INTO remwharead "
"VALUES(?, ?, ?, ?, ?, ?, ?);",
useRef(data.uri), useRef(data.archive_uri),
useRef(strdatetime), useRef(strtags), useRef(data.title),
useRef(data.description), useRef(data.fulltext);
insert.execute();
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
}
}
const list<Database::entry> Database::retrieve(
const time_point &start, const time_point &end) const
{
try
{
Database::entry entrybuf;
string datetime, strtags;
Statement select(*_session);
// bind() copies the value.
select << "SELECT * FROM remwharead WHERE datetime "
"BETWEEN ? AND ? ORDER BY datetime DESC;",
bind(timepoint_to_string(start, true)),
bind(timepoint_to_string(end, true)),
into(entrybuf.uri), into(entrybuf.archive_uri), into(datetime),
into(strtags), into(entrybuf.title), into(entrybuf.description),
into(entrybuf.fulltext), range(0, 1);
list<entry> entries;
while(!select.done() && select.execute() != 0)
{
entrybuf.datetime = string_to_timepoint(datetime, true);
vector<string> tags;
size_t pos = 0;
while (pos != string::npos)
{
const size_t newpos = strtags.find(',', pos);
const string tag = strtags.substr(pos, newpos - pos);
if (!tag.empty())
{
tags.push_back(tag);
}
pos = newpos;
if (pos != string::npos)
{
++pos;
}
}
entrybuf.tags = tags;
entries.push_back(entrybuf);
}
return entries;
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
}
return {};
cerr << "Error in " << __func__ << ": " << e.what() << endl;
}
}
Database::operator bool() const
{
return _connected;
}
bool operator ==(const Database::entry &a, const Database::entry &b)
{
return (a.datetime == b.datetime);
}
string Database::entry::fulltext_oneline() const
{
string oneline = fulltext;
size_t pos = 0;
while ((pos = oneline.find('\n', pos)) != string::npos)
{
oneline.replace(pos, 1, "\\n");
}
return oneline;
}
void Database::store(const Database::entry &data) const
{
try
{
const string strdatetime = timepoint_to_string(data.datetime, true);
string strtags = tags_to_string(data.tags);
Statement insert(*_session);
// useRef() uses the const reference.
insert << "INSERT INTO remwharead "
"VALUES(?, ?, ?, ?, ?, ?, ?);",
useRef(data.uri), useRef(data.archive_uri),
useRef(strdatetime), useRef(strtags), useRef(data.title),
useRef(data.description), useRef(data.fulltext);
insert.execute();
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
}
}
list<Database::entry> Database::retrieve(const time_point &start,
const time_point &end) const
{
try
{
Database::entry entrybuf;
string datetime;
string strtags;
Statement select(*_session);
// bind() copies the value.
select << "SELECT * FROM remwharead WHERE datetime "
"BETWEEN ? AND ? ORDER BY datetime DESC;",
bind(timepoint_to_string(start, true)),
bind(timepoint_to_string(end, true)),
into(entrybuf.uri), into(entrybuf.archive_uri), into(datetime),
into(strtags), into(entrybuf.title), into(entrybuf.description),
into(entrybuf.fulltext), range(0, 1);
list<entry> entries;
while(!select.done() && select.execute() != 0)
{
entrybuf.datetime = string_to_timepoint(datetime, true);
vector<string> tags;
size_t pos = 0;
while (pos != string::npos)
{
const size_t newpos = strtags.find(',', pos);
const string tag = strtags.substr(pos, newpos - pos);
if (!tag.empty())
{
tags.push_back(tag);
}
pos = newpos;
if (pos != string::npos)
{
++pos;
}
}
entrybuf.tags = tags;
entries.push_back(entrybuf);
}
return entries;
}
catch (std::exception &e)
{
cerr << "Error in " << __func__ << ": " << e.what() << endl;
}
return {};
}
size_t Database::remove(const string &uri)
{
Statement del(*_session);
del << "DELETE FROM remwharead WHERE uri = ?;", bind(uri);
return del.execute();
}
string Database::tags_to_string(const vector<string> &tags)
{
string strtags;
for (const string &tag : tags)
{
strtags += tag;
if (tag != *(tags.rbegin()))
{
strtags += ',';
}
}
return strtags;
}
fs::path Database::get_data_home()
{
fs::path path;
if (Environment::has("XDG_DATA_HOME"))
{
path = Environment::get("XDG_DATA_HOME") / fs::path("remwharead");
}
else if (Environment::has("HOME"))
{
path = Environment::get("HOME") / fs::path(".local/share/remwharead");
} // Else return empty path.
return path;
}
} // namespace remwharead

View File

@ -14,48 +14,53 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "time.hpp"
#include "sqlite.hpp"
#include <array>
#include <cstdint>
#include <ctime>
#include <iomanip>
#include <sstream>
#include <cstdint>
#include "time.hpp"
namespace remwharead
{
const time_point string_to_timepoint(const string &strtime, bool sqlite)
using std::array;
time_point string_to_timepoint(const string &strtime, bool sqlite)
{
std::stringstream sstime(strtime);
struct std::tm tm = {};
tm.tm_isdst = -1; // Don't convert to/from daylight saving time.
if (sqlite)
{
std::stringstream sstime(strtime);
struct std::tm tm = {};
tm.tm_isdst = -1; // Detect daylight saving time.
if (sqlite)
{
sstime >> std::get_time(&tm, "%Y-%m-%d %T");
}
else
{
sstime >> std::get_time(&tm, "%Y-%m-%dT%T");
}
std::time_t time = timelocal(&tm); // Assume time is local.
return system_clock::from_time_t(time);
sstime >> std::get_time(&tm, "%Y-%m-%d %T");
}
const string timepoint_to_string(const time_point &tp, bool sqlite)
else
{
constexpr std::uint16_t bufsize = 32;
std::time_t time = system_clock::to_time_t(tp);
std::tm *tm;
tm = std::localtime(&time);
char buffer[bufsize];
if (sqlite)
{
std::strftime(buffer, bufsize, "%F %T", tm);
}
else
{
std::strftime(buffer, bufsize, "%FT%T", tm);
}
return static_cast<const string>(buffer);
sstime >> std::get_time(&tm, "%Y-%m-%dT%T");
}
std::time_t time = mktime(&tm);
return system_clock::from_time_t(time);
}
string timepoint_to_string(const time_point &tp, bool sqlite)
{
constexpr std::uint16_t bufsize = 32;
std::time_t time = system_clock::to_time_t(tp);
std::tm *tm;
tm = std::localtime(&time);
array<char, bufsize> buffer = {};
if (sqlite)
{
std::strftime(buffer.begin(), bufsize, "%F %T", tm);
}
else
{
std::strftime(buffer.begin(), bufsize, "%FT%T", tm);
}
return buffer.begin();
}
} // namespace remwharead

View File

@ -1,5 +1,5 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
* Copyright © 2019, 2020 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -14,635 +14,406 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <sstream>
#include <cstdint>
#include <iostream>
#include <locale>
#include <codecvt>
#include <exception>
#include <vector>
#include <Poco/Net/HTTPClientSession.h>
#include <Poco/Net/HTTPSClientSession.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Net/HTTPResponse.h>
#include <Poco/StreamCopier.h>
#include <Poco/URI.h>
#include <Poco/Environment.h>
#include <Poco/Exception.h>
#include <Poco/RegularExpression.h>
#include "version.hpp"
#include "uri.hpp"
#include "curl_wrapper.hpp"
#include "version.hpp"
#include <Poco/RegularExpression.h>
#include <boost/locale.hpp>
#include <atomic>
#include <codecvt>
#include <cstdint>
#include <exception>
#include <iostream>
#include <iterator>
#include <locale>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
namespace remwharead
{
using std::array;
using std::istream;
using std::unique_ptr;
using std::make_unique;
using std::vector;
using std::cerr;
using std::endl;
using Poco::Net::HTTPClientSession;
using Poco::Net::HTTPSClientSession;
using Poco::Net::HTTPRequest;
using Poco::Net::HTTPResponse;
using Poco::Net::HTTPMessage;
using Poco::StreamCopier;
using Poco::Environment;
using RegEx = Poco::RegularExpression;
using std::array;
using std::exception;
using std::move;
using std::to_string;
using std::uint32_t;
using std::vector;
using RegEx = Poco::RegularExpression;
html_extract::operator bool()
html_extract::operator bool() const
{
return successful;
}
archive_answer::operator bool() const
{
return successful;
}
URI::URI(string uri)
: _uri(move(uri))
{
// FIXME: Only call locale-stuff once after getting rid of POCO.
// Set global locale with Boost extras. Needed for Boost functions.
// Uhm… I don't remember what I meant with the above. 🤦
const boost::locale::generator locgen;
const std::locale loc = locgen("");
std::locale::global(loc);
}
html_extract URI::get()
{
using namespace curl_wrapper;
try
{
return successful;
CURLWrapper curl;
_document = to_utf8(
curl.make_http_request(http_method::GET, _uri).body);
if (!_document.empty())
{
return {true, "", extract_title(), extract_description(),
strip_html()};
}
}
catch (const exception &e)
{
return {false, e.what(), "", "", ""};
}
archive_answer::operator bool()
return {false, "Unknown error.", "", "", ""};
}
string URI::extract_title() const
{
if (is_html())
{
return successful;
}
URI::URI(const string &uri)
:_uri(uri)
{
Poco::Net::initializeSSL();
try
const RegEx re_title("<title(?: [^>]+)?>([^<]+)", RegEx::RE_CASELESS);
vector<string> matches;
re_title.split(_document, matches);
if (matches.size() >= 2)
{
HTTPClientSession::ProxyConfig proxy;
const string env_proxy = Environment::get("http_proxy");
const RegEx re_proxy("^(?:https?://)?(?:([^:]+):?([^@]*)@)?"
"([^:/]+)(?::([\\d]{1,5}))?/?$");
vector<string> matches;
if (re_proxy.split(env_proxy, matches) >= 4)
{
proxy.username = matches[1];
proxy.password = matches[2];
proxy.host = matches[3];
if (!matches[4].empty())
{
const std::uint32_t &port = std::stoul(matches[4]);
if (port > 65535)
{
throw std::invalid_argument("Port number out of range");
}
proxy.port = port;
}
}
HTTPClientSession::setGlobalProxyConfig(proxy);
}
catch (const Poco::RegularExpressionException &e)
{
cerr << "Error: Proxy could not be set ("
<< e.displayText() << ")\n";
}
catch (const std::invalid_argument &e)
{
cerr << "Error: " << e.what() << endl;
}
catch (const std::exception &)
{
// No proxy found, no problem.
return remove_newlines(unescape_html(matches[1]));
}
}
URI::~URI()
return "";
}
string URI::extract_description() const
{
if (is_html())
{
Poco::Net::uninitializeSSL();
}
const html_extract URI::get()
{
try
const RegEx re_desc(R"(description"[^>]+content="([^"]+))",
RegEx::RE_CASELESS);
vector<string> matches;
re_desc.split(_document, matches);
if (matches.size() >= 2)
{
const string answer = make_request(_uri);
if (!answer.empty())
{
return
{
true,
"",
extract_title(answer),
extract_description(answer),
strip_html(answer)
};
}
}
catch (const Poco::Exception &e)
{
return { false, e.displayText(), "", "", "" };
}
return { false, "Unknown error.", "", "", "" };
}
const string URI::make_request(const string &uri, bool archive) const
{
Poco::URI poco_uri(uri);
string method =
archive ? HTTPRequest::HTTP_HEAD : HTTPRequest::HTTP_GET;
string path = poco_uri.getPathAndQuery();
if (path.empty())
{
path = "/";
}
unique_ptr<HTTPClientSession> session;
if (poco_uri.getScheme() == "https")
{
session = make_unique<HTTPSClientSession>(poco_uri.getHost(),
poco_uri.getPort());
}
else if (poco_uri.getScheme() == "http")
{
session = make_unique<HTTPClientSession>(poco_uri.getHost(),
poco_uri.getPort());
}
else
{
throw Poco::Exception("Protocol not supported.");
}
HTTPRequest request(method, path, HTTPMessage::HTTP_1_1);
request.set("User-Agent", string("remwharead/") + global::version);
HTTPResponse response;
session->sendRequest(request);
istream &rs = session->receiveResponse(response);
// Not using the constants because some are too new for Debian stretch.
switch (response.getStatus())
{
case 301: // HTTPResponse::HTTP_MOVED_PERMANENTLY
case 308: // HTTPResponse::HTTP_PERMANENT_REDIRECT
case 302: // HTTPResponse::HTTP_FOUND
case 303: // HTTPResponse::HTTP_SEE_OTHER
case 307: // HTTPResponse::HTTP_TEMPORARY_REDIRECT
{
string location = response.get("Location");
if (location.substr(0, 4) != "http")
{
location = poco_uri.getScheme() + "://" + poco_uri.getHost()
+ location;
}
return make_request(location);
}
case HTTPResponse::HTTP_OK:
{
string answer;
if (archive)
{
answer = response.get("Content-Location");
}
else
{
StreamCopier::copyToString(rs, answer);
}
return answer;
}
default:
{
throw Poco::Exception(response.getReason());
return "";
}
return cut_text(remove_newlines(unescape_html(matches[1])), 500);
}
}
const string URI::extract_title(const string &html)
{
const RegEx re_htmlfile(".*\\.(.?html?|xml|rss)$", RegEx::RE_CASELESS);
if (_uri.substr(0, 4) == "http" || re_htmlfile.match(_uri))
{
const RegEx re_title("<title>([^<]+)", RegEx::RE_CASELESS);
vector<string> matches;
re_title.split(html, matches);
if (matches.size() >= 2)
{
return remove_newlines(unescape_html(matches[1]));
}
}
return "";
}
return "";
string URI::strip_html() const
{
string out;
out = remove_html_tags(_document, "script"); // Remove JavaScript.
out = remove_html_tags(out, "style"); // Remove CSS.
out = remove_html_tags(out); // Remove tags.
size_t pos = 0;
while ((pos = out.find('\r', pos)) != std::string::npos) // Remove CR.
{
out.replace(pos, 1, "");
}
const string URI::extract_description(const string &html)
// Remove whitespace at eol.
RegEx("\\s+\n").subst(out, "\n", RegEx::RE_GLOBAL);
RegEx("\n{2,}").subst(out, "\n", RegEx::RE_GLOBAL); // Reduce newlines.
return unescape_html(out);
}
string URI::remove_html_tags(const string &html, const string &tag)
{
// NOTE: I did this with regex_replace before, but libstdc++ segfaulted.
string out;
if (tag.empty())
{
const RegEx re_htmlfile(".*\\.(.?html?|xml|rss)$", RegEx::RE_CASELESS);
if (_uri.substr(0, 4) == "http" || re_htmlfile.match(_uri))
{
const RegEx re_desc("description\"[^>]+content=\"([^\"]+)",
RegEx::RE_CASELESS);
vector<string> matches;
re_desc.split(html, matches);
if (matches.size() >= 2)
{
return remove_newlines(unescape_html(matches[1]));
}
}
return "";
}
const string URI::strip_html(const string &html)
{
string out;
out = remove_html_tags(html, "script"); // Remove JavaScript.
out = remove_html_tags(out, "style"); // Remove CSS.
out = remove_html_tags(out); // Remove tags.
size_t pos = 0;
while ((pos = out.find("\r", pos)) != std::string::npos) // Remove CR.
while (pos != std::string::npos)
{
out.replace(pos, 1, "");
size_t startpos = html.find('<', pos);
size_t endpos = html.find('>', startpos);
out += html.substr(pos, startpos - pos);
pos = endpos;
if (pos != std::string::npos)
{
++pos;
}
}
}
else
{
size_t pos = 0;
out = html;
while ((pos = out.find("<" + tag)) != std::string::npos)
{
size_t endpos = out.find("</" + tag, pos);
if (endpos == std::string::npos)
{
break;
}
endpos += 3 + tag.length(); // tag + </ + >
out.replace(pos, endpos - pos, "");
}
// Remove whitespace at eol.
RegEx("\\s+\n").subst(out, "\n", RegEx::RE_GLOBAL);
RegEx("\n{2,}").subst(out, "\n", RegEx::RE_GLOBAL); // Reduce newlines.
return unescape_html(out);
}
const string URI::remove_html_tags(const string &html, const string &tag)
return out;
}
string URI::unescape_html(string html)
{
// Used to convert int to utf-8 char.
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> u8c;
const RegEx re_entity("&#(x)?([[:alnum:]]{1,8});");
RegEx::MatchVec matches;
string::size_type pos = 0;
while (re_entity.match(html, pos, matches) != 0)
{
// NOTE: I did this with regex_replace before, but libstdc++ segfaulted.
string out;
if (tag.empty())
char32_t codepoint = 0;
const string number = html.substr(matches[2].offset, matches[2].length);
// 'x' in front of the number means it's hexadecimal, else decimal.
if (matches[1].length != 0)
{
size_t pos = 0;
while (pos != std::string::npos)
{
size_t startpos = html.find('<', pos);
size_t endpos = html.find('>', startpos);
out += html.substr(pos, startpos - pos);
pos = endpos;
if (pos != std::string::npos)
{
++pos;
}
}
codepoint = static_cast<char32_t>(std::stoul(number, nullptr, 16));
}
else
{
size_t pos = 0;
out = html;
while ((pos = out.find("<" + tag)) != std::string::npos)
{
size_t endpos = out.find("</" + tag, pos);
if (endpos == std::string::npos)
{
break;
}
endpos += 3 + tag.length(); // tag + </ + >
out.replace(pos, endpos - pos, "");
}
codepoint = static_cast<char32_t>(std::stoi(number, nullptr, 10));
}
return out;
const string unicode = u8c.to_bytes(codepoint);
html.replace(matches[0].offset, matches[0].length, unicode);
pos = matches[0].offset + unicode.length();
}
const string URI::unescape_html(string html)
// Source: https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_
// entity_references#Character_entity_references_in_HTML
const array<const std::pair<const string, const char32_t>, 258> names = {
{{"exclamation", 0x0021}, {"quot", 0x0022}, {"percent", 0x0025},
{"amp", 0x0026}, {"apos", 0x0027}, {"add", 0x002B},
{"lt", 0x003C}, {"equal", 0x003D}, {"gt", 0x003E},
{"nbsp", 0x00A0}, {"iexcl", 0x00A1}, {"cent", 0x00A2},
{"pound", 0x00A3}, {"curren", 0x00A4}, {"yen", 0x00A5},
{"brvbar", 0x00A6}, {"sect", 0x00A7}, {"uml", 0x00A8},
{"copy", 0x00A9}, {"ordf", 0x00AA}, {"laquo", 0x00AB},
{"not", 0x00AC}, {"shy", 0x00AD}, {"reg", 0x00AE},
{"macr", 0x00AF}, {"deg", 0x00B0}, {"plusmn", 0x00B1},
{"sup2", 0x00B2}, {"sup3", 0x00B3}, {"acute", 0x00B4},
{"micro", 0x00B5}, {"para", 0x00B6}, {"middot", 0x00B7},
{"cedil", 0x00B8}, {"sup1", 0x00B9}, {"ordm", 0x00BA},
{"raquo", 0x00BB}, {"frac14", 0x00BC}, {"frac12", 0x00BD},
{"frac34", 0x00BE}, {"iquest", 0x00BF}, {"Agrave", 0x00C0},
{"Aacute", 0x00C1}, {"Acirc", 0x00C2}, {"Atilde", 0x00C3},
{"Auml", 0x00C4}, {"Aring", 0x00C5}, {"AElig", 0x00C6},
{"Ccedil", 0x00C7}, {"Egrave", 0x00C8}, {"Eacute", 0x00C9},
{"Ecirc", 0x00CA}, {"Euml", 0x00CB}, {"Igrave", 0x00CC},
{"Iacute", 0x00CD}, {"Icirc", 0x00CE}, {"Iuml", 0x00CF},
{"ETH", 0x00D0}, {"Ntilde", 0x00D1}, {"Ograve", 0x00D2},
{"Oacute", 0x00D3}, {"Ocirc", 0x00D4}, {"Otilde", 0x00D5},
{"Ouml", 0x00D6}, {"times", 0x00D7}, {"Oslash", 0x00D8},
{"Ugrave", 0x00D9}, {"Uacute", 0x00DA}, {"Ucirc", 0x00DB},
{"Uuml", 0x00DC}, {"Yacute", 0x00DD}, {"THORN", 0x00DE},
{"szlig", 0x00DF}, {"agrave", 0x00E0}, {"aacute", 0x00E1},
{"acirc", 0x00E2}, {"atilde", 0x00E3}, {"auml", 0x00E4},
{"aring", 0x00E5}, {"aelig", 0x00E6}, {"ccedil", 0x00E7},
{"egrave", 0x00E8}, {"eacute", 0x00E9}, {"ecirc", 0x00EA},
{"euml", 0x00EB}, {"igrave", 0x00EC}, {"iacute", 0x00ED},
{"icirc", 0x00EE}, {"iuml", 0x00EF}, {"eth", 0x00F0},
{"ntilde", 0x00F1}, {"ograve", 0x00F2}, {"oacute", 0x00F3},
{"ocirc", 0x00F4}, {"otilde", 0x00F5}, {"ouml", 0x00F6},
{"divide", 0x00F7}, {"oslash", 0x00F8}, {"ugrave", 0x00F9},
{"uacute", 0x00FA}, {"ucirc", 0x00FB}, {"uuml", 0x00FC},
{"yacute", 0x00FD}, {"thorn", 0x00FE}, {"yuml", 0x00FF},
{"OElig", 0x0152}, {"oelig", 0x0153}, {"Scaron", 0x0160},
{"scaron", 0x0161}, {"Yuml", 0x0178}, {"fnof", 0x0192},
{"circ", 0x02C6}, {"tilde", 0x02DC}, {"Alpha", 0x0391},
{"Beta", 0x0392}, {"Gamma", 0x0393}, {"Delta", 0x0394},
{"Epsilon", 0x0395}, {"Zeta", 0x0396}, {"Eta", 0x0397},
{"Theta", 0x0398}, {"Iota", 0x0399}, {"Kappa", 0x039A},
{"Lambda", 0x039B}, {"Mu", 0x039C}, {"Nu", 0x039D},
{"Xi", 0x039E}, {"Omicron", 0x039F}, {"Pi", 0x03A0},
{"Rho", 0x03A1}, {"Sigma", 0x03A3}, {"Tau", 0x03A4},
{"Upsilon", 0x03A5}, {"Phi", 0x03A6}, {"Chi", 0x03A7},
{"Psi", 0x03A8}, {"Omega", 0x03A9}, {"alpha", 0x03B1},
{"beta", 0x03B2}, {"gamma", 0x03B3}, {"delta", 0x03B4},
{"epsilon", 0x03B5}, {"zeta", 0x03B6}, {"eta", 0x03B7},
{"theta", 0x03B8}, {"iota", 0x03B9}, {"kappa", 0x03BA},
{"lambda", 0x03BB}, {"mu", 0x03BC}, {"nu", 0x03BD},
{"xi", 0x03BE}, {"omicron", 0x03BF}, {"pi", 0x03C0},
{"rho", 0x03C1}, {"sigmaf", 0x03C2}, {"sigma", 0x03C3},
{"tau", 0x03C4}, {"upsilon", 0x03C5}, {"phi", 0x03C6},
{"chi", 0x03C7}, {"psi", 0x03C8}, {"omega", 0x03C9},
{"thetasym", 0x03D1}, {"upsih", 0x03D2}, {"piv", 0x03D6},
{"ensp", 0x2002}, {"emsp", 0x2003}, {"thinsp", 0x2009},
{"zwnj", 0x200C}, {"zwj", 0x200D}, {"lrm", 0x200E},
{"rlm", 0x200F}, {"ndash", 0x2013}, {"mdash", 0x2014},
{"horbar", 0x2015}, {"lsquo", 0x2018}, {"rsquo", 0x2019},
{"sbquo", 0x201A}, {"ldquo", 0x201C}, {"rdquo", 0x201D},
{"bdquo", 0x201E}, {"dagger", 0x2020}, {"Dagger", 0x2021},
{"bull", 0x2022}, {"hellip", 0x2026}, {"permil", 0x2030},
{"prime", 0x2032}, {"Prime", 0x2033}, {"lsaquo", 0x2039},
{"rsaquo", 0x203A}, {"oline", 0x203E}, {"frasl", 0x2044},
{"euro", 0x20AC}, {"image", 0x2111}, {"weierp", 0x2118},
{"real", 0x211C}, {"trade", 0x2122}, {"alefsym", 0x2135},
{"larr", 0x2190}, {"uarr", 0x2191}, {"rarr", 0x2192},
{"darr", 0x2193}, {"harr", 0x2194}, {"crarr", 0x21B5},
{"lArr", 0x21D0}, {"uArr", 0x21D1}, {"rArr", 0x21D2},
{"dArr", 0x21D3}, {"hArr", 0x21D4}, {"forall", 0x2200},
{"part", 0x2202}, {"exist", 0x2203}, {"empty", 0x2205},
{"nabla", 0x2207}, {"isin", 0x2208}, {"notin", 0x2209},
{"ni", 0x220B}, {"prod", 0x220F}, {"sum", 0x2211},
{"minus", 0x2212}, {"lowast", 0x2217}, {"radic", 0x221A},
{"prop", 0x221D}, {"infin", 0x221E}, {"ang", 0x2220},
{"and", 0x2227}, {"or", 0x2228}, {"cap", 0x2229},
{"cup", 0x222A}, {"int", 0x222B}, {"there4", 0x2234},
{"sim", 0x223C}, {"cong", 0x2245}, {"asymp", 0x2248},
{"ne", 0x2260}, {"equiv", 0x2261}, {"le", 0x2264},
{"ge", 0x2265}, {"sub", 0x2282}, {"sup", 0x2283},
{"nsub", 0x2284}, {"sube", 0x2286}, {"supe", 0x2287},
{"oplus", 0x2295}, {"otimes", 0x2297}, {"perp", 0x22A5},
{"sdot", 0x22C5}, {"lceil", 0x2308}, {"rceil", 0x2309},
{"lfloor", 0x230A}, {"rfloor", 0x230B}, {"lang", 0x2329},
{"rang", 0x232A}, {"loz", 0x25CA}, {"spades", 0x2660},
{"clubs", 0x2663}, {"hearts", 0x2665}, {"diams", 0x2666}}};
for (const auto &pair : names)
{
// Used to convert int to utf-8 char.
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> u8c;
const RegEx re_entity("&#(x)?([[:alnum:]]{1,8});");
RegEx::MatchVec matches;
string::size_type pos = 0;
while (re_entity.match(html, pos, matches) != 0)
{
char32_t codepoint = 0;
const string number = html.substr(matches[2].offset,
matches[2].length);
// 'x' in front of the number means it's hexadecimal, else decimal.
if (matches[1].length != 0)
{
codepoint = std::stoi(number, nullptr, 16);
}
else
{
codepoint = std::stoi(number, nullptr, 10);
}
const string unicode = u8c.to_bytes(codepoint);
html.replace(matches[0].offset, matches[0].length, unicode);
pos = matches[0].offset + unicode.length();
}
// Source: https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_
// entity_references#Character_entity_references_in_HTML
const array<const std::pair<const string, const char32_t>, 258> names =
{{
{ "exclamation", 0x0021 },
{ "quot", 0x0022 },
{ "percent", 0x0025 },
{ "amp", 0x0026 },
{ "apos", 0x0027 },
{ "add", 0x002B },
{ "lt", 0x003C },
{ "equal", 0x003D },
{ "gt", 0x003E },
{ "nbsp", 0x00A0 },
{ "iexcl", 0x00A1 },
{ "cent", 0x00A2 },
{ "pound", 0x00A3 },
{ "curren", 0x00A4 },
{ "yen", 0x00A5 },
{ "brvbar", 0x00A6 },
{ "sect", 0x00A7 },
{ "uml", 0x00A8 },
{ "copy", 0x00A9 },
{ "ordf", 0x00AA },
{ "laquo", 0x00AB },
{ "not", 0x00AC },
{ "shy", 0x00AD },
{ "reg", 0x00AE },
{ "macr", 0x00AF },
{ "deg", 0x00B0 },
{ "plusmn", 0x00B1 },
{ "sup2", 0x00B2 },
{ "sup3", 0x00B3 },
{ "acute", 0x00B4 },
{ "micro", 0x00B5 },
{ "para", 0x00B6 },
{ "middot", 0x00B7 },
{ "cedil", 0x00B8 },
{ "sup1", 0x00B9 },
{ "ordm", 0x00BA },
{ "raquo", 0x00BB },
{ "frac14", 0x00BC },
{ "frac12", 0x00BD },
{ "frac34", 0x00BE },
{ "iquest", 0x00BF },
{ "Agrave", 0x00C0 },
{ "Aacute", 0x00C1 },
{ "Acirc", 0x00C2 },
{ "Atilde", 0x00C3 },
{ "Auml", 0x00C4 },
{ "Aring", 0x00C5 },
{ "AElig", 0x00C6 },
{ "Ccedil", 0x00C7 },
{ "Egrave", 0x00C8 },
{ "Eacute", 0x00C9 },
{ "Ecirc", 0x00CA },
{ "Euml", 0x00CB },
{ "Igrave", 0x00CC },
{ "Iacute", 0x00CD },
{ "Icirc", 0x00CE },
{ "Iuml", 0x00CF },
{ "ETH", 0x00D0 },
{ "Ntilde", 0x00D1 },
{ "Ograve", 0x00D2 },
{ "Oacute", 0x00D3 },
{ "Ocirc", 0x00D4 },
{ "Otilde", 0x00D5 },
{ "Ouml", 0x00D6 },
{ "times", 0x00D7 },
{ "Oslash", 0x00D8 },
{ "Ugrave", 0x00D9 },
{ "Uacute", 0x00DA },
{ "Ucirc", 0x00DB },
{ "Uuml", 0x00DC },
{ "Yacute", 0x00DD },
{ "THORN", 0x00DE },
{ "szlig", 0x00DF },
{ "agrave", 0x00E0 },
{ "aacute", 0x00E1 },
{ "acirc", 0x00E2 },
{ "atilde", 0x00E3 },
{ "auml", 0x00E4 },
{ "aring", 0x00E5 },
{ "aelig", 0x00E6 },
{ "ccedil", 0x00E7 },
{ "egrave", 0x00E8 },
{ "eacute", 0x00E9 },
{ "ecirc", 0x00EA },
{ "euml", 0x00EB },
{ "igrave", 0x00EC },
{ "iacute", 0x00ED },
{ "icirc", 0x00EE },
{ "iuml", 0x00EF },
{ "eth", 0x00F0 },
{ "ntilde", 0x00F1 },
{ "ograve", 0x00F2 },
{ "oacute", 0x00F3 },
{ "ocirc", 0x00F4 },
{ "otilde", 0x00F5 },
{ "ouml", 0x00F6 },
{ "divide", 0x00F7 },
{ "oslash", 0x00F8 },
{ "ugrave", 0x00F9 },
{ "uacute", 0x00FA },
{ "ucirc", 0x00FB },
{ "uuml", 0x00FC },
{ "yacute", 0x00FD },
{ "thorn", 0x00FE },
{ "yuml", 0x00FF },
{ "OElig", 0x0152 },
{ "oelig", 0x0153 },
{ "Scaron", 0x0160 },
{ "scaron", 0x0161 },
{ "Yuml", 0x0178 },
{ "fnof", 0x0192 },
{ "circ", 0x02C6 },
{ "tilde", 0x02DC },
{ "Alpha", 0x0391 },
{ "Beta", 0x0392 },
{ "Gamma", 0x0393 },
{ "Delta", 0x0394 },
{ "Epsilon", 0x0395 },
{ "Zeta", 0x0396 },
{ "Eta", 0x0397 },
{ "Theta", 0x0398 },
{ "Iota", 0x0399 },
{ "Kappa", 0x039A },
{ "Lambda", 0x039B },
{ "Mu", 0x039C },
{ "Nu", 0x039D },
{ "Xi", 0x039E },
{ "Omicron", 0x039F },
{ "Pi", 0x03A0 },
{ "Rho", 0x03A1 },
{ "Sigma", 0x03A3 },
{ "Tau", 0x03A4 },
{ "Upsilon", 0x03A5 },
{ "Phi", 0x03A6 },
{ "Chi", 0x03A7 },
{ "Psi", 0x03A8 },
{ "Omega", 0x03A9 },
{ "alpha", 0x03B1 },
{ "beta", 0x03B2 },
{ "gamma", 0x03B3 },
{ "delta", 0x03B4 },
{ "epsilon", 0x03B5 },
{ "zeta", 0x03B6 },
{ "eta", 0x03B7 },
{ "theta", 0x03B8 },
{ "iota", 0x03B9 },
{ "kappa", 0x03BA },
{ "lambda", 0x03BB },
{ "mu", 0x03BC },
{ "nu", 0x03BD },
{ "xi", 0x03BE },
{ "omicron", 0x03BF },
{ "pi", 0x03C0 },
{ "rho", 0x03C1 },
{ "sigmaf", 0x03C2 },
{ "sigma", 0x03C3 },
{ "tau", 0x03C4 },
{ "upsilon", 0x03C5 },
{ "phi", 0x03C6 },
{ "chi", 0x03C7 },
{ "psi", 0x03C8 },
{ "omega", 0x03C9 },
{ "thetasym", 0x03D1 },
{ "upsih", 0x03D2 },
{ "piv", 0x03D6 },
{ "ensp", 0x2002 },
{ "emsp", 0x2003 },
{ "thinsp", 0x2009 },
{ "zwnj", 0x200C },
{ "zwj", 0x200D },
{ "lrm", 0x200E },
{ "rlm", 0x200F },
{ "ndash", 0x2013 },
{ "mdash", 0x2014 },
{ "horbar", 0x2015 },
{ "lsquo", 0x2018 },
{ "rsquo", 0x2019 },
{ "sbquo", 0x201A },
{ "ldquo", 0x201C },
{ "rdquo", 0x201D },
{ "bdquo", 0x201E },
{ "dagger", 0x2020 },
{ "Dagger", 0x2021 },
{ "bull", 0x2022 },
{ "hellip", 0x2026 },
{ "permil", 0x2030 },
{ "prime", 0x2032 },
{ "Prime", 0x2033 },
{ "lsaquo", 0x2039 },
{ "rsaquo", 0x203A },
{ "oline", 0x203E },
{ "frasl", 0x2044 },
{ "euro", 0x20AC },
{ "image", 0x2111 },
{ "weierp", 0x2118 },
{ "real", 0x211C },
{ "trade", 0x2122 },
{ "alefsym", 0x2135 },
{ "larr", 0x2190 },
{ "uarr", 0x2191 },
{ "rarr", 0x2192 },
{ "darr", 0x2193 },
{ "harr", 0x2194 },
{ "crarr", 0x21B5 },
{ "lArr", 0x21D0 },
{ "uArr", 0x21D1 },
{ "rArr", 0x21D2 },
{ "dArr", 0x21D3 },
{ "hArr", 0x21D4 },
{ "forall", 0x2200 },
{ "part", 0x2202 },
{ "exist", 0x2203 },
{ "empty", 0x2205 },
{ "nabla", 0x2207 },
{ "isin", 0x2208 },
{ "notin", 0x2209 },
{ "ni", 0x220B },
{ "prod", 0x220F },
{ "sum", 0x2211 },
{ "minus", 0x2212 },
{ "lowast", 0x2217 },
{ "radic", 0x221A },
{ "prop", 0x221D },
{ "infin", 0x221E },
{ "ang", 0x2220 },
{ "and", 0x2227 },
{ "or", 0x2228 },
{ "cap", 0x2229 },
{ "cup", 0x222A },
{ "int", 0x222B },
{ "there4", 0x2234 },
{ "sim", 0x223C },
{ "cong", 0x2245 },
{ "asymp", 0x2248 },
{ "ne", 0x2260 },
{ "equiv", 0x2261 },
{ "le", 0x2264 },
{ "ge", 0x2265 },
{ "sub", 0x2282 },
{ "sup", 0x2283 },
{ "nsub", 0x2284 },
{ "sube", 0x2286 },
{ "supe", 0x2287 },
{ "oplus", 0x2295 },
{ "otimes", 0x2297 },
{ "perp", 0x22A5 },
{ "sdot", 0x22C5 },
{ "lceil", 0x2308 },
{ "rceil", 0x2309 },
{ "lfloor", 0x230A },
{ "rfloor", 0x230B },
{ "lang", 0x2329 },
{ "rang", 0x232A },
{ "loz", 0x25CA },
{ "spades", 0x2660 },
{ "clubs", 0x2663 },
{ "hearts", 0x2665 },
{ "diams", 0x2666 }
}};
for (auto &pair : names)
{
const RegEx re('&' + pair.first + ';');
re.subst(html, u8c.to_bytes(pair.second), RegEx::RE_GLOBAL);
}
return html;
const RegEx re('&' + pair.first + ';');
re.subst(html, u8c.to_bytes(pair.second), RegEx::RE_GLOBAL);
}
const archive_answer URI::archive()
return html;
}
archive_answer URI::archive() const
{
using namespace curl_wrapper;
if (_uri.substr(0, 4) != "http")
{
if (_uri.substr(0, 4) != "http")
{
return { false, "Only HTTP(S) is archivable.", "" };
}
try
{
const string answer = make_request("https://web.archive.org/save/"
+ _uri, true);
if (!answer.empty())
{
return { true, "", "https://web.archive.org" + answer };
}
}
catch (const Poco::Exception &e)
{
return { false, e.displayText(), "" };
}
return { false, "Unknown error.", "" };
return {false, "Only HTTP(S) is archivable.", ""};
}
const string URI::remove_newlines(string text)
try
{
size_t posn = 0;
while ((posn = text.find('\n', posn)) != std::string::npos)
{
text.replace(posn, 1, " ");
CURLWrapper curl;
const auto answer =
curl.make_http_request(http_method::HEAD,
"https://web.archive.org/save/" + _uri);
size_t posr = posn - 1;
if (text[posr] == '\r')
if (answer)
{
string location{answer.get_header("location")};
if (location.empty())
{
text.replace(posr, 1, " ");
location = answer.get_header("content-location");
}
++posn;
if (!location.empty())
{
return {true, "", location};
}
return {false, "Could not extract location.", ""};
}
}
catch (const exception &e)
{
return {false, e.what(), ""};
}
return {false, "Unknown error.", ""};
}
string URI::remove_newlines(string text)
{
size_t posn = 0;
while ((posn = text.find('\n', posn)) != std::string::npos)
{
text.replace(posn, 1, " ");
size_t posr = posn - 1;
if (text[posr] == '\r')
{
text.replace(posr, 1, " ");
}
++posn;
}
return text;
}
string URI::cut_text(const string &text, const uint16_t n_chars)
{
if (text.size() > n_chars)
{
constexpr char suffix[] = " […]";
constexpr auto suffix_len = std::end(suffix) - std::begin(suffix) - 1;
if (n_chars <= suffix_len)
{
throw std::invalid_argument("n_chars has to be greater than "
+ std::to_string(suffix_len));
}
return text;
const size_t pos = text.rfind(' ', static_cast<size_t>(n_chars
- suffix_len));
return text.substr(0, pos) + suffix;
}
return text;
}
string URI::to_utf8(const string &str)
{
if (_encoding.empty())
{
detect_encoding();
}
if (_encoding == "utf-8")
{
return str;
}
return boost::locale::conv::to_utf<char>(str, _encoding);
}
void URI::detect_encoding()
{
const RegEx re_encoding(R"(<meta.+charset="([^";]+))", RegEx::RE_CASELESS);
vector<string> matches;
re_encoding.split(_document, matches);
if (matches.size() >= 2)
{
_encoding = boost::locale::to_lower(matches[1]);
}
}
bool URI::is_html() const
{
const RegEx re_htmlfile(".*\\.(.?html?|xml|rss)$", RegEx::RE_CASELESS);
return (_uri.substr(0, 4) == "http" || re_htmlfile.match(_uri));
}
} // namespace remwharead

View File

@ -1,9 +1,9 @@
#ifndef VERSION_HPP
#define VERSION_HPP
namespace global
namespace remwharead
{
static constexpr char version[] = "@PROJECT_VERSION@";
}
static constexpr char version[] = "@PROJECT_VERSION@";
} // namespace remwharead
#endif // VERSION_HPP
#endif // VERSION_HPP

79
tests/test_json.cpp Normal file
View File

@ -0,0 +1,79 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <exception>
#include <string>
#include <sstream>
#include <regex>
#include <chrono>
#include <catch.hpp>
#include "sqlite.hpp"
#include "export/json.hpp"
using namespace remwharead;
using std::string;
using std::chrono::system_clock;
using std::regex;
using std::regex_search;
SCENARIO ("The JSON export works correctly")
{
bool exception = false;
bool json_ok = true;
GIVEN ("One database entry")
{
Database::entry entry;
entry.uri = "https://example.com/page.html";
entry.tags = { "tag1", "tag2" };
entry.title = "Nice title";
entry.datetime = system_clock::time_point();
entry.fulltext = "Full text.";
entry.description = "Good description.";
try
{
std::ostringstream output;
Export::JSON({ entry }, output).print();
const string json = output.str();
const regex re(
R"(^\[\{"archive_uri":"",)"
R"("datetime":"\d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}",)"
R"("description":"Good description\.",)"
R"("fulltext":"Full text\.",)"
R"("tags":\["tag1","tag2"\],)"
R"("title":"Nice title",)"
R"("uri":"https:\\/\\/example\.com\\/page\.html"\}\]\n$)");
if (!regex_search(json, re))
{
json_ok = false;
}
}
catch (const std::exception &e)
{
exception = true;
}
THEN ("No exception is thrown")
AND_THEN ("Output looks okay")
{
REQUIRE_FALSE(exception);
REQUIRE(json_ok);
}
}
}

60
tests/test_link.cpp Normal file
View File

@ -0,0 +1,60 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <exception>
#include <string>
#include <sstream>
#include <catch.hpp>
#include "sqlite.hpp"
#include "export/link.hpp"
using namespace remwharead;
using std::string;
SCENARIO ("The Link export works correctly")
{
bool exception = false;
bool link_ok = true;
GIVEN ("One database entry")
{
Database::entry entry;
entry.uri = "https://example.com/page.html";
try
{
std::ostringstream output;
Export::Link({ entry }, output).print();
const string link = output.str();
if (link != (entry.uri + '\n'))
{
link_ok = false;
}
}
catch (const std::exception &e)
{
exception = true;
}
THEN ("No exception is thrown")
AND_THEN ("Output looks okay")
{
REQUIRE_FALSE(exception);
REQUIRE(link_ok);
}
}
}

68
tests/test_rofi.cpp Normal file
View File

@ -0,0 +1,68 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <exception>
#include <string>
#include <catch.hpp>
#include "sqlite.hpp"
#include "export/rofi.hpp"
using namespace remwharead;
using std::string;
SCENARIO ("The Rofi export works correctly")
{
bool exception = false;
bool rofi_ok = true;
GIVEN ("One database entry")
{
Database::entry entry;
entry.uri = "https://example.com/page.html";
entry.title = "Thoughtful title";
entry.tags = { "tag1", "tag2" };
const string expected = static_cast<char>(0x00) + string("markup-rows")
+ static_cast<char>(0x1f) + "true\n"
"Thoughtful title "
R"(<span size="small" weight="light" style="italic">(tag1,tag2))"
R"(</span> <span size="xx-small" weight="ultralight">)"
"https://example.com/page.html</span>\n";
try
{
std::ostringstream output;
Export::Rofi({ entry }, output).print();
const string rofi = output.str();
if (rofi != expected)
{
rofi_ok = false;
}
}
catch (const std::exception &e)
{
exception = true;
}
THEN ("No exception is thrown")
AND_THEN ("Output looks okay")
{
REQUIRE_FALSE(exception);
REQUIRE(rofi_ok);
}
}
}

89
tests/test_rss.cpp Normal file
View File

@ -0,0 +1,89 @@
/* This file is part of remwharead.
* Copyright © 2019 tastytea <tastytea@tastytea.de>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <exception>
#include <string>
#include <sstream>
#include <regex>
#include <chrono>
#include <catch.hpp>
#include "sqlite.hpp"
#include "export/rss.hpp"
using namespace remwharead;
using std::string;
using std::chrono::system_clock;
using std::regex;
using std::regex_search;
SCENARIO ("The RSS export works correctly")
{
bool exception = false;
bool rss_ok = true;
GIVEN ("One database entry")
{
Database::entry entry;
entry.uri = "https://example.com/page.html";
entry.tags = { "tag1", "tag2" };
entry.title = "Nice title";
entry.datetime = system_clock::time_point();
entry.fulltext = "Full text.";
entry.description = "Good description.";
try
{
std::ostringstream output;
Export::RSS({ entry }, output).print();
const string rss = output.str();
const regex re(
R"(^<rss version="2\.0" )"
R"(xmlns:atom="http://www\.w3\.org/2005/Atom"><channel>)"
R"(<title>Visited things</title><link/>)"
R"(<description>Export from remwharead\.</description>)"
R"(<generator>remwharead \d+\.\d+\.\d+</generator>)"
R"(<lastBuildDate>[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} )"
R"((\d{2}:){2}\d{2} [A-Z]+</lastBuildDate>)"
R"(<item><title>Nice title</title>)"
R"(<link>https://example\.com/page\.html</link>)"
R"(<guid isPermaLink="false">https://example\.com/page\.html )"
R"(at \d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}</guid>)"
R"(<pubDate>[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} )"
R"((\d{2}:){2}\d{2} [A-Z]+</pubDate>)"
R"(<description>&lt;p&gt;Good description\.&lt;/p&gt;)"
R"(&lt;p&gt;&lt;strong&gt;Tags:&lt;/strong&gt; )"
R"(tag1, tag2&lt;/p&gt;</description>)"
R"(</item></channel></rss>\n$)");
if (!regex_search(rss, re))
{
rss_ok = false;
}
}
catch (const std::exception &e)
{
exception = true;
}
THEN ("No exception is thrown")
AND_THEN ("Output looks okay")
{
REQUIRE_FALSE(exception);
REQUIRE(rss_ok);
}
}
}

View File

@ -17,7 +17,6 @@
#include <exception>
#include <string>
#include <chrono>
#include <vector>
#include <catch.hpp>
#include "sqlite.hpp"
#include "search.hpp"
@ -25,7 +24,6 @@
using namespace remwharead;
using std::string;
using std::chrono::system_clock;
using std::vector;
SCENARIO ("Searching works correctly")
{

View File

@ -32,41 +32,29 @@ SCENARIO ("URI works correctly")
explicit URITest(const string &)
: URI("") {}
URITest()
: URI("test.html") {}
: URI("test.html")
{
_document =
"<html><head><title>title</title>"
"<meta name=\"description\" content=\"description\" />"
"<body><p>A short <span style=\"\">sentence</span>.</p>"
"</body></head></html>";
}
bool test_title()
{
if (extract_title(_html) == "title")
{
return true;
}
return false;
return (extract_title() == "title");
}
bool test_description()
{
if (extract_description(_html) == "description")
{
return true;
}
return false;
return (extract_description() == "description");
}
bool test_fulltext()
{
if (strip_html(_html) == "titleA short sentence.")
{
return true;
}
return false;
return (strip_html() == "titleA short sentence.");
}
private:
const string _html =
"<html><head><title>title</title>"
"<meta name=\"description\" content=\"description\" />"
"<body><p>A short <span style=\"\">sentence</span>.</p>"
"</body></head></html>";
};
WHEN ("extract_title() is called")