From 3d25bb1a4bf05dc3ca5a38c5a53d24278572c6e2 Mon Sep 17 00:00:00 2001 From: tastytea Date: Mon, 16 Dec 2019 00:57:38 +0100 Subject: [PATCH 01/62] Only test with GCC 5 and 8. --- .drone.yml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/.drone.yml b/.drone.yml index 249f69c..a8ae3ab 100644 --- a/.drone.yml +++ b/.drone.yml @@ -34,35 +34,6 @@ pipeline: volumes: - /var/cache/debian-package-cache:/var/cache/apt/archives - gcc7: - image: debian:stretch-slim - pull: true - environment: - - LANG=C.utf8 - commands: - - rm /etc/apt/apt.conf.d/docker-clean - - apt-get update -q - - echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release - - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list - - echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >> /etc/apt/sources.list.d/ubuntu-toolchain-r.list - - apt-get install -qy gnupg - - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 0x60c317803a41ba51845e371a1e9377a2ba9ef27f - - gpg --armor --export 0x60c317803a41ba51845e371a1e9377a2ba9ef27f | apt-key add - - - apt-get update -q - - apt-get install -qy build-essential cmake pkg-config - - apt-get install -qy -t xenial g++-7 - - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100 - - update-alternatives --set g++ /usr/bin/g++-7 - - apt-get install -qy libcurl4-openssl-dev libjsoncpp-dev libboost-all-dev - - apt-get install -qy -t sid libcurlpp-dev - - dpkg -i mastodon-cpp.deb - - rm -rf build && mkdir -p build && cd build - - cmake .. - - make VERBOSE=1 - - make install DESTDIR=install - volumes: - - /var/cache/debian-package-cache:/var/cache/apt/archives - gcc5: image: debian:stretch-slim pull: true @@ -92,27 +63,6 @@ pipeline: volumes: - /var/cache/debian-package-cache:/var/cache/apt/archives - gcc6: - image: debian:stretch-slim - pull: true - environment: - - LANG=C.utf8 - commands: - - rm /etc/apt/apt.conf.d/docker-clean - - echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release - - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list - - apt-get update -q - - apt-get install -qy build-essential cmake pkg-config - - apt-get install -qy libcurl4-openssl-dev libjsoncpp-dev libboost-all-dev - - apt-get install -qy -t sid libcurlpp-dev - - dpkg -i mastodon-cpp.deb - - rm -rf build && mkdir -p build && cd build - - cmake .. - - make VERBOSE=1 - - make install DESTDIR=install - volumes: - - /var/cache/debian-package-cache:/var/cache/apt/archives - notify: image: drillster/drone-email pull: true From 2a6712e04cb71cdf7b6555be4b7ed0fbe53bfb5a Mon Sep 17 00:00:00 2001 From: tastytea Date: Mon, 16 Dec 2019 00:58:12 +0100 Subject: [PATCH 02/62] Replace README.md with README.adoc. --- README.adoc | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 90 --------------------------------------------- 2 files changed, 104 insertions(+), 90 deletions(-) create mode 100644 README.adoc delete mode 100644 README.md diff --git a/README.adoc b/README.adoc new file mode 100644 index 0000000..d8a2ac5 --- /dev/null +++ b/README.adoc @@ -0,0 +1,104 @@ +:uri-gcc: https://gcc.gnu.org/ +:uri-cmake: https://cmake.org/ +:uri-asciidoc: http://asciidoc.org/ +:uri-catch: https://github.com/catchorg/Catch2 +:uri-boost: https://www.boost.org/ +:uri-mastodon-cpp: https://schlomp.space/tastytea/mastodon-cpp +:uri-jsoncpp: https://github.com/open-source-parsers/jsoncpp + +*mastorss* dumps RSS feeds into a mastodon account. Supports RSS 2.0 but not +RSS 0.92. Does not support Atom at the moment. + +Each `` in feeds must have ``, `` and `<description>`. + +== Install + +=== Dependencies + +* Tested OS: Linux +* C++ compiler (tested: link:{uri-gcc}[gcc] 5 / 8) +* link:{uri-cmake}[cmake] (tested: 3.14 / 3.9) +* link:{uri-boost}[boost] (tested: 1.71 / 1.62) +* link:{uri-mastodon-cpp}[mastodon-cpp] (at least: 0.111.2) +* link:{uri-jsoncpp}[jsoncpp] (tested: 1.9 / 1.7) + +=== Get sourcecode + +==== Development version + +[source,shell] +-------------------------------------------------------------------------------- +git clone https://schlomp.space/tastytea/mastorss.git +-------------------------------------------------------------------------------- + +=== Compile + +[source,shell] +-------------------------------------------------------------------------------- +mkdir build +cd build/ +cmake .. +cmake --build . +-------------------------------------------------------------------------------- + +Install with `make install`. + +== Usage + +Put `watchwords.json` into `${XDG_CONFIG_HOME}/mastorss/` +footnote:[`${XDG_CONFIG_HOME}` is usually `~/.config/`]. Launch with profile +name. The first occurence of every watchword in an RSS item will be turned into +a hashtag. For profile-specific watchwords see the example in +`watchwords.json`. In the first run only the newest entry is posted. + +The profile can't be named "global". + +=== Example session + +[source,shellsession] +-------------------------------------------------------------------------------- +% mastorss example +-------------------------------------------------------------------------------- + +=== Example config file + +${HOME}/.config/mastorss/config-example.json + +[source,json] +-------------------------------------------------------------------------------- + { + "example": { + "access_token": "123abc", + "append": "#bot", + "feedurl": "https:\/\/example.com\/feed.rss", + "fixes": + [ + "delete this", + "<p>[Rr]ead more(\.{3}|…)</p>" + ], + "instance": "botsin.space", + "interval" : 600, + "max_size": 400, + "skip": + [ + "If the entry starts with this, skip it", + "Skip me too!" + ], + "titles_as_cw" : false, + "titles_only": false + } + } +-------------------------------------------------------------------------------- + +=== Error codes + +Same as +link:{uri-mastodon-cpp}/src/branch/master/README.adoc#_error_codes[mastodon-cpp], +plus: + +[%autowidth,cols=">,<"] +|=========================================================== +| Code | Explanation + +| 1 | Could not send post for unknown reasons +|=========================================================== diff --git a/README.md b/README.md deleted file mode 100644 index f771d19..0000000 --- a/README.md +++ /dev/null @@ -1,90 +0,0 @@ -**mastorss** dumps RSS feeds into a mastodon account. -Supports RSS 2.0 but not RSS 0.92. Does not support Atom at the moment. - -\<item\>s in feeds must have \<link\>, \<title\> and \<description\>. - -The documentation is far from complete, sorry. - -# Install - -## Dependencies - - * Tested OS: Linux - * C++ compiler (tested: gcc 5 / 6 / 7 / 8) - * [cmake](https://cmake.org/) (tested: 3.9 / 3.12) - * [boost](http://www.boost.org/) (tested: 1.65 / 1.62) - * [curlpp](http://www.curlpp.org/) (tested: 0.8) - * [mastodon-cpp](https://schlomp.space/tastytea/mastodon-cpp) (at least: - 0.105.0) - * [jsoncpp](https://github.com/open-source-parsers/jsoncpp) (tested: 1.8 / 1.7) - -## Get sourcecode - -### Development version - - git clone https://schlomp.space/tastytea/mastorss.git - -## Compile - - mkdir build - cd build/ - cmake .. - make - -## Install - -Install with `make install`. - -# Usage - -Put `watchwords.json` into `~/.config/mastorss/`. Launch with profile name. The -first occurence of every watchword in an RSS item will be turned into a hashtag. -For profile-specific watchwords see the example in `watchwords.json`. In the -first run only the newest entry is tooted. - -The profile can't be named "global". - -## Example config file - -${HOME}/.config/mastorss/config-example.json - - { - "example": { - "instance": "botsin.space", - "feedurl": "https:\/\/example.com\/feed.rss", - "access_token": "123abc", - "max_size": 400, - "titles_only": false, - "skip": - [ - "If the entry starts with this, skip it", - "Skip me too!" - ], - "fixes": - [ - "delete this", - "<p>[Rr]ead more(\.{3}|…)</p>" - ], - "append": "#bot" - } - } - - -## Error codes - -Same as -[mastodon-cpp](https://schlomp.space/tastytea/mastodon-cpp/src/branch/master/README.adoc#_error_codes), -plus: - -| Code | Explanation | -| --------: |:----------------------------------------| -| 1 | Could not send post for unknown reasons | - -# Copyright - -``` text -Copyright © 2018, 2019 tastytea <tastytea@tastytea.de>. -License GPLv3: GNU GPL version 3 <https://www.gnu.org/licenses/gpl-3.0.html>. -This program comes with ABSOLUTELY NO WARRANTY. This is free software, -and you are welcome to redistribute it under certain conditions. -``` From a2c942adbb368a3a1817fb7ecd2d98acf93663d7 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Mon, 16 Dec 2019 01:00:59 +0100 Subject: [PATCH 03/62] Add Code Of Conduct. --- CODE_OF_CONDUCT.adoc | 49 ++++++++++++++++++++++++++++++++++++++++++++ CONTRIBUTING.adoc | 29 ++++++++++++++++++++++++++ README.adoc | 6 ++++++ 3 files changed, 84 insertions(+) create mode 100644 CODE_OF_CONDUCT.adoc create mode 100644 CONTRIBUTING.adoc diff --git a/CODE_OF_CONDUCT.adoc b/CODE_OF_CONDUCT.adoc new file mode 100644 index 0000000..a561593 --- /dev/null +++ b/CODE_OF_CONDUCT.adoc @@ -0,0 +1,49 @@ +:contact-coc: tastytea@tastytea.de + +== Code of Conduct + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, education, ethnicity, gender identity and expression, level of +experience, nationality, personal appearance, race, religion, sex +characteristics, sexual identity and orientation or socio-economic status. + +=== Examples + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language. +* Being respectful of differing viewpoints and experiences. +* Gracefully accepting constructive criticism. +* Focusing on what is best for the community. +* Showing empathy towards other community members. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances. +* Trolling, insulting/derogatory comments, and personal attacks. +* Public or private harassment. +* Publishing others’ private information, such as a physical or electronic + address, without explicit permission. + +=== Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at {contact-coc}. + +All complaints will be reviewed and investigated and will result in a response +that is deemed necessary and appropriate to the circumstances. The project team +is obligated to maintain confidentiality with regard to the reporter of an +incident. + +=== Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.adoc b/CONTRIBUTING.adoc new file mode 100644 index 0000000..b64ebbc --- /dev/null +++ b/CONTRIBUTING.adoc @@ -0,0 +1,29 @@ +:project: mastorss +:uri-base: https://schlomp.space/tastytea/{project} +:uri-coc: {uri-base}/src/branch/main/CODE_OF_CONDUCT.adoc +:contact-email: tastytea@tastytea.de +:contact-xmpp: {contact-email} +:contact-fediverse: https://likeable.space/users/tastytea + +== How to contribute + +Read the link:{uri-coc}[Code of Conduct]. + +=== Reporting bugs or suggesting enhancements + +Before reporting a bug, please +https://schlomp.space/tastytea/{project}/issues[perform a search] to see if the +problem has already been reported. If it has, add a comment to the existing +issue instead of opening a new one. Same for enhancements. + +You can also contact me via mailto:{contact-email}[E-Mail], +link:xmpp:{contact-xmpp}[XMPP] or the {contact-fediverse}[Fediverse] if you +don't want to open an account. + +=== Pull requests + +Please use similar coding conventions as the rest of the project. The basic rule +to remember is to write code in the same style as the existing/surrounding code. + +You can also send me your patches via mailto:{contact-email}[E-Mail], ideally +using `git format-patch` or `git send-email`. diff --git a/README.adoc b/README.adoc index d8a2ac5..1856eac 100644 --- a/README.adoc +++ b/README.adoc @@ -1,3 +1,7 @@ +:toc: preamble + +:project: mastorss +:uri-base: https://schlomp.space/tastytea/{project} :uri-gcc: https://gcc.gnu.org/ :uri-cmake: https://cmake.org/ :uri-asciidoc: http://asciidoc.org/ @@ -102,3 +106,5 @@ plus: | 1 | Could not send post for unknown reasons |=========================================================== + +include::{uri-base}/raw/branch/main/CONTRIBUTING.adoc[] From b320cafa405b0319fbcc6f375e60defba4c7d619 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Mon, 16 Dec 2019 01:04:45 +0100 Subject: [PATCH 04/62] Add EditorConfig. --- .editorconfig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ad5e59f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,20 @@ +# Configuration file for EditorConfig. +# More information is available under <https://editorconfig.org/>. + +root = true + +[*] +indent_style = space +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +max_line_length = 80 + +[*.?pp] +indent_size = 4 +tab_width = 4 + +[{CMakeLists.txt,*.cmake}] +indent_size = 2 +tab_width = 2 From da57e4f1edf666cca1770460ae84058abe557597 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Mon, 16 Dec 2019 01:16:17 +0100 Subject: [PATCH 05/62] Update drone file. --- .drone.yml | 158 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 91 insertions(+), 67 deletions(-) diff --git a/.drone.yml b/.drone.yml index a8ae3ab..ffe7c10 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,73 +1,97 @@ -pipeline: - download: - image: plugins/download - pull: true - source: https://schlomp.space/attachments/e1c1e64b-1192-4037-aad4-95238ad648b0 +# -*- fill-column: 1000 -*- +kind: pipeline +name: build x86_64 + +volumes: +- name: debian-package-cache + host: + path: /var/cache/debian-package-cache + +trigger: + event: + exclude: + - tag + +steps: +- name: download + image: plugins/download + settings: + source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.2/libmastodon-cpp_0.111.2-0_amd64.deb destination: mastodon-cpp.deb - gcc8: - image: debian:stretch-slim - pull: true - environment: - - LANG=C.utf8 - commands: - - rm /etc/apt/apt.conf.d/docker-clean - - apt-get update -q - - echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release - - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list - - echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >> /etc/apt/sources.list.d/ubuntu-toolchain-r.list - - apt-get install -qy gnupg - - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 0x60c317803a41ba51845e371a1e9377a2ba9ef27f - - gpg --armor --export 0x60c317803a41ba51845e371a1e9377a2ba9ef27f | apt-key add - - - apt-get update -q - - apt-get install -qy build-essential cmake pkg-config - - apt-get install -qy -t xenial g++-8 - - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 100 - - update-alternatives --set g++ /usr/bin/g++-8 - - apt-get install -qy libcurl4-openssl-dev libjsoncpp-dev libboost-all-dev - - apt-get install -qy -t sid libcurlpp-dev - - dpkg -i mastodon-cpp.deb - - rm -rf build && mkdir -p build && cd build - - cmake .. - - make VERBOSE=1 - - make install DESTDIR=install - volumes: - - /var/cache/debian-package-cache:/var/cache/apt/archives +- name: gcc8 + image: debian:stretch-slim + pull: always + environment: + CXX: g++-8 + CXXFLAGS: -pipe -O2 + LANG: C.utf8 + commands: + - rm /etc/apt/apt.conf.d/docker-clean + - apt-get update -q + - echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release + - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list + - echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >> /etc/apt/sources.list.d/ubuntu-toolchain-r.list + - apt-get install -qy gnupg + - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 0x60c317803a41ba51845e371a1e9377a2ba9ef27f + - gpg --armor --export 0x60c317803a41ba51845e371a1e9377a2ba9ef27f | apt-key add - + - apt-get update -q + - apt-get install -qy build-essential cmake pkg-config + - apt-get install -qy -t xenial g++-8 + - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 100 + - update-alternatives --set g++ /usr/bin/g++-8 + - apt-get install -qy libcurl4-openssl-dev libjsoncpp-dev libboost-all-dev + - apt-get install -qy -t sid libcurlpp-dev + - apt-get -qy install ./mastodon-cpp.deb + - rm -rf build && mkdir -p build && cd build + - cmake .. + - make VERBOSE=1 + - make install DESTDIR=install + volumes: + - name: debian-package-cache + path: /var/cache/apt/archives - gcc5: - image: debian:stretch-slim - pull: true - environment: - - LANG=C.utf8 - commands: - - rm /etc/apt/apt.conf.d/docker-clean - - apt-get update -q - - echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release - - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list - - echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >> /etc/apt/sources.list.d/ubuntu-toolchain-r.list - - apt-get install -qy gnupg - - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 0x60c317803a41ba51845e371a1e9377a2ba9ef27f - - gpg --armor --export 0x60c317803a41ba51845e371a1e9377a2ba9ef27f | apt-key add - - - apt-get update -q - - apt-get install -qy build-essential cmake pkg-config - - apt-get install -qy -t xenial g++-5 - - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 100 - - update-alternatives --set g++ /usr/bin/g++-5 - - apt-get install -qy libcurl4-openssl-dev libjsoncpp-dev libboost-all-dev - - apt-get install -qy -t sid libcurlpp-dev - - dpkg -i mastodon-cpp.deb - - rm -rf build && mkdir -p build && cd build - - cmake .. - - make VERBOSE=1 - - make install DESTDIR=install - volumes: - - /var/cache/debian-package-cache:/var/cache/apt/archives +- name: gcc5 + image: debian:stretch-slim + pull: always + environment: + CXX: g++-8 + CXXFLAGS: -pipe -O2 + LANG: C.utf8 + commands: + - rm /etc/apt/apt.conf.d/docker-clean + - apt-get update -q + - echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release + - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list + - echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >> /etc/apt/sources.list.d/ubuntu-toolchain-r.list + - apt-get install -qy gnupg + - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 0x60c317803a41ba51845e371a1e9377a2ba9ef27f + - gpg --armor --export 0x60c317803a41ba51845e371a1e9377a2ba9ef27f | apt-key add - + - apt-get update -q + - apt-get install -qy build-essential cmake pkg-config + - apt-get install -qy -t xenial g++-5 + - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 100 + - update-alternatives --set g++ /usr/bin/g++-5 + - apt-get install -qy libcurl4-openssl-dev libjsoncpp-dev libboost-all-dev + - apt-get install -qy -t sid libcurlpp-dev + - apt-get install -qy ./mastodon-cpp.deb + - rm -rf build && mkdir -p build && cd build + - cmake .. + - make VERBOSE=1 + - make install DESTDIR=install + volumes: + - name: debian-package-cache + path: /var/cache/apt/archives - notify: - image: drillster/drone-email - pull: true +- name: notify + image: drillster/drone-email + pull: always + settings: host: cryptoparty-celle.de - secrets: [ email_username, email_password ] from: drone@tzend.de - when: - status: [ changed, failure ] + username: + from_secret: email_username + password: + from_secret: email_password + when: + status: [ changed, failure ] From 232180257539f908831856058956dd36d5164fb8 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Mon, 16 Dec 2019 15:21:06 +0100 Subject: [PATCH 06/62] Rewrite root CMake recipe. --- CMakeLists.txt | 67 ++++++++++++++++++----------------------- cmake/debug_flags.cmake | 58 +++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 37 +++++++++++++++++++++++ 3 files changed, 125 insertions(+), 37 deletions(-) create mode 100644 cmake/debug_flags.cmake create mode 100644 src/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f5bcebb..7d75ad2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,47 +1,40 @@ -cmake_minimum_required (VERSION 3.6) +# Support version 3.9 and above, but use policy settings up to 3.14. +# 3.9 is needed for project description. +cmake_minimum_required(VERSION 3.9...3.14) +# Ranges are supported from 3.12, set policy to current for < 3.12. +if(${CMAKE_VERSION} VERSION_LESS 3.12) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) +endif() + +# Global build options. +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "The type of build.") +option(BUILD_SHARED_LIBS "Build shared libraries." YES) + project (mastorss - VERSION 0.9.0 - LANGUAGES CXX -) + VERSION 0.9.9999 + DESCRIPTION "Another RSS to Mastodon bot." + LANGUAGES CXX) -include(GNUInstallDirs) -find_package(CURL REQUIRED) -find_package(Boost REQUIRED COMPONENTS system filesystem) -find_package(PkgConfig REQUIRED) -pkg_check_modules(CURLPP REQUIRED curlpp) -pkg_check_modules(JSONCPP REQUIRED jsoncpp) +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") -set(CMAKE_CXX_STANDARD 14) +# Project build options. +option(WITH_MAN "Compile and install manpage." YES) +option(WITH_TESTS "Compile tests." NO) + +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -include_directories(${PROJECT_SOURCE_DIR}/src) -include_directories(${PROJECT_BINARY_DIR}) +include(debug_flags) -include_directories(${CURL_INCLUDE_DIR}) -include_directories(${CURLPP_INCLUDE_DIR}) -include_directories(${JSONCPP_INCLUDE_DIR}) -include_directories(${Boost_INCLUDE_DIRS}) +add_subdirectory(src) -link_directories(${CURL_LIBRARY_DIRS}) -link_directories(${CURLPP_LIBRARY_DIRS}) -link_directories(${JSONCPP_LIBRARY_DIRS}) -link_directories(${Boost_LIBRARY_DIRS}) +if(WITH_MAN) + add_subdirectory(man) +endif() -add_definitions(${Boost_DEFINITIONS}) +# if(WITH_TESTS) +# add_subdirectory(tests) +# endif() -# Write version in header -configure_file ( - "${PROJECT_SOURCE_DIR}/src/version.hpp.in" - "${PROJECT_BINARY_DIR}/version.hpp" -) - -file(GLOB sources src/*.cpp) -add_executable(mastorss ${sources}) -target_link_libraries(mastorss - mastodon-cpp ${CURLPP_LIBRARIES} - ${JSONCPP_LIBRARIES} ${Boost_LIBRARIES} stdc++fs) -install(TARGETS mastorss DESTINATION ${CMAKE_INSTALL_BINDIR}) - -install(FILES watchwords.json - DESTINATION ${CMAKE_INSTALL_DATADIR}/mastorss) +# include(cmake/packages.cmake) diff --git a/cmake/debug_flags.cmake b/cmake/debug_flags.cmake new file mode 100644 index 0000000..0009039 --- /dev/null +++ b/cmake/debug_flags.cmake @@ -0,0 +1,58 @@ +# Set compiler flags for Debug builds. +# Only has an effect on GCC/Clang >= 5.0. + +set(DEBUG_CXXFLAGS "") +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang" + AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5") + list(APPEND DEBUG_CXXFLAGS + "-Wall" + "-Wextra" + "-Wpedantic" + "-Wuninitialized" + "-Wshadow" + "-Wnon-virtual-dtor" + "-Wconversion" + "-Wsign-conversion" + "-Wold-style-cast" + "-Wzero-as-null-pointer-constant" + "-Wmissing-declarations" + "-Wcast-align" + "-Wunused" + "-Woverloaded-virtual" + "-Wdouble-promotion" + "-Wformat=2" + "-ftrapv" + "-fsanitize=undefined" + "-g" + "-Og" + "-fno-omit-frame-pointer") + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + list(APPEND DEBUG_CXXFLAGS + "-Wlogical-op" + "-Wuseless-cast") + if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "6") + list(APPEND DEBUG_CXXFLAGS + "-Wmisleading-indentation" + "-Wduplicated-cond" + "-Wnull-dereference") + if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7") + list(APPEND DEBUG_CXXFLAGS + "-Wduplicated-branches") + endif() + endif() + endif() + add_compile_options("$<$<CONFIG:Debug>:${DEBUG_CXXFLAGS}>") + + set(DEBUG_LDFLAGS + "-fsanitize=undefined") + # add_link_options was introduced in version 3.13. + if(${CMAKE_VERSION} VERSION_LESS 3.13) + set(CMAKE_SHARED_LINKER_FLAGS_DEBUG "${DEBUG_LDFLAGS}") + else() + add_link_options("$<$<CONFIG:Debug>:${DEBUG_LDFLAGS}>") + endif() +else() + message(STATUS + "No additional compiler flags were set, " + "because your compiler was not anticipated.") +endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..9d605e6 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,37 @@ +include(GNUInstallDirs) +find_package(CURL REQUIRED) +find_package(Boost REQUIRED COMPONENTS system filesystem) +find_package(PkgConfig REQUIRED) +pkg_check_modules(CURLPP REQUIRED curlpp) +pkg_check_modules(JSONCPP REQUIRED jsoncpp) + +include_directories(${PROJECT_SOURCE_DIR}/src) +include_directories(${PROJECT_BINARY_DIR}) + +include_directories(${CURL_INCLUDE_DIR}) +include_directories(${CURLPP_INCLUDE_DIR}) +include_directories(${JSONCPP_INCLUDE_DIR}) +include_directories(${Boost_INCLUDE_DIRS}) + +link_directories(${CURL_LIBRARY_DIRS}) +link_directories(${CURLPP_LIBRARY_DIRS}) +link_directories(${JSONCPP_LIBRARY_DIRS}) +link_directories(${Boost_LIBRARY_DIRS}) + +add_definitions(${Boost_DEFINITIONS}) + +# Write version in header +configure_file ( + "${PROJECT_SOURCE_DIR}/src/version.hpp.in" + "${PROJECT_BINARY_DIR}/version.hpp" +) + +file(GLOB sources *.cpp) +add_executable(mastorss ${sources}) +target_link_libraries(mastorss + mastodon-cpp ${CURLPP_LIBRARIES} + ${JSONCPP_LIBRARIES} ${Boost_LIBRARIES} stdc++fs) +install(TARGETS mastorss DESTINATION ${CMAKE_INSTALL_BINDIR}) + +install(FILES watchwords.json + DESTINATION ${CMAKE_INSTALL_DATADIR}/mastorss) From 5495acbfa296cfee34b9ceb98dfef25fd0bb9d40 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Mon, 16 Dec 2019 15:23:06 +0100 Subject: [PATCH 07/62] Add manpage (skeleton). --- README.adoc | 2 +- man/CMakeLists.txt | 13 +++++++++++ man/build_manpage.sh | 13 +++++++++++ man/mastorss.1.adoc | 53 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 man/CMakeLists.txt create mode 100755 man/build_manpage.sh create mode 100644 man/mastorss.1.adoc diff --git a/README.adoc b/README.adoc index 1856eac..eb03ff0 100644 --- a/README.adoc +++ b/README.adoc @@ -104,7 +104,7 @@ plus: |=========================================================== | Code | Explanation -| 1 | Could not send post for unknown reasons +| 1 | Could not send post for unknown reasons. |=========================================================== include::{uri-base}/raw/branch/main/CONTRIBUTING.adoc[] diff --git a/man/CMakeLists.txt b/man/CMakeLists.txt new file mode 100644 index 0000000..fdd941b --- /dev/null +++ b/man/CMakeLists.txt @@ -0,0 +1,13 @@ +include(GNUInstallDirs) + +add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.1" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + DEPENDS "${PROJECT_NAME}.1.adoc" + COMMAND "${PROJECT_SOURCE_DIR}/man/build_manpage.sh" + ARGS "${PROJECT_VERSION}") + +add_custom_target(man ALL DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.1") + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.1 + DESTINATION ${CMAKE_INSTALL_MANDIR}/man1) diff --git a/man/build_manpage.sh b/man/build_manpage.sh new file mode 100755 index 0000000..c845d94 --- /dev/null +++ b/man/build_manpage.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +name="mastorss" + +if [ -n "${1}" ]; then + dir=$(dirname "${0}") + version=${1} + cp -vf "${dir}/${name}.1.adoc" . + sed -Ei "s/(Revision: +)[0-9]+\.[0-9]\.[0-9]/\1${version}/" "${name}.1.adoc" + a2x --doctype manpage --format manpage --no-xmllint "${name}.1.adoc" +else + echo "usage: ${0} VERSION" >&2 +fi diff --git a/man/mastorss.1.adoc b/man/mastorss.1.adoc new file mode 100644 index 0000000..7216dd8 --- /dev/null +++ b/man/mastorss.1.adoc @@ -0,0 +1,53 @@ += mastorss(1) +:doctype: manpage +:Author: tastytea +:Email: tastytea@tastytea.de +:Date: 2019-12-16 +:Revision: 0.0.0 +:man source: mastorss +:man manual: General Commands Manual + +== NAME + +mastorss - Another RSS to Mastodon bot. + +== SYNOPSIS + +*mastorss* profile + +// == DESCRIPTION + +// == OPTIONS + +// == EXAMPLES + +// == PROTOCOL SUPPORT + +// Currently only HTTP and HTTPS are supported. + +// == PROXY SUPPORT + +== FILES + +* *Configuration file directory*: `${XDG_CONFIG_HOME}/mastorss/` + +`${XDG_CONFIG_HOME}` is usually `~/.config`. + +== ERROR CODES + +[cols=">,<"] +|=========================================================== +| Code | Explanation + +| 1 | Could not send post for unknown reasons. +|=========================================================== + +== SEE ALSO + +*crontab*(1), *crontab*(5) + +== REPORTING BUGS + +Bugtracker: https://schlomp.space/tastytea/mastorss/issues + +E-mail: tastytea@tastytea.de From 90184869f3204b11b827070969d6690d7a8f9e4e Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Thu, 19 Dec 2019 15:19:48 +0100 Subject: [PATCH 08/62] Require mastodon-cpp-0.111.3. --- .drone.yml | 2 +- README.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index ffe7c10..920b304 100644 --- a/.drone.yml +++ b/.drone.yml @@ -16,7 +16,7 @@ steps: - name: download image: plugins/download settings: - source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.2/libmastodon-cpp_0.111.2-0_amd64.deb + source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.3/libmastodon-cpp_0.111.3-0_amd64.deb destination: mastodon-cpp.deb - name: gcc8 diff --git a/README.adoc b/README.adoc index eb03ff0..405a45d 100644 --- a/README.adoc +++ b/README.adoc @@ -23,7 +23,7 @@ Each `<item>` in feeds must have `<link>`, `<title>` and `<description>`. * C++ compiler (tested: link:{uri-gcc}[gcc] 5 / 8) * link:{uri-cmake}[cmake] (tested: 3.14 / 3.9) * link:{uri-boost}[boost] (tested: 1.71 / 1.62) -* link:{uri-mastodon-cpp}[mastodon-cpp] (at least: 0.111.2) +* link:{uri-mastodon-cpp}[mastodon-cpp] (at least: 0.111.3) * link:{uri-jsoncpp}[jsoncpp] (tested: 1.9 / 1.7) === Get sourcecode From 8870c37a42af9467e066e3f88c4bda1cf8393a39 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Fri, 20 Dec 2019 01:11:40 +0100 Subject: [PATCH 09/62] Delete old sources. --- src/config.cpp | 194 ------------------------------------------- src/http.cpp | 87 -------------------- src/mastorss.cpp | 164 ------------------------------------ src/mastorss.hpp | 43 ---------- src/parse.cpp | 210 ----------------------------------------------- 5 files changed, 698 deletions(-) delete mode 100644 src/config.cpp delete mode 100644 src/http.cpp delete mode 100644 src/mastorss.cpp delete mode 100644 src/mastorss.hpp delete mode 100644 src/parse.cpp diff --git a/src/config.cpp b/src/config.cpp deleted file mode 100644 index c0edeaa..0000000 --- a/src/config.cpp +++ /dev/null @@ -1,194 +0,0 @@ -/* This file is part of mastorss. - * Copyright © 2018, 2019 tastytea <tastytea@tastytea.de> - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <iostream> -#include <string> -#include <cstdint> -#include <fstream> -#include <sstream> -#include <experimental/filesystem> -#include <jsoncpp/json/json.h> -#include <mastodon-cpp/mastodon-cpp.hpp> -#include "mastorss.hpp" - -using std::cout; -using std::cerr; -using std::cin; -using std::string; -namespace fs = std::experimental::filesystem; - -std::uint16_t read_config(string &instance, string &access_token, string &feedurl) -{ - bool config_changed = false; - - // Read config file, get access token - std::ifstream file(filepath + "config-" + profile + ".json"); - if (file.is_open()) - { - std::stringstream json; - json << file.rdbuf(); - file.close(); - json >> config; - - instance = config[profile]["instance"].asString(); - access_token = config[profile]["access_token"].asString(); - feedurl = config[profile]["feedurl"].asString(); - if (!config[profile]["max_size"].isNull()) - { - max_size = config[profile]["max_size"].asUInt(); - } - else - { - config[profile]["max_size"] = max_size; - config_changed = true; - } - } - else - { - cout << "Config file not found. Building new one.\n"; - fs::create_directory(filepath); - } - - if (instance.empty()) - { - cout << "Instance: "; - std::getline(cin, instance); - config[profile]["instance"] = instance; - config_changed = true; - } - if (access_token.empty()) - { - cout << "No access token found.\n"; - string client_id, client_secret, url; - Mastodon::API masto(instance, ""); - Mastodon::return_call ret - = masto.register_app1("mastorss", - "urn:ietf:wg:oauth:2.0:oob", - "write", - "https://schlomp.space/tastytea/mastorss", - client_id, - client_secret, - url); - if (!ret) - { - string code; - cout << "Visit " << url << " to authorize this application.\n"; - cout << "Insert code: "; - std::getline(cin, code); - - ret = masto.register_app2(client_id, - client_secret, - "urn:ietf:wg:oauth:2.0:oob", - code, - access_token); - if (!ret) - { - config[profile]["access_token"] = access_token; - config_changed = true; - } - else - { - cerr << "Error code: " << ret.error_code << '\n'; - return ret.error_code; - } - } - else - { - cerr << "Error code: " << ret.error_code << '\n'; - return ret.error_code; - } - - } - if (feedurl.empty()) - { - cout << "feedurl: "; - std::getline(cin, feedurl); - config[profile]["feedurl"] = feedurl; - config_changed = true; - } - if (config[profile]["titles_only"].isNull()) - { - string titles_only; - cout << "post only titles? [y/n]: "; - std::getline(cin, titles_only); - if (titles_only[0] == 'y') - { - config[profile]["titles_only"] = true; - } - else - { - config[profile]["titles_only"] = false; - } - config_changed = true; - } - if (config[profile]["titles_as_cw"].isNull()) - { - string titles_as_cw; - cout << "Use titles as CW? [y/n]: "; - std::getline(cin, titles_as_cw); - if (titles_as_cw[0] == 'y') - { - config[profile]["titles_as_cw"] = true; - } - else - { - config[profile]["titles_as_cw"] = false; - } - config_changed = true; - } - if (config[profile]["append"].isNull()) - { - string append; - cout << "Append this string to each post []: "; - std::getline(cin, append); - config[profile]["append"] = append; - config_changed = true; - } - if (config[profile]["interval"].isNull()) - { - string interval; - cout << "Interval between posts in seconds [60]: "; - std::getline(cin, interval); - if (interval.empty()) - { - interval = "60"; - } - config[profile]["interval"] = Json::Value::UInt64(std::stoul(interval)); - config_changed = true; - } - if (config_changed) - { - write_config(); - } - - return 0; -} - - -bool write_config() -{ - std::ofstream outfile(filepath + "config-" + profile + ".json"); - if (outfile.is_open()) - { - outfile.write(config.toStyledString().c_str(), - config.toStyledString().length()); - outfile.close(); - - return true; - } - - return false; -} diff --git a/src/http.cpp b/src/http.cpp deleted file mode 100644 index 71c0bd8..0000000 --- a/src/http.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* This file is part of mastorss. - * Copyright © 2018, 2019 tastytea <tastytea@tastytea.de> - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <string> -#include <cstdint> -#include <iostream> -#include <istream> -#include <ostream> -#include <sstream> -#include <curlpp/cURLpp.hpp> -#include <curlpp/Easy.hpp> -#include <curlpp/Options.hpp> -#include <curlpp/Exception.hpp> -#include <curlpp/Infos.hpp> -#include "mastorss.hpp" - -using std::string; -using std::cerr; - -namespace curlopts = curlpp::options; - -void curlpp_init() -{ - curlpp::initialize(); -} - -std::uint16_t http_get(const string &feedurl, string &answer, - const string &useragent) -{ - try - { - std::ostringstream oss; - curlpp::Easy request; - request.setOpt<curlopts::Url>(feedurl); - request.setOpt<curlopts::UserAgent>(useragent); - request.setOpt<curlopts::HttpHeader>( - { - "Connection: close", - }); - request.setOpt<curlopts::FollowLocation>(true); - request.setOpt<curlopts::WriteStream>(&oss); - - request.perform(); - std::uint16_t ret = curlpp::infos::ResponseCode::get(request); - if (ret == 200 || ret == 302 || ret == 307) - { // OK or Found or Temporary Redirect - answer = oss.str(); - } - else if (ret == 301 || ret == 308) - { // Moved Permanently or Permanent Redirect - // FIXME: The new URL should be passed back somehow - answer = oss.str(); - } - else - { - return ret; - } - - return 0; - } - // TODO: More error codes - catch (curlpp::RuntimeError &e) - { - cerr << "RUNTIME ERROR: " << e.what() << std::endl; - return 0xffff; - } - catch (curlpp::LogicError &e) - { - cerr << "LOGIC ERROR: " << e.what() << std::endl; - return 0xffff; - } - - return 0; -} diff --git a/src/mastorss.cpp b/src/mastorss.cpp deleted file mode 100644 index 28e33a8..0000000 --- a/src/mastorss.cpp +++ /dev/null @@ -1,164 +0,0 @@ -/* This file is part of mastorss. - * Copyright © 2018, 2019 tastytea <tastytea@tastytea.de> - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <iostream> -#include <vector> -#include <string> -#include <cstdlib> // getenv() -#include <cstdint> -#include <thread> -#include <chrono> -#include <jsoncpp/json/json.h> -#include <mastodon-cpp/mastodon-cpp.hpp> -#include <mastodon-cpp/easy/all.hpp> -#include "version.hpp" -#include "mastorss.hpp" - -using namespace Mastodon; - -using std::cout; -using std::cerr; -using std::cin; -using std::endl; -using std::string; -using std::this_thread::sleep_for; -using std::chrono::seconds; - -// Initialize global variables -std::uint16_t max_size = 500; -const string filepath = string(getenv("HOME")) + "/.config/mastorss/"; -Json::Value config; -std::string profile; - -int main(int argc, char *argv[]) -{ - if (argc < 2) - { - cerr << "usage: " << argv[0] << " <profile> [max size]\n"; - return 10; - } - - if (argc == 3) - { - max_size = std::stoi(argv[2]); - } - - string instance = ""; - string access_token = ""; - string feedurl = ""; - profile = argv[1]; - std::uint_fast16_t ret; - string answer; - std::vector<Mastodon::Easy::Status> entries; - - read_config(instance, access_token, feedurl); - curlpp_init(); - - ret = http_get(feedurl, answer, "mastorss/" + (string)global::version); - if (ret != 0) - { - std::cerr << "Error code: " << ret << '\n'; - std::cerr << answer << '\n'; - return ret; - } - entries = parse_feed(answer); - - string last_entry = config[profile]["last_entry"].asString(); - if (last_entry.empty()) - { - // If no last_entry is stored in the config file, - // make last_entry the second-newest entry. - last_entry = entries.at(1).content(); - } - config[profile]["last_entry"] = entries.front().content(); - - bool new_content = false; - for (auto rit = entries.rbegin(); rit != entries.rend(); ++rit) - { - if (!new_content && (*rit).content().compare(last_entry) == 0) - { - // If the last entry is found in entries, - // start tooting in the next loop. - new_content = true; - continue; - } - else if (!new_content) - { - continue; - } - - Easy::return_entity<Easy::Status> ret_status; - Mastodon::Easy::API masto(instance, access_token); - - ret_status = masto.send_post(*rit); - - if (!ret_status) - { - const uint8_t err = ret_status.error_code; - switch (err) - { - case 110: - { - cerr << "Error " << err << ": Timeout\n"; - break; - } - case 111: - { - cerr << "Error " << err << ": Connection refused\n"; - cerr << "HTTP Error " << ret_status.http_error_code << endl; - break; - } - case 113: - { - cerr << "Error " << err << ": Could not reach host.\n"; - break; - } - case 192: - case 193: - { - cerr << "Error " << err << ": curlpp error\n"; - break; - } - default: - { - cerr << "Error " << err << '\n'; - cerr << "HTTP status " << ret_status.http_error_code << endl; - } - } - - cerr << ret_status.entity.to_string() << '\n'; - return ret; - } - - if (!ret_status.entity.valid()) - { - cerr << "Could not send post for unknown reasons.\n"; - cerr << "Please file a bug at " - "<https://schlomp.space/tastytea/mastorss/issues>.\n"; - return 1; - } - - if (rit != entries.rend()) - { // Only sleep if this is not the last entry - sleep_for(seconds(config[profile]["interval"].asUInt64())); - } - } - - // Write the new last_entry only if no error happened. - write_config(); - - return 0; -} diff --git a/src/mastorss.hpp b/src/mastorss.hpp deleted file mode 100644 index 7d9a6e8..0000000 --- a/src/mastorss.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* This file is part of mastorss. - * Copyright © 2018, 2019 tastytea <tastytea@tastytea.de> - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#ifndef mastorss_HPP -#define mastorss_HPP - -#include <cstdint> -#include <string> -#include <vector> -#include <jsoncpp/json/json.h> -#include <mastodon-cpp/easy/easy.hpp> - -using std::string; - -extern std::uint16_t max_size; -extern const string filepath; -extern Json::Value config; -extern std::string profile; - -std::uint16_t read_config(string &instance, string &access_token, string &feedurl); -bool write_config(); - -std::vector<Mastodon::Easy::Status> parse_feed(const string &xml); -void individual_fixes(string &str); - -std::uint16_t http_get(const string &feedurl, - string &answer, const string &useragent = ""); -void curlpp_init(); - -#endif // mastorss_HPP diff --git a/src/parse.cpp b/src/parse.cpp deleted file mode 100644 index a6fcd58..0000000 --- a/src/parse.cpp +++ /dev/null @@ -1,210 +0,0 @@ -/* This file is part of mastorss. - * Copyright © 2018, 2019 tastytea <tastytea@tastytea.de> - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <iostream> -#include <vector> -#include <string> -#include <cstdint> -#include <regex> -#include <sstream> -#include <locale> -#include <codecvt> -#include <fstream> -#include <algorithm> -#include <iterator> -#include <jsoncpp/json/json.h> -#include <boost/property_tree/ptree.hpp> -#include <boost/property_tree/xml_parser.hpp> -#include <mastodon-cpp/mastodon-cpp.hpp> -#include <mastodon-cpp/easy/all.hpp> -#include "mastorss.hpp" - -using std::cerr; -using std::string; -namespace pt = boost::property_tree; - -std::vector<Mastodon::Easy::Status> parse_feed(const string &xml) -{ - Json::Value list; - std::vector<string> watchwords; - - std::ifstream file(filepath + "watchwords.json"); - if (file.is_open()) - { - std::stringstream json; - json << file.rdbuf(); - file.close(); - json >> list; - } - else - { - cerr << "WARNING: " << filepath << "watchwords.json not found or not readable.\n"; - } - - // Read profile-specific hashtags or fail silently - const Json::Value &tags_profile = list[profile]["tags"]; - std::transform(tags_profile.begin(), tags_profile.end(), - std::back_inserter(watchwords), - [](const Json::Value &value) - { return value.asString(); }); - - // Read global hashtags or fail silently - const Json::Value &tags_global = list["global"]["tags"]; - std::transform(tags_global.begin(), tags_global.end(), - std::back_inserter(watchwords), - [](const Json::Value &value) - { return value.asString(); }); - - pt::ptree rss; - std::istringstream iss(xml); - pt::read_xml(iss, rss); - std::vector<Mastodon::Easy::Status> ret; - - for (const pt::ptree::value_type &chanchild : rss.get_child("rss.channel")) - { - if (chanchild.second.size() > 0) - { - if (string(chanchild.first.data()).compare("item") == 0) - { - string title = chanchild.second.get_child("title").data(); - string link = chanchild.second.get_child("link").data(); - string desc = chanchild.second.get_child("description").data(); - - Mastodon::Easy::Status status; - string content = ""; - if (config[profile]["titles_as_cw"].asBool()) - { - status.spoiler_text(Mastodon::unescape_html(title)); - } - else - { - content = title; - } - if (!config[profile]["titles_only"].asBool()) - { - if (!content.empty()) - { - content += "\n\n"; - } - content += desc; - - // Shrink overly long texts, to speed up replace operations - if (content.length() > 2000) - { - content.resize(2000); - } - } - - bool skipthis = false; - try - { - // Skip entries beginning with this text - for (const Json::Value &v : config[profile]["skip"]) - { - const string skip = v.asString(); - if (!skip.empty()) - { - if (title.compare(0, skip.length(), skip) == 0) - { - skipthis = true; - break; - } - } - } - } - catch (const std::exception &e) - { - // Node not found, no problem - } - if (skipthis) - { - continue; - } - - content = Mastodon::unescape_html(content); - - // Try to turn the HTML into human-readable text - std::regex reparagraph("<p>"); - std::regex recdata1("<!\\[CDATA\\["); - std::regex recdata2("\\]\\]>"); - std::regex restrip("<[^>]*>"); - - individual_fixes(content); - - content = std::regex_replace(content, reparagraph, "\n\n"); - content = std::regex_replace(content, recdata1, ""); - content = std::regex_replace(content, recdata2, ""); - content = std::regex_replace(content, restrip, ""); - // remove \r - content = std::regex_replace(content, std::regex("\\r"), ""); - // replace NO-BREAK SPACE with space (UTF-8: 0xc2a0) - content = std::regex_replace(content, std::regex("\u00a0"), " "); - // remove whitespace between newlines - content = std::regex_replace(content, std::regex("\\n[ \t]+\\n"), ""); - // remove excess newlines - content = std::regex_replace(content, std::regex("\\n{3,}"), "\n\n"); - - for (const string &hashtag : watchwords) - { - std::regex rehashtag("([[:space:][:punct:]]|^)(" + hashtag - + ")([[:space:][:punct:]]|$)", - std::regex_constants::icase); - content = std::regex_replace(content, rehashtag, "$1#$2$3", - std::regex_constants::format_first_only); - } - // Why is this necessary? Why does ##hashtag happen? - content = std::regex_replace(content, std::regex("##"), "#"); - - uint16_t appendix_size = config[profile]["append"].asString().length(); - if ((status.spoiler_text().size() + content.size() + link.size() + appendix_size) - > static_cast<std::uint16_t>(max_size - 4)) - { - content.resize((max_size - status.spoiler_text().size() - - link.size() - appendix_size - 4)); - content.resize(content.rfind(' ')); // Cut at word boundary - content += " […]"; - } - // Remove trailing newlines - while (content.back() == '\n' || - content.back() == '\r') - { - content.resize(content.length() - 1); - } - - content += "\n\n" + link; - - if (!config[profile]["append"].empty()) - { - content += "\n\n" + config[profile]["append"].asString(); - } - status.content(content); - ret.push_back(status); - } - } - } - - return ret; -} - -// Read regular expressions from the config file and delete all matches. -void individual_fixes(string &str) -{ - for (const Json::Value &v : config[profile]["fixes"]) - { - std::regex refix(v.asString()); - str = std::regex_replace(str, refix, ""); - } -} From e8b0e83bc81c42b31818ab8c865eda91fdc15a14 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Fri, 20 Dec 2019 01:12:14 +0100 Subject: [PATCH 10/62] Update version template. --- src/version.hpp.in | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/version.hpp.in b/src/version.hpp.in index cec7915..ec2ea02 100644 --- a/src/version.hpp.in +++ b/src/version.hpp.in @@ -1,9 +1,13 @@ -#ifndef VERSION_HPP -#define VERSION_HPP +#ifndef MASTORSS_VERSION_HPP +#define MASTORSS_VERSION_HPP -namespace global +#include <string_view> + +namespace mastorss { - static constexpr char version[] = "@PROJECT_VERSION@"; -} +using std::string_view; -#endif // VERSION_HPP +static constexpr string_view version = "@PROJECT_VERSION@"; +} // namespace mastorss + +#endif // MASTORSS_VERSION_HPP From 39ad18efc042e9d095f197b021c4652362f9e4a9 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Fri, 20 Dec 2019 01:13:19 +0100 Subject: [PATCH 11/62] Update CMake config for src/. --- src/CMakeLists.txt | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9d605e6..dca891e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,37 +1,25 @@ include(GNUInstallDirs) -find_package(CURL REQUIRED) -find_package(Boost REQUIRED COMPONENTS system filesystem) -find_package(PkgConfig REQUIRED) -pkg_check_modules(CURLPP REQUIRED curlpp) -pkg_check_modules(JSONCPP REQUIRED jsoncpp) -include_directories(${PROJECT_SOURCE_DIR}/src) -include_directories(${PROJECT_BINARY_DIR}) +# The minimum versions should be in Debian oldstable, if possible. +find_package(Boost 1.62 REQUIRED COMPONENTS system filesystem) +find_package(jsoncpp 1.7 REQUIRED CONFIG) +find_package(CURL 7.52 REQUIRED) +find_package(Threads REQUIRED) +find_package(restclient-cpp 0.5 REQUIRED CONFIG) +find_package(mastodon-cpp REQUIRED CONFIG) -include_directories(${CURL_INCLUDE_DIR}) -include_directories(${CURLPP_INCLUDE_DIR}) -include_directories(${JSONCPP_INCLUDE_DIR}) -include_directories(${Boost_INCLUDE_DIRS}) - -link_directories(${CURL_LIBRARY_DIRS}) -link_directories(${CURLPP_LIBRARY_DIRS}) -link_directories(${JSONCPP_LIBRARY_DIRS}) -link_directories(${Boost_LIBRARY_DIRS}) - -add_definitions(${Boost_DEFINITIONS}) - -# Write version in header +# Write version in header. configure_file ( - "${PROJECT_SOURCE_DIR}/src/version.hpp.in" - "${PROJECT_BINARY_DIR}/version.hpp" -) + "${PROJECT_SOURCE_DIR}/src/version.hpp.in" + "${PROJECT_BINARY_DIR}/version.hpp") + +include_directories("${PROJECT_BINARY_DIR}") file(GLOB sources *.cpp) add_executable(mastorss ${sources}) target_link_libraries(mastorss - mastodon-cpp ${CURLPP_LIBRARIES} - ${JSONCPP_LIBRARIES} ${Boost_LIBRARIES} stdc++fs) -install(TARGETS mastorss DESTINATION ${CMAKE_INSTALL_BINDIR}) + mastodon-cpp::mastodon-cpp Boost::system Boost::filesystem restclient-cpp) +install(TARGETS mastorss DESTINATION "${CMAKE_INSTALL_BINDIR}") install(FILES watchwords.json - DESTINATION ${CMAKE_INSTALL_DATADIR}/mastorss) + DESTINATION "${CMAKE_INSTALL_DATADIR}/mastorss") From 7bba87575f6fd688baa8786e4aa1bd4c9892704f Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Fri, 20 Dec 2019 01:14:23 +0100 Subject: [PATCH 12/62] Print --help, --version and implement downloading. --- man/mastorss.1.adoc | 42 ++++++++++++++++++----- src/document.cpp | 75 ++++++++++++++++++++++++++++++++++++++++ src/document.hpp | 44 ++++++++++++++++++++++++ src/exceptions.cpp | 43 +++++++++++++++++++++++ src/exceptions.hpp | 49 ++++++++++++++++++++++++++ src/main.cpp | 84 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 329 insertions(+), 8 deletions(-) create mode 100644 src/document.cpp create mode 100644 src/document.hpp create mode 100644 src/exceptions.cpp create mode 100644 src/exceptions.hpp create mode 100644 src/main.cpp diff --git a/man/mastorss.1.adoc b/man/mastorss.1.adoc index 7216dd8..83ca3ea 100644 --- a/man/mastorss.1.adoc +++ b/man/mastorss.1.adoc @@ -2,7 +2,7 @@ :doctype: manpage :Author: tastytea :Email: tastytea@tastytea.de -:Date: 2019-12-16 +:Date: 2019-12-20 :Revision: 0.0.0 :man source: mastorss :man manual: General Commands Manual @@ -13,19 +13,44 @@ mastorss - Another RSS to Mastodon bot. == SYNOPSIS -*mastorss* profile +*mastorss* [--help|--version] <profile> // == DESCRIPTION -// == OPTIONS +== OPTIONS + +*--help*:: +Show help message. + +*--version*:: +Show version, copyright and license. // == EXAMPLES -// == PROTOCOL SUPPORT +== PROTOCOL SUPPORT -// Currently only HTTP and HTTPS are supported. +Currently only HTTP and HTTPS are supported. -// == PROXY SUPPORT +// == PROXY SERVERS + +// Since mastorss is built on libcurl, it respects the same proxy environment +// variables. See *curl*(1), section _ENVIRONMENT_. + +// .Tunnel connections through tor. +// ================================================================================ +// [source,shell] +// -------------------------------------------------------------------------------- +// ALL_PROXY="socks4a://[::1]:9050" mastorss example +// -------------------------------------------------------------------------------- +// ================================================================================ + +== PROXY SERVERS + +mastorss supports HTTP proxies set via the environment variable +_http_proxy_. Accepted formats are: _\http://[user[:password]@]host[:port]/_ or +_[user[:password]@]host[:port]_. No SOCKS proxy support yet, sorry. + +Example: `http_proxy="http://localhost:3128/" mastorss` == FILES @@ -39,12 +64,13 @@ mastorss - Another RSS to Mastodon bot. |=========================================================== | Code | Explanation -| 1 | Could not send post for unknown reasons. +| 1 | No profile specified. +| 2 | Network error. |=========================================================== == SEE ALSO -*crontab*(1), *crontab*(5) +*curl*(1), *crontab*(1), *crontab*(5) == REPORTING BUGS diff --git a/src/document.cpp b/src/document.cpp new file mode 100644 index 0000000..0bf97e9 --- /dev/null +++ b/src/document.cpp @@ -0,0 +1,75 @@ +/* This file is part of mastorss. + * Copyright © 2019 tastytea <tastytea@tastytea.de> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "document.hpp" +#include "exceptions.hpp" +#include "version.hpp" + +#include <restclient-cpp/connection.h> +#include <restclient-cpp/restclient.h> + +#include <string> +#include <utility> + +using namespace mastorss; +using std::string; +using std::move; + +Document::Document(string uri) + : _uri{move(uri)} +{ + RestClient::init(); + + download(); +} + +Document::~Document() +{ + RestClient::disable(); +} + +void Document::download() +{ + RestClient::Connection connection(_uri); + connection.SetUserAgent(string("mastorss/").append(version)); + connection.FollowRedirects(true, 10); + + RestClient::Response response{connection.get("")}; + + switch (response.code) + { + case 200: + { + _raw_doc = response.body; + break; + } + case 301: + case 308: + { + // TODO(tastytea): Handle permanent redirections. + throw std::runtime_error{"Permanent redirect, " + "no solution implemented yet."}; + } + case -1: + { + throw CURLException{errno}; + } + default: + { + throw HTTPException{response.code}; + } + } +} diff --git a/src/document.hpp b/src/document.hpp new file mode 100644 index 0000000..1a5ddf1 --- /dev/null +++ b/src/document.hpp @@ -0,0 +1,44 @@ +/* This file is part of mastorss. + * Copyright © 2019 tastytea <tastytea@tastytea.de> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef MASTORSS_DOCUMENT_HPP +#define MASTORSS_DOCUMENT_HPP + +#include <string> + +namespace mastorss +{ +using std::string; + +class Document +{ +public: + explicit Document(string uri); + ~Document(); + Document(const Document &other) = default; + Document &operator=(const Document &other) = delete; + Document(Document &&other) = default; + Document &operator=(Document &&other) = delete; + + void download(); + +private: + const string _uri; + string _raw_doc; +}; +} // namespace mastorss + +#endif // MASTORSS_DOCUMENT_HPP diff --git a/src/exceptions.cpp b/src/exceptions.cpp new file mode 100644 index 0000000..58458b5 --- /dev/null +++ b/src/exceptions.cpp @@ -0,0 +1,43 @@ +/* This file is part of mastorss. + * Copyright © 2019 tastytea <tastytea@tastytea.de> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "exceptions.hpp" + +#include <string> + +using namespace mastorss; +using std::string; +using std::to_string; + +HTTPException::HTTPException(const int error) + : error_code{static_cast<uint16_t>(error)} +{} + +const char *HTTPException::what() const noexcept +{ + static const string error_string{"HTTP error: " + to_string(error_code)}; + return error_string.c_str(); +} + +CURLException::CURLException(const int error) + : error_code{static_cast<uint16_t>(error)} +{} + +const char *CURLException::what() const noexcept +{ + static const string error_string{"libCURL error: " + to_string(error_code)}; + return error_string.c_str(); +} diff --git a/src/exceptions.hpp b/src/exceptions.hpp new file mode 100644 index 0000000..aac1c9d --- /dev/null +++ b/src/exceptions.hpp @@ -0,0 +1,49 @@ +/* This file is part of mastorss. + * Copyright © 2019 tastytea <tastytea@tastytea.de> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef MASTORSS_EXCEPTIONS_HPP +#define MASTORSS_EXCEPTIONS_HPP + +#include <cstdint> +#include <exception> + +namespace mastorss +{ +using std::uint16_t; +using std::exception; + +class HTTPException : public exception +{ +public: + const uint16_t error_code; + + explicit HTTPException(const int error); + + virtual const char *what() const noexcept; +}; + +class CURLException : public exception +{ +public: + const uint16_t error_code; + + explicit CURLException(const int error); + + virtual const char *what() const noexcept; +}; +} // namespace mastorss + +#endif // MASTORSS_EXCEPTIONS_HPP diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..422259e --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,84 @@ +#include "version.hpp" +#include "document.hpp" +#include "exceptions.hpp" + +#include <iostream> +#include <string_view> +#include <vector> + +using namespace mastorss; +using std::cout; +using std::cerr; +using std::string_view; +using std::vector; + +namespace mastorss +{ +namespace error +{ +constexpr int noprofile = 1; +constexpr int network = 2; +} // namespace error + +void print_version(); +void print_help(const string_view &command); + +void print_version() +{ + cout << "mastorss " << version << "\n" + "Copyright (C) 2019 tastytea <tastytea@tastytea.de>\n" + "License GPLv3: GNU GPL version 3 " + "<https://www.gnu.org/licenses/gpl-3.0.html>.\n" + "This program comes with ABSOLUTELY NO WARRANTY. " + "This is free software,\n" + "and you are welcome to redistribute it under certain conditions.\n"; +} + +void print_help(const string_view &command) +{ + cerr << "Usage: " << command << " [--version] <profile>\n"; +} +} // namespace mastorss + +int main(int argc, char *argv[]) +{ + const vector<string_view> args(argv, argv + argc); + + if (args.size() == 1) + { + print_help(args[0]); + return error::noprofile; + } + + if (args.size() > 1) + { + if (args[1] == "--version") + { + print_version(); + } + else if (args[1] == "--help") + { + print_help(args[0]); + } + else + { + try + { + Document doc("https://ip.tastytea.de/"); + } + catch (const HTTPException &e) + { + cerr << e.what() << '\n'; + return error::network; + } + catch (const CURLException &e) + { + cerr << e.what() << '\n'; + return error::network; + } + + } + } + + return 0; +} From bf29df39e9c17be0151a3c8531b7b6aa783fb18d Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Tue, 24 Dec 2019 18:53:00 +0100 Subject: [PATCH 13/62] Read and parse config. --- man/mastorss.1.adoc | 3 + src/config.cpp | 243 ++++++++++++++++++++++++++++++++++++++++++++ src/config.hpp | 83 +++++++++++++++ src/exceptions.cpp | 24 ++++- src/exceptions.hpp | 35 ++++++- src/main.cpp | 28 ++++- 6 files changed, 407 insertions(+), 9 deletions(-) create mode 100644 src/config.cpp create mode 100644 src/config.hpp diff --git a/man/mastorss.1.adoc b/man/mastorss.1.adoc index 83ca3ea..15c6d92 100644 --- a/man/mastorss.1.adoc +++ b/man/mastorss.1.adoc @@ -66,6 +66,9 @@ Example: `http_proxy="http://localhost:3128/" mastorss` | 1 | No profile specified. | 2 | Network error. +| 3 | File error. +| 4 | Mastodon error. +| 9 | Unknown error. |=========================================================== == SEE ALSO diff --git a/src/config.cpp b/src/config.cpp new file mode 100644 index 0000000..7248ae8 --- /dev/null +++ b/src/config.cpp @@ -0,0 +1,243 @@ +/* This file is part of mastorss. + * Copyright © 2019 tastytea <tastytea@tastytea.de> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "config.hpp" +#include "exceptions.hpp" + +#include <boost/log/trivial.hpp> +#include <mastodon-cpp/mastodon-cpp.hpp> + +#include <cstdlib> +#include <fstream> +#include <iostream> +#include <sstream> +#include <stdexcept> +#include <utility> + +namespace mastorss +{ +using std::getenv; +using std::ifstream; +using std::ofstream; +using std::cin; +using std::cout; +using std::stringstream; +using std::runtime_error; +using std::getline; +using std::move; + +std::ostream &operator <<(std::ostream &out, const ProfileData &data) +{ + out << "access_token: \"" << data.access_token << "\", " + << "append: \"" << data.append << "\", " + << "feedurl: \"" << data.feedurl << "\", " + << "fixes: ["; + for (const auto &fix : data.fixes) + { + out << '"' << fix << '"'; + if (fix != *data.fixes.rbegin()) + { + out << ", "; + } + } + out << "], " + << "instance: \"" << data.instance << "\", " + << "interval: " << data.interval << ", " + << "max_size: " << data.max_size << ", " + << "skip: ["; + for (const auto &skip : data.skip) + { + out << '"' << skip << '"'; + if (skip != *data.skip.rbegin()) + { + out << ", "; + } + } + out << "], " + << "titles_as_cw: " << data.titles_as_cw << ", " + << "titles_only: " << data.titles_only; + + return out; +} + +Config::Config(string profile) + :_profile{move(profile)} +{ + const fs::path filename = get_filename(); + BOOST_LOG_TRIVIAL(debug) << "Config filename is: " << filename; + + ifstream file{filename}; + if (file.good()) + { + stringstream rawjson; + rawjson << file.rdbuf(); + rawjson >> _json; + } + else + { + generate(); + } + + parse(); +} + +fs::path Config::get_filename() const +{ + char *envdir = getenv("XDG_CONFIG_HOME"); + fs::path dir; + + if (envdir != nullptr) + { + dir = envdir; + } + else + { + envdir = getenv("HOME"); + if (envdir != nullptr) + { + dir = fs::path{envdir} /= ".config"; + } + else + { + throw FileException{"Couldn't find configuration directory."}; + } + } + + return (dir /= "mastorss") /= "config-" + _profile + ".json"; +} + +void Config::generate() +{ + Json::Value newjson; + string line; + + cout << "Instance (domain): "; + getline(cin, line); + newjson[_profile]["instance"] = line; + + newjson[_profile]["access_token"] = get_access_token(line); + + cout << "URL of the feed: "; + std::getline(cin, line); + newjson[_profile]["feedurl"] = line; + + cout << "Post only titles? [y/n]: "; + std::getline(cin, line); + if (line[0] == 'y') + { + newjson[_profile]["titles_as_cw"] = true; + } + else + { + newjson[_profile]["titles_as_cw"] = false; + } + + cout << "Append this string to each post: "; + std::getline(cin, line); + newjson[_profile]["append"] = line; + + cout << "Interval between posts in seconds [30]: "; + std::getline(cin, line); + if (line.empty()) + { + line = "30"; + } + newjson[_profile]["interval"] = std::stoul(line); + + cout << "Maximum size of posts [500]: "; + std::getline(cin, line); + if (line.empty()) + { + line = "500"; + } + newjson[_profile]["max_size"] = std::stoul(line); + + _json = newjson; + ofstream file{get_filename()}; + if (file.good()) + { + file << newjson.toStyledString(); + } + + BOOST_LOG_TRIVIAL(debug) << "Wrote config file."; +} + +string Config::get_access_token(const string &instance) const +{ + string client_id; + string client_secret; + string url; + Mastodon::API masto(instance, ""); + + Mastodon::return_call ret + { + masto.register_app1("mastorss", "urn:ietf:wg:oauth:2.0:oob", + "write", + "https://schlomp.space/tastytea/mastorss", + client_id, client_secret, url) + }; + + if (ret) + { + string code; + string access_token; + + cout << "Visit " << url << " to authorize this application.\n"; + cout << "Insert code: "; + std::getline(cin, code); + ret = masto.register_app2(client_id, client_secret, + "urn:ietf:wg:oauth:2.0:oob", + code, access_token); + if (ret) + { + BOOST_LOG_TRIVIAL(debug) << "Got access token: " << access_token; + return access_token; + } + } + + throw MastodonException{ret.error_code}; +} + +void Config::parse() +{ + data.access_token = _json[_profile]["access_token"].asString(); + data.append = _json[_profile]["append"].asString(); + data.feedurl = _json[_profile]["feedurl"].asString(); + for (const auto &fix : _json[_profile]["fixes"]) + { + data.fixes.push_back(fix.asString()); + } + data.instance = _json[_profile]["instance"].asString(); + if (!_json[_profile]["interval"].isNull()) + { + data.interval = + static_cast<uint32_t>(_json[_profile]["interval"].asUInt64()); + } + if (!_json[_profile]["max_size"].isNull()) + { + data.max_size = + static_cast<uint32_t>(_json[_profile]["max_size"].asUInt64()); + } + for (const auto &skip : _json[_profile]["skip"]) + { + data.skip.push_back(skip.asString()); + } + data.titles_as_cw = _json[_profile]["titles_as_cw"].asBool(); + data.titles_only = _json[_profile]["titles_only"].asBool(); + + BOOST_LOG_TRIVIAL(debug) << "Read config: " << data; +} +} // namespace mastorss diff --git a/src/config.hpp b/src/config.hpp new file mode 100644 index 0000000..3e0bbd1 --- /dev/null +++ b/src/config.hpp @@ -0,0 +1,83 @@ +/* This file is part of mastorss. + * Copyright © 2019 tastytea <tastytea@tastytea.de> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef MASTORSS_CONFIG_HPP +#define MASTORSS_CONFIG_HPP + +#include <boost/filesystem.hpp> +#include <jsoncpp/json/json.h> + +#include <cstdint> +#include <string> +#include <string_view> +#include <vector> + +namespace mastorss +{ +namespace fs = boost::filesystem; +using std::uint32_t; +using std::string; +using std::string_view; +using std::vector; + +/*! + * @brief The configuration for a profile as data structure. + * + * @since 0.10.0 + */ +struct ProfileData +{ + string access_token; + string append; + string feedurl; + vector<string> fixes; + string instance; + uint32_t interval{30}; + uint32_t max_size{500}; + vector<string> skip; + bool titles_as_cw{false}; + bool titles_only{false}; + + friend std::ostream &operator <<(std::ostream &out, + const ProfileData &data); +}; + +/*! + * @brief A configuration file. + * + * @since 0.10.0 + */ +class Config +{ +public: + explicit Config(string profile); + + ProfileData data; + +private: + const string _profile; + Json::Value _json; + + [[nodiscard]] + fs::path get_filename() const; + void generate(); + [[nodiscard]] + string get_access_token(const string &instance) const; + void parse(); +}; +} // namespace mastorss + +#endif // MASTORSS_CONFIG_HPP diff --git a/src/exceptions.cpp b/src/exceptions.cpp index 58458b5..1a2587e 100644 --- a/src/exceptions.cpp +++ b/src/exceptions.cpp @@ -16,11 +16,11 @@ #include "exceptions.hpp" -#include <string> +#include <utility> using namespace mastorss; -using std::string; using std::to_string; +using std::move; HTTPException::HTTPException(const int error) : error_code{static_cast<uint16_t>(error)} @@ -41,3 +41,23 @@ const char *CURLException::what() const noexcept static const string error_string{"libCURL error: " + to_string(error_code)}; return error_string.c_str(); } + +MastodonException::MastodonException(const int error) + : error_code{static_cast<uint16_t>(error)} +{} + +const char *MastodonException::what() const noexcept +{ + static const string error_string{"Mastodon error: " + + to_string(error_code)}; + return error_string.c_str(); +} + +FileException::FileException(string message) + : _message{move(message)} +{} + +const char *FileException::what() const noexcept +{ + return _message.c_str(); +} diff --git a/src/exceptions.hpp b/src/exceptions.hpp index aac1c9d..555e7a4 100644 --- a/src/exceptions.hpp +++ b/src/exceptions.hpp @@ -19,20 +19,23 @@ #include <cstdint> #include <exception> +#include <string> namespace mastorss { using std::uint16_t; using std::exception; +using std::string; class HTTPException : public exception { public: const uint16_t error_code; - explicit HTTPException(const int error); + explicit HTTPException(int error); - virtual const char *what() const noexcept; + [[nodiscard]] + const char *what() const noexcept override; }; class CURLException : public exception @@ -40,9 +43,33 @@ class CURLException : public exception public: const uint16_t error_code; - explicit CURLException(const int error); + explicit CURLException(int error); - virtual const char *what() const noexcept; + [[nodiscard]] + const char *what() const noexcept override; +}; + +class MastodonException : public exception +{ +public: + const uint16_t error_code; + + explicit MastodonException(int error); + + [[nodiscard]] + const char *what() const noexcept override; +}; + +class FileException : public exception +{ +public: + explicit FileException(string message); + + [[nodiscard]] + const char *what() const noexcept override; + +private: + const string _message; }; } // namespace mastorss diff --git a/src/main.cpp b/src/main.cpp index 422259e..4e5c6d3 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,6 +1,7 @@ -#include "version.hpp" +#include "config.hpp" #include "document.hpp" #include "exceptions.hpp" +#include "version.hpp" #include <iostream> #include <string_view> @@ -18,6 +19,9 @@ namespace error { constexpr int noprofile = 1; constexpr int network = 2; +constexpr int file = 3; +constexpr int mastodon = 4; +constexpr int unknown = 9; } // namespace error void print_version(); @@ -62,9 +66,23 @@ int main(int argc, char *argv[]) } else { + const string_view profile = args[1]; + BOOST_LOG_TRIVIAL(debug) << "Using profile: " << profile; + try { - Document doc("https://ip.tastytea.de/"); + Config cfg(profile.data()); + Document doc(cfg.data.feedurl); + } + catch (const FileException &e) + { + cerr << e.what() << '\n'; + return error::file; + } + catch (const MastodonException &e) + { + cerr << e.what() << '\n'; + return error::mastodon; } catch (const HTTPException &e) { @@ -76,7 +94,11 @@ int main(int argc, char *argv[]) cerr << e.what() << '\n'; return error::network; } - + catch (const runtime_error &e) + { + cerr << e.what() << '\n'; + return error::unknown; + } } } From c631e5c4fea63df28ae01dac2483c3f91eaff2ea Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Tue, 24 Dec 2019 18:53:45 +0100 Subject: [PATCH 14/62] Add debug logging. --- man/mastorss.1.adoc | 14 +++++++++++++- src/CMakeLists.txt | 9 +++++++-- src/document.cpp | 6 +++++- src/main.cpp | 21 ++++++++++++++++++++- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/man/mastorss.1.adoc b/man/mastorss.1.adoc index 15c6d92..4d15d64 100644 --- a/man/mastorss.1.adoc +++ b/man/mastorss.1.adoc @@ -2,7 +2,7 @@ :doctype: manpage :Author: tastytea :Email: tastytea@tastytea.de -:Date: 2019-12-20 +:Date: 2019-12-24 :Revision: 0.0.0 :man source: mastorss :man manual: General Commands Manual @@ -71,6 +71,18 @@ Example: `http_proxy="http://localhost:3128/" mastorss` | 9 | Unknown error. |=========================================================== +== DEBUGGING + +Define the variable `MASTORSS_DEBUG` to enable debug output. + +.Debug mastorss while using the profile “example”. +================================================================================ +[source,shell] +-------------------------------------------------------------------------------- +MASTORSS_DEBUG=1 mastorss example +-------------------------------------------------------------------------------- +================================================================================ + == SEE ALSO *curl*(1), *crontab*(1), *crontab*(5) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dca891e..e1ac829 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,7 @@ include(GNUInstallDirs) # The minimum versions should be in Debian oldstable, if possible. -find_package(Boost 1.62 REQUIRED COMPONENTS system filesystem) +find_package(Boost 1.62 REQUIRED COMPONENTS system filesystem log) find_package(jsoncpp 1.7 REQUIRED CONFIG) find_package(CURL 7.52 REQUIRED) find_package(Threads REQUIRED) @@ -18,7 +18,12 @@ include_directories("${PROJECT_BINARY_DIR}") file(GLOB sources *.cpp) add_executable(mastorss ${sources}) target_link_libraries(mastorss - mastodon-cpp::mastodon-cpp Boost::system Boost::filesystem restclient-cpp) + PRIVATE + jsoncpp restclient-cpp mastodon-cpp::mastodon-cpp + Boost::filesystem Boost::log) +if(BUILD_SHARED_LIBS) + target_compile_definitions(mastorss PRIVATE "BOOST_ALL_DYN_LINK=1") +endif() install(TARGETS mastorss DESTINATION "${CMAKE_INSTALL_BINDIR}") install(FILES watchwords.json diff --git a/src/document.cpp b/src/document.cpp index 0bf97e9..bd7531b 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -18,13 +18,15 @@ #include "exceptions.hpp" #include "version.hpp" +#include <boost/log/trivial.hpp> #include <restclient-cpp/connection.h> #include <restclient-cpp/restclient.h> #include <string> #include <utility> -using namespace mastorss; +namespace mastorss +{ using std::string; using std::move; @@ -54,6 +56,7 @@ void Document::download() case 200: { _raw_doc = response.body; + BOOST_LOG_TRIVIAL(debug) << "Downloaded feed: " << _uri; break; } case 301: @@ -73,3 +76,4 @@ void Document::download() } } } +} // namespace mastorss diff --git a/src/main.cpp b/src/main.cpp index 4e5c6d3..0f6d1de 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3,13 +3,21 @@ #include "exceptions.hpp" #include "version.hpp" +#include <boost/log/core.hpp> +#include <boost/log/trivial.hpp> +#include <boost/log/utility/setup/console.hpp> + +#include <cstdlib> #include <iostream> +#include <stdexcept> #include <string_view> #include <vector> using namespace mastorss; +using std::getenv; using std::cout; using std::cerr; +using std::runtime_error; using std::string_view; using std::vector; @@ -40,7 +48,7 @@ void print_version() void print_help(const string_view &command) { - cerr << "Usage: " << command << " [--version] <profile>\n"; + cerr << "Usage: " << command << " [--version|--help] <profile>\n"; } } // namespace mastorss @@ -48,6 +56,17 @@ int main(int argc, char *argv[]) { const vector<string_view> args(argv, argv + argc); + if (getenv("MASTORSS_DEBUG") == nullptr) + { + boost::log::core::get()->set_filter + (boost::log::trivial::severity >= boost::log::trivial::info); + } + else + { + boost::log::core::get()->set_filter + (boost::log::trivial::severity >= boost::log::trivial::debug); + } + if (args.size() == 1) { print_help(args[0]); From a369e9617757d0acb0de1c9f010ec6eef87ab717 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Tue, 24 Dec 2019 22:35:49 +0100 Subject: [PATCH 15/62] Add last_guid to ProfileData. --- src/config.cpp | 2 ++ src/config.hpp | 1 + 2 files changed, 3 insertions(+) diff --git a/src/config.cpp b/src/config.cpp index 7248ae8..6eb25d9 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -56,6 +56,7 @@ std::ostream &operator <<(std::ostream &out, const ProfileData &data) out << "], " << "instance: \"" << data.instance << "\", " << "interval: " << data.interval << ", " + << "last_guid: \"" << data.last_guid << "\", " << "max_size: " << data.max_size << ", " << "skip: ["; for (const auto &skip : data.skip) @@ -226,6 +227,7 @@ void Config::parse() data.interval = static_cast<uint32_t>(_json[_profile]["interval"].asUInt64()); } + data.last_guid = _json[_profile]["last_guid"].asString(); if (!_json[_profile]["max_size"].isNull()) { data.max_size = diff --git a/src/config.hpp b/src/config.hpp index 3e0bbd1..eee785f 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -46,6 +46,7 @@ struct ProfileData vector<string> fixes; string instance; uint32_t interval{30}; + string last_guid; uint32_t max_size{500}; vector<string> skip; bool titles_as_cw{false}; From 73542eff48151e33d9971416c080407da26dfde5 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Tue, 24 Dec 2019 23:24:24 +0100 Subject: [PATCH 16/62] Update description. --- README.adoc | 4 ++-- man/mastorss.1.adoc | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.adoc b/README.adoc index 405a45d..d3ae9af 100644 --- a/README.adoc +++ b/README.adoc @@ -10,8 +10,8 @@ :uri-mastodon-cpp: https://schlomp.space/tastytea/mastodon-cpp :uri-jsoncpp: https://github.com/open-source-parsers/jsoncpp -*mastorss* dumps RSS feeds into a mastodon account. Supports RSS 2.0 but not -RSS 0.92. Does not support Atom at the moment. +*mastorss* reads RSS feeds and posts the items via the Mastodon API. Does not +support Atom at the moment. Each `<item>` in feeds must have `<link>`, `<title>` and `<description>`. diff --git a/man/mastorss.1.adoc b/man/mastorss.1.adoc index 4d15d64..3db60fc 100644 --- a/man/mastorss.1.adoc +++ b/man/mastorss.1.adoc @@ -15,7 +15,10 @@ mastorss - Another RSS to Mastodon bot. *mastorss* [--help|--version] <profile> -// == DESCRIPTION +== DESCRIPTION + +*mastorss* reads RSS feeds and posts the items via the Mastodon API. Does not +support Atom at the moment. == OPTIONS From 91acfb0bc3a98a5fbb0762e99feb9bd9f735195d Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 02:08:16 +0100 Subject: [PATCH 17/62] Update versions in rteadme. --- README.adoc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.adoc b/README.adoc index d3ae9af..9f516a1 100644 --- a/README.adoc +++ b/README.adoc @@ -20,11 +20,11 @@ Each `<item>` in feeds must have `<link>`, `<title>` and `<description>`. === Dependencies * Tested OS: Linux -* C++ compiler (tested: link:{uri-gcc}[gcc] 5 / 8) -* link:{uri-cmake}[cmake] (tested: 3.14 / 3.9) -* link:{uri-boost}[boost] (tested: 1.71 / 1.62) +* C++ compiler (tested: link:{uri-gcc}[gcc] 9) +* link:{uri-cmake}[cmake] (at least: 3.9) +* link:{uri-boost}[boost] (at least: 1.62) * link:{uri-mastodon-cpp}[mastodon-cpp] (at least: 0.111.3) -* link:{uri-jsoncpp}[jsoncpp] (tested: 1.9 / 1.7) +* link:{uri-jsoncpp}[jsoncpp] (at least: 1.7) === Get sourcecode @@ -51,9 +51,9 @@ Install with `make install`. Put `watchwords.json` into `${XDG_CONFIG_HOME}/mastorss/` footnote:[`${XDG_CONFIG_HOME}` is usually `~/.config/`]. Launch with profile -name. The first occurence of every watchword in an RSS item will be turned into -a hashtag. For profile-specific watchwords see the example in -`watchwords.json`. In the first run only the newest entry is posted. +name. The first occurence of every watchword in an RSS item will be turned into +a hashtag. For profile-specific watchwords see the example in +`watchwords.json`. In the first run only the newest entry is posted. The profile can't be named "global". From 2bb03dd10470ba150047ff2ef3a036d6450dc855 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 02:08:53 +0100 Subject: [PATCH 18/62] Extract and parse <item>s from RSS feeds. --- src/document.cpp | 103 +++++++++++++++++++++++++++++++++++++++++++++-- src/document.hpp | 36 ++++++++++++++++- src/main.cpp | 11 +++-- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/src/document.cpp b/src/document.cpp index bd7531b..665a6b3 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -19,19 +19,28 @@ #include "version.hpp" #include <boost/log/trivial.hpp> +#include <boost/property_tree/xml_parser.hpp> +#include <boost/regex.hpp> +#include <mastodon-cpp/mastodon-cpp.hpp> #include <restclient-cpp/connection.h> #include <restclient-cpp/restclient.h> +#include <list> +#include <sstream> #include <string> #include <utility> namespace mastorss { +using boost::regex; +using boost::regex_replace; +using std::list; +using std::istringstream; using std::string; using std::move; -Document::Document(string uri) - : _uri{move(uri)} +Document::Document(const ProfileData &data) + : _data{data} { RestClient::init(); @@ -45,7 +54,7 @@ Document::~Document() void Document::download() { - RestClient::Connection connection(_uri); + RestClient::Connection connection(_data.feedurl); connection.SetUserAgent(string("mastorss/").append(version)); connection.FollowRedirects(true, 10); @@ -56,7 +65,7 @@ void Document::download() case 200: { _raw_doc = response.body; - BOOST_LOG_TRIVIAL(debug) << "Downloaded feed: " << _uri; + BOOST_LOG_TRIVIAL(debug) << "Downloaded feed: " << _data.feedurl; break; } case 301: @@ -76,4 +85,90 @@ void Document::download() } } } + +void Document::parse() +{ + pt::ptree tree; + istringstream iss{_raw_doc}; + pt::read_xml(iss, tree); + + if (tree.front().first == "rss") + { + parse_rss(tree); + } +} + +void Document::parse_rss(const pt::ptree &tree) +{ + for (const auto &child : tree.get_child("rss.channel")) + { + if (child.first == "item") + { + const auto &rssitem = child.second; + + string guid{rssitem.get<string>("guid")}; + if (guid.empty()) // We hope either <guid> or <link> are present. + { + guid = rssitem.get<string>("link"); + } + if (guid == _data.last_guid) + { + break; + } + + bool skipthis{false}; + string title = rssitem.get<string>("title"); + for (const auto &skip : _data.skip) + { + if (title.substr(0, skip.length()) == skip) + { + skipthis = true; + break; + } + } + if (skipthis) + { + BOOST_LOG_TRIVIAL(debug) << "Skipped GUID: " << guid; + continue; + } + + Item item; + item.description = remove_html(rssitem.get<string>("description")); + item.guid = move(guid); + item.link = rssitem.get<string>("link"); + item.title = move(title); + new_items.push_back(item); + + BOOST_LOG_TRIVIAL(debug) << "Found GUID: " << item.guid; + } + } +} + +string Document::remove_html(string html) const +{ + html = Mastodon::unescape_html(html); // Decode HTML entities. + + html = regex_replace(html, regex{"<p>"}, "\n\n"); + + const list re_list + { + regex{R"(<!\[CDATA\[)"}, // CDATA beginning. + regex{R"(\]\]>)"}, // CDATA end. + regex{"<[^>]+>"}, // HTML tags. + regex{R"(\r)"}, // Carriage return. + regex{"\\n[ \\t\u00a0]+\\n"}, // Whitespace between newlines. + regex{R"(^\n+)"} // Newlines at the beginning. + }; + for (const regex &re : re_list) + { + html = regex_replace(html, re, ""); + } + + // Remove excess newlines. + html = regex_replace(html, regex{R"(\n{3,})"}, "\n\n"); + // Replace single newlines with spaces (?<= is lookbehind, ?= is lookahead). + html = regex_replace(html, regex{R"((?<=[^\n])\n(?=[^\n]))"}, " "); + + return html; +} } // namespace mastorss diff --git a/src/document.hpp b/src/document.hpp index 1a5ddf1..b69267b 100644 --- a/src/document.hpp +++ b/src/document.hpp @@ -17,27 +17,59 @@ #ifndef MASTORSS_DOCUMENT_HPP #define MASTORSS_DOCUMENT_HPP +#include "config.hpp" + +#include <boost/property_tree/ptree.hpp> + #include <string> +#include <vector> namespace mastorss { +namespace pt = boost::property_tree; using std::string; +using std::vector; +/*! + * @brief An Item of a feed. + * + * @since 0.10.0 + */ +struct Item +{ + string description; + string guid; + string link; + string title; +}; + +/*! + * @brief A feed. + * + * @since 0.10.0 + */ class Document { public: - explicit Document(string uri); + explicit Document(const ProfileData &data); ~Document(); Document(const Document &other) = default; Document &operator=(const Document &other) = delete; Document(Document &&other) = default; Document &operator=(Document &&other) = delete; + vector<Item> new_items; + void download(); + void parse(); private: - const string _uri; + const ProfileData &_data; string _raw_doc; + + void parse_rss(const pt::ptree &tree); + [[nodiscard]] + string remove_html(string html) const; }; } // namespace mastorss diff --git a/src/main.cpp b/src/main.cpp index 0f6d1de..4da5a97 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -85,13 +85,18 @@ int main(int argc, char *argv[]) } else { - const string_view profile = args[1]; + const string_view profile{args[1]}; BOOST_LOG_TRIVIAL(debug) << "Using profile: " << profile; try { - Config cfg(profile.data()); - Document doc(cfg.data.feedurl); + Config cfg{profile.data()}; + Document doc{cfg.data}; + doc.parse(); + for (const auto &item : doc.new_items) + { + cout << "--\n" << item.description.substr(0, 200) << "\n"; + } } catch (const FileException &e) { From 27ac10b821ce4e8462ff78e690aaa865f6167871 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 02:09:22 +0100 Subject: [PATCH 19/62] Typo in manpage. --- man/mastorss.1.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/man/mastorss.1.adoc b/man/mastorss.1.adoc index 3db60fc..16189a5 100644 --- a/man/mastorss.1.adoc +++ b/man/mastorss.1.adoc @@ -2,7 +2,7 @@ :doctype: manpage :Author: tastytea :Email: tastytea@tastytea.de -:Date: 2019-12-24 +:Date: 2019-12-25 :Revision: 0.0.0 :man source: mastorss :man manual: General Commands Manual @@ -70,7 +70,7 @@ Example: `http_proxy="http://localhost:3128/" mastorss` | 1 | No profile specified. | 2 | Network error. | 3 | File error. -| 4 | Mastodon error. +| 4 | Mastodon API error. | 9 | Unknown error. |=========================================================== From e1a973f11faa6b776cb169fd0a31bcab5d13682d Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 02:42:27 +0100 Subject: [PATCH 20/62] Apply user-fixes. --- src/document.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/document.cpp b/src/document.cpp index 665a6b3..6ddfe25 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -133,7 +133,17 @@ void Document::parse_rss(const pt::ptree &tree) } Item item; - item.description = remove_html(rssitem.get<string>("description")); + item.description = + [&] + { + string desc = + remove_html(rssitem.get<string>("description")); + for (const auto &fix : _data.fixes) + { + desc = regex_replace(desc, regex{fix}, ""); + } + return desc; + }(); item.guid = move(guid); item.link = rssitem.get<string>("link"); item.title = move(title); From f6b5ccbe22b3db14fa245b000bbfaa7ef0e50c50 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 02:42:47 +0100 Subject: [PATCH 21/62] Add a bit of logging. --- src/document.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/document.cpp b/src/document.cpp index 6ddfe25..dc631a8 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -94,6 +94,7 @@ void Document::parse() if (tree.front().first == "rss") { + BOOST_LOG_TRIVIAL(debug) << "RSS detected."; parse_rss(tree); } } @@ -179,6 +180,8 @@ string Document::remove_html(string html) const // Replace single newlines with spaces (?<= is lookbehind, ?= is lookahead). html = regex_replace(html, regex{R"((?<=[^\n])\n(?=[^\n]))"}, " "); + BOOST_LOG_TRIVIAL(debug) << "Converted HTML to text."; + return html; } } // namespace mastorss From b76f79632c9205905647bb1d3bedb32c2e2d15d3 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 03:29:30 +0100 Subject: [PATCH 22/62] Document usage and configuration in manpage. --- README.adoc | 46 +----------------- man/mastorss.1.adoc | 111 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 104 insertions(+), 53 deletions(-) diff --git a/README.adoc b/README.adoc index 9f516a1..c651d5e 100644 --- a/README.adoc +++ b/README.adoc @@ -2,6 +2,7 @@ :project: mastorss :uri-base: https://schlomp.space/tastytea/{project} +:uri-branch-main: {uri-base}/src/branch/main :uri-gcc: https://gcc.gnu.org/ :uri-cmake: https://cmake.org/ :uri-asciidoc: http://asciidoc.org/ @@ -49,50 +50,7 @@ Install with `make install`. == Usage -Put `watchwords.json` into `${XDG_CONFIG_HOME}/mastorss/` -footnote:[`${XDG_CONFIG_HOME}` is usually `~/.config/`]. Launch with profile -name. The first occurence of every watchword in an RSS item will be turned into -a hashtag. For profile-specific watchwords see the example in -`watchwords.json`. In the first run only the newest entry is posted. - -The profile can't be named "global". - -=== Example session - -[source,shellsession] --------------------------------------------------------------------------------- -% mastorss example --------------------------------------------------------------------------------- - -=== Example config file - -${HOME}/.config/mastorss/config-example.json - -[source,json] --------------------------------------------------------------------------------- - { - "example": { - "access_token": "123abc", - "append": "#bot", - "feedurl": "https:\/\/example.com\/feed.rss", - "fixes": - [ - "delete this", - "<p>[Rr]ead more(\.{3}|…)</p>" - ], - "instance": "botsin.space", - "interval" : 600, - "max_size": 400, - "skip": - [ - "If the entry starts with this, skip it", - "Skip me too!" - ], - "titles_as_cw" : false, - "titles_only": false - } - } --------------------------------------------------------------------------------- +See link:{uri-branch-main}/man/mastorss.1.adoc[manpage]. === Error codes diff --git a/man/mastorss.1.adoc b/man/mastorss.1.adoc index 16189a5..db32774 100644 --- a/man/mastorss.1.adoc +++ b/man/mastorss.1.adoc @@ -28,7 +28,93 @@ Show help message. *--version*:: Show version, copyright and license. -// == EXAMPLES +== USAGE + +Put `watchwords.json` into `${XDG_CONFIG_HOME}/mastorss/`. Launch with profile +name. The first occurence of every watchword in an RSS item will be turned into +a hashtag. For profile-specific watchwords see the example in +`watchwords.json`. In the first run only the newest entry is posted. + +The profile is the identifier for a feed and can't be named "global". + +.Launch mastorss with the profile “example”. +================================================================================ +[source,shellsession] +-------------------------------------------------------------------------------- +% mastorss example +-------------------------------------------------------------------------------- +================================================================================ + +=== Configuration + +If the profile does not exist yet, a configuration will be created interactively +and then saved to `${XDG_CONFIG_HOME}/mastorss/config-<profile>.json`. + +*access_token*:: +The API token needed to communicate with the Mastodon API on the _instance_ +you're using to post. + +*append*:: +This string will be appended to every post. + +*feedurl*:: +The URI of the source feed. + +*fixes*:: +Array of regular expressions that should be deleted from the text. For +information about the syntax see *perlre*(1). + +*instance*:: +Hostname of the instance you're using to post. + +*interval*:: +Time to wait between posts. + +*max_size*:: +Maximum number of characters a post can have. + +*skip*:: +Array of strings with titles of posts that should be skipped. Matched against +the beginning of the title; this means: “Good Day” will match “Good Day in +Happytown” but not “A Good Day in Happytown”. + +*titles_as_cw*:: +If true, the title will be used as the subject (also known as spoiler warning or +content warning) of the post. + +*titles_only*:: +If true, only post titles, no descriptions. + +== EXAMPLES + +=== Configuration file + +[source,json] +-------------------------------------------------------------------------------- +{ + "example" : + { + "access_token" : "123abc", + "append" : "#bot", + "feedurl" : "https://example.com/feed.rss", + "fixes" : + [ + "delete this", + "<p>[Rr]ead more(\.{3}|…)</p>" + ], + "instance" : "newsbots.eu", + "interval" : 600, + "max_size" : 500, + "skip" : + [ + "If the title starts with this, skip it", + "Skip me too!" + ], + "titles_as_cw" : true, + "titles_only" : false + } +} +-------------------------------------------------------------------------------- == PROTOCOL SUPPORT @@ -50,14 +136,21 @@ Currently only HTTP and HTTPS are supported. == PROXY SERVERS mastorss supports HTTP proxies set via the environment variable -_http_proxy_. Accepted formats are: _\http://[user[:password]@]host[:port]/_ or -_[user[:password]@]host[:port]_. No SOCKS proxy support yet, sorry. +_http_proxy_. Accepted format: _\http://[user[:password]@]host[:port]/_ No SOCKS +proxy support yet, sorry. -Example: `http_proxy="http://localhost:3128/" mastorss` +.Use mastorss with a proxy server +================================================================================ +[source,shellsession] +-------------------------------------------------------------------------------- +% http_proxy="http://localhost:3128/" mastorss example +-------------------------------------------------------------------------------- +================================================================================ == FILES -* *Configuration file directory*: `${XDG_CONFIG_HOME}/mastorss/` +*Configuration file directory*:: +`${XDG_CONFIG_HOME}/mastorss/` `${XDG_CONFIG_HOME}` is usually `~/.config`. @@ -86,12 +179,12 @@ MASTORSS_DEBUG=1 mastorss example -------------------------------------------------------------------------------- ================================================================================ -== SEE ALSO - -*curl*(1), *crontab*(1), *crontab*(5) - == REPORTING BUGS Bugtracker: https://schlomp.space/tastytea/mastorss/issues E-mail: tastytea@tastytea.de + +== SEE ALSO + +*perlre*(1), *crontab*(1), *crontab*(5) From f1be8c49fdb72e58f1219064f33cd57468705913 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 05:39:58 +0100 Subject: [PATCH 23/62] Delete sentence about RSS requirements from readme. --- README.adoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.adoc b/README.adoc index c651d5e..4a9d227 100644 --- a/README.adoc +++ b/README.adoc @@ -14,8 +14,6 @@ *mastorss* reads RSS feeds and posts the items via the Mastodon API. Does not support Atom at the moment. -Each `<item>` in feeds must have `<link>`, `<title>` and `<description>`. - == Install === Dependencies From 8a0c69b1e8eecbfd52d9a3e01b74154388a14405 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 05:40:32 +0100 Subject: [PATCH 24/62] Add restclient-cpp to dependencies in readme. --- README.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.adoc b/README.adoc index 4a9d227..3f74e94 100644 --- a/README.adoc +++ b/README.adoc @@ -10,6 +10,7 @@ :uri-boost: https://www.boost.org/ :uri-mastodon-cpp: https://schlomp.space/tastytea/mastodon-cpp :uri-jsoncpp: https://github.com/open-source-parsers/jsoncpp +:uri-restclient-cpp: http://code.mrtazz.com/restclient-cpp/ *mastorss* reads RSS feeds and posts the items via the Mastodon API. Does not support Atom at the moment. @@ -24,6 +25,7 @@ support Atom at the moment. * link:{uri-boost}[boost] (at least: 1.62) * link:{uri-mastodon-cpp}[mastodon-cpp] (at least: 0.111.3) * link:{uri-jsoncpp}[jsoncpp] (at least: 1.7) +* link:{uri-restclient-cpp}[restclient-cpp] (at least: 0.5.1) === Get sourcecode From ed14492b461bd460aec27acc92d1952f6a9c1e9a Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 06:25:11 +0100 Subject: [PATCH 25/62] Add method write() to Config. --- src/config.cpp | 20 +++++++++++++------- src/config.hpp | 2 ++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index 6eb25d9..f089161 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -167,13 +167,8 @@ void Config::generate() newjson[_profile]["max_size"] = std::stoul(line); _json = newjson; - ofstream file{get_filename()}; - if (file.good()) - { - file << newjson.toStyledString(); - } - - BOOST_LOG_TRIVIAL(debug) << "Wrote config file."; + BOOST_LOG_TRIVIAL(debug) << "Generated configuration."; + write(); } string Config::get_access_token(const string &instance) const @@ -242,4 +237,15 @@ void Config::parse() BOOST_LOG_TRIVIAL(debug) << "Read config: " << data; } + +void Config::write() const +{ + ofstream file{get_filename()}; + if (file.good()) + { + file << _json.toStyledString(); + } + + BOOST_LOG_TRIVIAL(debug) << "Wrote config file."; +} } // namespace mastorss diff --git a/src/config.hpp b/src/config.hpp index eee785f..b378cf7 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -68,6 +68,8 @@ public: ProfileData data; + void write() const; + private: const string _profile; Json::Value _json; From d08bb95fbdc4dacc64f46a939db39522f663e518 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 06:26:20 +0100 Subject: [PATCH 26/62] Pass whole Config object to Document. --- src/document.cpp | 5 +++-- src/document.hpp | 5 +++-- src/main.cpp | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/document.cpp b/src/document.cpp index dc631a8..b811dcb 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -39,8 +39,9 @@ using std::istringstream; using std::string; using std::move; -Document::Document(const ProfileData &data) - : _data{data} +Document::Document(Config &cfg) + : _cfg{cfg} + , _data{cfg.data} { RestClient::init(); diff --git a/src/document.hpp b/src/document.hpp index b69267b..31b91e4 100644 --- a/src/document.hpp +++ b/src/document.hpp @@ -51,7 +51,7 @@ struct Item class Document { public: - explicit Document(const ProfileData &data); + explicit Document(Config &cfg); ~Document(); Document(const Document &other) = default; Document &operator=(const Document &other) = delete; @@ -64,7 +64,8 @@ public: void parse(); private: - const ProfileData &_data; + Config &_cfg; + ProfileData &_data; string _raw_doc; void parse_rss(const pt::ptree &tree); diff --git a/src/main.cpp b/src/main.cpp index 4da5a97..cb30232 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -91,8 +91,9 @@ int main(int argc, char *argv[]) try { Config cfg{profile.data()}; - Document doc{cfg.data}; + Document doc{cfg}; doc.parse(); + for (const auto &item : doc.new_items) { cout << "--\n" << item.description.substr(0, 200) << "\n"; From 4491af53db7f53a23df43d1d790222202da92603 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 06:28:31 +0100 Subject: [PATCH 27/62] Handle HTTP redirects. --- src/document.cpp | 49 +++++++++++++++++++++++++++++++++++++++++------- src/document.hpp | 4 ++++ 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/document.cpp b/src/document.cpp index b811dcb..15203c3 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -23,7 +23,6 @@ #include <boost/regex.hpp> #include <mastodon-cpp/mastodon-cpp.hpp> #include <restclient-cpp/connection.h> -#include <restclient-cpp/restclient.h> #include <list> #include <sstream> @@ -53,11 +52,11 @@ Document::~Document() RestClient::disable(); } -void Document::download() +void Document::download(const string &uri) { - RestClient::Connection connection(_data.feedurl); + RestClient::Connection connection{uri}; connection.SetUserAgent(string("mastorss/").append(version)); - connection.FollowRedirects(true, 10); + connection.FollowRedirects(false); RestClient::Response response{connection.get("")}; @@ -72,9 +71,30 @@ void Document::download() case 301: case 308: { - // TODO(tastytea): Handle permanent redirections. - throw std::runtime_error{"Permanent redirect, " - "no solution implemented yet."}; + _data.feedurl = extract_location(response.headers); + if (_data.feedurl.empty()) + { + throw HTTPException{response.code}; + } + + BOOST_LOG_TRIVIAL(debug) << "Feed has new location: " << _data.feedurl; + _cfg.write(); + download(); + break; + } + case 302: + case 303: + case 307: + { + string newuri{extract_location(response.headers)}; + if (newuri.empty()) + { + throw HTTPException{response.code}; + } + + BOOST_LOG_TRIVIAL(debug) << "Feed redirect: " << _data.feedurl; + download(move(newuri)); + break; } case -1: { @@ -87,6 +107,11 @@ void Document::download() } } +void Document::download() +{ + download(_data.feedurl); +} + void Document::parse() { pt::ptree tree; @@ -185,4 +210,14 @@ string Document::remove_html(string html) const return html; } + +string Document::extract_location(const RestClient::HeaderFields &headers) const +{ + string location{headers.at("Location")}; + if (location.empty()) + { + location = headers.at("location"); + } + return location; +} } // namespace mastorss diff --git a/src/document.hpp b/src/document.hpp index 31b91e4..7a9c422 100644 --- a/src/document.hpp +++ b/src/document.hpp @@ -20,6 +20,7 @@ #include "config.hpp" #include <boost/property_tree/ptree.hpp> +#include <restclient-cpp/restclient.h> #include <string> #include <vector> @@ -61,6 +62,7 @@ public: vector<Item> new_items; void download(); + void download(const string &uri); void parse(); private: @@ -71,6 +73,8 @@ private: void parse_rss(const pt::ptree &tree); [[nodiscard]] string remove_html(string html) const; + [[nodiscard]] + string extract_location(const RestClient::HeaderFields &headers) const; }; } // namespace mastorss From 1bbebe2977384c5f4f3f7a84b238c6279792c6f5 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 07:02:19 +0100 Subject: [PATCH 28/62] =?UTF-8?q?Actually=20write=20new=20config.=20?= =?UTF-8?q?=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config.cpp | 47 ++++++++++++++++++++++++++++++++++------------- src/config.hpp | 2 +- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index f089161..1cd112b 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -29,6 +29,7 @@ namespace mastorss { +using std::stoul; using std::getenv; using std::ifstream; using std::ofstream; @@ -86,13 +87,12 @@ Config::Config(string profile) stringstream rawjson; rawjson << file.rdbuf(); rawjson >> _json; + parse(); } else { generate(); } - - parse(); } fs::path Config::get_filename() const @@ -122,33 +122,43 @@ fs::path Config::get_filename() const void Config::generate() { - Json::Value newjson; string line; cout << "Instance (domain): "; getline(cin, line); - newjson[_profile]["instance"] = line; + data.instance = line; - newjson[_profile]["access_token"] = get_access_token(line); + data.access_token = get_access_token(line); cout << "URL of the feed: "; std::getline(cin, line); - newjson[_profile]["feedurl"] = line; + data.feedurl = line; cout << "Post only titles? [y/n]: "; std::getline(cin, line); if (line[0] == 'y') { - newjson[_profile]["titles_as_cw"] = true; + data.titles_only = true; } else { - newjson[_profile]["titles_as_cw"] = false; + data.titles_only = false; + } + + cout << "Post titles as cw? [y/n]: "; + std::getline(cin, line); + if (line[0] == 'y') + { + data.titles_as_cw = true; + } + else + { + data.titles_as_cw = false; } cout << "Append this string to each post: "; std::getline(cin, line); - newjson[_profile]["append"] = line; + data.append = line; cout << "Interval between posts in seconds [30]: "; std::getline(cin, line); @@ -156,7 +166,7 @@ void Config::generate() { line = "30"; } - newjson[_profile]["interval"] = std::stoul(line); + data.interval = static_cast<uint32_t>(stoul(line)); cout << "Maximum size of posts [500]: "; std::getline(cin, line); @@ -164,9 +174,8 @@ void Config::generate() { line = "500"; } - newjson[_profile]["max_size"] = std::stoul(line); + data.max_size = static_cast<uint32_t>(stoul(line)); - _json = newjson; BOOST_LOG_TRIVIAL(debug) << "Generated configuration."; write(); } @@ -238,8 +247,20 @@ void Config::parse() BOOST_LOG_TRIVIAL(debug) << "Read config: " << data; } -void Config::write() const +void Config::write() { + _json[_profile]["access_token"] = data.access_token; + _json[_profile]["append"] = data.append; + _json[_profile]["feedurl"] = data.feedurl; + // Leave fixes. + _json[_profile]["instance"] = data.instance; + _json[_profile]["interval"] = data.interval; + _json[_profile]["last_guid"] = data.last_guid; + _json[_profile]["max_size"] = data.max_size; + // Leave skip. + _json[_profile]["titles_as_cw"] = data.titles_as_cw; + _json[_profile]["titles_as_cw"] = data.titles_only; + ofstream file{get_filename()}; if (file.good()) { diff --git a/src/config.hpp b/src/config.hpp index b378cf7..d7ed3f2 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -68,7 +68,7 @@ public: ProfileData data; - void write() const; + void write(); private: const string _profile; From d9b8b4407f7a9fc4a73c2f9f20b5ccf15563a7e6 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 07:08:41 +0100 Subject: [PATCH 29/62] Formatting. --- src/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.cpp b/src/config.cpp index 1cd112b..afdfbbf 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -235,7 +235,7 @@ void Config::parse() if (!_json[_profile]["max_size"].isNull()) { data.max_size = - static_cast<uint32_t>(_json[_profile]["max_size"].asUInt64()); + static_cast<uint32_t>(_json[_profile]["max_size"].asUInt64()); } for (const auto &skip : _json[_profile]["skip"]) { From eca12c08312531634f926c891e7b3b902f721a96 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 07:12:32 +0100 Subject: [PATCH 30/62] Create config dir if it doesn't exist. --- src/config.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/config.cpp b/src/config.cpp index afdfbbf..bb4a879 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -117,7 +117,13 @@ fs::path Config::get_filename() const } } - return (dir /= "mastorss") /= "config-" + _profile + ".json"; + dir /= "mastorss"; + if (fs::create_directories(dir)) + { + BOOST_LOG_TRIVIAL(debug) << "Created config dir: " << dir; + } + + return dir /= "config-" + _profile + ".json"; } void Config::generate() From d0ea315e086c6f039a272759a6f375d1626efbbb Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 19:28:48 +0100 Subject: [PATCH 31/62] Remove useless move. --- src/document.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/document.cpp b/src/document.cpp index 15203c3..118de6c 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -86,14 +86,14 @@ void Document::download(const string &uri) case 303: case 307: { - string newuri{extract_location(response.headers)}; + const string newuri{extract_location(response.headers)}; if (newuri.empty()) { throw HTTPException{response.code}; } BOOST_LOG_TRIVIAL(debug) << "Feed redirect: " << _data.feedurl; - download(move(newuri)); + download(newuri); break; } case -1: From 359bcacd1f5297fc26605c17e6b0f3f4edc31fd9 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 19:31:07 +0100 Subject: [PATCH 32/62] Clarify redirect log messages. --- src/document.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/document.cpp b/src/document.cpp index 118de6c..6ef7d08 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -77,7 +77,8 @@ void Document::download(const string &uri) throw HTTPException{response.code}; } - BOOST_LOG_TRIVIAL(debug) << "Feed has new location: " << _data.feedurl; + BOOST_LOG_TRIVIAL(debug) << "Feed has new location (permanent): " + << _data.feedurl; _cfg.write(); download(); break; @@ -92,7 +93,8 @@ void Document::download(const string &uri) throw HTTPException{response.code}; } - BOOST_LOG_TRIVIAL(debug) << "Feed redirect: " << _data.feedurl; + BOOST_LOG_TRIVIAL(debug) << "Feed has new location (temporary): " + << _data.feedurl; download(newuri); break; } From 06b989a0e32f9a4ff3a9f1ebf28351b440e650a6 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 20:10:43 +0100 Subject: [PATCH 33/62] Fix drone recipe. --- .drone.yml | 56 +++++++++-------------------------------------------- README.adoc | 2 +- 2 files changed, 10 insertions(+), 48 deletions(-) diff --git a/.drone.yml b/.drone.yml index 920b304..fe3ccc0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,14 +13,14 @@ trigger: - tag steps: -- name: download +- name: download mastodon-cpp image: plugins/download settings: - source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.3/libmastodon-cpp_0.111.3-0_amd64.deb + source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.4/libmastodon-cpp_0.111.4-0_buster_amd64.deb destination: mastodon-cpp.deb - name: gcc8 - image: debian:stretch-slim + image: debian:buster-slim pull: always environment: CXX: g++-8 @@ -28,20 +28,14 @@ steps: LANG: C.utf8 commands: - rm /etc/apt/apt.conf.d/docker-clean + - alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get' - apt-get update -q - - echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release - - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list - - echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >> /etc/apt/sources.list.d/ubuntu-toolchain-r.list - - apt-get install -qy gnupg - - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 0x60c317803a41ba51845e371a1e9377a2ba9ef27f - - gpg --armor --export 0x60c317803a41ba51845e371a1e9377a2ba9ef27f | apt-key add - + - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/debian/ buster main" > /etc/apt/sources.list.d/restclient-cpp.list + - apt-get install -qy debian-archive-keyring curl gnupg apt-transport-https + - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - - apt-get install -qy build-essential cmake pkg-config - - apt-get install -qy -t xenial g++-8 - - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 100 - - update-alternatives --set g++ /usr/bin/g++-8 - - apt-get install -qy libcurl4-openssl-dev libjsoncpp-dev libboost-all-dev - - apt-get install -qy -t sid libcurlpp-dev + - apt-get install -qy build-essential cmake asciidoc + - apt-get install -qy libboost-filesystem-dev libboost-log-dev libjsoncpp-dev catch restclient-cpp - apt-get -qy install ./mastodon-cpp.deb - rm -rf build && mkdir -p build && cd build - cmake .. @@ -51,38 +45,6 @@ steps: - name: debian-package-cache path: /var/cache/apt/archives -- name: gcc5 - image: debian:stretch-slim - pull: always - environment: - CXX: g++-8 - CXXFLAGS: -pipe -O2 - LANG: C.utf8 - commands: - - rm /etc/apt/apt.conf.d/docker-clean - - apt-get update -q - - echo "APT::Default-Release \"stretch\";" >> /etc/apt/apt.conf.d/00default_release - - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list - - echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" >> /etc/apt/sources.list.d/ubuntu-toolchain-r.list - - apt-get install -qy gnupg - - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 0x60c317803a41ba51845e371a1e9377a2ba9ef27f - - gpg --armor --export 0x60c317803a41ba51845e371a1e9377a2ba9ef27f | apt-key add - - - apt-get update -q - - apt-get install -qy build-essential cmake pkg-config - - apt-get install -qy -t xenial g++-5 - - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 100 - - update-alternatives --set g++ /usr/bin/g++-5 - - apt-get install -qy libcurl4-openssl-dev libjsoncpp-dev libboost-all-dev - - apt-get install -qy -t sid libcurlpp-dev - - apt-get install -qy ./mastodon-cpp.deb - - rm -rf build && mkdir -p build && cd build - - cmake .. - - make VERBOSE=1 - - make install DESTDIR=install - volumes: - - name: debian-package-cache - path: /var/cache/apt/archives - - name: notify image: drillster/drone-email pull: always diff --git a/README.adoc b/README.adoc index 3f74e94..bf33676 100644 --- a/README.adoc +++ b/README.adoc @@ -20,7 +20,7 @@ support Atom at the moment. === Dependencies * Tested OS: Linux -* C++ compiler (tested: link:{uri-gcc}[gcc] 9) +* C++ compiler (tested: link:{uri-gcc}[gcc] 8/9) * link:{uri-cmake}[cmake] (at least: 3.9) * link:{uri-boost}[boost] (at least: 1.62) * link:{uri-mastodon-cpp}[mastodon-cpp] (at least: 0.111.3) From 75005cdae0ec74709ee710337f3683a3cec34c2b Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 21:40:48 +0100 Subject: [PATCH 34/62] Drop version requirement for jsoncpp. --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e1ac829..f01338e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,8 +1,8 @@ include(GNUInstallDirs) # The minimum versions should be in Debian oldstable, if possible. -find_package(Boost 1.62 REQUIRED COMPONENTS system filesystem log) -find_package(jsoncpp 1.7 REQUIRED CONFIG) +find_package(Boost 1.62 REQUIRED COMPONENTS filesystem log) +find_package(jsoncpp REQUIRED CONFIG) # 1.7.4 (Debian buster) has no version. find_package(CURL 7.52 REQUIRED) find_package(Threads REQUIRED) find_package(restclient-cpp 0.5 REQUIRED CONFIG) From 9882a15704fc6981c263e42f460318c6b298e8e4 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 21:45:50 +0100 Subject: [PATCH 35/62] Add libcurl dependency to drone recipe. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index fe3ccc0..09d2648 100644 --- a/.drone.yml +++ b/.drone.yml @@ -35,7 +35,7 @@ steps: - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - apt-get install -qy build-essential cmake asciidoc - - apt-get install -qy libboost-filesystem-dev libboost-log-dev libjsoncpp-dev catch restclient-cpp + - apt-get install -qy libboost-filesystem-dev libboost-log-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp - apt-get -qy install ./mastodon-cpp.deb - rm -rf build && mkdir -p build && cd build - cmake .. From 255afcea027cb45e770c164050cb1b65ef30f553 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 22:50:25 +0100 Subject: [PATCH 36/62] Try to find restclient-cpp if its CMake config is not found. --- src/CMakeLists.txt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f01338e..6a73789 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,9 +5,22 @@ find_package(Boost 1.62 REQUIRED COMPONENTS filesystem log) find_package(jsoncpp REQUIRED CONFIG) # 1.7.4 (Debian buster) has no version. find_package(CURL 7.52 REQUIRED) find_package(Threads REQUIRED) -find_package(restclient-cpp 0.5 REQUIRED CONFIG) +find_package(restclient-cpp 0.5 CONFIG) find_package(mastodon-cpp REQUIRED CONFIG) +if(NOT ${restclient-cpp_FOUND}) + find_file(restclient_h NAMES "restclient-cpp/restclient.h" + PATHS "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}") + if("${restclient_h}" STREQUAL "restclient_h-NOTFOUND") + message(FATAL_ERROR "Could not find restclient-cpp.") + else() + message(WARNING + "Your distribution of restclient-cpp doesn't contain the *Config.cmake " + "recipes, but the files seem to be in the standard directories. " + "Let's hope this works.") + endif() +endif() + # Write version in header. configure_file ( "${PROJECT_SOURCE_DIR}/src/version.hpp.in" From 092a34a1abd6de87f4905f944d3f75bfc22576cb Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 23:00:13 +0100 Subject: [PATCH 37/62] Add POCO to dependencies. --- .drone.yml | 2 +- README.adoc | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 09d2648..de10c79 100644 --- a/.drone.yml +++ b/.drone.yml @@ -35,7 +35,7 @@ steps: - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - apt-get install -qy build-essential cmake asciidoc - - apt-get install -qy libboost-filesystem-dev libboost-log-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp + - apt-get install -qy libboost-filesystem-dev libboost-log-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev - apt-get -qy install ./mastodon-cpp.deb - rm -rf build && mkdir -p build && cd build - cmake .. diff --git a/README.adoc b/README.adoc index bf33676..ea15e8b 100644 --- a/README.adoc +++ b/README.adoc @@ -11,6 +11,7 @@ :uri-mastodon-cpp: https://schlomp.space/tastytea/mastodon-cpp :uri-jsoncpp: https://github.com/open-source-parsers/jsoncpp :uri-restclient-cpp: http://code.mrtazz.com/restclient-cpp/ +:uri-poco: https://pocoproject.org/ *mastorss* reads RSS feeds and posts the items via the Mastodon API. Does not support Atom at the moment. @@ -24,6 +25,7 @@ support Atom at the moment. * link:{uri-cmake}[cmake] (at least: 3.9) * link:{uri-boost}[boost] (at least: 1.62) * link:{uri-mastodon-cpp}[mastodon-cpp] (at least: 0.111.3) +** link:{uri-poco}[POCO] (at least: 1.7) * link:{uri-jsoncpp}[jsoncpp] (at least: 1.7) * link:{uri-restclient-cpp}[restclient-cpp] (at least: 0.5.1) From 711e3199af4c24e60c18f6ff2cb8b479d6cf3952 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 23:05:48 +0100 Subject: [PATCH 38/62] Add Boost::regex to dependencies. --- .drone.yml | 2 +- src/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index de10c79..40b8a08 100644 --- a/.drone.yml +++ b/.drone.yml @@ -35,7 +35,7 @@ steps: - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - apt-get install -qy build-essential cmake asciidoc - - apt-get install -qy libboost-filesystem-dev libboost-log-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev + - apt-get install -qy libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev - apt-get -qy install ./mastodon-cpp.deb - rm -rf build && mkdir -p build && cd build - cmake .. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6a73789..e7bb453 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,7 @@ include(GNUInstallDirs) # The minimum versions should be in Debian oldstable, if possible. -find_package(Boost 1.62 REQUIRED COMPONENTS filesystem log) +find_package(Boost 1.62 REQUIRED COMPONENTS filesystem log regex) find_package(jsoncpp REQUIRED CONFIG) # 1.7.4 (Debian buster) has no version. find_package(CURL 7.52 REQUIRED) find_package(Threads REQUIRED) @@ -33,7 +33,7 @@ add_executable(mastorss ${sources}) target_link_libraries(mastorss PRIVATE jsoncpp restclient-cpp mastodon-cpp::mastodon-cpp - Boost::filesystem Boost::log) + Boost::filesystem Boost::log Boost::regex) if(BUILD_SHARED_LIBS) target_compile_definitions(mastorss PRIVATE "BOOST_ALL_DYN_LINK=1") endif() From 42039bcc92529436ceba0d2e1b2a507c195e9d37 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Wed, 25 Dec 2019 23:14:42 +0100 Subject: [PATCH 39/62] Add libcurl4 to Drone dependencies. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 40b8a08..27a0bd5 100644 --- a/.drone.yml +++ b/.drone.yml @@ -35,7 +35,7 @@ steps: - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - apt-get install -qy build-essential cmake asciidoc - - apt-get install -qy libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev + - apt-get install -qy libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev libcurl4 restclient-cpp libpoco-dev - apt-get -qy install ./mastodon-cpp.deb - rm -rf build && mkdir -p build && cd build - cmake .. From 330849df7e2d6409710dec76a4f638d28f0019e9 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Thu, 26 Dec 2019 03:30:49 +0100 Subject: [PATCH 40/62] Drone: Switch to Ubuntu bionic. --- .drone.yml | 52 ++++++++++++++++++++++++++++++++++++++++++---------- README.adoc | 2 +- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/.drone.yml b/.drone.yml index 27a0bd5..8e3d227 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,30 +13,62 @@ trigger: - tag steps: -- name: download mastodon-cpp +# - name: download mastodon-cpp for buster +# image: plugins/download +# settings: +# source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.5/libmastodon-cpp_0.111.5-0_buster_amd64.deb +# destination: mastodon-cpp_buster.deb + +- name: download mastodon-cpp for bionic image: plugins/download settings: - source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.4/libmastodon-cpp_0.111.4-0_buster_amd64.deb - destination: mastodon-cpp.deb + source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.5/libmastodon-cpp_0.111.5-0_bionic_amd64.deb + destination: mastodon-cpp_bionic.deb -- name: gcc8 - image: debian:buster-slim +# - name: GCC 8 on Debian buster +# image: debian:buster-slim +# pull: always +# environment: +# CXX: g++-8 +# CXXFLAGS: -pipe -O2 +# LANG: C.utf8 +# commands: +# - rm /etc/apt/apt.conf.d/docker-clean +# - alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get' +# - apt-get update -q +# - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/debian/ buster main" > /etc/apt/sources.list.d/restclient-cpp.list +# - apt-get install -qy debian-archive-keyring curl gnupg apt-transport-https +# - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - +# - apt-get update -q +# - apt-get install -qy build-essential cmake asciidoc +# - apt-get install -qy libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev libcurl4 restclient-cpp libpoco-dev +# - apt-get -qy install ./mastodon-cpp_buster.deb +# - rm -rf build && mkdir -p build && cd build +# - cmake .. +# - make VERBOSE=1 +# - make install DESTDIR=install +# volumes: +# - name: debian-package-cache +# path: /var/cache/apt/archives + +- name: GCC 7 on Ubuntu bionic + image: ubuntu:bionic pull: always environment: - CXX: g++-8 + CXX: g++-7 CXXFLAGS: -pipe -O2 LANG: C.utf8 commands: - rm /etc/apt/apt.conf.d/docker-clean - alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get' - apt-get update -q - - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/debian/ buster main" > /etc/apt/sources.list.d/restclient-cpp.list - - apt-get install -qy debian-archive-keyring curl gnupg apt-transport-https + - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/ubuntu/ bionic main" > /etc/apt/sources.list.d/restclient-cpp.list + - apt-get install -qy curl gnupg apt-transport-https - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - apt-get install -qy build-essential cmake asciidoc - - apt-get install -qy libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev libcurl4 restclient-cpp libpoco-dev - - apt-get -qy install ./mastodon-cpp.deb + - apt-get install -qy libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch restclient-cpp libpoco-dev + - apt-get -qy install ./mastodon-cpp_bionic.deb - rm -rf build && mkdir -p build && cd build - cmake .. - make VERBOSE=1 diff --git a/README.adoc b/README.adoc index ea15e8b..a2f6e60 100644 --- a/README.adoc +++ b/README.adoc @@ -21,7 +21,7 @@ support Atom at the moment. === Dependencies * Tested OS: Linux -* C++ compiler (tested: link:{uri-gcc}[gcc] 8/9) +* C++ compiler (tested: link:{uri-gcc}[gcc] 7/9) * link:{uri-cmake}[cmake] (at least: 3.9) * link:{uri-boost}[boost] (at least: 1.62) * link:{uri-mastodon-cpp}[mastodon-cpp] (at least: 0.111.3) From 608c21eccdb9ff2c60bc0a991e46a29bab7216bd Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Thu, 26 Dec 2019 03:37:51 +0100 Subject: [PATCH 41/62] Drone: Replace -qy with -qq. --- .drone.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.drone.yml b/.drone.yml index 8e3d227..0c98df4 100644 --- a/.drone.yml +++ b/.drone.yml @@ -37,12 +37,12 @@ steps: # - alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get' # - apt-get update -q # - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/debian/ buster main" > /etc/apt/sources.list.d/restclient-cpp.list -# - apt-get install -qy debian-archive-keyring curl gnupg apt-transport-https +# - apt-get install -qq debian-archive-keyring curl gnupg apt-transport-https # - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - # - apt-get update -q -# - apt-get install -qy build-essential cmake asciidoc -# - apt-get install -qy libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev libcurl4 restclient-cpp libpoco-dev -# - apt-get -qy install ./mastodon-cpp_buster.deb +# - apt-get install -qq build-essential cmake asciidoc +# - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev libcurl4 restclient-cpp libpoco-dev +# - apt-get -qq install ./mastodon-cpp_buster.deb # - rm -rf build && mkdir -p build && cd build # - cmake .. # - make VERBOSE=1 @@ -63,12 +63,12 @@ steps: - alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get' - apt-get update -q - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/ubuntu/ bionic main" > /etc/apt/sources.list.d/restclient-cpp.list - - apt-get install -qy curl gnupg apt-transport-https + - apt-get install -qq curl gnupg apt-transport-https - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - - apt-get install -qy build-essential cmake asciidoc - - apt-get install -qy libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch restclient-cpp libpoco-dev - - apt-get -qy install ./mastodon-cpp_bionic.deb + - apt-get install -qq build-essential cmake asciidoc + - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch restclient-cpp libpoco-dev + - apt-get -qq install ./mastodon-cpp_bionic.deb - rm -rf build && mkdir -p build && cd build - cmake .. - make VERBOSE=1 From cb69331bfc6f4d4bd125dd504b900d65f629f9df Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Thu, 26 Dec 2019 03:41:44 +0100 Subject: [PATCH 42/62] Drone: Set DEBIAN_FRONTEND to noninteractive. --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index 0c98df4..dea384f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -31,6 +31,7 @@ steps: # environment: # CXX: g++-8 # CXXFLAGS: -pipe -O2 +# DEBIAN_FRONTEND: noninteractive # LANG: C.utf8 # commands: # - rm /etc/apt/apt.conf.d/docker-clean @@ -57,6 +58,7 @@ steps: environment: CXX: g++-7 CXXFLAGS: -pipe -O2 + DEBIAN_FRONTEND: noninteractive LANG: C.utf8 commands: - rm /etc/apt/apt.conf.d/docker-clean From 47c5bb82b56654866089a75b4dc30d941b1aea96 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Thu, 26 Dec 2019 03:45:32 +0100 Subject: [PATCH 43/62] Drone: Add libcurl4-openssl-dev to Ubuntu build. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index dea384f..3d3768f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -69,7 +69,7 @@ steps: - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - apt-get install -qq build-essential cmake asciidoc - - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch restclient-cpp libpoco-dev + - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev - apt-get -qq install ./mastodon-cpp_bionic.deb - rm -rf build && mkdir -p build && cd build - cmake .. From 66292f7b150d73e8f055d20fb67141c72158fbdf Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Thu, 26 Dec 2019 03:56:46 +0100 Subject: [PATCH 44/62] Convert paths to C strings in fstream initializers. --- src/config.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index bb4a879..664cce8 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -81,7 +81,7 @@ Config::Config(string profile) const fs::path filename = get_filename(); BOOST_LOG_TRIVIAL(debug) << "Config filename is: " << filename; - ifstream file{filename}; + ifstream file{filename.c_str()}; if (file.good()) { stringstream rawjson; @@ -267,7 +267,7 @@ void Config::write() _json[_profile]["titles_as_cw"] = data.titles_as_cw; _json[_profile]["titles_as_cw"] = data.titles_only; - ofstream file{get_filename()}; + ofstream file{get_filename().c_str()}; if (file.good()) { file << _json.toStyledString(); From a11b9e03022ec47c70886771068cc76a4a0719fd Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Thu, 26 Dec 2019 04:10:32 +0100 Subject: [PATCH 45/62] Drone: Add libcurl3 to Ubuntu build. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 3d3768f..0eb465e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -69,7 +69,7 @@ steps: - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - apt-get install -qq build-essential cmake asciidoc - - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev + - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev libcurl3 restclient-cpp libpoco-dev - apt-get -qq install ./mastodon-cpp_bionic.deb - rm -rf build && mkdir -p build && cd build - cmake .. From 9386681b693ccd5516952bdc3e6a7db0e4b12651 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Thu, 26 Dec 2019 04:14:29 +0100 Subject: [PATCH 46/62] Drone: Re-enabled buster build. --- .drone.yml | 64 +++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.drone.yml b/.drone.yml index 0eb465e..4991577 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,11 +13,11 @@ trigger: - tag steps: -# - name: download mastodon-cpp for buster -# image: plugins/download -# settings: -# source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.5/libmastodon-cpp_0.111.5-0_buster_amd64.deb -# destination: mastodon-cpp_buster.deb +- name: download mastodon-cpp for buster + image: plugins/download + settings: + source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.5/libmastodon-cpp_0.111.5-0_buster_amd64.deb + destination: mastodon-cpp_buster.deb - name: download mastodon-cpp for bionic image: plugins/download @@ -25,32 +25,32 @@ steps: source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.5/libmastodon-cpp_0.111.5-0_bionic_amd64.deb destination: mastodon-cpp_bionic.deb -# - name: GCC 8 on Debian buster -# image: debian:buster-slim -# pull: always -# environment: -# CXX: g++-8 -# CXXFLAGS: -pipe -O2 -# DEBIAN_FRONTEND: noninteractive -# LANG: C.utf8 -# commands: -# - rm /etc/apt/apt.conf.d/docker-clean -# - alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get' -# - apt-get update -q -# - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/debian/ buster main" > /etc/apt/sources.list.d/restclient-cpp.list -# - apt-get install -qq debian-archive-keyring curl gnupg apt-transport-https -# - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - -# - apt-get update -q -# - apt-get install -qq build-essential cmake asciidoc -# - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev libcurl4 restclient-cpp libpoco-dev -# - apt-get -qq install ./mastodon-cpp_buster.deb -# - rm -rf build && mkdir -p build && cd build -# - cmake .. -# - make VERBOSE=1 -# - make install DESTDIR=install -# volumes: -# - name: debian-package-cache -# path: /var/cache/apt/archives +- name: GCC 8 on Debian buster + image: debian:buster-slim + pull: always + environment: + CXX: g++-8 + CXXFLAGS: -pipe -O2 + DEBIAN_FRONTEND: noninteractive + LANG: C.utf8 + commands: + - rm /etc/apt/apt.conf.d/docker-clean + - alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get' + - apt-get update -q + - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/debian/ buster main" > /etc/apt/sources.list.d/restclient-cpp.list + - apt-get install -qq debian-archive-keyring curl gnupg apt-transport-https + - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - + - apt-get update -q + - apt-get install -qq build-essential cmake asciidoc + - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev + - apt-get -qq install ./mastodon-cpp_buster.deb + - rm -rf build && mkdir -p build && cd build + - cmake .. + - make VERBOSE=1 + - make install DESTDIR=install + volumes: + - name: debian-package-cache + path: /var/cache/apt/archives - name: GCC 7 on Ubuntu bionic image: ubuntu:bionic @@ -69,7 +69,7 @@ steps: - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - apt-get update -q - apt-get install -qq build-essential cmake asciidoc - - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev libcurl3 restclient-cpp libpoco-dev + - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev - apt-get -qq install ./mastodon-cpp_bionic.deb - rm -rf build && mkdir -p build && cd build - cmake .. From 35afb2800e5d99dcec568dc1c5b243b9666d4e9d Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 05:53:49 +0100 Subject: [PATCH 47/62] Change ProfileData::max_size to size_t. --- src/config.cpp | 5 ++--- src/config.hpp | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index 664cce8..22d87a9 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -180,7 +180,7 @@ void Config::generate() { line = "500"; } - data.max_size = static_cast<uint32_t>(stoul(line)); + data.max_size = stoul(line); BOOST_LOG_TRIVIAL(debug) << "Generated configuration."; write(); @@ -240,8 +240,7 @@ void Config::parse() data.last_guid = _json[_profile]["last_guid"].asString(); if (!_json[_profile]["max_size"].isNull()) { - data.max_size = - static_cast<uint32_t>(_json[_profile]["max_size"].asUInt64()); + data.max_size = _json[_profile]["max_size"].asUInt64(); } for (const auto &skip : _json[_profile]["skip"]) { diff --git a/src/config.hpp b/src/config.hpp index d7ed3f2..58fd99d 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -47,7 +47,7 @@ struct ProfileData string instance; uint32_t interval{30}; string last_guid; - uint32_t max_size{500}; + size_t max_size{500}; vector<string> skip; bool titles_as_cw{false}; bool titles_only{false}; From ebf5a2cd00392752d5cf6de9ce4f933b4c79c668 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 05:54:52 +0100 Subject: [PATCH 48/62] Fix return code initialization in access token generation. Brace-initialization won't work here because return_call is a struct without initializer. --- src/config.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index 22d87a9..72383c1 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -36,7 +36,6 @@ using std::ofstream; using std::cin; using std::cout; using std::stringstream; -using std::runtime_error; using std::getline; using std::move; @@ -193,13 +192,10 @@ string Config::get_access_token(const string &instance) const string url; Mastodon::API masto(instance, ""); - Mastodon::return_call ret - { - masto.register_app1("mastorss", "urn:ietf:wg:oauth:2.0:oob", - "write", - "https://schlomp.space/tastytea/mastorss", - client_id, client_secret, url) - }; + auto ret = masto.register_app1("mastorss", "urn:ietf:wg:oauth:2.0:oob", + "write", + "https://schlomp.space/tastytea/mastorss", + client_id, client_secret, url); if (ret) { From 8b0c2500f9150b1ed38db618503ad00ebc4b1372 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 06:27:56 +0100 Subject: [PATCH 49/62] Fix some initializations. --- src/document.cpp | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/document.cpp b/src/document.cpp index 6ef7d08..0d54260 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -146,7 +146,7 @@ void Document::parse_rss(const pt::ptree &tree) } bool skipthis{false}; - string title = rssitem.get<string>("title"); + string title{rssitem.get<string>("title")}; for (const auto &skip : _data.skip) { if (title.substr(0, skip.length()) == skip) @@ -162,17 +162,16 @@ void Document::parse_rss(const pt::ptree &tree) } Item item; - item.description = - [&] + item.description = [&] + { + string desc + {remove_html(rssitem.get<string>("description"))}; + for (const auto &fix : _data.fixes) { - string desc = - remove_html(rssitem.get<string>("description")); - for (const auto &fix : _data.fixes) - { - desc = regex_replace(desc, regex{fix}, ""); - } - return desc; - }(); + desc = regex_replace(desc, regex{fix}, ""); + } + return desc; + }(); item.guid = move(guid); item.link = rssitem.get<string>("link"); item.title = move(title); From 527c2347ea99d764af36395bc5228aa1ad30b653 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 06:47:56 +0100 Subject: [PATCH 50/62] Fix bug where titles_as_cw would get the value of titles_only. --- src/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.cpp b/src/config.cpp index 72383c1..f617a89 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -260,7 +260,7 @@ void Config::write() _json[_profile]["max_size"] = data.max_size; // Leave skip. _json[_profile]["titles_as_cw"] = data.titles_as_cw; - _json[_profile]["titles_as_cw"] = data.titles_only; + _json[_profile]["titles_only"] = data.titles_only; ofstream file{get_filename().c_str()}; if (file.good()) From 9a3fb420d7df5d260f454693d12ed58d654b8679 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 06:48:35 +0100 Subject: [PATCH 51/62] Begin with the oldest item. --- src/document.cpp | 2 +- src/document.hpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/document.cpp b/src/document.cpp index 0d54260..6456823 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -175,7 +175,7 @@ void Document::parse_rss(const pt::ptree &tree) item.guid = move(guid); item.link = rssitem.get<string>("link"); item.title = move(title); - new_items.push_back(item); + new_items.push_front(item); BOOST_LOG_TRIVIAL(debug) << "Found GUID: " << item.guid; } diff --git a/src/document.hpp b/src/document.hpp index 7a9c422..0da2092 100644 --- a/src/document.hpp +++ b/src/document.hpp @@ -23,13 +23,13 @@ #include <restclient-cpp/restclient.h> #include <string> -#include <vector> +#include <list> namespace mastorss { namespace pt = boost::property_tree; using std::string; -using std::vector; +using std::list; /*! * @brief An Item of a feed. @@ -59,7 +59,7 @@ public: Document(Document &&other) = default; Document &operator=(Document &&other) = delete; - vector<Item> new_items; + list<Item> new_items; void download(); void download(const string &uri); From 4168e77648d979bb3ebb5a8a3a1b592a50964698 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 06:54:05 +0100 Subject: [PATCH 52/62] Only add one new item if it is the first run. --- src/document.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/document.cpp b/src/document.cpp index 6456823..51343e7 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -178,6 +178,11 @@ void Document::parse_rss(const pt::ptree &tree) new_items.push_front(item); BOOST_LOG_TRIVIAL(debug) << "Found GUID: " << item.guid; + + if (_data.last_guid.empty()) + { + break; + } } } } From 74369d6ab5f99655761f1849ae8011eae61e84a1 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 06:57:23 +0100 Subject: [PATCH 53/62] =?UTF-8?q?Rename=20=5Fdata=20=E2=86=92=20=5Fprofile?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/document.cpp | 22 +++++++++++----------- src/document.hpp | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/document.cpp b/src/document.cpp index 51343e7..9716080 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -40,7 +40,7 @@ using std::move; Document::Document(Config &cfg) : _cfg{cfg} - , _data{cfg.data} + , _profile{cfg.data} { RestClient::init(); @@ -65,20 +65,20 @@ void Document::download(const string &uri) case 200: { _raw_doc = response.body; - BOOST_LOG_TRIVIAL(debug) << "Downloaded feed: " << _data.feedurl; + BOOST_LOG_TRIVIAL(debug) << "Downloaded feed: " << _profile.feedurl; break; } case 301: case 308: { - _data.feedurl = extract_location(response.headers); - if (_data.feedurl.empty()) + _profile.feedurl = extract_location(response.headers); + if (_profile.feedurl.empty()) { throw HTTPException{response.code}; } BOOST_LOG_TRIVIAL(debug) << "Feed has new location (permanent): " - << _data.feedurl; + << _profile.feedurl; _cfg.write(); download(); break; @@ -94,7 +94,7 @@ void Document::download(const string &uri) } BOOST_LOG_TRIVIAL(debug) << "Feed has new location (temporary): " - << _data.feedurl; + << _profile.feedurl; download(newuri); break; } @@ -111,7 +111,7 @@ void Document::download(const string &uri) void Document::download() { - download(_data.feedurl); + download(_profile.feedurl); } void Document::parse() @@ -140,14 +140,14 @@ void Document::parse_rss(const pt::ptree &tree) { guid = rssitem.get<string>("link"); } - if (guid == _data.last_guid) + if (guid == _profile.last_guid) { break; } bool skipthis{false}; string title{rssitem.get<string>("title")}; - for (const auto &skip : _data.skip) + for (const auto &skip : _profile.skip) { if (title.substr(0, skip.length()) == skip) { @@ -166,7 +166,7 @@ void Document::parse_rss(const pt::ptree &tree) { string desc {remove_html(rssitem.get<string>("description"))}; - for (const auto &fix : _data.fixes) + for (const auto &fix : _profile.fixes) { desc = regex_replace(desc, regex{fix}, ""); } @@ -179,7 +179,7 @@ void Document::parse_rss(const pt::ptree &tree) BOOST_LOG_TRIVIAL(debug) << "Found GUID: " << item.guid; - if (_data.last_guid.empty()) + if (_profile.last_guid.empty()) { break; } diff --git a/src/document.hpp b/src/document.hpp index 0da2092..325015e 100644 --- a/src/document.hpp +++ b/src/document.hpp @@ -67,7 +67,7 @@ public: private: Config &_cfg; - ProfileData &_data; + ProfileData &_profile; string _raw_doc; void parse_rss(const pt::ptree &tree); From ec8bc6508cfefb59ee6089dc9239709015f7958a Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 07:12:28 +0100 Subject: [PATCH 54/62] Add operator != to Item. To be able to determine if the last item is visited in main(). --- src/document.cpp | 5 +++++ src/document.hpp | 2 ++ 2 files changed, 7 insertions(+) diff --git a/src/document.cpp b/src/document.cpp index 9716080..fbe0216 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -38,6 +38,11 @@ using std::istringstream; using std::string; using std::move; +bool operator !=(const Item &a, const Item &b) +{ + return a.guid != b.guid; +} + Document::Document(Config &cfg) : _cfg{cfg} , _profile{cfg.data} diff --git a/src/document.hpp b/src/document.hpp index 325015e..6dd842f 100644 --- a/src/document.hpp +++ b/src/document.hpp @@ -42,6 +42,8 @@ struct Item string guid; string link; string title; + + friend bool operator !=(const Item &a, const Item &b); }; /*! From 477bf265d4f89bb997c95c89944cb9436a82dcc1 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 07:13:30 +0100 Subject: [PATCH 55/62] Add MastoAPI. --- src/mastoapi.cpp | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ src/mastoapi.hpp | 40 ++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 src/mastoapi.cpp create mode 100644 src/mastoapi.hpp diff --git a/src/mastoapi.cpp b/src/mastoapi.cpp new file mode 100644 index 0000000..1e00aff --- /dev/null +++ b/src/mastoapi.cpp @@ -0,0 +1,89 @@ +/* This file is part of mastorss. + * Copyright © 2019 tastytea <tastytea@tastytea.de> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "exceptions.hpp" +#include "mastoapi.hpp" + +#include <boost/log/trivial.hpp> + +#include <string> + +namespace mastorss +{ +using std::string; + +MastoAPI::MastoAPI(const ProfileData &data) + : _profile{data} + , _masto{_profile.instance, _profile.access_token} +{ +} + +void MastoAPI::post_item(const Item &item) +{ + string status{[&] + { + if (_profile.titles_as_cw) + { + if (_profile.titles_only) + { + return string{}; + } + return item.description; + } + + string s{item.title}; + if (!_profile.titles_only) + { + s.append("\n\n" + item.description); + } + return s; + }()}; + status.append("\n\n" + item.link); + + if (!_profile.append.empty()) + { + const size_t len{status.size()}; + const size_t len_append{_profile.append.size() + 2}; + const size_t len_max{_profile.max_size}; + + if ((len + len_append) > len_max) + { + status.resize(len_max - len_append); + } + status.append("\n\n" + _profile.append); + } + + Mastodon::parameters params{{"status", {status}}}; + if (_profile.titles_as_cw) + { + params.push_back({"spoiler_text", {item.title}}); + } + + const auto ret = _masto.post(Mastodon::API::v1::statuses, params); + if (!ret) + { + if (ret.http_error_code != 200) + { + throw HTTPException{ret.http_error_code}; + } + else + { + throw MastodonException{ret.error_code}; + } + } + BOOST_LOG_TRIVIAL(debug) << "Posted status with GUID: " << item.guid; +} +} // namespace mastorss diff --git a/src/mastoapi.hpp b/src/mastoapi.hpp new file mode 100644 index 0000000..6b9a041 --- /dev/null +++ b/src/mastoapi.hpp @@ -0,0 +1,40 @@ +/* This file is part of mastorss. + * Copyright © 2019 tastytea <tastytea@tastytea.de> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef MASTORSS_MASTOAPI_HPP +#define MASTORSS_MASTOAPI_HPP + +#include "config.hpp" +#include "document.hpp" + +#include <mastodon-cpp/mastodon-cpp.hpp> + +namespace mastorss +{ +class MastoAPI +{ +public: + explicit MastoAPI(const ProfileData &data); + + void post_item(const Item &item); + +private: + const ProfileData &_profile; + Mastodon::API _masto; +}; +} // namespace mastorss + +#endif // MASTORSS_MASTOAPI_HPP From 8bfd9218cb530bb83edb1d04d24616d31f920382 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 07:13:56 +0100 Subject: [PATCH 56/62] Post new items. --- src/main.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index cb30232..f6c4d60 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,25 +1,30 @@ #include "config.hpp" #include "document.hpp" #include "exceptions.hpp" +#include "mastoapi.hpp" #include "version.hpp" #include <boost/log/core.hpp> #include <boost/log/trivial.hpp> #include <boost/log/utility/setup/console.hpp> +#include <chrono> #include <cstdlib> #include <iostream> #include <stdexcept> #include <string_view> +#include <thread> #include <vector> using namespace mastorss; +using std::chrono::seconds; using std::getenv; using std::cout; using std::cerr; using std::runtime_error; using std::string_view; using std::vector; +using std::this_thread::sleep_for; namespace mastorss { @@ -94,9 +99,19 @@ int main(int argc, char *argv[]) Document doc{cfg}; doc.parse(); - for (const auto &item : doc.new_items) + MastoAPI masto{cfg.data}; + if (!doc.new_items.empty()) { - cout << "--\n" << item.description.substr(0, 200) << "\n"; + for (const auto &item : doc.new_items) + { + masto.post_item(item); + cfg.data.last_guid = item.guid; + if (item != *doc.new_items.rbegin()) + { // Don't sleep if this is the last item. + sleep_for(seconds(cfg.data.interval)); + } + } + cfg.write(); } } catch (const FileException &e) From 2fca8008685a4a6371e1141b25443088eb1b7830 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 07:16:15 +0100 Subject: [PATCH 57/62] Add log message for first run. --- src/document.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/document.cpp b/src/document.cpp index fbe0216..e91c908 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -186,6 +186,7 @@ void Document::parse_rss(const pt::ptree &tree) if (_profile.last_guid.empty()) { + BOOST_LOG_TRIVIAL(debug) << "This is the first run."; break; } } From 9e58e319124ce67ef2863d6fb1776ff561ff3cd2 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 07:27:09 +0100 Subject: [PATCH 58/62] Remove option for tests. It's not really good testable. --- CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d75ad2..6d684f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,6 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") # Project build options. option(WITH_MAN "Compile and install manpage." YES) -option(WITH_TESTS "Compile tests." NO) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -33,8 +32,4 @@ if(WITH_MAN) add_subdirectory(man) endif() -# if(WITH_TESTS) -# add_subdirectory(tests) -# endif() - # include(cmake/packages.cmake) From 3ffd73d68a8138607fd4b7d8dab6a853fda481f4 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 07:37:58 +0100 Subject: [PATCH 59/62] CI: Compile restclient-cpp as a workaround. --- .drone.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.drone.yml b/.drone.yml index 4991577..3133636 100644 --- a/.drone.yml +++ b/.drone.yml @@ -25,6 +25,13 @@ steps: source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.5/libmastodon-cpp_0.111.5-0_bionic_amd64.deb destination: mastodon-cpp_bionic.deb +# Workaround until the debian packages are fixed. +- name: download restclient-cpp source + image: plugins/download + settings: + source: https://github.com/mrtazz/restclient-cpp/archive/0.5.1.tar.gz + destination: restclient-cpp.tar.gz + - name: GCC 8 on Debian buster image: debian:buster-slim pull: always @@ -44,6 +51,12 @@ steps: - apt-get install -qq build-essential cmake asciidoc - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev - apt-get -qq install ./mastodon-cpp_buster.deb + - tar -xf restclient-cpp.tar.gz + - cd restclient-cpp-* + - cmake -DCMAKE_INSTALL_PREFIX=/usr . + - cmake --build . + - make install + - cd .. - rm -rf build && mkdir -p build && cd build - cmake .. - make VERBOSE=1 @@ -71,6 +84,12 @@ steps: - apt-get install -qq build-essential cmake asciidoc - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev - apt-get -qq install ./mastodon-cpp_bionic.deb + - tar -xf restclient-cpp.tar.gz + - cd restclient-cpp-* + - cmake -DCMAKE_INSTALL_PREFIX=/usr . + - cmake --build . + - make install + - cd -- - rm -rf build && mkdir -p build && cd build - cmake .. - make VERBOSE=1 From 8650c3cc6b36a1291c79d68370ddfb905c368c40 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 07:49:28 +0100 Subject: [PATCH 60/62] Convert size_t to Json::Value::UInt64 for jsoncpp. jsoncpp-1.7.4 can't convert it itself. --- README.adoc | 2 +- src/config.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index a2f6e60..5c2065c 100644 --- a/README.adoc +++ b/README.adoc @@ -21,7 +21,7 @@ support Atom at the moment. === Dependencies * Tested OS: Linux -* C++ compiler (tested: link:{uri-gcc}[gcc] 7/9) +* C++ compiler (tested: link:{uri-gcc}[gcc] 7/8/9) * link:{uri-cmake}[cmake] (at least: 3.9) * link:{uri-boost}[boost] (at least: 1.62) * link:{uri-mastodon-cpp}[mastodon-cpp] (at least: 0.111.3) diff --git a/src/config.cpp b/src/config.cpp index f617a89..a2d9e35 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -257,7 +257,8 @@ void Config::write() _json[_profile]["instance"] = data.instance; _json[_profile]["interval"] = data.interval; _json[_profile]["last_guid"] = data.last_guid; - _json[_profile]["max_size"] = data.max_size; + _json[_profile]["max_size"] + = static_cast<Json::Value::UInt64>(data.max_size); // Leave skip. _json[_profile]["titles_as_cw"] = data.titles_as_cw; _json[_profile]["titles_only"] = data.titles_only; From ca2b9a16e4e9461c47820ebcd65feecbf2a803f3 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 08:03:34 +0100 Subject: [PATCH 61/62] Move installation of watchwords.json to root CMakeLists.txt --- CMakeLists.txt | 5 +++++ src/CMakeLists.txt | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d684f8..49b3a66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,10 +26,15 @@ set(CMAKE_CXX_EXTENSIONS OFF) include(debug_flags) +include(GNUInstallDirs) + add_subdirectory(src) if(WITH_MAN) add_subdirectory(man) endif() +install(FILES watchwords.json + DESTINATION "${CMAKE_INSTALL_DATADIR}/mastorss") + # include(cmake/packages.cmake) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e7bb453..69106c5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -38,6 +38,3 @@ if(BUILD_SHARED_LIBS) target_compile_definitions(mastorss PRIVATE "BOOST_ALL_DYN_LINK=1") endif() install(TARGETS mastorss DESTINATION "${CMAKE_INSTALL_BINDIR}") - -install(FILES watchwords.json - DESTINATION "${CMAKE_INSTALL_DATADIR}/mastorss") From 8c77df1ee16181316faa10189a15941c72a9d987 Mon Sep 17 00:00:00 2001 From: tastytea <tastytea@tastytea.de> Date: Sat, 28 Dec 2019 08:15:32 +0100 Subject: [PATCH 62/62] CI: Disable Ubuntu build. CMake is too old for restclient-cpp. --- .drone.yml | 68 +++++++++++++++++++++++++----------------------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/.drone.yml b/.drone.yml index 3133636..987065e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -19,11 +19,11 @@ steps: source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.5/libmastodon-cpp_0.111.5-0_buster_amd64.deb destination: mastodon-cpp_buster.deb -- name: download mastodon-cpp for bionic - image: plugins/download - settings: - source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.5/libmastodon-cpp_0.111.5-0_bionic_amd64.deb - destination: mastodon-cpp_bionic.deb +# - name: download mastodon-cpp for bionic +# image: plugins/download +# settings: +# source: https://schlomp.space/tastytea/mastodon-cpp/releases/download/0.111.5/libmastodon-cpp_0.111.5-0_bionic_amd64.deb +# destination: mastodon-cpp_bionic.deb # Workaround until the debian packages are fixed. - name: download restclient-cpp source @@ -65,38 +65,32 @@ steps: - name: debian-package-cache path: /var/cache/apt/archives -- name: GCC 7 on Ubuntu bionic - image: ubuntu:bionic - pull: always - environment: - CXX: g++-7 - CXXFLAGS: -pipe -O2 - DEBIAN_FRONTEND: noninteractive - LANG: C.utf8 - commands: - - rm /etc/apt/apt.conf.d/docker-clean - - alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get' - - apt-get update -q - - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/ubuntu/ bionic main" > /etc/apt/sources.list.d/restclient-cpp.list - - apt-get install -qq curl gnupg apt-transport-https - - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - - - apt-get update -q - - apt-get install -qq build-essential cmake asciidoc - - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev - - apt-get -qq install ./mastodon-cpp_bionic.deb - - tar -xf restclient-cpp.tar.gz - - cd restclient-cpp-* - - cmake -DCMAKE_INSTALL_PREFIX=/usr . - - cmake --build . - - make install - - cd -- - - rm -rf build && mkdir -p build && cd build - - cmake .. - - make VERBOSE=1 - - make install DESTDIR=install - volumes: - - name: debian-package-cache - path: /var/cache/apt/archives +# - name: GCC 7 on Ubuntu bionic +# image: ubuntu:bionic +# pull: always +# environment: +# CXX: g++-7 +# CXXFLAGS: -pipe -O2 +# DEBIAN_FRONTEND: noninteractive +# LANG: C.utf8 +# commands: +# - rm /etc/apt/apt.conf.d/docker-clean +# - alias apt-get='rm -f /var/cache/apt/archives/lock && apt-get' +# - apt-get update -q +# - echo "deb https://packagecloud.io/mrtazz/restclient-cpp/ubuntu/ bionic main" > /etc/apt/sources.list.d/restclient-cpp.list +# - apt-get install -qq curl gnupg apt-transport-https +# - curl -L https://packagecloud.io/mrtazz/restclient-cpp/gpgkey | apt-key add - +# - apt-get update -q +# - apt-get install -qq build-essential cmake asciidoc +# - apt-get install -qq libboost-filesystem-dev libboost-log-dev libboost-regex-dev libjsoncpp-dev catch libcurl4-openssl-dev restclient-cpp libpoco-dev +# - apt-get -qq install ./mastodon-cpp_bionic.deb +# - rm -rf build && mkdir -p build && cd build +# - cmake .. +# - make VERBOSE=1 +# - make install DESTDIR=install +# volumes: +# - name: debian-package-cache +# path: /var/cache/apt/archives - name: notify image: drillster/drone-email