Fixed the fix in to_lowercase with ICU. Now it is really working!

Turns out I fixed the crash, but some unicode-characters were not lower-cased. I'm using ICU now. It is very easy, makes sense and is all-around pleasant.
2019-07-25 05:58:16 +02:00 · 2019-07-25 05:58:16 +02:00 · 2fd761ee72
parent f76a7f8250
commit 2fd761ee72
4 changed files with 16 additions and 39 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -31,7 +31,7 @@ steps:
  - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
  - apt-get update -q
  - apt-get install -qy g++-6 cmake pkg-config
-  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev asciidoc catch
+  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
  - apt-get install -qy -t sid libcurlpp-dev
  - locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8
  - rm -rf build && mkdir -p build && cd build
@ -62,7 +62,7 @@ steps:
  - apt-get update -q
  - apt-get install -qy -t xenial g++-5
  - apt-get install -qy cmake pkg-config
-  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev asciidoc catch
+  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
  - apt-get install -qy -t sid libcurlpp-dev
  - rm -rf build && mkdir -p build && cd build
  - cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES ..
@ -90,7 +90,7 @@ steps:
  - apt-get update -q
  - apt-get install -qy -t xenial g++-9
  - apt-get install -qy cmake pkg-config
-  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev asciidoc catch
+  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
  - apt-get install -qy -t sid libcurlpp-dev
  - rm -rf build && mkdir -p build && cd build
  - cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES ..
@ -112,7 +112,7 @@ steps:
  - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
  - apt-get update -q
  - apt-get install -qy clang cmake pkg-config
-  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev asciidoc catch
+  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
  - apt-get install -qy -t sid libcurlpp-dev
  - rm -rf build && mkdir -p build && cd build
  - cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES ..
@ -136,7 +136,7 @@ steps:
  - apt-get update -q
  - apt-get install -qy -t stretch-backports clang-6.0
  - apt-get install -qy cmake pkg-config
-  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev asciidoc catch
+  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
  - apt-get install -qy -t sid libcurlpp-dev
  - rm -rf build && mkdir -p build && cd build
  - cmake -DCMAKE_INSTALL_PREFIX=/usr -DWITH_MOZILLA=YES ..
@ -195,7 +195,7 @@ steps:
  - echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list.d/sid.list
  - apt-get update -q
  - apt-get install -qy g++-6 cmake pkg-config
-  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev asciidoc catch
+  - apt-get install -qy libcurl4-openssl-dev libxdg-basedir-dev libvsqlitepp-dev libboost-system-dev libboost-filesystem-dev libicu-dev asciidoc catch
  - apt-get install -qy -t sid libcurlpp-dev
  - apt-get install -qy build-essential file
  - rm -rf build && mkdir -p build && cd build
@ -206,32 +206,6 @@ steps:
  - name: debian-package-cache
    path: /var/cache/apt/archives

-# - name: rpm
-#   image: centos:7
-#   pull: always
-#   environment:
-#     CXX: /opt/rh/devtoolset-6/root/usr/bin/g++
-#     CXXFLAGS: -pipe -O2
-#   commands:
-#   - sed -i 's/keepcache=0/keepcache=1/' /etc/yum.conf
-#   - curl -s -o /var/cache/yum/epel-release-latest-7.noarch.rpm http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-#   - rpm -i /var/cache/yum/epel-release-latest-7.noarch.rpm
-#   - yum install -qy centos-release-scl
-#   - yum install -qy devtoolset-6
-#   - scl enable devtoolset-6 bash
-#   - yum install -qy libcurl-devel rpm-build libxdg-basedir-devel boost-system boost-filesystem asciidoc
-#   - yum --enablerepo=epel install -qy cmake3 vsqlite++-devel
-#   - curl -s -o /var/cache/yum/curlpp-devel-0.7.3-5.el6.x86_64.rpm https://download.fedoraproject.org/pub/epel/6/x86_64/Packages/c/curlpp-devel-0.7.3-5.el6.x86_64.rpm
-#   - curl -s -o /var/cache/yum/curlpp-0.7.3-5.el6.x86_64.rpm https://download.fedoraproject.org/pub/epel/6/x86_64/Packages/c/curlpp-0.7.3-5.el6.x86_64.rpm
-#   - yum localinstall -qy /var/cache/yum/curlpp-*
-#   - rm -rf build && mkdir -p build && cd build
-#   - cmake3 -DCMAKE_INSTALL_PREFIX=/usr -DWITH_RPM=YES ..
-#   - make package
-#   - cp -v remwharead-${DRONE_TAG}-0.x86_64.rpm ..
-#   volumes:
-#   - name: centos-package-cache
-#     path: /var/cache/yum
-
 - name: release
  image: plugins/gitea-release
  pull: always
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -1,6 +1,6 @@
 cmake_minimum_required (VERSION 3.2)
 project(remwharead
-  VERSION 0.6.0
+  VERSION 0.6.1
  LANGUAGES CXX
 )

@ -17,6 +17,7 @@ pkg_check_modules(LIBXDG_BASEDIR REQUIRED libxdg-basedir)
 # because the sqlite3 in Debian jessie doesn't come with a pkg-config module.
 pkg_check_modules(SQLITE3 sqlite3)
 pkg_check_modules(CURLPP REQUIRED curlpp)
+find_package(ICU COMPONENTS uc i18n REQUIRED)

 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@ -32,13 +33,16 @@ Include_directories(${PROJECT_BINARY_DIR})
 include_directories(${LIBXDG_BASEDIR_INCLUDE_DIRS})
 include_directories(${SQLITE3_INCLUDE_DIRS})
 include_directories(${CURLPP_INCLUDE_DIRS})
+include_directories(${ICU_INCLUDE_DIRS})

 link_directories(${LIBXDG_BASEDIR_LIBRARY_DIRS})
 link_directories(${SQLITE3_LIBRARY_DIRS})
 link_directories(${CURLPP_LIBRARY_DIRS})
+link_directories(${ICU_LIBRARY_DIRS})

 set(COMMON_LIBRARIES
-  ${LIBXDG_BASEDIR_LIBRARIES} vsqlitepp stdc++fs ${CURLPP_LIBRARIES})
+  ${LIBXDG_BASEDIR_LIBRARIES} vsqlitepp stdc++fs ${CURLPP_LIBRARIES}
+  ${ICU_LIBRARIES})

 # Write version in header
 configure_file(
--- a/README.adoc
+++ b/README.adoc
@ -55,6 +55,7 @@ only.
 * http://repo.or.cz/w/libxdg-basedir.git[libxdg-basedir] (tested: 1.2)
 * http://www.curlpp.org/[curlpp] (tested: 0.8 / 0.7)
 * http://vsqlite.virtuosic-bytes.com/[vsqlite++] (tested: 0.3)
+* http://www.icu-project.org/[ICU] (tested: 64.2 / 57.1)
 * Optional:
 ** Manpage: http://asciidoc.org/[asciidoc] (tested: 8.6)
 ** Tests: https://github.com/catchorg/Catch2[catch] (tested: 2.5 / 1.2)
--- a/src/search.cpp
+++ b/src/search.cpp
@ -17,7 +17,7 @@
 #include <regex>
 #include <algorithm>
 #include <locale>
-#include <codecvt>
+#include <unicode/unistr.h>
 #include "search.hpp"

 using std::regex;
@ -63,11 +63,9 @@ const vector<vector<string>> parse_expression(string expression)

 const string to_lowercase(const string &str)
 {
+    icu::UnicodeString uni(str.c_str());
    string out;
-    std::locale loc("");
-    const std::ctype<char>& ct = std::use_facet<std::ctype<char>>(loc);
-    std::transform(str.begin(), str.end(), std::back_inserter(out),
-                   std::bind1st(std::mem_fun(&std::ctype<char>::tolower), &ct));
+    uni.toLower().toUTF8String(out);
    return out;
 }