From d5cabfe5c65ac29d2f9c151b46c01a9fd885a9e0 Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Wed, 10 May 2017 02:58:03 +0900 Subject: [PATCH] Use CLD3 (#2949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compact Language Detector v3 (CLD3) is the successor of CLD2, which was used in the previous implementation. CLD3 includes improvements since CLD2, and supports newer compilers. On the other hand, it has additional requirements and cld3-ruby, the FFI of CLD3 for Ruby, is still new and may be still inmature. Though CLD3 is named after CLD2, it is implemented with a neural network model, different from the old implementation, which is based on a Naïve Bayesian classifier. CLD3 supports newer compilers, such as GCC 6. CLD2 is not compatible with GCC 6 because it assigns negative values to varibales typed unsigned. (see internal/cld_generated_cjk_uni_prop_80.cc) The support for GCC 6 and newer compilers are essential today, when some server operating system such as Ubuntu Server 16.10 has GCC 6 by default. On the one hand, CLD3 requires C++11 support. Environments with old compilers such as Ubuntu Server 14.04 needs to update the system or install a newer compiler. CLD3 needs protocol buffers as a new dependency. However,it is not considered problematic because major server operating systems, CentOS and Ubuntu Server provide them. The FFI cld3-ruby was written by me (Akihiko Odaki) for use in Mastodon. It is still new and may be inmature, but confirmed to pass existing tests. --- .travis.yml | 7 +++++-- Dockerfile | 4 +++- Gemfile | 2 +- Gemfile.lock | 6 +++--- Vagrantfile | 2 ++ app/lib/language_detector.rb | 9 +++++---- spec/lib/language_detector_spec.rb | 8 ++++---- 7 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 632074874..f18da476b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,9 @@ env: - LOCAL_DOMAIN=cb6e6126.ngrok.io - LOCAL_HTTPS=true - RAILS_ENV=test - - CXX=g++-4.8 - NOKOGIRI_USE_SYSTEM_LIBRARIES=true - PARALLEL_TEST_PROCESSORS=2 + - "PATH=$HOME:$PATH" addons: postgresql: 9.4 @@ -24,8 +24,10 @@ addons: - ubuntu-toolchain-r-test - trusty-media packages: - - g++-4.8 - ffmpeg + - g++-6 + - libprotobuf-dev + - protobuf-compiler rvm: - 2.3.4 @@ -43,6 +45,7 @@ install: before_script: - bundle exec rake parallel:create parallel:load_schema parallel:prepare - bundle exec rails assets:precompile + - ln -s /usr/bin/x86_64-linux-gnu-g++-6 "$HOME/g++" script: - bundle exec parallel_test spec/ --group-by filesize --type rspec diff --git a/Dockerfile b/Dockerfile index 3e4727258..e2926725a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ RUN echo "@edge https://nl.alpinelinux.org/alpine/edge/main" >> /etc/apk/reposit libxml2-dev \ libxslt-dev \ python \ - build-base" \ + build-base \ + protobuf-dev" \ && apk -U upgrade && apk add \ $BUILD_DEPS \ nodejs@edge \ @@ -29,6 +30,7 @@ RUN echo "@edge https://nl.alpinelinux.org/alpine/edge/main" >> /etc/apk/reposit file \ imagemagick@edge \ ca-certificates \ + protobuf \ && npm install -g npm@3 && npm install -g yarn \ && update-ca-certificates \ && rm -rf /tmp/* /var/cache/apk/* diff --git a/Gemfile b/Gemfile index 3651ad8b0..9c22b3c42 100644 --- a/Gemfile +++ b/Gemfile @@ -19,7 +19,7 @@ gem 'paperclip', '~> 5.1' gem 'paperclip-av-transcoder' gem 'addressable' -gem 'cld2', require: 'cld' +gem 'cld3', '~> 3.1.0' gem 'devise' gem 'devise-two-factor' gem 'doorkeeper' diff --git a/Gemfile.lock b/Gemfile.lock index 1baf6bec5..059fbab0e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -99,8 +99,8 @@ GEM rack-test (>= 0.5.4) xpath (~> 2.0) chunky_png (1.3.8) - cld2 (1.0.3) - ffi (~> 1.9.3) + cld3 (3.1.0) + ffi (>= 1.1.0, < 1.10.0) climate_control (0.1.0) cocaine (0.5.8) climate_control (>= 0.0.3, < 1.0) @@ -480,7 +480,7 @@ DEPENDENCIES capistrano-rbenv capistrano-yarn capybara - cld2 + cld3 (~> 3.1.0) devise devise-two-factor doorkeeper diff --git a/Vagrantfile b/Vagrantfile index b2ebbb83f..f2302b24f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -33,7 +33,9 @@ sudo apt-get install \ redis-tools \ postgresql \ postgresql-contrib \ + protobuf-compiler \ yarn \ + libprotobuf-dev \ libreadline-dev \ -y diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb index ca5cb2591..1c22a9ccc 100644 --- a/app/lib/language_detector.rb +++ b/app/lib/language_detector.rb @@ -6,6 +6,7 @@ class LanguageDetector def initialize(text, account = nil) @text = text @account = account + @identifier = CLD3::NNetLanguageIdentifier.new(1, 2048) end def to_iso_s @@ -15,15 +16,15 @@ class LanguageDetector private def detected_language_code - detected_language[:code].to_sym if detected_language_reliable? + result.language.to_sym if detected_language_reliable? end - def detected_language - @_detected_language ||= CLD.detect_language(text_without_urls) + def result + @result ||= @identifier.find_language(text_without_urls) end def detected_language_reliable? - detected_language[:reliable] + result.reliable? end def text_without_urls diff --git a/spec/lib/language_detector_spec.rb b/spec/lib/language_detector_spec.rb index 18ab5aee6..e543edd49 100644 --- a/spec/lib/language_detector_spec.rb +++ b/spec/lib/language_detector_spec.rb @@ -24,15 +24,15 @@ describe LanguageDetector do end describe 'when language can\'t be detected' do - it 'confirm language engine cant detect' do - result = CLD.detect_language('') - expect(result[:reliable]).to be false + it 'uses default locale when sent an empty document' do + result = described_class.new('').to_iso_s + expect(result).to eq :en end describe 'because of a URL' do it 'uses default locale when sent just a URL' do string = 'http://example.com/media/2kFTgOJLXhQf0g2nKB4' - cld_result = CLD.detect_language(string)[:code] + cld_result = CLD3::NNetLanguageIdentifier.new(0, 2048).find_language(string) expect(cld_result).not_to eq :en result = described_class.new(string).to_iso_s