Add user content translations with configurable backends (#19218)

app/controllers/api/v1/statuses/translations_controller.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class Api::V1::Statuses::TranslationsController < Api::BaseController
+  include Authorization
+
+  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }
+  before_action :set_status
+  before_action :set_translation
+
+  rescue_from TranslationService::NotConfiguredError, with: :not_found
+  rescue_from TranslationService::UnexpectedResponseError, TranslationService::QuotaExceededError, TranslationService::TooManyRequestsError, with: :service_unavailable
+
+  def create
+    render json: @translation, serializer: REST::TranslationSerializer
+  end
+
+  private
+
+  def set_status
+    @status = Status.find(params[:status_id])
+    authorize @status, :show?
+  rescue Mastodon::NotPermittedError
+    not_found
+  end
+
+  def set_translation
+    @translation = TranslateStatusService.new.call(@status, content_locale)
+  end
+end

app/javascript/mastodon/actions/statuses.js

@@ -34,6 +34,11 @@ export const STATUS_FETCH_SOURCE_REQUEST = 'STATUS_FETCH_SOURCE_REQUEST';
 export const STATUS_FETCH_SOURCE_SUCCESS = 'STATUS_FETCH_SOURCE_SUCCESS';
 export const STATUS_FETCH_SOURCE_FAIL    = 'STATUS_FETCH_SOURCE_FAIL';
 
+export const STATUS_TRANSLATE_REQUEST = 'STATUS_TRANSLATE_REQUEST';
+export const STATUS_TRANSLATE_SUCCESS = 'STATUS_TRANSLATE_SUCCESS';
+export const STATUS_TRANSLATE_FAIL    = 'STATUS_TRANSLATE_FAIL';
+export const STATUS_TRANSLATE_UNDO    = 'STATUS_TRANSLATE_UNDO';
+
 export function fetchStatusRequest(id, skipLoading) {
   return {
     type: STATUS_FETCH_REQUEST,
@@ -309,4 +314,36 @@ export function toggleStatusCollapse(id, isCollapsed) {
     id,
     isCollapsed,
   };
-}
+};
+
+export const translateStatus = id => (dispatch, getState) => {
+  dispatch(translateStatusRequest(id));
+
+  api(getState).post(`/api/v1/statuses/${id}/translate`).then(response => {
+    dispatch(translateStatusSuccess(id, response.data));
+  }).catch(error => {
+    dispatch(translateStatusFail(id, error));
+  });
+};
+
+export const translateStatusRequest = id => ({
+  type: STATUS_TRANSLATE_REQUEST,
+  id,
+});
+
+export const translateStatusSuccess = (id, translation) => ({
+  type: STATUS_TRANSLATE_SUCCESS,
+  id,
+  translation,
+});
+
+export const translateStatusFail = (id, error) => ({
+  type: STATUS_TRANSLATE_FAIL,
+  id,
+  error,
+});
+
+export const undoStatusTranslation = id => ({
+  type: STATUS_TRANSLATE_UNDO,
+  id,
+});

app/javascript/mastodon/components/status.js

@@ -85,6 +85,7 @@ class Status extends ImmutablePureComponent {
     onHeightChange: PropTypes.func,
     onToggleHidden: PropTypes.func,
     onToggleCollapsed: PropTypes.func,
+    onTranslate: PropTypes.func,
     muted: PropTypes.bool,
     hidden: PropTypes.bool,
     unread: PropTypes.bool,
@@ -171,6 +172,10 @@ class Status extends ImmutablePureComponent {
     this.props.onToggleCollapsed(this._properStatus(), isCollapsed);
   }
 
+  handleTranslate = () => {
+    this.props.onTranslate(this._properStatus());
+  }
+
   renderLoadingMediaGallery () {
     return <div className='media-gallery' style={{ height: '110px' }} />;
   }
@@ -512,7 +517,16 @@ class Status extends ImmutablePureComponent {
               </a>
             </div>
 
-            <StatusContent status={status} onClick={this.handleClick} expanded={!status.get('hidden')} showThread={showThread} onExpandedToggle={this.handleExpandedToggle} collapsable onCollapsedToggle={this.handleCollapsedToggle} />
+            <StatusContent
+              status={status}
+              onClick={this.handleClick}
+              expanded={!status.get('hidden')}
+              showThread={showThread}
+              onExpandedToggle={this.handleExpandedToggle}
+              onTranslate={this.handleTranslate}
+              collapsable
+              onCollapsedToggle={this.handleCollapsedToggle}
+            />
 
             {media}
 

app/javascript/mastodon/components/status_content.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import PropTypes from 'prop-types';
-import { FormattedMessage } from 'react-intl';
+import { FormattedMessage, injectIntl } from 'react-intl';
 import Permalink from './permalink';
 import classnames from 'classnames';
 import PollContainer from 'mastodon/containers/poll_container';
@@ -10,7 +10,8 @@ import { autoPlayGif } from 'mastodon/initial_state';
 
 const MAX_HEIGHT = 642; // 20px * 32 (+ 2px padding at the top)
 
-export default class StatusContent extends React.PureComponent {
+export default @injectIntl
+class StatusContent extends React.PureComponent {
 
   static contextTypes = {
     router: PropTypes.object,
@@ -21,9 +22,11 @@ export default class StatusContent extends React.PureComponent {
     expanded: PropTypes.bool,
     showThread: PropTypes.bool,
     onExpandedToggle: PropTypes.func,
+    onTranslate: PropTypes.func,
     onClick: PropTypes.func,
     collapsable: PropTypes.bool,
     onCollapsedToggle: PropTypes.func,
+    intl: PropTypes.object,
   };
 
   state = {
@@ -163,20 +166,26 @@ export default class StatusContent extends React.PureComponent {
     }
   }
 
+  handleTranslate = () => {
+    this.props.onTranslate();
+  }
+
   setRef = (c) => {
     this.node = c;
   }
 
   render () {
-    const { status } = this.props;
+    const { status, intl } = this.props;
 
     const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden;
     const renderReadMore = this.props.onClick && status.get('collapsed');
     const renderViewThread = this.props.showThread && status.get('in_reply_to_id') && status.get('in_reply_to_account_id') === status.getIn(['account', 'id']);
+    const renderTranslate = this.props.onTranslate && ['public', 'unlisted'].includes(status.get('visibility')) && intl.locale !== status.get('language');
+    const languageNames = new Intl.DisplayNames([intl.locale], { type: 'language' });
 
-    const content = { __html: status.get('contentHtml') };
+    const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') };
     const spoilerContent = { __html: status.get('spoilerHtml') };
-    const lang = status.get('language');
+    const lang = status.get('translation') ? intl.locale : status.get('language');
     const classNames = classnames('status__content', {
       'status__content--with-action': this.props.onClick && this.context.router,
       'status__content--with-spoiler': status.get('spoiler_text').length > 0,
@@ -195,6 +204,12 @@ export default class StatusContent extends React.PureComponent {
       </button>
     );
 
+    const translateButton = (
+      <button className='status__content__read-more-button' onClick={this.handleTranslate}>
+        {status.get('translation') ? <span><FormattedMessage id='status.translated_from' defaultMessage='Translated from {lang}' values={{ lang: languageNames.of(status.get('language')) }} /> · <FormattedMessage id='status.show_original' defaultMessage='Show original' /></span> : <FormattedMessage id='status.translate' defaultMessage='Translate' />}
+      </button>
+    );
+
     if (status.get('spoiler_text').length > 0) {
       let mentionsPlaceholder = '';
 
@@ -223,7 +238,7 @@ export default class StatusContent extends React.PureComponent {
           <div tabIndex={!hidden ? 0 : null} className={`status__content__text ${!hidden ? 'status__content__text--visible' : ''} translate`} lang={lang} dangerouslySetInnerHTML={content} />
 
           {!hidden && !!status.get('poll') && <PollContainer pollId={status.get('poll')} />}
-
+          {!hidden && renderTranslate && translateButton}
           {renderViewThread && showThreadButton}
         </div>
       );
@@ -233,7 +248,7 @@ export default class StatusContent extends React.PureComponent {
           <div className='status__content__text status__content__text--visible translate' lang={lang} dangerouslySetInnerHTML={content} />
 
           {!!status.get('poll') && <PollContainer pollId={status.get('poll')} />}
-
+          {renderTranslate && translateButton}
           {renderViewThread && showThreadButton}
         </div>,
       ];
@@ -249,7 +264,7 @@ export default class StatusContent extends React.PureComponent {
           <div className='status__content__text status__content__text--visible translate' lang={lang} dangerouslySetInnerHTML={content} />
 
           {!!status.get('poll') && <PollContainer pollId={status.get('poll')} />}
-
+          {renderTranslate && translateButton}
           {renderViewThread && showThreadButton}
         </div>
       );

app/javascript/mastodon/containers/status_container.js

@@ -25,6 +25,8 @@ import {
   revealStatus,
   toggleStatusCollapse,
   editStatus,
+  translateStatus,
+  undoStatusTranslation,
 } from '../actions/statuses';
 import {
   unmuteAccount,
@@ -150,6 +152,14 @@ const mapDispatchToProps = (dispatch, { intl, contextType }) => ({
     dispatch(editStatus(status.get('id'), history));
   },
 
+  onTranslate (status) {
+    if (status.get('translation')) {
+      dispatch(undoStatusTranslation(status.get('id')));
+    } else {
+      dispatch(translateStatus(status.get('id')));
+    }
+  },
+
   onDirect (account, router) {
     dispatch(directCompose(account, router));
   },

app/javascript/mastodon/features/status/components/detailed_status.js

@@ -37,6 +37,7 @@ class DetailedStatus extends ImmutablePureComponent {
     onOpenMedia: PropTypes.func.isRequired,
     onOpenVideo: PropTypes.func.isRequired,
     onToggleHidden: PropTypes.func.isRequired,
+    onTranslate: PropTypes.func.isRequired,
     measureHeight: PropTypes.bool,
     onHeightChange: PropTypes.func,
     domain: PropTypes.string.isRequired,
@@ -103,6 +104,11 @@ class DetailedStatus extends ImmutablePureComponent {
     window.open(href, 'mastodon-intent', 'width=445,height=600,resizable=no,menubar=no,status=no,scrollbars=yes');
   }
 
+  handleTranslate = () => {
+    const { onTranslate, status } = this.props;
+    onTranslate(status);
+  }
+
   render () {
     const status = (this.props.status && this.props.status.get('reblog')) ? this.props.status.get('reblog') : this.props.status;
     const outerStyle = { boxSizing: 'border-box' };
@@ -260,7 +266,12 @@ class DetailedStatus extends ImmutablePureComponent {
             <DisplayName account={status.get('account')} localDomain={this.props.domain} />
           </a>
 
-          <StatusContent status={status} expanded={!status.get('hidden')} onExpandedToggle={this.handleExpandedToggle} />
+          <StatusContent
+            status={status}
+            expanded={!status.get('hidden')}
+            onExpandedToggle={this.handleExpandedToggle}
+            onTranslate={this.handleTranslate}
+          />
 
           {media}
 

app/javascript/mastodon/features/status/index.js

@@ -32,6 +32,8 @@ import {
   editStatus,
   hideStatus,
   revealStatus,
+  translateStatus,
+  undoStatusTranslation,
 } from '../../actions/statuses';
 import {
   unblockAccount,
@@ -339,6 +341,16 @@ class Status extends ImmutablePureComponent {
     }
   }
 
+  handleTranslate = status => {
+    const { dispatch } = this.props;
+
+    if (status.get('translation')) {
+      dispatch(undoStatusTranslation(status.get('id')));
+    } else {
+      dispatch(translateStatus(status.get('id')));
+    }
+  }
+
   handleBlockClick = (status) => {
     const { dispatch } = this.props;
     const account = status.get('account');
@@ -558,6 +570,7 @@ class Status extends ImmutablePureComponent {
                   onOpenVideo={this.handleOpenVideo}
                   onOpenMedia={this.handleOpenMedia}
                   onToggleHidden={this.handleToggleHidden}
+                  onTranslate={this.handleTranslate}
                   domain={domain}
                   showMedia={this.state.showMedia}
                   onToggleMediaVisibility={this.handleToggleMediaVisibility}

app/javascript/mastodon/reducers/statuses.js

@@ -13,6 +13,8 @@ import {
   STATUS_REVEAL,
   STATUS_HIDE,
   STATUS_COLLAPSE,
+  STATUS_TRANSLATE_SUCCESS,
+  STATUS_TRANSLATE_UNDO,
 } from '../actions/statuses';
 import { TIMELINE_DELETE } from '../actions/timelines';
 import { STATUS_IMPORT, STATUSES_IMPORT } from '../actions/importer';
@@ -77,6 +79,10 @@ export default function statuses(state = initialState, action) {
     return state.setIn([action.id, 'collapsed'], action.isCollapsed);
   case TIMELINE_DELETE:
     return deleteStatus(state, action.id, action.references);
+  case STATUS_TRANSLATE_SUCCESS:
+    return state.setIn([action.id, 'translation'], fromJS(action.translation));
+  case STATUS_TRANSLATE_UNDO:
+    return state.deleteIn([action.id, 'translation']);
   default:
     return state;
   }

app/lib/translation_service.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class TranslationService
+  class Error < StandardError; end
+  class NotConfiguredError < Error; end
+  class TooManyRequestsError < Error; end
+  class QuotaExceededError < Error; end
+  class UnexpectedResponseError < Error; end
+
+  def self.configured
+    if ENV['DEEPL_API_KEY'].present?
+      TranslationService::DeepL.new(ENV.fetch('DEEPL_PLAN', 'free'), ENV['DEEPL_API_KEY'])
+    elsif ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
+      TranslationService::LibreTranslate.new(ENV['LIBRE_TRANSLATE_ENDPOINT'], ENV['LIBRE_TRANSLATE_API_KEY'])
+    else
+      raise NotConfiguredError
+    end
+  end
+
+  def translate(_text, _source_language, _target_language)
+    raise NotImplementedError
+  end
+end

app/lib/translation_service/deepl.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class TranslationService::DeepL < TranslationService
+  include JsonLdHelper
+
+  def initialize(plan, api_key)
+    super()
+
+    @plan    = plan
+    @api_key = api_key
+  end
+
+  def translate(text, source_language, target_language)
+    request(text, source_language, target_language).perform do |res|
+      case res.code
+      when 429
+        raise TooManyRequestsError
+      when 456
+        raise QuotaExceededError
+      when 200...300
+        transform_response(res.body_with_limit)
+      else
+        raise UnexpectedResponseError
+      end
+    end
+  end
+
+  private
+
+  def request(text, source_language, target_language)
+    req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language.upcase, target_lang: target_language, tag_handling: 'html' })
+    req.add_headers('Authorization': "DeepL-Auth-Key #{@api_key}")
+    req
+  end
+
+  def endpoint_url
+    if @plan == 'free'
+      'https://api-free.deepl.com/v2/translate'
+    else
+      'https://api.deepl.com/v2/translate'
+    end
+  end
+
+  def transform_response(str)
+    json = Oj.load(str, mode: :strict)
+
+    raise UnexpectedResponseError unless json.is_a?(Hash)
+
+    Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase)
+  rescue Oj::ParseError
+    raise UnexpectedResponseError
+  end
+end

app/lib/translation_service/libre_translate.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class TranslationService::LibreTranslate < TranslationService
+  def initialize(base_url, api_key)
+    super()
+
+    @base_url = base_url
+    @api_key  = api_key
+  end
+
+  def translate(text, source_language, target_language)
+    request(text, source_language, target_language).perform do |res|
+      case res.code
+      when 429
+        raise TooManyRequestsError
+      when 403
+        raise QuotaExceededError
+      when 200...300
+        transform_response(res.body_with_limit, source_language)
+      else
+        raise UnexpectedResponseError
+      end
+    end
+  end
+
+  private
+
+  def request(text, source_language, target_language)
+    req = Request.new(:post, "#{@base_url}/translate", body: Oj.dump(q: text, source: source_language, target: target_language, format: 'html', api_key: @api_key))
+    req.add_headers('Content-Type': 'application/json')
+    req
+  end
+
+  def transform_response(str, source_language)
+    json = Oj.load(str, mode: :strict)
+
+    raise UnexpectedResponseError unless json.is_a?(Hash)
+
+    Translation.new(text: json['translatedText'], detected_source_language: source_language)
+  rescue Oj::ParseError
+    raise UnexpectedResponseError
+  end
+end

app/lib/translation_service/translation.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class TranslationService::Translation < ActiveModelSerializers::Model
+  attributes :text, :detected_source_language
+end

app/serializers/rest/translation_serializer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class REST::TranslationSerializer < ActiveModel::Serializer
+  attributes :content, :detected_source_language
+
+  def content
+    object.text
+  end
+end

app/services/translate_status_service.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class TranslateStatusService < BaseService
+  CACHE_TTL = 1.day.freeze
+
+  def call(status, target_language)
+    raise Mastodon::NotPermittedError unless status.public_visibility? || status.unlisted_visibility?
+
+    @status = status
+    @target_language = target_language
+
+    Rails.cache.fetch("translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) { translation_backend.translate(@status.text, @status.language, @target_language) }
+  end
+
+  private
+
+  def translation_backend
+    TranslationService.configured
+  end
+
+  def content_hash
+    Digest::SHA256.base64digest(@status.text)
+  end
+end

config/initializers/inflections.rb

@@ -25,6 +25,7 @@ ActiveSupport::Inflector.inflections(:en) do |inflect|
   inflect.acronym 'REST'
   inflect.acronym 'URL'
   inflect.acronym 'ASCII'
+  inflect.acronym 'DeepL'
 
   inflect.singular 'data', 'data'
 end

config/routes.rb

@@ -393,6 +393,8 @@ Rails.application.routes.draw do
 
           resource :history, only: :show
           resource :source, only: :show
+
+          post :translate, to: 'translations#create'
         end
 
         member do