Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- `README.md`:
  Our README.md files are completely different. Discarded upstream changes.
- `app/javascript/core/admin.js`:
  Updating rails-ujs, no real conflict, but a comment to close to changed
  code. Various glitch-soc-only files have been updated to match those changes,
  though.
- `package.json`:
  No real conflict, just an additional dependency in glitch-soc that was too
  close to something updated upstream. Took upstream's changes.

Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:18.04 as build-dep
 SHELL ["bash", "-c"]
 
 # Install Node v12 (LTS)
-ENV NODE_VER="12.14.0"  
+ENV NODE_VER="12.16.1"
 RUN	ARCH= && \
     dpkgArch="$(dpkg --print-architecture)" && \
   case "${dpkgArch##*-}" in \

Gemfile

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 source 'https://rubygems.org'
-ruby '>= 2.4.0', '< 3.0.0'
+ruby '>= 2.5.0', '< 3.0.0'
 
 gem 'pkg-config', '~> 1.4'
 
@@ -35,7 +35,7 @@ gem 'browser'
 gem 'charlock_holmes', '~> 0.7.7'
 gem 'iso-639'
 gem 'chewy', '~> 5.1'
-gem 'cld3', '~> 3.2.6'
+gem 'cld3', '~> 3.3.0'
 gem 'devise', '~> 4.7'
 gem 'devise-two-factor', '~> 3.1'
 
@@ -84,7 +84,7 @@ gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock'
 gem 'rqrcode', '~> 1.1'
 gem 'ruby-progressbar', '~> 1.10'
 gem 'sanitize', '~> 5.1'
-gem 'sidekiq', '~> 5.2'
+gem 'sidekiq', '~> 6.0'
 gem 'sidekiq-scheduler', '~> 3.0'
 gem 'sidekiq-unique-jobs', '~> 6.0'
 gem 'sidekiq-bulk', '~>0.2.0'
@@ -145,7 +145,7 @@ group :development do
   gem 'brakeman', '~> 4.7', require: false
   gem 'bundler-audit', '~> 0.6', require: false
 
-  gem 'capistrano', '~> 3.11'
+  gem 'capistrano', '~> 3.12'
   gem 'capistrano-rails', '~> 1.4'
   gem 'capistrano-rbenv', '~> 2.1'
   gem 'capistrano-yarn', '~> 2.0'

Gemfile.lock

@@ -128,7 +128,7 @@ GEM
       bundler (>= 1.2.0, < 3)
       thor (~> 0.18)
     byebug (11.1.1)
-    capistrano (3.11.2)
+    capistrano (3.12.1)
       airbrussh (>= 1.0.0)
       i18n
       rake (>= 10.0.0)
@@ -160,7 +160,7 @@ GEM
       elasticsearch (>= 2.0.0)
       elasticsearch-dsl
     chunky_png (1.3.11)
-    cld3 (3.2.6)
+    cld3 (3.3.0)
       ffi (>= 1.1.0, < 1.12.0)
     climate_control (0.2.0)
     cocaine (0.5.8)
@@ -214,7 +214,7 @@ GEM
     encryptor (3.0.0)
     equatable (0.6.1)
     erubi (1.9.0)
-    et-orbi (1.1.6)
+    et-orbi (1.2.3)
       tzinfo
     excon (0.71.0)
     fabrication (2.21.0)
@@ -241,8 +241,8 @@ GEM
       fog-json (>= 1.0)
       ipaddress (>= 0.8)
     formatador (0.2.5)
-    fugit (1.1.6)
-      et-orbi (~> 1.1, >= 1.1.6)
+    fugit (1.3.3)
+      et-orbi (~> 1.1, >= 1.1.8)
       raabro (~> 1.1)
     fuubar (2.5.0)
       rspec-core (~> 3.0)
@@ -265,8 +265,8 @@ GEM
       railties (>= 4.0.1)
     hamster (3.0.0)
       concurrent-ruby (~> 1.0)
-    hashdiff (1.0.0)
-    hashie (3.6.0)
+    hashdiff (1.0.1)
+    hashie (4.1.0)
     highline (2.0.3)
     hiredis (0.6.3)
     hkdf (0.3.0)
@@ -304,9 +304,9 @@ GEM
     jmespath (1.4.0)
     json (2.3.0)
     json-canonicalization (0.2.0)
-    json-ld (3.1.0)
+    json-ld (3.1.1)
       htmlentities (~> 4.3)
-      json-canonicalization (~> 0.1)
+      json-canonicalization (~> 0.2)
       link_header (~> 0.0, >= 0.0.8)
       multi_json (~> 1.14)
       rack (~> 2.0)
@@ -384,8 +384,8 @@ GEM
       sidekiq (>= 3.5)
       statsd-ruby (~> 1.4, >= 1.4.0)
     oj (3.10.3)
-    omniauth (1.9.0)
-      hashie (>= 3.4.6, < 3.7.0)
+    omniauth (1.9.1)
+      hashie (>= 3.4.6)
       rack (>= 1.6.2, < 3)
     omniauth-cas (1.1.1)
       addressable (~> 2.3)
@@ -445,7 +445,7 @@ GEM
       rack (>= 1.0, < 3)
     rack-cors (1.1.1)
       rack (>= 2.0.0)
-    rack-protection (2.0.7)
+    rack-protection (2.0.8.1)
       rack
     rack-proxy (0.6.5)
       rack
@@ -556,32 +556,34 @@ GEM
     ruby-progressbar (1.10.1)
     ruby-saml (1.9.0)
       nokogiri (>= 1.5.10)
-    rufus-scheduler (3.5.2)
-      fugit (~> 1.1, >= 1.1.5)
+    rufus-scheduler (3.6.0)
+      fugit (~> 1.1, >= 1.1.6)
     safe_yaml (1.0.5)
     sanitize (5.1.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
-    sidekiq (5.2.7)
-      connection_pool (~> 2.2, >= 2.2.2)
-      rack (>= 1.5.0)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
+    sidekiq (6.0.4)
+      connection_pool (>= 2.2.2)
+      rack (>= 2.0.0)
+      rack-protection (>= 2.0.0)
+      redis (>= 4.1.0)
     sidekiq-bulk (0.2.0)
       sidekiq
-    sidekiq-scheduler (3.0.0)
+    sidekiq-scheduler (3.0.1)
+      e2mmap
       redis (>= 3, < 5)
       rufus-scheduler (~> 3.2)
       sidekiq (>= 3)
+      thwait
       tilt (>= 1.4.0)
-    sidekiq-unique-jobs (6.0.18)
+    sidekiq-unique-jobs (6.0.20)
       concurrent-ruby (~> 1.0, >= 1.0.5)
       sidekiq (>= 4.0, < 7.0)
       thor (~> 0)
     simple-navigation (4.1.0)
       activesupport (>= 2.3.2)
-    simple_form (5.0.1)
+    simple_form (5.0.2)
       actionpack (>= 5.0)
       activemodel (>= 5.0)
     simplecov (0.18.2)
@@ -595,7 +597,7 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
-    sshkit (1.20.0)
+    sshkit (1.21.0)
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
     stackprof (0.2.15)
@@ -640,7 +642,7 @@ GEM
     uniform_notifier (1.13.0)
     warden (1.2.8)
       rack (>= 2.0.6)
-    webmock (3.8.0)
+    webmock (3.8.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -675,14 +677,14 @@ DEPENDENCIES
   browser
   bullet (~> 6.1)
   bundler-audit (~> 0.6)
-  capistrano (~> 3.11)
+  capistrano (~> 3.12)
   capistrano-rails (~> 1.4)
   capistrano-rbenv (~> 2.1)
   capistrano-yarn (~> 2.0)
   capybara (~> 3.31)
   charlock_holmes (~> 0.7.7)
   chewy (~> 5.1)
-  cld3 (~> 3.2.6)
+  cld3 (~> 3.3.0)
   climate_control (~> 0.2)
   concurrent-ruby
   connection_pool
@@ -767,7 +769,7 @@ DEPENDENCIES
   rubocop-rails (~> 2.4)
   ruby-progressbar (~> 1.10)
   sanitize (~> 5.1)
-  sidekiq (~> 5.2)
+  sidekiq (~> 6.0)
   sidekiq-bulk (~> 0.2.0)
   sidekiq-scheduler (~> 3.0)
   sidekiq-unique-jobs (~> 6.0)

app/chewy/statuses_index.rb

@@ -47,6 +47,11 @@ class StatusesIndex < Chewy::Index
       data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
     end
 
+    crutch :bookmarks do |collection|
+      data = ::Bookmark.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
+      data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+    end
+
     root date_detection: false do
       field :id, type: 'long'
       field :account_id, type: 'long'

app/controllers/admin/email_domain_blocks_controller.rb

@@ -6,12 +6,12 @@ module Admin
 
     def index
       authorize :email_domain_block, :index?
-      @email_domain_blocks = EmailDomainBlock.page(params[:page])
+      @email_domain_blocks = EmailDomainBlock.where(parent_id: nil).includes(:children).order(id: :desc).page(params[:page])
     end
 
     def new
       authorize :email_domain_block, :create?
-      @email_domain_block = EmailDomainBlock.new
+      @email_domain_block = EmailDomainBlock.new(domain: params[:_domain])
     end
 
     def create
@@ -21,6 +21,28 @@ module Admin
 
       if @email_domain_block.save
         log_action :create, @email_domain_block
+
+        if @email_domain_block.with_dns_records?
+          hostnames = []
+          ips       = []
+
+          Resolv::DNS.open do |dns|
+            dns.timeouts = 1
+
+            hostnames = dns.getresources(@email_domain_block.domain, Resolv::DNS::Resource::IN::MX).to_a.map { |e| e.exchange.to_s }
+
+            ([@email_domain_block.domain] + hostnames).uniq.each do |hostname|
+              ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::A).to_a.map { |e| e.address.to_s })
+              ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::AAAA).to_a.map { |e| e.address.to_s })
+            end
+          end
+
+          (hostnames + ips).each do |hostname|
+            another_email_domain_block = EmailDomainBlock.new(domain: hostname, parent: @email_domain_block)
+            log_action :create, another_email_domain_block if another_email_domain_block.save
+          end
+        end
+
         redirect_to admin_email_domain_blocks_path, notice: I18n.t('admin.email_domain_blocks.created_msg')
       else
         render :new
@@ -41,7 +63,7 @@ module Admin
     end
 
     def resource_params
-      params.require(:email_domain_block).permit(:domain)
+      params.require(:email_domain_block).permit(:domain, :with_dns_records)
     end
   end
 end

app/controllers/admin/warning_presets_controller.rb

@@ -7,7 +7,7 @@ module Admin
     def index
       authorize :account_warning_preset, :index?
 
-      @warning_presets = AccountWarningPreset.all
+      @warning_presets = AccountWarningPreset.alphabetic
       @warning_preset  = AccountWarningPreset.new
     end
 
@@ -19,7 +19,7 @@ module Admin
       if @warning_preset.save
         redirect_to admin_warning_presets_path
       else
-        @warning_presets = AccountWarningPreset.all
+        @warning_presets = AccountWarningPreset.alphabetic
         render :index
       end
     end
@@ -52,7 +52,7 @@ module Admin
     end
 
     def warning_preset_params
-      params.require(:account_warning_preset).permit(:text)
+      params.require(:account_warning_preset).permit(:title, :text)
     end
   end
 end

app/controllers/api/v1/accounts/follower_accounts_controller.rb

@@ -25,7 +25,7 @@ class Api::V1::Accounts::FollowerAccountsController < Api::BaseController
   end
 
   def hide_results?
-    (@account.user_hides_network? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
+    (@account.hides_followers? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
   end
 
   def default_accounts

app/controllers/api/v1/accounts/following_accounts_controller.rb

@@ -25,7 +25,7 @@ class Api::V1::Accounts::FollowingAccountsController < Api::BaseController
   end
 
   def hide_results?
-    (@account.user_hides_network? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
+    (@account.hides_following? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
   end
 
   def default_accounts

app/controllers/api/v1/media_controller.rb

@@ -3,25 +3,42 @@
 class Api::V1::MediaController < Api::BaseController
   before_action -> { doorkeeper_authorize! :write, :'write:media' }
   before_action :require_user!
+  before_action :set_media_attachment, except: [:create]
+  before_action :check_processing, except: [:create]
 
   def create
-    @media = current_account.media_attachments.create!(media_params)
-    render json: @media, serializer: REST::MediaAttachmentSerializer
+    @media_attachment = current_account.media_attachments.create!(media_attachment_params)
+    render json: @media_attachment, serializer: REST::MediaAttachmentSerializer
   rescue Paperclip::Errors::NotIdentifiedByImageMagickError
     render json: file_type_error, status: 422
   rescue Paperclip::Error
     render json: processing_error, status: 500
   end
 
+  def show
+    render json: @media_attachment, serializer: REST::MediaAttachmentSerializer, status: status_code_for_media_attachment
+  end
+
   def update
-    @media = current_account.media_attachments.where(status_id: nil).find(params[:id])
-    @media.update!(media_params)
-    render json: @media, serializer: REST::MediaAttachmentSerializer
+    @media_attachment.update!(media_attachment_params)
+    render json: @media_attachment, serializer: REST::MediaAttachmentSerializer, status: status_code_for_media_attachment
   end
 
   private
 
-  def media_params
+  def status_code_for_media_attachment
+    @media_attachment.not_processed? ? 206 : 200
+  end
+
+  def set_media_attachment
+    @media_attachment = current_account.media_attachments.unattached.find(params[:id])
+  end
+
+  def check_processing
+    render json: processing_error, status: 422 if @media_attachment.processing_failed?
+  end
+
+  def media_attachment_params
     params.permit(:file, :description, :focus)
   end
 

app/controllers/api/v2/media_controller.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class Api::V2::MediaController < Api::V1::MediaController
+  def create
+    @media_attachment = current_account.media_attachments.create!({ delay_processing: true }.merge(media_attachment_params))
+    render json: @media_attachment, serializer: REST::MediaAttachmentSerializer, status: 202
+  rescue Paperclip::Errors::NotIdentifiedByImageMagickError
+    render json: file_type_error, status: 422
+  rescue Paperclip::Error
+    render json: processing_error, status: 500
+  end
+end

app/controllers/follower_accounts_controller.rb

@@ -29,7 +29,8 @@ class FollowerAccountsController < ApplicationController
         render json: collection_presenter,
                serializer: ActivityPub::CollectionSerializer,
                adapter: ActivityPub::Adapter,
-               content_type: 'application/activity+json'
+               content_type: 'application/activity+json',
+               fields: restrict_fields_to
       end
     end
   end
@@ -72,4 +73,12 @@ class FollowerAccountsController < ApplicationController
       )
     end
   end
+
+  def restrict_fields_to
+    if page_requested? || !@account.user_hides_network?
+      # Return all fields
+    else
+      %i(id type totalItems)
+    end
+  end
 end

app/controllers/following_accounts_controller.rb

@@ -29,7 +29,8 @@ class FollowingAccountsController < ApplicationController
         render json: collection_presenter,
                serializer: ActivityPub::CollectionSerializer,
                adapter: ActivityPub::Adapter,
-               content_type: 'application/activity+json'
+               content_type: 'application/activity+json',
+               fields: restrict_fields_to
       end
     end
   end
@@ -72,4 +73,12 @@ class FollowingAccountsController < ApplicationController
       )
     end
   end
+
+  def restrict_fields_to
+    if page_requested? || !@account.user_hides_network?
+      # Return all fields
+    else
+      %i(id type totalItems)
+    end
+  end
 end

app/javascript/core/admin.js

@@ -1,6 +1,6 @@
 //  This file will be loaded on admin pages, regardless of theme.
 
-import { delegate } from 'rails-ujs';
+import { delegate } from '@rails/ujs';
 import ready from '../mastodon/ready';
 
 const batchCheckboxClassName = '.batch-checkbox input[type="checkbox"]';

app/javascript/core/public.js

@@ -3,7 +3,7 @@
 import createHistory from 'history/createBrowserHistory';
 import ready from '../mastodon/ready';
 
-const { delegate } = require('rails-ujs');
+const { delegate } = require('@rails/ujs');
 const { length } = require('stringz');
 
 delegate(document, '.webapp-btn', 'click', ({ target, button }) => {

app/javascript/core/settings.js

@@ -1,7 +1,7 @@
 //  This file will be loaded on settings pages, regardless of theme.
 
 import escapeTextContentForBrowser from 'escape-html';
-const { delegate } = require('rails-ujs');
+const { delegate } = require('@rails/ujs');
 import emojify from '../mastodon/features/emoji/emoji';
 
 delegate(document, '#account_display_name', 'input', ({ target }) => {

app/javascript/flavours/glitch/packs/common.js

@@ -1,4 +1,4 @@
-import { start } from 'rails-ujs';
+import { start } from '@rails/ujs';
 
 start();
 

app/javascript/flavours/glitch/packs/public.js

@@ -5,7 +5,7 @@ import loadKeyboardExtensions from 'flavours/glitch/util/load_keyboard_extension
 function main() {
   const IntlMessageFormat = require('intl-messageformat').default;
   const { timeAgoString } = require('flavours/glitch/components/relative_timestamp');
-  const { delegate } = require('rails-ujs');
+  const { delegate } = require('@rails/ujs');
   const emojify = require('flavours/glitch/util/emoji').default;
   const { getLocale } = require('locales');
   const { messages } = getLocale();

app/javascript/flavours/glitch/packs/settings.js

@@ -3,7 +3,7 @@ import ready from 'flavours/glitch/util/ready';
 import loadKeyboardExtensions from 'flavours/glitch/util/load_keyboard_extensions';
 
 function main() {
-  const { delegate } = require('rails-ujs');
+  const { delegate } = require('@rails/ujs');
 
   delegate(document, '.sidebar__toggle__icon', 'click', () => {
     const target = document.querySelector('.sidebar ul');

app/javascript/flavours/glitch/util/log_out.js

@@ -1,4 +1,4 @@
-import Rails from 'rails-ujs';
+import Rails from '@rails/ujs';
 import { signOutLink } from 'flavours/glitch/util/backend_links';
 
 export const logOut = () => {

app/javascript/mastodon/actions/compose.js

@@ -230,12 +230,31 @@ export function uploadCompose(files) {
         // Account for disparity in size of original image and resized data
         total += file.size - f.size;
 
-        return api(getState).post('/api/v1/media', data, {
+        return api(getState).post('/api/v2/media', data, {
           onUploadProgress: function({ loaded }){
             progress[i] = loaded;
             dispatch(uploadComposeProgress(progress.reduce((a, v) => a + v, 0), total));
           },
-        }).then(({ data }) => dispatch(uploadComposeSuccess(data, f)));
+        }).then(({ status, data }) => {
+          // If server-side processing of the media attachment has not completed yet,
+          // poll the server until it is, before showing the media attachment as uploaded
+
+          if (status === 200) {
+            dispatch(uploadComposeSuccess(data, f));
+          } else if (status === 202) {
+            const poll = () => {
+              api(getState).get(`/api/v1/media/${data.id}`).then(response => {
+                if (response.status === 200) {
+                  dispatch(uploadComposeSuccess(response.data, f));
+                } else if (response.status === 206) {
+                  setTimeout(() => poll(), 1000);
+                }
+              }).catch(error => dispatch(uploadComposeFail(error)));
+            };
+
+            poll();
+          }
+        });
       }).catch(error => dispatch(uploadComposeFail(error)));
     };
   };

app/javascript/mastodon/common.js

@@ -1,4 +1,4 @@
-import Rails from 'rails-ujs';
+import Rails from '@rails/ujs';
 
 export function start() {
   require('font-awesome/css/font-awesome.css');

app/javascript/mastodon/components/domain.js

@@ -5,7 +5,7 @@ import { defineMessages, injectIntl } from 'react-intl';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 
 const messages = defineMessages({
-  unblockDomain: { id: 'account.unblock_domain', defaultMessage: 'Unhide {domain}' },
+  unblockDomain: { id: 'account.unblock_domain', defaultMessage: 'Unblock domain {domain}' },
 });
 
 export default @injectIntl

app/javascript/mastodon/components/scrollable_list.js

@@ -82,15 +82,19 @@ export default class ScrollableList extends PureComponent {
   lastScrollWasSynthetic = false;
   scrollToTopOnMouseIdle = false;
 
+  _getScrollingElement = () => {
+    if (this.props.bindToDocument) {
+      return (document.scrollingElement || document.body);
+    } else {
+      return this.node;
+    }
+  }
+
   setScrollTop = newScrollTop => {
     if (this.getScrollTop() !== newScrollTop) {
       this.lastScrollWasSynthetic = true;
 
-      if (this.props.bindToDocument) {
-        document.scrollingElement.scrollTop = newScrollTop;
-      } else {
-        this.node.scrollTop = newScrollTop;
-      }
+      this._getScrollingElement().scrollTop = newScrollTop;
     }
   };
 
@@ -151,15 +155,15 @@ export default class ScrollableList extends PureComponent {
   }
 
   getScrollTop = () => {
-    return this.props.bindToDocument ? document.scrollingElement.scrollTop : this.node.scrollTop;
+    return this._getScrollingElement().scrollTop;
   }
 
   getScrollHeight = () => {
-    return this.props.bindToDocument ? document.scrollingElement.scrollHeight : this.node.scrollHeight;
+    return this._getScrollingElement().scrollHeight;
   }
 
   getClientHeight = () => {
-    return this.props.bindToDocument ? document.scrollingElement.clientHeight : this.node.clientHeight;
+    return this._getScrollingElement().clientHeight;
   }
 
   updateScrollBottom = (snapshot) => {

app/javascript/mastodon/containers/domain_container.js

@@ -6,7 +6,7 @@ import Domain from '../components/domain';
 import { openModal } from '../actions/modal';
 
 const messages = defineMessages({
-  blockDomainConfirm: { id: 'confirmations.domain_block.confirm', defaultMessage: 'Hide entire domain' },
+  blockDomainConfirm: { id: 'confirmations.domain_block.confirm', defaultMessage: 'Block entire domain' },
 });
 
 const makeMapStateToProps = () => {

app/javascript/mastodon/features/account/components/header.js

@@ -39,7 +39,7 @@ const messages = defineMessages({
   favourites: { id: 'navigation_bar.favourites', defaultMessage: 'Favourites' },
   lists: { id: 'navigation_bar.lists', defaultMessage: 'Lists' },
   blocks: { id: 'navigation_bar.blocks', defaultMessage: 'Blocked users' },
-  domain_blocks: { id: 'navigation_bar.domain_blocks', defaultMessage: 'Hidden domains' },
+  domain_blocks: { id: 'navigation_bar.domain_blocks', defaultMessage: 'Blocked domains' },
   mutes: { id: 'navigation_bar.mutes', defaultMessage: 'Muted users' },
   endorse: { id: 'account.endorse', defaultMessage: 'Feature on profile' },
   unendorse: { id: 'account.unendorse', defaultMessage: 'Don\'t feature on profile' },
@@ -142,7 +142,7 @@ class Header extends ImmutablePureComponent {
     if (me !== account.get('id') && account.getIn(['relationship', 'muting'])) {
       info.push(<span key='muted' className='relationship-tag'><FormattedMessage id='account.muted' defaultMessage='Muted' /></span>);
     } else if (me !== account.get('id') && account.getIn(['relationship', 'domain_blocking'])) {
-      info.push(<span key='domain_blocked' className='relationship-tag'><FormattedMessage id='account.domain_blocked' defaultMessage='Domain hidden' /></span>);
+      info.push(<span key='domain_blocked' className='relationship-tag'><FormattedMessage id='account.domain_blocked' defaultMessage='Domain blocked' /></span>);
     }
 
     if (me !== account.get('id')) {

app/javascript/mastodon/features/compose/components/action_bar.js

@@ -16,6 +16,7 @@ const messages = defineMessages({
   mutes: { id: 'navigation_bar.mutes', defaultMessage: 'Muted users' },
   filters: { id: 'navigation_bar.filters', defaultMessage: 'Muted words' },
   logout: { id: 'navigation_bar.logout', defaultMessage: 'Logout' },
+  bookmarks: { id: 'navigation_bar.bookmarks', defaultMessage: 'Bookmarks' },
 });
 
 export default @injectIntl
@@ -42,6 +43,7 @@ class ActionBar extends React.PureComponent {
     menu.push(null);
     menu.push({ text: intl.formatMessage(messages.follow_requests), to: '/follow_requests' });
     menu.push({ text: intl.formatMessage(messages.favourites), to: '/favourites' });
+    menu.push({ text: intl.formatMessage(messages.bookmarks), to: '/bookmarks' });
     menu.push({ text: intl.formatMessage(messages.lists), to: '/lists' });
     menu.push(null);
     menu.push({ text: intl.formatMessage(messages.mutes), to: '/mutes' });

app/javascript/mastodon/features/domain_blocks/index.js

@@ -13,8 +13,8 @@ import { fetchDomainBlocks, expandDomainBlocks } from '../../actions/domain_bloc
 import ScrollableList from '../../components/scrollable_list';
 
 const messages = defineMessages({
-  heading: { id: 'column.domain_blocks', defaultMessage: 'Hidden domains' },
-  unblockDomain: { id: 'account.unblock_domain', defaultMessage: 'Unhide {domain}' },
+  heading: { id: 'column.domain_blocks', defaultMessage: 'Blocked domains' },
+  unblockDomain: { id: 'account.unblock_domain', defaultMessage: 'Unblock domain {domain}' },
 });
 
 const mapStateToProps = state => ({
@@ -55,7 +55,7 @@ class Blocks extends ImmutablePureComponent {
       );
     }
 
-    const emptyMessage = <FormattedMessage id='empty_column.domain_blocks' defaultMessage='There are no hidden domains yet.' />;
+    const emptyMessage = <FormattedMessage id='empty_column.domain_blocks' defaultMessage='There are no blocked domains yet.' />;
 
     return (
       <Column bindToDocument={!multiColumn} icon='minus-circle' heading={intl.formatMessage(messages.heading)}>

app/javascript/mastodon/features/getting_started/components/announcements.js

@@ -389,7 +389,7 @@ class Announcements extends ImmutablePureComponent {
   _markAnnouncementAsRead () {
     const { dismissAnnouncement, announcements } = this.props;
     const { index } = this.state;
-    const announcement = announcements.get(index);
+    const announcement = announcements.get(index) || announcements.get(index - 1);
     if (!announcement.get('read')) dismissAnnouncement(announcement.get('id'));
   }
 
@@ -407,7 +407,7 @@ class Announcements extends ImmutablePureComponent {
 
   render () {
     const { announcements, intl } = this.props;
-    const { index } = this.state;
+    const index = this.state.index < announcements.size ? this.state.index : announcements.size - 1;
 
     if (announcements.isEmpty()) {
       return null;

app/javascript/mastodon/features/status/components/detailed_status.js

@@ -166,7 +166,7 @@ export default class DetailedStatus extends ImmutablePureComponent {
       reblogIcon = 'lock';
     }
 
-    if (status.get('visibility') === 'private') {
+    if (['private', 'direct'].includes(status.get('visibility'))) {
       reblogLink = <Icon id={reblogIcon} />;
     } else if (this.context.router) {
       reblogLink = (

app/javascript/mastodon/locales/defaultMessages.json

@@ -523,7 +523,7 @@
   {
     "descriptors": [
       {
-        "defaultMessage": "Hide entire domain",
+        "defaultMessage": "Block entire domain",
         "id": "confirmations.domain_block.confirm"
       },
       {
@@ -737,7 +737,7 @@
         "id": "navigation_bar.blocks"
       },
       {
-        "defaultMessage": "Hidden domains",
+        "defaultMessage": "Blocked domains",
         "id": "navigation_bar.domain_blocks"
       },
       {
@@ -773,7 +773,7 @@
         "id": "account.muted"
       },
       {
-        "defaultMessage": "Domain hidden",
+        "defaultMessage": "Domain blocked",
         "id": "account.domain_blocked"
       },
       {
@@ -917,6 +917,10 @@
       {
         "defaultMessage": "Logout",
         "id": "navigation_bar.logout"
+      },
+      {
+        "defaultMessage": "Bookmarks",
+        "id": "navigation_bar.bookmarks"
       }
     ],
     "path": "app/javascript/mastodon/features/compose/components/action_bar.json"
@@ -1466,7 +1470,7 @@
   {
     "descriptors": [
       {
-        "defaultMessage": "Hidden domains",
+        "defaultMessage": "Blocked domains",
         "id": "column.domain_blocks"
       },
       {
@@ -1474,7 +1478,7 @@
         "id": "account.unblock_domain"
       },
       {
-        "defaultMessage": "There are no hidden domains yet.",
+        "defaultMessage": "There are no blocked domains yet.",
         "id": "empty_column.domain_blocks"
       }
     ],
@@ -2957,4 +2961,4 @@
     ],
     "path": "app/javascript/mastodon/features/video/index.json"
   }
-]
+]

app/javascript/mastodon/locales/en.json

@@ -7,7 +7,7 @@
   "account.blocked": "Blocked",
   "account.cancel_follow_request": "Cancel follow request",
   "account.direct": "Direct message @{name}",
-  "account.domain_blocked": "Domain hidden",
+  "account.domain_blocked": "Domain blocked",
   "account.edit_profile": "Edit profile",
   "account.endorse": "Feature on profile",
   "account.follow": "Follow",
@@ -57,7 +57,7 @@
   "column.community": "Local timeline",
   "column.direct": "Direct messages",
   "column.directory": "Browse profiles",
-  "column.domain_blocks": "Hidden domains",
+  "column.domain_blocks": "Blocked domains",
   "column.favourites": "Favourites",
   "column.follow_requests": "Follow requests",
   "column.home": "Home",
@@ -107,7 +107,7 @@
   "confirmations.delete.message": "Are you sure you want to delete this status?",
   "confirmations.delete_list.confirm": "Delete",
   "confirmations.delete_list.message": "Are you sure you want to permanently delete this list?",
-  "confirmations.domain_block.confirm": "Hide entire domain",
+  "confirmations.domain_block.confirm": "Block entire domain",
   "confirmations.domain_block.message": "Are you really, really sure you want to block the entire {domain}? In most cases a few targeted blocks or mutes are sufficient and preferable. You will not see content from that domain in any public timelines or your notifications. Your followers from that domain will be removed.",
   "confirmations.logout.confirm": "Log out",
   "confirmations.logout.message": "Are you sure you want to log out?",
@@ -150,7 +150,7 @@
   "empty_column.bookmarked_statuses": "You don't have any bookmarked toots yet. When you bookmark one, it will show up here.",
   "empty_column.community": "The local timeline is empty. Write something publicly to get the ball rolling!",
   "empty_column.direct": "You don't have any direct messages yet. When you send or receive one, it will show up here.",
-  "empty_column.domain_blocks": "There are no hidden domains yet.",
+  "empty_column.domain_blocks": "There are no blocked domains yet.",
   "empty_column.favourited_statuses": "You don't have any favourite toots yet. When you favourite one, it will show up here.",
   "empty_column.favourites": "No one has favourited this toot yet. When someone does, they will show up here.",
   "empty_column.follow_requests": "You don't have any follow requests yet. When you receive one, it will show up here.",
@@ -269,7 +269,7 @@
   "navigation_bar.compose": "Compose new toot",
   "navigation_bar.direct": "Direct messages",
   "navigation_bar.discover": "Discover",
-  "navigation_bar.domain_blocks": "Hidden domains",
+  "navigation_bar.domain_blocks": "Blocked domains",
   "navigation_bar.edit_profile": "Edit profile",
   "navigation_bar.favourites": "Favourites",
   "navigation_bar.filters": "Muted words",

app/javascript/mastodon/utils/log_out.js

@@ -1,4 +1,4 @@
-import Rails from 'rails-ujs';
+import Rails from '@rails/ujs';
 
 export const logOut = () => {
   const form = document.createElement('form');

app/javascript/packs/public.js

@@ -8,7 +8,7 @@ start();
 function main() {
   const IntlMessageFormat = require('intl-messageformat').default;
   const { timeAgoString } = require('../mastodon/components/relative_timestamp');
-  const { delegate } = require('rails-ujs');
+  const { delegate } = require('@rails/ujs');
   const emojify = require('../mastodon/features/emoji/emoji').default;
   const { getLocale } = require('../mastodon/locales');
   const { messages } = getLocale();

app/lib/language_detector.rb

@@ -52,8 +52,10 @@ class LanguageDetector
 
   def detect_language_code(text)
     return if unreliable_input?(text)
+
     result = @identifier.find_language(text)
-    iso6391(result.language.to_s).to_sym if result.reliable?
+
+    iso6391(result.language.to_s).to_sym if result&.reliable?
   end
 
   def iso6391(bcp47)

app/models/account.rb

@@ -46,6 +46,7 @@
 #  silenced_at             :datetime
 #  suspended_at            :datetime
 #  trust_level             :integer
+#  hide_collections        :boolean
 #
 
 class Account < ApplicationRecord
@@ -325,6 +326,14 @@ class Account < ApplicationRecord
     save!
   end
 
+  def hides_followers?
+    hide_collections? || user_hides_network?
+  end
+
+  def hides_following?
+    hide_collections? || user_hides_network?
+  end
+
   def object_type
     :person
   end

app/models/account_warning_preset.rb

@@ -8,8 +8,11 @@
 #  text       :text             default(""), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
+#  title      :string           default(""), not null
 #
 
 class AccountWarningPreset < ApplicationRecord
   validates :text, presence: true
+
+  scope :alphabetic, -> { order(title: :asc, text: :asc) }
 end

app/models/admin/account_action.rb

@@ -62,8 +62,6 @@ class Admin::AccountAction
 
   def process_action!
     case type
-    when 'none'
-      handle_resolve!
     when 'disable'
       handle_disable!
     when 'silence'
@@ -105,16 +103,6 @@ class Admin::AccountAction
     end
   end
 
-  def handle_resolve!
-    if with_report? && report.account_id == -99 && target_account.trust_level == Account::TRUST_LEVELS[:untrusted]
-      # This is an automated report and it is being dismissed, so it's
-      # a false positive, in which case update the account's trust level
-      # to prevent further spam checks
-
-      target_account.update(trust_level: Account::TRUST_LEVELS[:trusted])
-    end
-  end
-
   def handle_disable!
     authorize(target_account.user, :disable?)
     log_action(:disable, target_account.user)

app/models/concerns/attachmentable.rb

@@ -74,7 +74,7 @@ module Attachmentable
     self.class.attachment_definitions.each_key do |attachment_name|
       attachment = send(attachment_name)
 
-      next if attachment.blank? || attachment.queued_for_write[:original].blank?
+      next if attachment.blank? || attachment.queued_for_write[:original].blank? || attachment.options[:preserve_files]
 
       attachment.instance_write :file_name, SecureRandom.hex(8) + File.extname(attachment.instance_read(:file_name))
     end

app/models/email_domain_block.rb

@@ -7,13 +7,27 @@
 #  domain     :string           default(""), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
+#  parent_id  :bigint(8)
 #
 
 class EmailDomainBlock < ApplicationRecord
   include DomainNormalizable
 
+  belongs_to :parent, class_name: 'EmailDomainBlock', optional: true
+  has_many :children, class_name: 'EmailDomainBlock', foreign_key: :parent_id, inverse_of: :parent, dependent: :destroy
+
   validates :domain, presence: true, uniqueness: true, domain: true
 
+  def with_dns_records=(val)
+    @with_dns_records = ActiveModel::Type::Boolean.new.cast(val)
+  end
+
+  def with_dns_records?
+    @with_dns_records
+  end
+
+  alias with_dns_records with_dns_records?
+
   def self.block?(email)
     _, domain = email.split('@', 2)
 

app/models/media_attachment.rb

@@ -19,12 +19,14 @@
 #  description         :text
 #  scheduled_status_id :bigint(8)
 #  blurhash            :string
+#  processing          :integer
 #
 
 class MediaAttachment < ApplicationRecord
   self.inheritance_column = nil
 
   enum type: [:image, :gifv, :video, :unknown, :audio]
+  enum processing: [:queued, :in_progress, :complete, :failed], _prefix: true
 
   MAX_DESCRIPTION_LENGTH = 1_500
 
@@ -55,47 +57,6 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
-  VIDEO_STYLES = {
-    small: {
-      convert_options: {
-        output: {
-          'loglevel' => 'fatal',
-          vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
-        },
-      },
-      format: 'png',
-      time: 0,
-      file_geometry_parser: FastGeometryParser,
-      blurhash: BLURHASH_OPTIONS,
-    },
-
-    original: {
-      keep_same_format: true,
-      convert_options: {
-        output: {
-          'loglevel' => 'fatal',
-          'map_metadata' => '-1',
-          'c:v' => 'copy',
-          'c:a' => 'copy',
-        },
-      },
-    },
-  }.freeze
-
-  AUDIO_STYLES = {
-    original: {
-      format: 'mp3',
-      content_type: 'audio/mpeg',
-      convert_options: {
-        output: {
-          'loglevel' => 'fatal',
-          'map_metadata' => '-1',
-          'q:a' => 2,
-        },
-      },
-    },
-  }.freeze
-
   VIDEO_FORMAT = {
     format: 'mp4',
     content_type: 'video/mp4',
@@ -116,6 +77,54 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
+  VIDEO_PASSTHROUGH_OPTIONS = {
+    video_codecs: ['h264'],
+    audio_codecs: ['aac', nil],
+    colorspaces: ['yuv420p'],
+    options: {
+      format: 'mp4',
+      convert_options: {
+        output: {
+          'loglevel' => 'fatal',
+          'map_metadata' => '-1',
+          'c:v' => 'copy',
+          'c:a' => 'copy',
+        },
+      },
+    },
+  }.freeze
+
+  VIDEO_STYLES = {
+    small: {
+      convert_options: {
+        output: {
+          'loglevel' => 'fatal',
+          vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
+        },
+      },
+      format: 'png',
+      time: 0,
+      file_geometry_parser: FastGeometryParser,
+      blurhash: BLURHASH_OPTIONS,
+    },
+
+    original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS),
+  }.freeze
+
+  AUDIO_STYLES = {
+    original: {
+      format: 'mp3',
+      content_type: 'audio/mpeg',
+      convert_options: {
+        output: {
+          'loglevel' => 'fatal',
+          'map_metadata' => '-1',
+          'q:a' => 2,
+        },
+      },
+    },
+  }.freeze
+
   VIDEO_CONVERTED_STYLES = {
     small: VIDEO_STYLES[:small],
     original: VIDEO_FORMAT,
@@ -124,6 +133,9 @@ class MediaAttachment < ApplicationRecord
   IMAGE_LIMIT = (ENV['MAX_IMAGE_SIZE'] || 10.megabytes).to_i
   VIDEO_LIMIT = (ENV['MAX_VIDEO_SIZE'] || 40.megabytes).to_i
 
+  MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px
+  MAX_VIDEO_FRAME_RATE   = 60
+
   belongs_to :account,          inverse_of: :media_attachments, optional: true
   belongs_to :status,           inverse_of: :media_attachments, optional: true
   belongs_to :scheduled_status, inverse_of: :media_attachments, optional: true
@@ -156,6 +168,10 @@ class MediaAttachment < ApplicationRecord
     remote_url.blank?
   end
 
+  def not_processed?
+    processing.present? && !processing_complete?
+  end
+
   def needs_redownload?
     file.blank? && remote_url.present?
   end
@@ -203,12 +219,21 @@ class MediaAttachment < ApplicationRecord
     "#{x},#{y}"
   end
 
+  attr_writer :delay_processing
+
+  def delay_processing?
+    @delay_processing
+  end
+
+  after_commit :enqueue_processing, on: :create
   after_commit :reset_parent_cache, on: :update
 
   before_create :prepare_description, unless: :local?
   before_create :set_shortcode
+  before_create :set_processing
 
   before_post_process :set_type_and_extension
+  before_post_process :check_video_dimensions
 
   before_save :set_meta
 
@@ -277,6 +302,21 @@ class MediaAttachment < ApplicationRecord
     end
   end
 
+  def set_processing
+    self.processing = delay_processing? ? :queued : :complete
+  end
+
+  def check_video_dimensions
+    return unless (video? || gifv?) && file.queued_for_write[:original].present?
+
+    movie = FFMPEG::Movie.new(file.queued_for_write[:original].path)
+
+    return unless movie.valid?
+
+    raise Mastodon::DimensionsValidationError, "#{movie.width}x#{movie.height} videos are not supported" if movie.width * movie.height > MAX_VIDEO_MATRIX_LIMIT
+    raise Mastodon::DimensionsValidationError, "#{movie.frame_rate.to_i}fps videos are not supported" if movie.frame_rate > MAX_VIDEO_FRAME_RATE
+  end
+
   def set_meta
     meta = populate_meta
 
@@ -322,9 +362,11 @@ class MediaAttachment < ApplicationRecord
     }.compact
   end
 
-  def reset_parent_cache
-    return if status_id.nil?
+  def enqueue_processing
+    PostProcessMediaWorker.perform_async(id) if delay_processing?
+  end
 
-    Rails.cache.delete("statuses/#{status_id}")
+  def reset_parent_cache
+    Rails.cache.delete("statuses/#{status_id}") if status_id.present?
   end
 end

app/models/report.rb

@@ -59,6 +59,14 @@ class Report < ApplicationRecord
   end
 
   def resolve!(acting_account)
+    if account_id == -99 && target_account.trust_level == Account::TRUST_LEVELS[:untrusted]
+      # This is an automated report and it is being dismissed, so it's
+      # a false positive, in which case update the account's trust level
+      # to prevent further spam checks
+
+      target_account.update(trust_level: Account::TRUST_LEVELS[:trusted])
+    end
+
     RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] }
     update!(action_taken: true, action_taken_by_account_id: acting_account.id)
   end

app/models/status.rb

@@ -148,10 +148,12 @@ class Status < ApplicationRecord
       ids += mentions.where(account: Account.local).pluck(:account_id)
       ids += favourites.where(account: Account.local).pluck(:account_id)
       ids += reblogs.where(account: Account.local).pluck(:account_id)
+      ids += bookmarks.where(account: Account.local).pluck(:account_id)
     else
       ids += preloaded.mentions[id] || []
       ids += preloaded.favourites[id] || []
       ids += preloaded.reblogs[id] || []
+      ids += preloaded.bookmarks[id] || []
     end
 
     ids.uniq

app/serializers/rest/media_attachment_serializer.rb

@@ -12,7 +12,9 @@ class REST::MediaAttachmentSerializer < ActiveModel::Serializer
   end
 
   def url
-    if object.needs_redownload?
+    if object.not_processed?
+      nil
+    elsif object.needs_redownload?
       media_proxy_url(object.id, :original)
     else
       full_asset_url(object.file.url(:original))

app/services/activitypub/process_account_service.rb

@@ -94,6 +94,7 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.statuses_count    = outbox_total_items    if outbox_total_items.present?
     @account.following_count   = following_total_items if following_total_items.present?
     @account.followers_count   = followers_total_items if followers_total_items.present?
+    @account.hide_collections  = following_private? || followers_private?
     @account.moved_to_account  = @json['movedTo'].present? ? moved_account : nil
   end
 
@@ -166,26 +167,36 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def outbox_total_items
-    collection_total_items('outbox')
+    collection_info('outbox').first
   end
 
   def following_total_items
-    collection_total_items('following')
+    collection_info('following').first
   end
 
   def followers_total_items
-    collection_total_items('followers')
+    collection_info('followers').first
   end
 
-  def collection_total_items(type)
-    return if @json[type].blank?
+  def following_private?
+    !collection_info('following').last
+  end
+
+  def followers_private?
+    !collection_info('followers').last
+  end
+
+  def collection_info(type)
+    return [nil, nil] if @json[type].blank?
     return @collections[type] if @collections.key?(type)
 
     collection = fetch_resource_without_id_validation(@json[type])
 
-    @collections[type] = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil
+    total_items = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil
+    has_first_page = collection.is_a?(Hash) && collection['first'].present?
+    @collections[type] = [total_items, has_first_page]
   rescue HTTP::Error, OpenSSL::SSL::SSLError
-    @collections[type] = nil
+    @collections[type] = [nil, nil]
   end
 
   def moved_account

app/services/fetch_resource_service.rb

@@ -5,6 +5,8 @@ class FetchResourceService < BaseService
 
   ACCEPT_HEADER = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", text/html;q=0.1'
 
+  attr_reader :response_code
+
   def call(url)
     return if url.blank?
 
@@ -27,6 +29,7 @@ class FetchResourceService < BaseService
   end
 
   def process_response(response, terminal = false)
+    @response_code = response.code
     return nil if response.code != 200
 
     if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)

app/services/post_status_service.rb

@@ -110,6 +110,7 @@ class PostStatusService < BaseService
     @media = @account.media_attachments.where(status_id: nil).where(id: @options[:media_ids].take(4).map(&:to_i))
 
     raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:audio_or_video?)
+    raise Mastodon::ValidationError, I18n.t('media_attachments.validations.not_ready') if @media.any?(&:not_processed?)
   end
 
   def language_from_option(str)

app/services/resolve_url_service.rb

@@ -12,7 +12,7 @@ class ResolveURLService < BaseService
       process_local_url
     elsif !fetched_resource.nil?
       process_url
-    elsif @on_behalf_of.present?
+    else
       process_url_from_db
     end
   end
@@ -30,6 +30,8 @@ class ResolveURLService < BaseService
   end
 
   def process_url_from_db
+    return unless @on_behalf_of.present? && [401, 403, 404].include?(fetch_resource_service.response_code)
+
     # It may happen that the resource is a private toot, and thus not fetchable,
     # but we can return the toot if we already know about it.
     status = Status.find_by(uri: @url) || Status.find_by(url: @url)
@@ -40,7 +42,11 @@ class ResolveURLService < BaseService
   end
 
   def fetched_resource
-    @fetched_resource ||= FetchResourceService.new.call(@url)
+    @fetched_resource ||= fetch_resource_service.call(@url)
+  end
+
+  def fetch_resource_service
+    @_fetch_resource_service ||= FetchResourceService.new
   end
 
   def resource_url

app/views/admin/account_actions/new.html.haml

@@ -21,7 +21,7 @@
 
     - unless @warning_presets.empty?
       .fields-group
-        = f.input :warning_preset_id, collection: @warning_presets, label_method: :text, wrapper: :with_block_label
+        = f.input :warning_preset_id, collection: @warning_presets, label_method: ->(warning_preset) { [warning_preset.title.presence, truncate(warning_preset.text)].compact.join(' - ') }, wrapper: :with_block_label
 
     .fields-group
       = f.input :text, as: :text, wrapper: :with_block_label, hint: t('simple_form.hints.admin_account_action.text_html', path: admin_warning_presets_path)

app/views/admin/accounts/show.html.haml

@@ -96,10 +96,17 @@
               = table_link_to 'angle-double-down', t('admin.accounts.demote'), demote_admin_account_role_path(@account.id), method: :post, data: { confirm: t('admin.accounts.are_you_sure') } if can?(:demote, @account.user)
 
           %tr
-            %th= t('admin.accounts.email')
-            %td= @account.user_email
+            %th{ rowspan: can?(:create, :email_domain_block) ? 3 : 2 }= t('admin.accounts.email')
+            %td{ rowspan: can?(:create, :email_domain_block) ? 3 : 2 }= @account.user_email
             %td= table_link_to 'edit', t('admin.accounts.change_email.label'), admin_account_change_email_path(@account.id) if can?(:change_email, @account.user)
 
+          %tr
+            %td= table_link_to 'search', t('admin.accounts.search_same_email_domain'), admin_accounts_path(email: "%@#{@account.user_email.split('@').last}")
+
+          - if can?(:create, :email_domain_block)
+            %tr
+              %td= table_link_to 'ban', t('admin.accounts.add_email_domain_block'), new_admin_email_domain_block_path(_domain: @account.user_email.split('@').last)
+
           - if @account.user_unconfirmed_email.present?
             %tr
               %th= t('admin.accounts.unconfirmed_email')
@@ -204,7 +211,7 @@
         = link_to t('admin.accounts.perform_full_suspension'), new_admin_account_action_path(@account.id, type: 'suspend'), class: 'button button--destructive' if can?(:suspend, @account)
 
       - unless @account.local?
-        - if DomainBlock.where(domain: @account.domain).exists?
+        - if DomainBlock.rule_for(@account.domain)
           = link_to t('admin.domain_blocks.view'), admin_instance_path(@account.domain), class: 'button'
         - else
           = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @account.domain), class: 'button button--destructive'

app/views/admin/email_domain_blocks/_email_domain_block.html.haml

@@ -3,3 +3,13 @@
     %samp= email_domain_block.domain
   %td
     = table_link_to 'trash', t('admin.email_domain_blocks.delete'), admin_email_domain_block_path(email_domain_block), method: :delete
+
+- email_domain_block.children.each do |child_email_domain_block|
+  %tr
+    %td
+      %samp= child_email_domain_block.domain
+      %span.muted-hint
+        = surround '(', ')' do
+          = t('admin.email_domain_blocks.from_html', domain: content_tag(:samp, email_domain_block.domain))
+    %td
+      = table_link_to 'trash', t('admin.email_domain_blocks.delete'), admin_email_domain_block_path(child_email_domain_block), method: :delete

app/views/admin/email_domain_blocks/new.html.haml

@@ -5,7 +5,10 @@
   = render 'shared/error_messages', object: @email_domain_block
 
   .fields-group
-    = f.input :domain, wrapper: :with_label, label: t('admin.email_domain_blocks.domain')
+    = f.input :domain, wrapper: :with_block_label, label: t('admin.email_domain_blocks.domain')
+
+  .fields-group
+    = f.input :with_dns_records, as: :boolean, wrapper: :with_label
 
   .actions
     = f.button :button, t('.create'), type: :submit

app/views/admin/warning_presets/_warning_preset.html.haml

@@ -0,0 +1,10 @@
+.announcements-list__item
+  = link_to edit_admin_warning_preset_path(warning_preset), class: 'announcements-list__item__title' do
+    = warning_preset.title.presence || truncate(warning_preset.text)
+
+  .announcements-list__item__action-bar
+    .announcements-list__item__meta
+      = truncate(warning_preset.text)
+
+    %div
+      = table_link_to 'trash', t('admin.warning_presets.delete'), admin_warning_preset_path(warning_preset), method: :delete, data: { confirm: t('admin.accounts.are_you_sure') } if can?(:destroy, warning_preset)

app/views/admin/warning_presets/edit.html.haml

@@ -4,6 +4,9 @@
 = simple_form_for @warning_preset, url: admin_warning_preset_path(@warning_preset) do |f|
   = render 'shared/error_messages', object: @warning_preset
 
+  .fields-group
+    = f.input :title, wrapper: :with_block_label
+
   .fields-group
     = f.input :text, wrapper: :with_block_label
 

app/views/admin/warning_presets/index.html.haml

@@ -5,6 +5,9 @@
   = simple_form_for @warning_preset, url: admin_warning_presets_path do |f|
     = render 'shared/error_messages', object: @warning_preset
 
+    .fields-group
+      = f.input :title, wrapper: :with_block_label
+
     .fields-group
       = f.input :text, wrapper: :with_block_label
 
@@ -13,18 +16,9 @@
 
   %hr.spacer/
 
-- unless @warning_presets.empty?
-  .table-wrapper
-    %table.table
-      %thead
-        %tr
-          %th= t('simple_form.labels.account_warning_preset.text')
-          %th
-      %tbody
-        - @warning_presets.each do |preset|
-          %tr
-            %td
-              = Formatter.instance.linkify(preset.text)
-            %td
-              = table_link_to 'pencil', t('admin.warning_presets.edit'), edit_admin_warning_preset_path(preset)
-              = table_link_to 'trash', t('admin.warning_presets.delete'), admin_warning_preset_path(preset), method: :delete, data: { confirm: t('admin.accounts.are_you_sure') }
+- if @warning_presets.empty?
+  %div.muted-hint.center-text
+    = t 'admin.warning_presets.empty'
+- else
+  .announcements-list
+    = render partial: 'warning_preset', collection: @warning_presets

app/workers/backup_worker.rb

@@ -9,8 +9,12 @@ class BackupWorker
     backup_id = msg['args'].first
 
     ActiveRecord::Base.connection_pool.with_connection do
-      backup = Backup.find(backup_id)
-      backup&.destroy
+      begin
+        backup = Backup.find(backup_id)
+        backup.destroy
+      rescue ActiveRecord::RecordNotFound
+        true
+      end
     end
   end
 

app/workers/post_process_media_worker.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class PostProcessMediaWorker
+  include Sidekiq::Worker
+
+  sidekiq_options retry: 1, dead: false
+
+  sidekiq_retries_exhausted do |msg|
+    media_attachment_id = msg['args'].first
+
+    ActiveRecord::Base.connection_pool.with_connection do
+      begin
+        media_attachment = MediaAttachment.find(media_attachment_id)
+        media_attachment.processing = :failed
+        media_attachment.save
+      rescue ActiveRecord::RecordNotFound
+        true
+      end
+    end
+
+    Sidekiq.logger.error("Processing media attachment #{media_attachment_id} failed with #{msg['error_message']}")
+  end
+
+  def perform(media_attachment_id)
+    media_attachment = MediaAttachment.find(media_attachment_id)
+    media_attachment.processing = :in_progress
+    media_attachment.save
+    media_attachment.file.reprocess_original!
+    media_attachment.processing = :complete
+    media_attachment.save
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

config/application.rb

@@ -7,6 +7,7 @@ require 'rails/all'
 Bundler.require(*Rails.groups)
 
 require_relative '../app/lib/exceptions'
+require_relative '../lib/paperclip/attachment_extensions'
 require_relative '../lib/paperclip/lazy_thumbnail'
 require_relative '../lib/paperclip/gif_transcoder'
 require_relative '../lib/paperclip/video_transcoder'

config/deploy.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-lock '3.11.2'
+lock '3.12.1'
 
 set :repo_url, ENV.fetch('REPO', 'https://github.com/tootsuite/mastodon.git')
 set :branch, ENV.fetch('BRANCH', 'master')

config/initializers/sidekiq.rb

@@ -19,4 +19,4 @@ Sidekiq.configure_client do |config|
   config.redis = redis_params
 end
 
-Sidekiq::Logging.logger.level = ::Logger.const_get(ENV.fetch('RAILS_LOG_LEVEL', 'info').upcase.to_s)
+Sidekiq.logger.level = ::Logger.const_get(ENV.fetch('RAILS_LOG_LEVEL', 'info').upcase.to_s)

config/locales/ar.yml

@@ -562,7 +562,6 @@ ar:
     warning_presets:
       add_new: إضافة واحد جديد
       delete: حذف
-      edit: تعديل
       edit_preset: تعديل نموذج التحذير
       title: إدارة نماذج التحذير
   admin_mailer:

config/locales/ca.yml

@@ -573,7 +573,6 @@ ca:
     warning_presets:
       add_new: Afegeix-ne un de nou
       delete: Esborra
-      edit: Edita
       edit_preset: Edita l'avís predeterminat
       title: Gestiona les configuracions predefinides dels avisos
   admin_mailer:

config/locales/co.yml

@@ -573,7 +573,6 @@ co:
     warning_presets:
       add_new: Aghjunghje
       delete: Sguassà
-      edit: Cambià
       edit_preset: Cambià a preselezzione d'avertimentu
       title: Amministrà e preselezzione d'avertimentu
   admin_mailer:

config/locales/cs.yml

@@ -588,7 +588,6 @@ cs:
     warning_presets:
       add_new: Přidat nové
       delete: Smazat
-      edit: Upravit
       edit_preset: Upravit předlohu pro varování
       title: Spravovat předlohy pro varování
   admin_mailer:

config/locales/cy.yml

@@ -605,7 +605,6 @@ cy:
     warning_presets:
       add_new: Ychwanegu newydd
       delete: Dileu
-      edit: Golygu
       edit_preset: Golygu rhagosodiad rhybudd
       title: Rheoli rhagosodiadau rhybudd
   admin_mailer:

config/locales/da.yml

@@ -489,7 +489,6 @@ da:
       most_recent: Seneste
     warning_presets:
       delete: Slet
-      edit: Rediger
   admin_mailer:
     new_report:
       body: "%{reporter} har anmeldt %{target}"

config/locales/de.yml

@@ -573,7 +573,6 @@ de:
     warning_presets:
       add_new: Neu hinzufügen
       delete: Löschen
-      edit: Bearbeiten
       edit_preset: Warnungsvorlage bearbeiten
       title: Warnungsvorlagen verwalten
   admin_mailer:

config/locales/el.yml

@@ -573,7 +573,6 @@ el:
     warning_presets:
       add_new: Πρόσθεση νέου
       delete: Διαγραφή
-      edit: Ενημέρωση
       edit_preset: Ενημέρωση προκαθορισμένης προειδοποίησης
       title: Διαχείριση προκαθορισμένων προειδοποιήσεων
   admin_mailer:

config/locales/en.yml

@@ -92,6 +92,7 @@ en:
       delete: Delete
       destroyed_msg: Moderation note successfully destroyed!
     accounts:
+      add_email_domain_block: Blacklist e-mail domain
       approve: Approve
       approve_all: Approve all
       are_you_sure: Are you sure?
@@ -172,6 +173,7 @@ en:
         staff: Staff
         user: User
       search: Search
+      search_same_email_domain: Other users with the same e-mail domain
       search_same_ip: Other users with the same IP
       shared_inbox_url: Shared inbox URL
       show:
@@ -359,6 +361,7 @@ en:
       destroyed_msg: Successfully deleted e-mail domain from blacklist
       domain: Domain
       empty: No e-mail domains currently blacklisted.
+      from_html: from %{domain}
       new:
         create: Add domain
         title: New e-mail blacklist entry
@@ -589,7 +592,6 @@ en:
     warning_presets:
       add_new: Add new
       delete: Delete
-      edit: Edit
       edit_preset: Edit warning preset
       title: Manage warning presets
   admin_mailer:
@@ -867,6 +869,7 @@ en:
   media_attachments:
     validations:
       images_and_video: Cannot attach a video to a status that already contains images
+      not_ready: Cannot attach files that have not finished processing. Try again in a moment!
       too_many: Cannot attach more than 4 files
   migrations:
     acct: Moved to

config/locales/en_GB.yml

@@ -474,7 +474,6 @@ en_GB:
     warning_presets:
       add_new: Add new
       delete: Delete
-      edit: Edit
       edit_preset: Edit warning preset
       title: Manage warning presets
   admin_mailer:

config/locales/eo.yml

@@ -557,7 +557,6 @@ eo:
     warning_presets:
       add_new: Aldoni novan
       delete: Forigi
-      edit: Redakti
       edit_preset: Redakti avertan antaŭagordon
       title: Administri avertajn antaŭagordojn
   admin_mailer:

config/locales/es-AR.yml

@@ -573,7 +573,6 @@ es-AR:
     warning_presets:
       add_new: Agregar nuevo
       delete: Eliminar
-      edit: Editar
       edit_preset: Editar preajuste de advertencia
       title: Administrar preajustes de advertencia
   admin_mailer:

config/locales/es.yml

@@ -573,7 +573,6 @@ es:
     warning_presets:
       add_new: Añadir nuevo
       delete: Borrar
-      edit: Editar
       edit_preset: Editar aviso predeterminado
       title: Editar configuración predeterminada de avisos
   admin_mailer:

config/locales/et.yml

@@ -576,7 +576,6 @@ et:
     warning_presets:
       add_new: Lisa uus
       delete: Kustuta
-      edit: Redigeeri
       edit_preset: Redigeeri hoiatuse eelseadistust
       title: Halda hoiatuste eelseadistusi
   admin_mailer:

config/locales/eu.yml

@@ -573,7 +573,6 @@ eu:
     warning_presets:
       add_new: Gehitu berria
       delete: Ezabatu
-      edit: Editatu
       edit_preset: Editatu abisu aurre-ezarpena
       title: Kudeatu abisu aurre-ezarpenak
   admin_mailer:

config/locales/fa.yml

@@ -575,7 +575,6 @@ fa:
     warning_presets:
       add_new: افزودن تازه
       delete: زدودن
-      edit: ویرایش
       edit_preset: ویرایش هشدار پیش‌فرض
       title: مدیریت هشدارهای پیش‌فرض
   admin_mailer:

config/locales/fr.yml

@@ -573,7 +573,6 @@ fr:
     warning_presets:
       add_new: Ajouter un nouveau
       delete: Effacer
-      edit: Éditer
       edit_preset: Éditer les avertissements prédéfinis
       title: Gérer les avertissements prédéfinis
   admin_mailer:

config/locales/gl.yml

@@ -573,7 +573,6 @@ gl:
     warning_presets:
       add_new: Engadir novo
       delete: Eliminar
-      edit: Editar
       edit_preset: Editar aviso preestablecido
       title: Xestionar avisos preestablecidos
   admin_mailer:

config/locales/hu.yml

@@ -575,7 +575,6 @@ hu:
     warning_presets:
       add_new: Új hozzáadása
       delete: Törlés
-      edit: Szerkesztés
       edit_preset: Figyelmeztetés szerkesztése
       title: Figyelmeztetések
   admin_mailer:

config/locales/id.yml

@@ -565,7 +565,6 @@ id:
     warning_presets:
       add_new: Tambah baru
       delete: Hapus
-      edit: Sunting
       edit_preset: Sunting preset peringatan
       title: Kelola preset peringatan
   admin_mailer:

config/locales/is.yml

@@ -573,7 +573,6 @@ is:
     warning_presets:
       add_new: Bæta við nýju
       delete: Eyða
-      edit: Breyta
       edit_preset: Breyta forstilltri aðvörun
       title: Sýsla með forstilltar aðvaranir
   admin_mailer:

config/locales/it.yml

@@ -573,7 +573,6 @@ it:
     warning_presets:
       add_new: Aggiungi nuovo
       delete: Cancella
-      edit: Modifica
       edit_preset: Modifica avviso predefinito
       title: Gestisci avvisi predefiniti
   admin_mailer:

config/locales/ja.yml

@@ -565,7 +565,6 @@ ja:
     warning_presets:
       add_new: 追加
       delete: 削除
-      edit: 編集
       edit_preset: プリセット警告文を編集
       title: プリセット警告文を管理
   admin_mailer:

config/locales/kab.yml

@@ -330,7 +330,6 @@ kab:
     warning_presets:
       add_new: Rnu amaynut
       delete: Kkes
-      edit: Ẓreg
   admin_mailer:
     new_report:
       subject: Aneqqis amaynut i %{instance} (#%{id})

config/locales/kk.yml

@@ -555,7 +555,6 @@ kk:
     warning_presets:
       add_new: Add nеw
       delete: Deletе
-      edit: Еdit
       edit_preset: Edit warning prеset
       title: Manage warning presеts
   admin_mailer:

config/locales/ko.yml

@@ -567,7 +567,6 @@ ko:
     warning_presets:
       add_new: 새로 추가
       delete: 삭제
-      edit: 편집
       edit_preset: 경고 틀 수정
       title: 경고 틀 관리
   admin_mailer:

config/locales/lt.yml

@@ -410,7 +410,6 @@ lt:
     warning_presets:
       add_new: Pridėti naują
       delete: Ištrinti
-      edit: Keisti
       edit_preset: Keisti įspėjimo nustatymus
       title: Valdyti įspėjimo nustatymus
   admin_mailer:

config/locales/nl.yml

@@ -573,7 +573,6 @@ nl:
     warning_presets:
       add_new: Nieuwe toevoegen
       delete: Verwijderen
-      edit: Bewerken
       edit_preset: Voorinstelling van waarschuwing bewerken
       title: Voorinstellingen van waarschuwingen beheren
   admin_mailer:

config/locales/nn.yml

@@ -566,7 +566,6 @@ nn:
     warning_presets:
       add_new: Legg til ny
       delete: Slett
-      edit: Rediger
       edit_preset: Endr åtvaringsoppsett
       title: Handsam åtvaringsoppsett
   admin_mailer:

config/locales/no.yml

@@ -561,7 +561,6 @@
     warning_presets:
       add_new: Legg til ny
       delete: Slett
-      edit: Rediger
   admin_mailer:
     new_pending_account:
       body: Detaljer om den nye kontoen er nedenfor. Du kan godkjenne eller avvise denne søknaden.

config/locales/oc.yml

@@ -558,7 +558,6 @@ oc:
     warning_presets:
       add_new: N’ajustar un nòu
       delete: Escafar
-      edit: Modificar
       edit_preset: Modificar lo tèxt predefinit d’avertiment
       title: Gerir los tèxtes predefinits
   admin_mailer:

config/locales/pl.yml

@@ -541,7 +541,6 @@ pl:
     warning_presets:
       add_new: Dodaj nowy
       delete: Usuń
-      edit: Edytuj
       edit_preset: Edytuj szablon ostrzeżenia
       title: Zarządzaj szablonami ostrzeżeń
   admin_mailer:

config/locales/pt-BR.yml

@@ -573,7 +573,6 @@ pt-BR:
     warning_presets:
       add_new: Adicionar novo
       delete: Excluir
-      edit: Editar
       edit_preset: Editar o aviso pré-definido
       title: Gerenciar os avisos pré-definidos
   admin_mailer:

config/locales/pt-PT.yml

@@ -573,7 +573,6 @@ pt-PT:
     warning_presets:
       add_new: Adicionar novo
       delete: Apagar
-      edit: Editar
       edit_preset: Editar o aviso predefinido
       title: Gerir os avisos predefinidos
   admin_mailer:

config/locales/ru.yml

@@ -594,7 +594,6 @@ ru:
     warning_presets:
       add_new: Добавить
       delete: Удалить
-      edit: Изменить
       edit_preset: Удалить шаблон предупреждения
       title: Управление шаблонами предупреждений
   admin_mailer:

config/locales/simple_form.en.yml

@@ -8,6 +8,7 @@ en:
         acct: Specify the username@domain of the account you want to move to
       account_warning_preset:
         text: You can use toot syntax, such as URLs, hashtags and mentions
+        title: Optional. Not visible to the recipient
       admin_account_action:
         include_statuses: The user will see which toots have caused the moderation action or warning
         send_email_notification: The user will receive an explanation of what happened with their account
@@ -58,6 +59,9 @@ en:
         whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word
       domain_allow:
         domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored
+      email_domain_block:
+        domain: This can be the domain name that shows up in the e-mail address, the MX record that domain resolves to, or IP of the server that MX record resolves to. Those will be checked upon user sign-up and the sign-up will be rejected.
+        with_dns_records: An attempt to resolve the given domain's DNS records will be made and the results will also be blacklisted
       featured_tag:
         name: 'You might want to use one of these:'
       form_challenge:
@@ -83,6 +87,7 @@ en:
         acct: Handle of the new account
       account_warning_preset:
         text: Preset text
+        title: Title
       admin_account_action:
         include_statuses: Include reported toots in the e-mail
         send_email_notification: Notify the user per e-mail
@@ -163,6 +168,8 @@ en:
         username: Username
         username_or_email: Username or Email
         whole_word: Whole word
+      email_domain_block:
+        with_dns_records: Include MX records and IPs of the domain
       featured_tag:
         name: Hashtag
       interactions:

config/locales/sk.yml

@@ -574,7 +574,6 @@ sk:
     warning_presets:
       add_new: Pridaj nové
       delete: Vymaž
-      edit: Uprav
       edit_preset: Uprav varovnú predlohu
       title: Spravuj varovné predlohy
   admin_mailer:

config/locales/sl.yml

@@ -485,7 +485,6 @@ sl:
     warning_presets:
       add_new: Dodaj novo
       delete: Izbriši
-      edit: Uredi
       edit_preset: Uredi prednastavitev opozoril
       title: Upravljaj prednastavitev opozoril
   admin_mailer:

config/locales/sq.yml

@@ -415,7 +415,6 @@ sq:
     warning_presets:
       add_new: Shtoni të ri
       delete: Fshije
-      edit: Përpunoni
       edit_preset: Përpunoni sinjalizim të paracaktuar
       title: Administroni sinjalizime të paracaktuara
   admin_mailer:

config/locales/sr.yml

@@ -431,7 +431,6 @@ sr:
     warning_presets:
       add_new: Додај нови
       delete: Избриши
-      edit: Уреди
       edit_preset: Уреди пресет упозорења
       title: Управљај пресетима упозорења
   admin_mailer:

config/locales/sv.yml

@@ -448,7 +448,6 @@ sv:
     warning_presets:
       add_new: Lägg till ny
       delete: Radera
-      edit: Redigera
   admin_mailer:
     new_report:
       body: "%{reporter} har rapporterat %{target}"