Change delivery failure tracking to work with hostnames instead of URLs (#13437)

This commit is contained in:
Eugen Rochko 2020-04-15 20:33:24 +02:00 committed by GitHub
parent 5524258da9
commit 5edff32733
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 103 additions and 44 deletions

View file

@ -49,7 +49,7 @@ class ActivityPub::InboxesController < ActivityPub::BaseController
ResolveAccountWorker.perform_async(signed_request_account.acct) ResolveAccountWorker.perform_async(signed_request_account.acct)
end end
DeliveryFailureTracker.track_inverse_success!(signed_request_account) DeliveryFailureTracker.reset!(signed_request_account.inbox_url)
end end
def process_payload def process_payload

View file

@ -19,7 +19,7 @@ module Admin
@followers_count = Follow.where(target_account: Account.where(domain: params[:id])).count @followers_count = Follow.where(target_account: Account.where(domain: params[:id])).count
@reports_count = Report.where(target_account: Account.where(domain: params[:id])).count @reports_count = Report.where(target_account: Account.where(domain: params[:id])).count
@blocks_count = Block.where(target_account: Account.where(domain: params[:id])).count @blocks_count = Block.where(target_account: Account.where(domain: params[:id])).count
@available = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url) @available = DeliveryFailureTracker.available?(params[:id])
@media_storage = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size) @media_storage = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
@private_comment = @domain_block&.private_comment @private_comment = @domain_block&.private_comment
@public_comment = @domain_block&.public_comment @public_comment = @domain_block&.public_comment

View file

@ -3,47 +3,53 @@
class DeliveryFailureTracker class DeliveryFailureTracker
FAILURE_DAYS_THRESHOLD = 7 FAILURE_DAYS_THRESHOLD = 7
def initialize(inbox_url) def initialize(url_or_host)
@inbox_url = inbox_url @host = url_or_host.start_with?('https://') || url_or_host.start_with?('http://') ? Addressable::URI.parse(url_or_host).normalized_host : url_or_host
end end
def track_failure! def track_failure!
Redis.current.sadd(exhausted_deliveries_key, today) Redis.current.sadd(exhausted_deliveries_key, today)
Redis.current.sadd('unavailable_inboxes', @inbox_url) if reached_failure_threshold? UnavailableDomain.create(domain: @host) if reached_failure_threshold?
end end
def track_success! def track_success!
Redis.current.del(exhausted_deliveries_key) Redis.current.del(exhausted_deliveries_key)
Redis.current.srem('unavailable_inboxes', @inbox_url) UnavailableDomain.find_by(domain: @host)&.destroy
end end
def days def days
Redis.current.scard(exhausted_deliveries_key) || 0 Redis.current.scard(exhausted_deliveries_key) || 0
end end
class << self def available?
def filter(arr) !UnavailableDomain.where(domain: @host).exists?
arr.reject(&method(:unavailable?)) end
end
def unavailable?(url) alias reset! track_success!
Redis.current.sismember('unavailable_inboxes', url)
class << self
def without_unavailable(urls)
unavailable_domains_map = Rails.cache.fetch('unavailable_domains') { UnavailableDomain.pluck(:domain).each_with_object({}) { |domain, hash| hash[domain] = true } }
urls.reject do |url|
host = Addressable::URI.parse(url).normalized_host
unavailable_domains_map[host]
end
end end
def available?(url) def available?(url)
!unavailable?(url) new(url).available?
end end
def track_inverse_success!(from_account) def reset!(url)
new(from_account.inbox_url).track_success! if from_account.inbox_url.present? new(url).reset!
new(from_account.shared_inbox_url).track_success! if from_account.shared_inbox_url.present?
end end
end end
private private
def exhausted_deliveries_key def exhausted_deliveries_key
"exhausted_deliveries:#{@inbox_url}" "exhausted_deliveries:#{@host}"
end end
def today def today

View file

@ -413,7 +413,7 @@ class Account < ApplicationRecord
def inboxes def inboxes
urls = reorder(nil).where(protocol: :activitypub).pluck(Arel.sql("distinct coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url)")) urls = reorder(nil).where(protocol: :activitypub).pluck(Arel.sql("distinct coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url)"))
DeliveryFailureTracker.filter(urls) DeliveryFailureTracker.without_unavailable(urls)
end end
def search_for(terms, limit = 10, offset = 0) def search_for(terms, limit = 10, offset = 0)

View file

@ -14,7 +14,7 @@
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# published_at :datetime # published_at :datetime
# status_ids :bigint is an Array # status_ids :bigint(8) is an Array
# #
class Announcement < ApplicationRecord class Announcement < ApplicationRecord

View file

@ -27,7 +27,7 @@ class Relay < ApplicationRecord
payload = Oj.dump(follow_activity(activity_id)) payload = Oj.dump(follow_activity(activity_id))
update!(state: :pending, follow_activity_id: activity_id) update!(state: :pending, follow_activity_id: activity_id)
DeliveryFailureTracker.new(inbox_url).track_success! DeliveryFailureTracker.track_success!(inbox_url)
ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url) ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url)
end end
@ -36,7 +36,7 @@ class Relay < ApplicationRecord
payload = Oj.dump(unfollow_activity(activity_id)) payload = Oj.dump(unfollow_activity(activity_id))
update!(state: :idle, follow_activity_id: nil) update!(state: :idle, follow_activity_id: nil)
DeliveryFailureTracker.new(inbox_url).track_success! DeliveryFailureTracker.track_success!(inbox_url)
ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url) ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url)
end end

View file

@ -0,0 +1,22 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: unavailable_domains
#
# id :bigint(8) not null, primary key
# domain :string default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
#
class UnavailableDomain < ApplicationRecord
include DomainNormalizable
after_commit :reset_cache!
private
def reset_cache!
Rails.cache.delete('unavailable_domains')
end
end

View file

@ -171,12 +171,12 @@
%th= t('admin.accounts.inbox_url') %th= t('admin.accounts.inbox_url')
%td %td
= @account.inbox_url = @account.inbox_url
= fa_icon DeliveryFailureTracker.unavailable?(@account.inbox_url) ? 'times' : 'check' = fa_icon DeliveryFailureTracker.available?(@account.inbox_url) ? 'check' : 'times'
%tr %tr
%th= t('admin.accounts.shared_inbox_url') %th= t('admin.accounts.shared_inbox_url')
%td %td
= @account.shared_inbox_url = @account.shared_inbox_url
= fa_icon DeliveryFailureTracker.unavailable?(@account.shared_inbox_url) ? 'times' : 'check' = fa_icon DeliveryFailureTracker.available?(@account.shared_inbox_url) ? 'check': 'times'
%div{ style: 'overflow: hidden' } %div{ style: 'overflow: hidden' }
%div{ style: 'float: right' } %div{ style: 'float: right' }

View file

@ -12,7 +12,7 @@ class ActivityPub::DeliveryWorker
HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
def perform(json, source_account_id, inbox_url, options = {}) def perform(json, source_account_id, inbox_url, options = {})
return if DeliveryFailureTracker.unavailable?(inbox_url) return unless DeliveryFailureTracker.available?(inbox_url)
@options = options.with_indifferent_access @options = options.with_indifferent_access
@json = json @json = json

View file

@ -0,0 +1,9 @@
class CreateUnavailableDomains < ActiveRecord::Migration[5.2]
def change
create_table :unavailable_domains do |t|
t.string :domain, default: '', null: false, index: { unique: true }
t.timestamps
end
end
end

View file

@ -0,0 +1,16 @@
class MigrateUnavailableInboxes < ActiveRecord::Migration[5.2]
disable_ddl_transaction!
def up
urls = Redis.current.smembers('unavailable_inboxes')
urls.each do |url|
host = Addressable::URI.parse(url).normalized_host
UnavailableDomain.create(domain: host)
end
Redis.current.del(*(['unavailable_inboxes'] + Redis.current.keys('exhausted_deliveries:*')))
end
def down; end
end

View file

@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_03_12_185443) do ActiveRecord::Schema.define(version: 2020_04_07_202420) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
@ -769,6 +769,13 @@ ActiveRecord::Schema.define(version: 2020_03_12_185443) do
t.index ["uri"], name: "index_tombstones_on_uri" t.index ["uri"], name: "index_tombstones_on_uri"
end end
create_table "unavailable_domains", force: :cascade do |t|
t.string "domain", default: "", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["domain"], name: "index_unavailable_domains_on_domain", unique: true
end
create_table "user_invite_requests", force: :cascade do |t| create_table "user_invite_requests", force: :cascade do |t|
t.bigint "user_id" t.bigint "user_id"
t.text "text" t.text "text"

View file

@ -0,0 +1,3 @@
Fabricator(:unavailable_domain) do
domain { Faker::Internet.domain }
end

View file

@ -22,11 +22,11 @@ describe DeliveryFailureTracker do
describe '#track_failure!' do describe '#track_failure!' do
it 'marks URL as unavailable after 7 days of being called' do it 'marks URL as unavailable after 7 days of being called' do
6.times { |i| Redis.current.sadd('exhausted_deliveries:http://example.com/inbox', i) } 6.times { |i| Redis.current.sadd('exhausted_deliveries:example.com', i) }
subject.track_failure! subject.track_failure!
expect(subject.days).to eq 7 expect(subject.days).to eq 7
expect(described_class.unavailable?('http://example.com/inbox')).to be true expect(described_class.available?('http://example.com/inbox')).to be false
end end
it 'repeated calls on the same day do not count' do it 'repeated calls on the same day do not count' do
@ -37,35 +37,27 @@ describe DeliveryFailureTracker do
end end
end end
describe '.filter' do describe '.without_unavailable' do
before do before do
Redis.current.sadd('unavailable_inboxes', 'http://example.com/unavailable/inbox') Fabricate(:unavailable_domain, domain: 'foo.bar')
end end
it 'removes URLs that are unavailable' do it 'removes URLs that are unavailable' do
result = described_class.filter(['http://example.com/good/inbox', 'http://example.com/unavailable/inbox']) results = described_class.without_unavailable(['http://example.com/good/inbox', 'http://foo.bar/unavailable/inbox'])
expect(result).to include('http://example.com/good/inbox') expect(results).to include('http://example.com/good/inbox')
expect(result).to_not include('http://example.com/unavailable/inbox') expect(results).to_not include('http://foo.bar/unavailable/inbox')
end end
end end
describe '.track_inverse_success!' do describe '.reset!' do
let(:from_account) { Fabricate(:account, inbox_url: 'http://example.com/inbox', shared_inbox_url: 'http://example.com/shared/inbox') }
before do before do
Redis.current.sadd('unavailable_inboxes', 'http://example.com/inbox') Fabricate(:unavailable_domain, domain: 'foo.bar')
Redis.current.sadd('unavailable_inboxes', 'http://example.com/shared/inbox') described_class.reset!('https://foo.bar/inbox')
described_class.track_inverse_success!(from_account)
end end
it 'marks inbox URL as available again' do it 'marks inbox URL as available again' do
expect(described_class.available?('http://example.com/inbox')).to be true expect(described_class.available?('http://foo.bar/inbox')).to be true
end
it 'marks shared inbox URL as available again' do
expect(described_class.available?('http://example.com/shared/inbox')).to be true
end end
end end
end end

View file

@ -0,0 +1,4 @@
require 'rails_helper'
RSpec.describe UnavailableDomain, type: :model do
end