diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb
index d5960c3ad..ba086449c 100644
--- a/app/services/notify_service.rb
+++ b/app/services/notify_service.rb
@@ -54,7 +54,7 @@ class NotifyService < BaseService
   end
 
   def response_to_recipient?
-    @notification.target_status.in_reply_to_account_id == @recipient.id
+    @notification.target_status.in_reply_to_account_id == @recipient.id && @notification.target_status.thread&.direct_visibility?
   end
 
   def optional_non_following_and_direct?
diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb
index bb7601e76..1435ec917 100644
--- a/spec/services/notify_service_spec.rb
+++ b/spec/services/notify_service_spec.rb
@@ -82,10 +82,19 @@ RSpec.describe NotifyService do
         is_expected.to_not change(Notification, :count)
       end
 
-      context 'if the message chain initiated by recipient' do
+      context 'if the message chain initiated by recipient, but is not direct message' do
         let(:reply_to) { Fabricate(:status, account: recipient) }
         let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) }
 
+        it 'does not notify' do
+          is_expected.to_not change(Notification, :count)
+        end
+      end
+
+      context 'if the message chain initiated by recipient and is direct message' do
+        let(:reply_to) { Fabricate(:status, account: recipient, visibility: :direct) }
+        let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) }
+
         it 'does notify' do
           is_expected.to change(Notification, :count)
         end