diff --git a/src/input_common/keyboard.cpp b/src/input_common/keyboard.cpp
index 078374be54..98d4f30a2f 100644
--- a/src/input_common/keyboard.cpp
+++ b/src/input_common/keyboard.cpp
@@ -2,6 +2,7 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include <algorithm>
 #include <atomic>
 #include <list>
 #include <mutex>
@@ -12,10 +13,9 @@ namespace InputCommon {
 
 class KeyButton final : public Input::ButtonDevice {
 public:
-    explicit KeyButton(std::shared_ptr<KeyButtonList> key_button_list_)
-        : key_button_list(std::move(key_button_list_)) {}
+    explicit KeyButton(std::atomic<bool>& status_) : status(status_) {}
 
-    ~KeyButton() override;
+    ~KeyButton() override = default;
 
     bool GetStatus() const override {
         return status.load();
@@ -24,40 +24,40 @@ public:
     friend class KeyButtonList;
 
 private:
-    std::shared_ptr<KeyButtonList> key_button_list;
-    std::atomic<bool> status{false};
+    std::atomic<bool>& status;
 };
 
 struct KeyButtonPair {
+    explicit KeyButtonPair(int key_code_) : key_code(key_code_) {}
     int key_code;
-    KeyButton* key_button;
+    std::atomic<bool> status{false};
 };
 
 class KeyButtonList {
 public:
-    void AddKeyButton(int key_code, KeyButton* key_button) {
+    KeyButtonPair& AddKeyButton(int key_code) {
         std::lock_guard guard{mutex};
-        list.push_back(KeyButtonPair{key_code, key_button});
-    }
-
-    void RemoveKeyButton(const KeyButton* key_button) {
-        std::lock_guard guard{mutex};
-        list.remove_if(
-            [key_button](const KeyButtonPair& pair) { return pair.key_button == key_button; });
+        auto it = std::find_if(list.begin(), list.end(), [key_code](const KeyButtonPair& pair) {
+            return pair.key_code == key_code;
+        });
+        if (it == list.end()) {
+            return list.emplace_back(key_code);
+        }
+        return *it;
     }
 
     void ChangeKeyStatus(int key_code, bool pressed) {
         std::lock_guard guard{mutex};
-        for (const KeyButtonPair& pair : list) {
+        for (KeyButtonPair& pair : list) {
             if (pair.key_code == key_code)
-                pair.key_button->status.store(pressed);
+                pair.status.store(pressed);
         }
     }
 
     void ChangeAllKeyStatus(bool pressed) {
         std::lock_guard guard{mutex};
-        for (const KeyButtonPair& pair : list) {
-            pair.key_button->status.store(pressed);
+        for (KeyButtonPair& pair : list) {
+            pair.status.store(pressed);
         }
     }
 
@@ -68,15 +68,10 @@ private:
 
 Keyboard::Keyboard() : key_button_list{std::make_shared<KeyButtonList>()} {}
 
-KeyButton::~KeyButton() {
-    key_button_list->RemoveKeyButton(this);
-}
-
 std::unique_ptr<Input::ButtonDevice> Keyboard::Create(const Common::ParamPackage& params) {
     int key_code = params.Get("code", 0);
-    std::unique_ptr<KeyButton> button = std::make_unique<KeyButton>(key_button_list);
-    key_button_list->AddKeyButton(key_code, button.get());
-    return std::move(button);
+    auto& pair = key_button_list->AddKeyButton(key_code);
+    return std::make_unique<KeyButton>(pair.status);
 }
 
 void Keyboard::PressKey(int key_code) {