diff --git a/src/audio_core/hle/hle.cpp b/src/audio_core/hle/hle.cpp
index db50c0bff6..4fa4145e28 100644
--- a/src/audio_core/hle/hle.cpp
+++ b/src/audio_core/hle/hle.cpp
@@ -24,7 +24,7 @@ static constexpr u64 audio_frame_ticks = 1310252ull; ///< Units: ARM11 cycles
 
 struct DspHle::Impl final {
 public:
-    explicit Impl(DspHle& parent);
+    explicit Impl(DspHle& parent, Memory::MemorySystem& memory);
     ~Impl();
 
     DspState GetDspState() const;
@@ -69,9 +69,13 @@ private:
     std::weak_ptr<DSP_DSP> dsp_dsp;
 };
 
-DspHle::Impl::Impl(DspHle& parent_) : parent(parent_) {
+DspHle::Impl::Impl(DspHle& parent_, Memory::MemorySystem& memory) : parent(parent_) {
     dsp_memory.raw_memory.fill(0);
 
+    for (auto& source : sources) {
+        source.SetMemory(memory);
+    }
+
     Core::Timing& timing = Core::System::GetInstance().CoreTiming();
     tick_event =
         timing.RegisterEvent("AudioCore::DspHle::tick_event", [this](u64, s64 cycles_late) {
@@ -335,7 +339,7 @@ void DspHle::Impl::AudioTickCallback(s64 cycles_late) {
     timing.ScheduleEvent(audio_frame_ticks - cycles_late, tick_event);
 }
 
-DspHle::DspHle() : impl(std::make_unique<Impl>(*this)) {}
+DspHle::DspHle(Memory::MemorySystem& memory) : impl(std::make_unique<Impl>(*this, memory)) {}
 DspHle::~DspHle() = default;
 
 DspState DspHle::GetDspState() const {
diff --git a/src/audio_core/hle/hle.h b/src/audio_core/hle/hle.h
index a957ee2f46..70abddbd59 100644
--- a/src/audio_core/hle/hle.h
+++ b/src/audio_core/hle/hle.h
@@ -13,11 +13,15 @@
 #include "core/hle/service/dsp/dsp_dsp.h"
 #include "core/memory.h"
 
+namespace Memory {
+class MemorySystem;
+}
+
 namespace AudioCore {
 
 class DspHle final : public DspInterface {
 public:
-    DspHle();
+    explicit DspHle(Memory::MemorySystem& memory);
     ~DspHle();
 
     DspState GetDspState() const override;
diff --git a/src/audio_core/hle/source.cpp b/src/audio_core/hle/source.cpp
index cbf99ec9da..8d42b0b52b 100644
--- a/src/audio_core/hle/source.cpp
+++ b/src/audio_core/hle/source.cpp
@@ -45,6 +45,10 @@ void Source::Reset() {
     state = {};
 }
 
+void Source::SetMemory(Memory::MemorySystem& memory) {
+    memory_system = &memory;
+}
+
 void Source::ParseConfig(SourceConfiguration::Configuration& config,
                          const s16_le (&adpcm_coeffs)[16]) {
     if (!config.dirty_raw) {
diff --git a/src/audio_core/hle/source.h b/src/audio_core/hle/source.h
index 2e24c5eab9..5f9f5b924d 100644
--- a/src/audio_core/hle/source.h
+++ b/src/audio_core/hle/source.h
@@ -14,6 +14,10 @@
 #include "audio_core/interpolate.h"
 #include "common/common_types.h"
 
+namespace Memory {
+class MemorySystem;
+}
+
 namespace AudioCore {
 namespace HLE {
 
@@ -35,6 +39,9 @@ public:
     /// Resets internal state.
     void Reset();
 
+    /// Sets the memory system to read data from
+    void SetMemory(Memory::MemorySystem& memory);
+
     /**
      * This is called once every audio frame. This performs per-source processing every frame.
      * @param config The new configuration we've got for this Source from the application.
@@ -56,6 +63,7 @@ public:
 
 private:
     const std::size_t source_id;
+    Memory::MemorySystem* memory_system;
     StereoFrame16 current_frame;
 
     using Format = SourceConfiguration::Configuration::Format;