diff --git a/src/audio_core/hle/source.cpp b/src/audio_core/hle/source.cpp
index 54a28bee39..d3fbfd0cb7 100644
--- a/src/audio_core/hle/source.cpp
+++ b/src/audio_core/hle/source.cpp
@@ -298,9 +298,9 @@ void Source::ParseConfig(SourceConfiguration::Configuration& config,
                         b.buffer_id,
                         state.mono_or_stereo,
                         state.format,
-                        true,
-                        {}, // 0 in u32_dsp
-                        false,
+                        true,  // from_queue
+                        0,     // play_position
+                        false, // has_played
                     });
                 }
                 LOG_TRACE(Audio_DSP, "enqueuing queued {} addr={:#010x} len={} id={}", i,
@@ -321,7 +321,11 @@ void Source::ParseConfig(SourceConfiguration::Configuration& config,
 void Source::GenerateFrame() {
     current_frame.fill({});
 
-    if (state.current_buffer.empty() && !DequeueBuffer()) {
+    if (state.current_buffer.empty()) {
+        // TODO(SachinV): Should dequeue happen at the end of the frame generation?
+        if (DequeueBuffer()) {
+            return;
+        }
         state.enabled = false;
         state.buffer_update = true;
         state.last_buffer_id = state.current_buffer_id;
@@ -330,8 +334,6 @@ void Source::GenerateFrame() {
     }
 
     std::size_t frame_position = 0;
-
-    state.current_sample_number = state.next_sample_number;
     while (frame_position < current_frame.size()) {
         if (state.current_buffer.empty() && !DequeueBuffer()) {
             break;
@@ -358,7 +360,7 @@ void Source::GenerateFrame() {
     }
     // TODO(jroweboy): Keep track of frame_position independently so that it doesn't lose precision
     // over time
-    state.next_sample_number += static_cast<u32>(frame_position * state.rate_multiplier);
+    state.current_sample_number += static_cast<u32>(frame_position * state.rate_multiplier);
 
     state.filters.ProcessFrame(current_frame);
 }
@@ -409,7 +411,6 @@ bool Source::DequeueBuffer() {
 
     // the first playthrough starts at play_position, loops start at the beginning of the buffer
     state.current_sample_number = (!buf.has_played) ? buf.play_position : 0;
-    state.next_sample_number = state.current_sample_number;
     state.current_buffer_physical_address = buf.physical_address;
     state.current_buffer_id = buf.buffer_id;
     state.last_buffer_id = 0;
@@ -420,8 +421,17 @@ bool Source::DequeueBuffer() {
         state.input_queue.push(buf);
     }
 
-    LOG_TRACE(Audio_DSP, "source_id={} buffer_id={} from_queue={} current_buffer.size()={}",
-              source_id, buf.buffer_id, buf.from_queue, state.current_buffer.size());
+    // Because our interpolation consumes samples instead of using an index,
+    // let's just consume the samples up to the current sample number.
+    state.current_buffer.erase(
+        state.current_buffer.begin(),
+        std::next(state.current_buffer.begin(), state.current_sample_number));
+
+    LOG_TRACE(Audio_DSP,
+              "source_id={} buffer_id={} from_queue={} current_buffer.size()={}, "
+              "buf.has_played={}, buf.play_position={}",
+              source_id, buf.buffer_id, buf.from_queue, state.current_buffer.size(), buf.has_played,
+              buf.play_position);
     return true;
 }
 
diff --git a/src/audio_core/hle/source.h b/src/audio_core/hle/source.h
index a3af06d27d..d7114952b6 100644
--- a/src/audio_core/hle/source.h
+++ b/src/audio_core/hle/source.h
@@ -87,8 +87,8 @@ private:
         Format format;
 
         bool from_queue;
-        u32_dsp play_position; // = 0;
-        bool has_played;       // = false;
+        u32 play_position; // = 0;
+        bool has_played;   // = false;
 
     private:
         template <class Archive>
@@ -136,7 +136,6 @@ private:
         // Current buffer
 
         u32 current_sample_number = 0;
-        u32 next_sample_number = 0;
         PAddr current_buffer_physical_address = 0;
         AudioInterp::StereoBuffer16 current_buffer = {};
 
@@ -171,7 +170,6 @@ private:
             ar& mono_or_stereo;
             ar& format;
             ar& current_sample_number;
-            ar& next_sample_number;
             ar& current_buffer_physical_address;
             ar& current_buffer;
             ar& buffer_update;