diff --git a/src/audio_core/dsp_interface.cpp b/src/audio_core/dsp_interface.cpp
index 4948eb958c..170bb4e4aa 100644
--- a/src/audio_core/dsp_interface.cpp
+++ b/src/audio_core/dsp_interface.cpp
@@ -66,18 +66,22 @@ void DspInterface::OutputSample(std::array<s16, 2> sample) {
 }
 
 void DspInterface::OutputCallback(s16* buffer, std::size_t num_frames) {
-    std::size_t frames_written;
+    std::size_t frames_written = 0;
     if (perform_time_stretching) {
         const std::vector<s16> in{fifo.Pop()};
         const std::size_t num_in{in.size() / 2};
         frames_written = time_stretcher.Process(in.data(), num_in, buffer, num_frames);
-    } else if (flushing_time_stretcher) {
-        time_stretcher.Flush();
-        frames_written = time_stretcher.Process(nullptr, 0, buffer, num_frames);
-        frames_written += fifo.Pop(buffer, num_frames - frames_written);
-        flushing_time_stretcher = false;
     } else {
-        frames_written = fifo.Pop(buffer, num_frames);
+        if (flushing_time_stretcher) {
+            time_stretcher.Flush();
+            frames_written = time_stretcher.Process(nullptr, 0, buffer, num_frames);
+            flushing_time_stretcher = false;
+
+            // Make sure any frames that did not fit are cleared from the time stretcher,
+            // so that they do not bleed into the next time the stretcher is enabled.
+            time_stretcher.Clear();
+        }
+        frames_written += fifo.Pop(buffer, num_frames - frames_written);
     }
 
     if (frames_written > 0) {
diff --git a/src/audio_core/time_stretch.cpp b/src/audio_core/time_stretch.cpp
index 38003ac3fc..2cdccdd899 100644
--- a/src/audio_core/time_stretch.cpp
+++ b/src/audio_core/time_stretch.cpp
@@ -18,8 +18,7 @@
 
 namespace AudioCore {
 
-TimeStretcher::TimeStretcher()
-    : sample_rate(native_sample_rate), sound_touch(std::make_unique<soundtouch::SoundTouch>()) {
+TimeStretcher::TimeStretcher() : sound_touch(std::make_unique<soundtouch::SoundTouch>()) {
     sound_touch->setChannels(2);
     sound_touch->setSampleRate(native_sample_rate);
     sound_touch->setPitch(1.0);
@@ -30,16 +29,15 @@ TimeStretcher::~TimeStretcher() = default;
 
 void TimeStretcher::SetOutputSampleRate(unsigned int sample_rate) {
     sound_touch->setSampleRate(sample_rate);
-    sample_rate = native_sample_rate;
 }
 
 std::size_t TimeStretcher::Process(const s16* in, std::size_t num_in, s16* out,
                                    std::size_t num_out) {
-    const double time_delta = static_cast<double>(num_out) / sample_rate; // seconds
+    const double time_delta = static_cast<double>(num_out) / native_sample_rate; // seconds
     double current_ratio = static_cast<double>(num_in) / static_cast<double>(num_out);
 
     const double max_latency = 0.25; // seconds
-    const double max_backlog = sample_rate * max_latency;
+    const double max_backlog = native_sample_rate * max_latency;
     const double backlog_fullness = sound_touch->numSamples() / max_backlog;
     if (backlog_fullness > 4.0) {
         // Too many samples in backlog: Don't push anymore on
diff --git a/src/audio_core/time_stretch.h b/src/audio_core/time_stretch.h
index 85495fa270..f7a582784a 100644
--- a/src/audio_core/time_stretch.h
+++ b/src/audio_core/time_stretch.h
@@ -34,7 +34,6 @@ public:
     void Flush();
 
 private:
-    unsigned int sample_rate;
     std::unique_ptr<soundtouch::SoundTouch> sound_touch;
     double stretch_ratio = 1.0;
 };