diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
index 982031c9d..f9dd36e36 100644
--- a/.github/workflows/build-image.yml
+++ b/.github/workflows/build-image.yml
@@ -49,8 +49,10 @@ jobs:
           images: |
             tootsuite/mastodon
             ghcr.io/mastodon/mastodon
+          # Only tag with latest when ran against the latest stable branch
+          # This needs to be updated after each minor version release
           flavor: |
-            latest=auto
+            latest=${{ startsWith(github.ref, 'refs/tags/v4.1.') && 'auto' || 'false' }}
           tags: |
             type=edge,branch=main
             type=pep440,pattern={{raw}}