diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index d58c6242ee..185d9cce2a 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -79,37 +79,22 @@ void UnmapRegion(PageTable& page_table, VAddr base, u32 size) {
 }
 
 /**
- * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned)
- * using a VMA from the current process
+ * Gets the pointer for virtual memory where the page is marked as RasterizerCachedMemory.
+ * This is used to access the memory where the page pointer is nullptr due to rasterizer cache.
+ * Since the cache only happens on linear heap or VRAM, we know the exact physical address and
+ * pointer of such virtual address
  */
-static u8* GetPointerFromVMA(const Kernel::Process& process, VAddr vaddr) {
-    u8* direct_pointer = nullptr;
-
-    auto& vm_manager = process.vm_manager;
-
-    auto it = vm_manager.FindVMA(vaddr);
-    ASSERT(it != vm_manager.vma_map.end());
-
-    auto& vma = it->second;
-    switch (vma.type) {
-    case Kernel::VMAType::BackingMemory:
-        direct_pointer = vma.backing_memory;
-        break;
-    case Kernel::VMAType::Free:
-        return nullptr;
-    default:
-        UNREACHABLE();
+static u8* GetPointerForRasterizerCache(VAddr addr) {
+    if (addr >= LINEAR_HEAP_VADDR && addr < LINEAR_HEAP_VADDR_END) {
+        return fcram.data() + (addr - LINEAR_HEAP_VADDR);
     }
-
-    return direct_pointer + (vaddr - vma.base);
-}
-
-/**
- * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned)
- * using a VMA from the current process.
- */
-static u8* GetPointerFromVMA(VAddr vaddr) {
-    return GetPointerFromVMA(*Core::System::GetInstance().Kernel().GetCurrentProcess(), vaddr);
+    if (addr >= NEW_LINEAR_HEAP_VADDR && addr < NEW_LINEAR_HEAP_VADDR_END) {
+        return fcram.data() + (addr - NEW_LINEAR_HEAP_VADDR);
+    }
+    if (addr >= VRAM_VADDR && addr < VRAM_VADDR_END) {
+        return vram.data() + (addr - VRAM_VADDR);
+    }
+    UNREACHABLE();
 }
 
 /**
@@ -159,7 +144,7 @@ T Read(const VAddr vaddr) {
         RasterizerFlushVirtualRegion(vaddr, sizeof(T), FlushMode::Flush);
 
         T value;
-        std::memcpy(&value, GetPointerFromVMA(vaddr), sizeof(T));
+        std::memcpy(&value, GetPointerForRasterizerCache(vaddr), sizeof(T));
         return value;
     }
     case PageType::Special:
@@ -195,7 +180,7 @@ void Write(const VAddr vaddr, const T data) {
         break;
     case PageType::RasterizerCachedMemory: {
         RasterizerFlushVirtualRegion(vaddr, sizeof(T), FlushMode::Invalidate);
-        std::memcpy(GetPointerFromVMA(vaddr), &data, sizeof(T));
+        std::memcpy(GetPointerForRasterizerCache(vaddr), &data, sizeof(T));
         break;
     }
     case PageType::Special:
@@ -242,7 +227,7 @@ u8* GetPointer(const VAddr vaddr) {
     }
 
     if (current_page_table->attributes[vaddr >> PAGE_BITS] == PageType::RasterizerCachedMemory) {
-        return GetPointerFromVMA(vaddr);
+        return GetPointerForRasterizerCache(vaddr);
     }
 
     LOG_ERROR(HW_Memory, "unknown GetPointer @ 0x{:08x}", vaddr);
@@ -364,16 +349,9 @@ void RasterizerMarkRegionCached(PAddr start, u32 size, bool cached) {
                     // space, for example, a system module need not have a VRAM mapping.
                     break;
                 case PageType::RasterizerCachedMemory: {
-                    u8* pointer = GetPointerFromVMA(vaddr & ~PAGE_MASK);
-                    if (pointer == nullptr) {
-                        // It's possible that this function has been called while updating the
-                        // pagetable after unmapping a VMA. In that case the underlying VMA will no
-                        // longer exist, and we should just leave the pagetable entry blank.
-                        page_type = PageType::Unmapped;
-                    } else {
-                        page_type = PageType::Memory;
-                        current_page_table->pointers[vaddr >> PAGE_BITS] = pointer;
-                    }
+                    page_type = PageType::Memory;
+                    current_page_table->pointers[vaddr >> PAGE_BITS] =
+                        GetPointerForRasterizerCache(vaddr & ~PAGE_MASK);
                     break;
                 }
                 default:
@@ -501,7 +479,7 @@ void ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_
         case PageType::RasterizerCachedMemory: {
             RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
                                          FlushMode::Flush);
-            std::memcpy(dest_buffer, GetPointerFromVMA(process, current_vaddr), copy_amount);
+            std::memcpy(dest_buffer, GetPointerForRasterizerCache(current_vaddr), copy_amount);
             break;
         }
         default:
@@ -570,7 +548,7 @@ void WriteBlock(const Kernel::Process& process, const VAddr dest_addr, const voi
         case PageType::RasterizerCachedMemory: {
             RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
                                          FlushMode::Invalidate);
-            std::memcpy(GetPointerFromVMA(process, current_vaddr), src_buffer, copy_amount);
+            std::memcpy(GetPointerForRasterizerCache(current_vaddr), src_buffer, copy_amount);
             break;
         }
         default:
@@ -624,7 +602,7 @@ void ZeroBlock(const Kernel::Process& process, const VAddr dest_addr, const std:
         case PageType::RasterizerCachedMemory: {
             RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
                                          FlushMode::Invalidate);
-            std::memset(GetPointerFromVMA(process, current_vaddr), 0, copy_amount);
+            std::memset(GetPointerForRasterizerCache(current_vaddr), 0, copy_amount);
             break;
         }
         default:
@@ -677,7 +655,8 @@ void CopyBlock(const Kernel::Process& process, VAddr dest_addr, VAddr src_addr,
         case PageType::RasterizerCachedMemory: {
             RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
                                          FlushMode::Flush);
-            WriteBlock(process, dest_addr, GetPointerFromVMA(process, current_vaddr), copy_amount);
+            WriteBlock(process, dest_addr, GetPointerForRasterizerCache(current_vaddr),
+                       copy_amount);
             break;
         }
         default:
@@ -732,7 +711,7 @@ void CopyBlock(const Kernel::Process& src_process, const Kernel::Process& dest_p
         case PageType::RasterizerCachedMemory: {
             RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
                                          FlushMode::Flush);
-            WriteBlock(dest_process, dest_addr, GetPointerFromVMA(src_process, current_vaddr),
+            WriteBlock(dest_process, dest_addr, GetPointerForRasterizerCache(current_vaddr),
                        copy_amount);
             break;
         }