diff --git a/core/memory.cc b/core/memory.cc
index 3b1639ea49..9b6a16558d 100644
--- a/core/memory.cc
+++ b/core/memory.cc
@@ -837,9 +837,6 @@ void configure(std::vector<resource::memory> m,
         cpu_mem.replace_memory_backing(sys_alloc);
     }
     cpu_mem.resize(total, sys_alloc);
-    if (hugetlbfs_path) {
-        cpu_mem.init_virt_to_phys_map();
-    }
     size_t pos = 0;
     for (auto&& x : m) {
 #ifdef HAVE_NUMA
@@ -852,6 +849,9 @@ void configure(std::vector<resource::memory> m,
 #endif
         pos += x.bytes;
     }
+    if (hugetlbfs_path) {
+        cpu_mem.init_virt_to_phys_map();
+    }
 }
 
 statistics stats() {