diff --git a/src/buildstream/_elementsources.py b/src/buildstream/_elementsources.py
index 39241493a..9ed26499d 100644
--- a/src/buildstream/_elementsources.py
+++ b/src/buildstream/_elementsources.py
@@ -47,6 +47,8 @@ def __init__(self, context: Context, project: "Project", plugin: Plugin):
         self._cache_key = None  # Our cached cache key
         self._proto = None  # The cached Source proto
 
+        self._cache = {}
+
     # get_project():
     #
     # Return the project associated with this object
@@ -462,6 +464,10 @@ def _stage(self, *, stop=None):
             if source == stop:
                 break
 
+            if source in self._cache:
+                vdir = CasBasedDirectory(cas, digest=self._cache[source])
+                continue
+
             if source._directory:
                 vsubdir = vdir.open_directory(source._directory.lstrip(os.path.sep), create=True)
             else:
@@ -483,6 +489,8 @@ def _stage(self, *, stop=None):
                 source_dir = self._sourcecache.export(source)
                 vsubdir.import_files(source_dir, collect_result=False)
 
+            self._cache[source] = vsubdir._get_digest()
+
         return vdir
 
     # Context manager that stages sources in a cas based or temporary file