diff --git a/autoapi/_mapper.py b/autoapi/_mapper.py
index e798f142..4e1c0935 100644
--- a/autoapi/_mapper.py
+++ b/autoapi/_mapper.py
@@ -394,7 +394,9 @@ def _need_to_load(self, files):
         if self.app.env.config_status != sphinx.environment.CONFIG_OK:
             return True
 
-        return last_files != files or not last_mtime or last_mtime < this_mtime
+        return (
+            set(last_files) != set(files) or not last_mtime or last_mtime < this_mtime
+        )
 
     def _find_files(self, patterns, dirs, ignore):
         for dir_ in dirs:
diff --git a/docs/changes/502.misc.rst b/docs/changes/502.misc.rst
new file mode 100644
index 00000000..6a2459f3
--- /dev/null
+++ b/docs/changes/502.misc.rst
@@ -0,0 +1 @@
+Make _need_to_load not assume that file ordering remained the same